@uniweb/build 0.3.2 → 0.4.1

package/src/i18n/freeform.js

@@ -0,0 +1,336 @@
+/**
+ * Free-form Translation Support
+ *
+ * Enables complete content replacement for locales using markdown files,
+ * as an alternative to hash-based string merging. Free-form translations
+ * allow translators to completely reword sections rather than translating
+ * element-by-element.
+ *
+ * Directory structure:
+ *   locales/freeform/{locale}/
+ *     pages/{pageRoute}/{stableId}.md      - By route
+ *     page-ids/{pageId}/{stableId}.md      - By page ID (stable)
+ *     collections/{collectionName}/{slug}.md - Collection items
+ *
+ * Resolution order for sections:
+ *   1. page-ids/{pageId}/{stableId}.md (if page has id:)
+ *   2. pages/{pageRoute}/{stableId}.md
+ *   3. Return null (fall back to granular translation)
+ */
+
+import { readFile, readdir, stat } from 'fs/promises'
+import { existsSync } from 'fs'
+import { join, relative, dirname } from 'path'
+import yaml from 'js-yaml'
+
+// Try to import content-reader for markdown → ProseMirror conversion
+let markdownToProseMirror
+try {
+  const contentReader = await import('@uniweb/content-reader')
+  markdownToProseMirror = contentReader.markdownToProseMirror
+} catch {
+  // Simplified fallback - just wraps content as text
+  markdownToProseMirror = (markdown) => ({
+    type: 'doc',
+    content: [
+      {
+        type: 'paragraph',
+        content: [{ type: 'text', text: markdown.trim() }]
+      }
+    ]
+  })
+}
+
+/**
+ * Parse YAML frontmatter from markdown content
+ * @param {string} content - Raw markdown content
+ * @returns {{ frontmatter: Object, body: string }}
+ */
+function parseFrontmatter(content) {
+  if (!content.trim().startsWith('---')) {
+    return { frontmatter: {}, body: content }
+  }
+
+  const parts = content.split('---\n')
+  if (parts.length < 3) {
+    return { frontmatter: {}, body: content }
+  }
+
+  try {
+    const frontmatter = yaml.load(parts[1]) || {}
+    const body = parts.slice(2).join('---\n')
+    return { frontmatter, body }
+  } catch {
+    return { frontmatter: {}, body: content }
+  }
+}
+
+/**
+ * Normalize route for filesystem path
+ * Removes leading slash, replaces remaining slashes with path separators
+ * @param {string} route - Page route (e.g., '/about/team')
+ * @returns {string} Normalized path (e.g., 'about/team')
+ */
+function normalizeRouteForPath(route) {
+  if (route === '/') return ''
+  return route.replace(/^\//, '').replace(/\//g, '/')
+}
+
+/**
+ * Load free-form translation for a section
+ *
+ * Resolution order:
+ *   1. page-ids/{pageId}/{stableId}.md (if page has id:)
+ *   2. pages/{pageRoute}/{stableId}.md
+ *   3. Return null (fall back to granular)
+ *
+ * @param {Object} section - Section object with stableId
+ * @param {Object} page - Page object with route and optional id
+ * @param {string} locale - Locale code (e.g., 'es', 'fr')
+ * @param {string} localesDir - Path to locales directory
+ * @returns {Promise<Object|null>} Parsed translation { content } or null
+ */
+export async function loadFreeformTranslation(section, page, locale, localesDir) {
+  const stableId = section.stableId
+  if (!stableId) return null
+
+  const freeformDir = join(localesDir, 'freeform', locale)
+  if (!existsSync(freeformDir)) return null
+
+  const candidates = []
+
+  // 1. Try page-ids path (if page has stable id)
+  if (page.id) {
+    candidates.push(join(freeformDir, 'page-ids', page.id, `${stableId}.md`))
+  }
+
+  // 2. Try pages path (by route)
+  const routePath = normalizeRouteForPath(page.route)
+  if (routePath) {
+    candidates.push(join(freeformDir, 'pages', routePath, `${stableId}.md`))
+  } else {
+    // Root page
+    candidates.push(join(freeformDir, 'pages', `${stableId}.md`))
+  }
+
+  // Try each candidate in order
+  for (const filePath of candidates) {
+    if (!existsSync(filePath)) continue
+
+    try {
+      const content = await readFile(filePath, 'utf-8')
+      const { frontmatter, body } = parseFrontmatter(content)
+
+      // Convert markdown body to ProseMirror
+      const proseMirrorContent = markdownToProseMirror(body)
+
+      return {
+        content: proseMirrorContent,
+        frontmatter,
+        filePath,
+        relativePath: relative(join(localesDir, 'freeform', locale), filePath)
+      }
+    } catch (err) {
+      console.warn(`[i18n] Failed to load free-form translation ${filePath}: ${err.message}`)
+      return null
+    }
+  }
+
+  return null
+}
+
+/**
+ * Load free-form translation for a collection item
+ *
+ * Path: collections/{collectionName}/{slug}.md
+ *
+ * @param {Object} item - Collection item with slug
+ * @param {string} collectionName - Name of the collection
+ * @param {string} locale - Locale code
+ * @param {string} localesDir - Path to locales directory
+ * @returns {Promise<Object|null>} Parsed translation { frontmatter, content } or null
+ */
+export async function loadFreeformCollectionItem(item, collectionName, locale, localesDir) {
+  const slug = item.slug
+  if (!slug) return null
+
+  const freeformDir = join(localesDir, 'freeform', locale)
+  if (!existsSync(freeformDir)) return null
+
+  const filePath = join(freeformDir, 'collections', collectionName, `${slug}.md`)
+  if (!existsSync(filePath)) return null
+
+  try {
+    const content = await readFile(filePath, 'utf-8')
+    const { frontmatter, body } = parseFrontmatter(content)
+
+    // Convert markdown body to ProseMirror (if body exists)
+    const proseMirrorContent = body.trim() ? markdownToProseMirror(body) : null
+
+    return {
+      frontmatter: Object.keys(frontmatter).length > 0 ? frontmatter : null,
+      content: proseMirrorContent,
+      filePath,
+      relativePath: relative(join(localesDir, 'freeform', locale), filePath)
+    }
+  } catch (err) {
+    console.warn(`[i18n] Failed to load free-form collection item ${filePath}: ${err.message}`)
+    return null
+  }
+}
+
+/**
+ * Recursively discover all markdown files in a directory
+ * @param {string} dir - Directory to scan
+ * @param {string} baseDir - Base directory for relative paths
+ * @returns {Promise<string[]>} Array of relative paths
+ */
+async function discoverMarkdownFiles(dir, baseDir) {
+  const files = []
+
+  if (!existsSync(dir)) return files
+
+  const entries = await readdir(dir, { withFileTypes: true })
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name)
+
+    if (entry.isDirectory()) {
+      const subFiles = await discoverMarkdownFiles(fullPath, baseDir)
+      files.push(...subFiles)
+    } else if (entry.isFile() && entry.name.endsWith('.md')) {
+      files.push(relative(baseDir, fullPath))
+    }
+  }
+
+  return files
+}
+
+/**
+ * Discover all free-form translation files for a locale
+ *
+ * Used by status commands to show what translations exist.
+ *
+ * @param {string} locale - Locale code
+ * @param {string} localesDir - Path to locales directory
+ * @returns {Promise<Object>} { pages: string[], pageIds: string[], collections: string[] }
+ */
+export async function discoverFreeformTranslations(locale, localesDir) {
+  const freeformDir = join(localesDir, 'freeform', locale)
+
+  const result = {
+    pages: [],
+    pageIds: [],
+    collections: []
+  }
+
+  if (!existsSync(freeformDir)) return result
+
+  // Discover pages translations
+  const pagesDir = join(freeformDir, 'pages')
+  if (existsSync(pagesDir)) {
+    result.pages = await discoverMarkdownFiles(pagesDir, pagesDir)
+  }
+
+  // Discover page-ids translations
+  const pageIdsDir = join(freeformDir, 'page-ids')
+  if (existsSync(pageIdsDir)) {
+    result.pageIds = await discoverMarkdownFiles(pageIdsDir, pageIdsDir)
+  }
+
+  // Discover collection translations
+  const collectionsDir = join(freeformDir, 'collections')
+  if (existsSync(collectionsDir)) {
+    result.collections = await discoverMarkdownFiles(collectionsDir, collectionsDir)
+  }
+
+  return result
+}
+
+/**
+ * Get metadata for a free-form translation file
+ *
+ * @param {string} filePath - Full path to translation file
+ * @returns {Promise<Object>} { mtime, size }
+ */
+export async function getFreeformFileMeta(filePath) {
+  if (!existsSync(filePath)) return null
+
+  const stats = await stat(filePath)
+  return {
+    mtime: stats.mtime.toISOString(),
+    size: stats.size
+  }
+}
+
+/**
+ * Parse a free-form translation file path to extract metadata
+ *
+ * @param {string} relativePath - Path relative to locale's freeform dir
+ * @returns {Object} { type, pageRoute?, pageId?, collectionName?, stableId, slug? }
+ */
+export function parseFreeformPath(relativePath) {
+  const parts = relativePath.split('/')
+
+  if (parts[0] === 'pages') {
+    // pages/about/hero.md → { type: 'page', pageRoute: '/about', stableId: 'hero' }
+    const stableId = parts[parts.length - 1].replace('.md', '')
+    const routeParts = parts.slice(1, -1)
+    const pageRoute = routeParts.length > 0 ? '/' + routeParts.join('/') : '/'
+    return { type: 'page', pageRoute, stableId }
+  }
+
+  if (parts[0] === 'page-ids') {
+    // page-ids/installation/intro.md → { type: 'pageId', pageId: 'installation', stableId: 'intro' }
+    const stableId = parts[parts.length - 1].replace('.md', '')
+    const pageId = parts.slice(1, -1).join('/')
+    return { type: 'pageId', pageId, stableId }
+  }
+
+  if (parts[0] === 'collections') {
+    // collections/articles/getting-started.md → { type: 'collection', collectionName: 'articles', slug: 'getting-started' }
+    const slug = parts[parts.length - 1].replace('.md', '')
+    const collectionName = parts[1]
+    return { type: 'collection', collectionName, slug }
+  }
+
+  return { type: 'unknown', relativePath }
+}
+
+/**
+ * Build the expected free-form translation path for a section
+ *
+ * @param {Object} section - Section with stableId
+ * @param {Object} page - Page with route and optional id
+ * @param {boolean} preferPageId - Whether to prefer page-ids/ over pages/
+ * @returns {string} Relative path (e.g., 'pages/about/hero.md')
+ */
+export function buildFreeformPath(section, page, preferPageId = true) {
+  const stableId = section.stableId
+  if (!stableId) return null
+
+  // Prefer page-ids if page has stable id
+  if (preferPageId && page.id) {
+    return `page-ids/${page.id}/${stableId}.md`
+  }
+
+  // Fall back to route-based path
+  const routePath = normalizeRouteForPath(page.route)
+  if (routePath) {
+    return `pages/${routePath}/${stableId}.md`
+  }
+
+  // Root page
+  return `pages/${stableId}.md`
+}
+
+/**
+ * Build the expected free-form translation path for a collection item
+ *
+ * @param {string} collectionName - Name of the collection
+ * @param {string} slug - Item slug
+ * @returns {string} Relative path (e.g., 'collections/articles/getting-started.md')
+ */
+export function buildFreeformCollectionPath(collectionName, slug) {
+  return `collections/${collectionName}/${slug}.md`
+}

package/src/i18n/index.js

@@ -24,6 +24,30 @@ import {
 } from './collections.js'
 import { generateSearchIndex, isSearchEnabled } from '../search/index.js'
 
+// Free-form translation support
+import {
+  loadFreeformTranslation,
+  loadFreeformCollectionItem,
+  discoverFreeformTranslations,
+  getFreeformFileMeta,
+  parseFreeformPath,
+  buildFreeformPath,
+  buildFreeformCollectionPath
+} from './freeform.js'
+import {
+  computeSourceHash,
+  loadManifest as loadFreeformManifest,
+  saveManifest as saveFreeformManifest,
+  recordHash,
+  checkStaleness,
+  updateHash,
+  removeManifestEntries,
+  renameManifestEntries,
+  getStaleTranslations,
+  getOrphanedTranslations,
+  getUnregisteredTranslations
+} from './freeform-manifest.js'
+
 export {
   // Hash utilities
   computeHash,
@@ -49,7 +73,29 @@ export {
 
   // Locale resolution
   getAvailableLocales,
-  resolveLocales
+  resolveLocales,
+
+  // Free-form translation functions
+  loadFreeformTranslation,
+  loadFreeformCollectionItem,
+  discoverFreeformTranslations,
+  getFreeformFileMeta,
+  parseFreeformPath,
+  buildFreeformPath,
+  buildFreeformCollectionPath,
+
+  // Free-form manifest functions
+  computeSourceHash,
+  loadFreeformManifest,
+  saveFreeformManifest,
+  recordHash,
+  checkStaleness,
+  updateHash,
+  removeManifestEntries,
+  renameManifestEntries,
+  getStaleTranslations,
+  getOrphanedTranslations,
+  getUnregisteredTranslations
 }
 
 /**
@@ -278,7 +324,8 @@ export async function getTranslationStatus(siteRoot, options = {}) {
  * Build translated site content for all locales
  * @param {string} siteRoot - Site root directory
  * @param {Object} options - Options
- * @param {boolean} [options.generateSearchIndex=true] - Generate search indexes for each locale
+ * @param {boolean} [options.generateSearchIndexes=true] - Generate search indexes for each locale
+ * @param {boolean} [options.freeformEnabled=true] - Enable free-form translation support
  * @returns {Object} Map of locale to output paths
  */
 export async function buildLocalizedContent(siteRoot, options = {}) {
@@ -287,7 +334,8 @@ export async function buildLocalizedContent(siteRoot, options = {}) {
     locales = [],
     outputDir = join(siteRoot, 'dist'),
     fallbackToSource = true,
-    generateSearchIndexes = true
+    generateSearchIndexes = true,
+    freeformEnabled = true
   } = options
 
   const localesPath = join(siteRoot, localesDir)
@@ -309,10 +357,29 @@ export async function buildLocalizedContent(siteRoot, options = {}) {
       translations = JSON.parse(localeRaw)
     }
 
-    // Merge translations
-    const translated = mergeTranslations(siteContent, translations, {
-      fallbackToSource
-    })
+    // Check if free-form translations exist for this locale
+    const freeformDir = join(localesPath, 'freeform', locale)
+    const hasFreeform = freeformEnabled && existsSync(freeformDir)
+
+    // Merge translations (with free-form support if enabled)
+    let translated
+    if (hasFreeform) {
+      // Use async merge with free-form support
+      translated = await mergeTranslations(siteContent, translations, {
+        fallbackToSource,
+        locale,
+        localesDir: localesPath,
+        freeformEnabled: true
+      })
+
+      // Check for stale/orphaned free-form translations and warn
+      await warnAboutFreeformIssues(locale, freeformDir, siteContent)
+    } else {
+      // Use sync merge (original behavior)
+      translated = mergeTranslations(siteContent, translations, {
+        fallbackToSource
+      })
+    }
 
     // Mark the active locale in the translated content
     translated.config = { ...translated.config, activeLocale: locale }
@@ -346,6 +413,63 @@ export async function buildLocalizedContent(siteRoot, options = {}) {
   return outputs
 }
 
+/**
+ * Check for stale/orphaned free-form translations and emit warnings
+ * @param {string} locale - Locale code
+ * @param {string} freeformDir - Path to locale's freeform directory
+ * @param {Object} siteContent - Site content for building source hashes
+ */
+async function warnAboutFreeformIssues(locale, freeformDir, siteContent) {
+  try {
+    // Build map of valid source hashes
+    const sourceHashes = {}
+    const validPaths = new Set()
+
+    for (const page of siteContent.pages || []) {
+      for (const section of page.sections || []) {
+        if (section.stableId) {
+          const path = buildFreeformPath(section, page)
+          if (path) {
+            validPaths.add(path)
+            // Compute source hash for staleness check
+            if (section.content) {
+              sourceHashes[path] = computeSourceHash(section.content)
+            }
+          }
+        }
+      }
+    }
+
+    // Check for stale translations
+    const stale = await getStaleTranslations(freeformDir, sourceHashes)
+    for (const item of stale) {
+      console.warn(`[i18n] Free-form translation stale: ${locale}/${item.path} (source changed ${item.recordedDate})`)
+    }
+
+    // Check for orphaned translations
+    const orphaned = await getOrphanedTranslations(freeformDir, validPaths)
+    for (const item of orphaned) {
+      console.warn(`[i18n] Free-form translation orphaned: ${locale}/${item.path}`)
+    }
+
+    // Check for unregistered translations (new files)
+    const discovered = await discoverFreeformTranslations(locale, dirname(dirname(freeformDir)))
+    const allPaths = [...discovered.pages, ...discovered.pageIds, ...discovered.collections]
+    const unregistered = await getUnregisteredTranslations(freeformDir, allPaths)
+    for (const path of unregistered) {
+      // Auto-register new free-form translations
+      const sourcePath = validPaths.has(path) ? path : null
+      if (sourcePath && sourceHashes[sourcePath]) {
+        await recordHash(freeformDir, path, sourceHashes[sourcePath])
+        console.log(`[i18n] Free-form translation registered: ${locale}/${path} (new file)`)
+      }
+    }
+  } catch (err) {
+    // Non-fatal: just log and continue
+    console.warn(`[i18n] Could not check free-form translation status for ${locale}: ${err.message}`)
+  }
+}
+
 /**
  * Format translation status for console output
  * @param {Object} status - Status from getTranslationStatus

package/src/i18n/merge.js

@@ -3,20 +3,55 @@
  *
  * Takes site-content.json and locale translations,
  * produces translated site-content for each locale.
+ *
+ * Supports two translation modes:
+ * 1. Hash-based (granular): Translate individual strings by hash lookup
+ * 2. Free-form: Complete content replacement from markdown files
+ *
+ * Free-form translations are checked first, falling back to hash-based
+ * when no free-form translation exists.
  */
 
 import { computeHash } from './hash.js'
+import { loadFreeformTranslation } from './freeform.js'
 
 /**
  * Merge translations into site content for a specific locale
+ *
  * @param {Object} siteContent - Original site-content.json
  * @param {Object} translations - Locale translations (hash -> translation)
  * @param {Object} options - Merge options
- * @returns {Object} Translated site content
+ * @param {boolean} [options.fallbackToSource=true] - Return source if no translation
+ * @param {string} [options.locale] - Locale code for free-form lookups
+ * @param {string} [options.localesDir] - Path to locales directory
+ * @param {boolean} [options.freeformEnabled=false] - Enable free-form translation lookup
+ * @returns {Object|Promise<Object>} Translated site content (async if freeformEnabled)
  */
 export function mergeTranslations(siteContent, translations, options = {}) {
-  const { fallbackToSource = true } = options
+  const {
+    fallbackToSource = true,
+    locale = null,
+    localesDir = null,
+    freeformEnabled = false
+  } = options
 
+  // If free-form is enabled, use async version
+  if (freeformEnabled && locale && localesDir) {
+    return mergeTranslationsAsync(siteContent, translations, {
+      fallbackToSource,
+      locale,
+      localesDir
+    })
+  }
+
+  // Sync version (original behavior)
+  return mergeTranslationsSync(siteContent, translations, fallbackToSource)
+}
+
+/**
+ * Synchronous merge (original behavior, no free-form)
+ */
+function mergeTranslationsSync(siteContent, translations, fallbackToSource) {
   // Deep clone to avoid mutating original
   const translated = JSON.parse(JSON.stringify(siteContent))
 
@@ -28,7 +63,35 @@ export function mergeTranslations(siteContent, translations, options = {}) {
 
     // Translate section content
     for (const section of page.sections || []) {
-      translateSection(section, pageRoute, translations, fallbackToSource)
+      translateSectionSync(section, pageRoute, translations, fallbackToSource)
+    }
+  }
+
+  return translated
+}
+
+/**
+ * Asynchronous merge with free-form support
+ */
+async function mergeTranslationsAsync(siteContent, translations, options) {
+  const { fallbackToSource, locale, localesDir } = options
+
+  // Deep clone to avoid mutating original
+  const translated = JSON.parse(JSON.stringify(siteContent))
+
+  for (const page of translated.pages || []) {
+    const pageRoute = page.route || '/'
+
+    // Translate page metadata (always hash-based)
+    translatePageMeta(page, pageRoute, translations, fallbackToSource)
+
+    // Translate section content (with free-form check)
+    for (const section of page.sections || []) {
+      await translateSectionAsync(section, page, translations, {
+        fallbackToSource,
+        locale,
+        localesDir
+      })
     }
   }
 
@@ -77,9 +140,9 @@ function translatePageMeta(page, pageRoute, translations, fallbackToSource) {
 }
 
 /**
- * Translate a section's content
+ * Translate a section's content (synchronous, hash-based only)
  */
-function translateSection(section, pageRoute, translations, fallbackToSource) {
+function translateSectionSync(section, pageRoute, translations, fallbackToSource) {
   const sectionId = section.id || 'unknown'
   const context = { page: pageRoute, section: sectionId }
 
@@ -89,7 +152,40 @@ function translateSection(section, pageRoute, translations, fallbackToSource) {
 
   // Recursively translate subsections
   for (const subsection of section.subsections || []) {
-    translateSection(subsection, pageRoute, translations, fallbackToSource)
+    translateSectionSync(subsection, pageRoute, translations, fallbackToSource)
+  }
+}
+
+/**
+ * Translate a section's content (async, with free-form check)
+ *
+ * Resolution order:
+ * 1. Check for free-form translation (complete replacement)
+ * 2. Fall back to hash-based translation (element-by-element)
+ */
+async function translateSectionAsync(section, page, translations, options) {
+  const { fallbackToSource, locale, localesDir } = options
+  const pageRoute = page.route || '/'
+  const sectionId = section.id || 'unknown'
+  const context = { page: pageRoute, section: sectionId }
+
+  // Check for free-form translation first
+  const freeform = await loadFreeformTranslation(section, page, locale, localesDir)
+
+  if (freeform) {
+    // Complete content replacement
+    section.content = freeform.content
+    // Note: We could also merge frontmatter into params here if needed
+  } else {
+    // Fall back to hash-based translation
+    if (section.content?.type === 'doc') {
+      translateProseMirrorDoc(section.content, context, translations, fallbackToSource)
+    }
+  }
+
+  // Recursively translate subsections
+  for (const subsection of section.subsections || []) {
+    await translateSectionAsync(subsection, page, translations, options)
   }
 }