champollion 0.3.4 → 0.4.0

package/lib/watch.js

@@ -20,7 +20,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { resolveConfig } from './config.js';
-import { detectFormatFromDir, getExtension } from './format.js';
+import { discoverLocaleLayout } from './locale-layout.js';
 import { runSync } from './sync.js';
 import { output } from './output.js';
 
@@ -53,13 +53,27 @@ import { output } from './output.js';
 async function startWatch(options = {}) {
   const { cwd = process.cwd(), cliArgs = {} } = options;
   const config = resolveConfig(cliArgs, cwd);
-  const format = config.format !== 'auto'
-    ? config.format
-    : detectFormatFromDir(config.localesDir);
-  const ext = getExtension(format);
-  const inputLocale = config.inputLocale;
-  const sourceFile = `${inputLocale}${ext}`;
-  const sourcePath = path.join(config.localesDir, sourceFile);
+  // Every source file of the layout is watched: one for a flat project,
+  // one per namespace for a folder-per-locale project (en/common.json,
+  // en/admin.json, …). Docusaurus watches its source locale's JSON the
+  // same way the flat lane always did (i18n/<source>.json does not exist
+  // there, so it keeps the historical single-path behaviour).
+  let sourcePaths;
+  let sourceLabel;
+  if (config.format === 'docusaurus') {
+    sourcePaths = [path.join(config.localesDir, `${config.inputLocale}.json`)];
+    sourceLabel = `${config.inputLocale}.json`;
+  } else {
+    const layout = discoverLocaleLayout(config, { cwd });
+    sourcePaths = layout.sourceFiles.map(f => f.path);
+    sourceLabel = layout.namespaced
+      ? `${layout.sourceFiles.length} source file(s) (${layout.display})`
+      : layout.sourceFiles[0].rel;
+    if (sourcePaths.length === 0) {
+      throw new Error(`No ${layout.format} source files found for ${config.inputLocale} (${layout.display}) — nothing to watch.`);
+    }
+  }
+  const sourceFile = sourceLabel;
 
   output.info(`Watching ${sourceFile} for changes...`);
 
@@ -101,9 +115,11 @@ async function startWatch(options = {}) {
     }
   }
 
-  // Watch the source file using stat polling (fs.watchFile).
-  // On change, debounce for 500ms then sync.
-  fs.watchFile(sourcePath, { interval: 500 }, (curr, prev) => {
+  // Watch the source file(s) using stat polling (fs.watchFile).
+  // On change, debounce for 500ms then sync. A namespace file ADDED after
+  // the watcher started is picked up by the next sync, but is not itself
+  // watched until watch mode restarts.
+  const onChange = (curr, prev) => {
     // Only react to actual content changes (mtime changed)
     if (curr.mtimeMs === prev.mtimeMs) return;
 
@@ -120,7 +136,10 @@ async function startWatch(options = {}) {
       // inside doSync() and logged, never thrown.
       doSync();
     }, 500);
-  });
+  };
+  for (const sourcePath of sourcePaths) {
+    fs.watchFile(sourcePath, { interval: 500 }, onChange);
+  }
 
   // Signal that the watcher is active and ready for file changes.
   // Tests use this line to know when it's safe to modify the source file.
@@ -135,7 +154,7 @@ async function startWatch(options = {}) {
     // if startWatch were ever called more than once in the same process.
     process.once('SIGINT', () => {
       if (debounceTimer) clearTimeout(debounceTimer);
-      fs.unwatchFile(sourcePath);
+      for (const sourcePath of sourcePaths) fs.unwatchFile(sourcePath);
       resolve(); // Allow the promise to resolve so cleanup can happen
       process.exit(0);
     });

package/lib/xliff.js

@@ -20,6 +20,15 @@
  *   third-party tools with extensions, inline markup, etc. If a user
  *   imports XLIFF from memoQ, we extract <target> text and ignore the rest.
  *
+ * UNIT IDS are locale keys. A gettext key with a context is
+ * `msgctxt + U+0004 + msgid` (lib/po.js), and XML 1.0 cannot carry U+0004
+ * at all — not even as a character reference — so the id writes it as "␄"
+ * (U+2404, the visible symbol for that character, the same form reports
+ * and `--force-keys` use) and import maps it back. Newlines, tabs and CRs
+ * in an id (multi-line msgids) are written as character references: a
+ * literal newline in an attribute is turned into a space by every XML
+ * parser, which would silently rename the key in a CAT tool round trip.
+ *
  * ZERO DEPENDENCIES. Uses regex-based XML parsing because:
  *   1. Our XLIFF output is predictable and well-formed
  *   2. We only need to extract source/target text from <trans-unit> elements
@@ -66,7 +75,7 @@ function exportXLIFF({ sourceLocale, targetLocale, sourceFlat, targetFlat, origi
     const body = exempt ? sourceValue : (hasTarget ? targetValue : '');
 
     units.push(
-      `      <trans-unit id="${escapeXML(key)}"${exempt ? ' translate="no"' : ''} xml:space="preserve">` +
+      `      <trans-unit id="${encodeUnitId(key)}"${exempt ? ' translate="no"' : ''} xml:space="preserve">` +
       `\n        <source>${escapeXML(sourceValue)}</source>` +
       `\n        <target state="${state}">${escapeXML(body)}</target>` +
       `\n      </trans-unit>`
@@ -125,7 +134,7 @@ function importXLIFF(xliffString) {
   let match;
 
   while ((match = unitPattern.exec(xliffString)) !== null) {
-    const unitId = unescapeXML(match[1]);
+    const unitId = decodeUnitId(match[1]);
     const unitContent = match[0];
 
     // Extract target content (if present and non-empty)
@@ -157,14 +166,44 @@ function escapeXML(str) {
     .replace(/'/g, '&apos;');
 }
 
+/** gettext's msgctxt separator, and how an XLIFF id writes it. */
+const CONTEXT_SEPARATOR = '\u0004';
+const CONTEXT_SEPARATOR_VISIBLE = '\u2404';
+
+/**
+ * A locale key as an XLIFF unit id (see "UNIT IDS" above).
+ *
+ * @param {string} key
+ * @returns {string} Attribute-safe id
+ */
+function encodeUnitId(key) {
+  return escapeXML(key.split(CONTEXT_SEPARATOR).join(CONTEXT_SEPARATOR_VISIBLE))
+    .replace(/\n/g, '&#10;')
+    .replace(/\r/g, '&#13;')
+    .replace(/\t/g, '&#9;');
+}
+
+/**
+ * The locale key an XLIFF unit id names.
+ *
+ * @param {string} raw - Attribute value as found in the file
+ * @returns {string}
+ */
+function decodeUnitId(raw) {
+  return unescapeXML(raw).split(CONTEXT_SEPARATOR_VISIBLE).join(CONTEXT_SEPARATOR);
+}
+
 /**
- * Unescape XML entities back to raw characters.
+ * Unescape XML entities back to raw characters (named entities and
+ * numeric character references).
  *
  * @param {string} str - XML-escaped string
  * @returns {string} Raw string
  */
 function unescapeXML(str) {
   return str
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
+    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
     .replace(/&apos;/g, "'")
     .replace(/&quot;/g, '"')
     .replace(/&gt;/g, '>')
@@ -181,4 +220,6 @@ export {
   importXLIFF,
   escapeXML,
   unescapeXML,
+  encodeUnitId,
+  decodeUnitId,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "champollion",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "Research-grade translation engine for i18n projects. Pluggable methods, per-pair quality tiers, and deterministic script converters. Supports JSON (next-intl, i18next), TOML, and YAML (Hugo).",
   "type": "module",
   "bin": {

package/shared/CORPORA-CARDS.md

@@ -54,6 +54,8 @@ A pair-specific, community-curated dataset used for official Champollion evaluat
 
 The `eval-` prefix makes it instantly clear this is a benchmark, not just a dataset catalogue entry.
 
+The `{segment}` part is optional: the schema's id pattern only requires `eval-`/`ref-` plus lower-case kebab-case, and some cards carry none (`eval-in22-gen-v1`). Cards written by `champollion register-corpus` are `eval-{src}-{tgt}-{name}[-{role}]-v1`: `{name}` is the slugged `--name` (the publisher only when the name has no a–z/0–9 letters), and `{role}` (`test` / `dev` / `train`) appears only when the author passed `--role`. No role is ever guessed, and an id already registered is never re-derived.
+
 ---
 
 ## Official Champollion Evaluation