champollion 0.3.3 → 0.4.0

package/lib/watch.js

@@ -20,7 +20,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { resolveConfig } from './config.js';
-import { detectFormatFromDir, getExtension } from './format.js';
+import { discoverLocaleLayout } from './locale-layout.js';
 import { runSync } from './sync.js';
 import { output } from './output.js';
 
@@ -53,13 +53,27 @@ import { output } from './output.js';
 async function startWatch(options = {}) {
   const { cwd = process.cwd(), cliArgs = {} } = options;
   const config = resolveConfig(cliArgs, cwd);
-  const format = config.format !== 'auto'
-    ? config.format
-    : detectFormatFromDir(config.localesDir);
-  const ext = getExtension(format);
-  const inputLocale = config.inputLocale;
-  const sourceFile = `${inputLocale}${ext}`;
-  const sourcePath = path.join(config.localesDir, sourceFile);
+  // Every source file of the layout is watched: one for a flat project,
+  // one per namespace for a folder-per-locale project (en/common.json,
+  // en/admin.json, …). Docusaurus watches its source locale's JSON the
+  // same way the flat lane always did (i18n/<source>.json does not exist
+  // there, so it keeps the historical single-path behaviour).
+  let sourcePaths;
+  let sourceLabel;
+  if (config.format === 'docusaurus') {
+    sourcePaths = [path.join(config.localesDir, `${config.inputLocale}.json`)];
+    sourceLabel = `${config.inputLocale}.json`;
+  } else {
+    const layout = discoverLocaleLayout(config, { cwd });
+    sourcePaths = layout.sourceFiles.map(f => f.path);
+    sourceLabel = layout.namespaced
+      ? `${layout.sourceFiles.length} source file(s) (${layout.display})`
+      : layout.sourceFiles[0].rel;
+    if (sourcePaths.length === 0) {
+      throw new Error(`No ${layout.format} source files found for ${config.inputLocale} (${layout.display}) — nothing to watch.`);
+    }
+  }
+  const sourceFile = sourceLabel;
 
   output.info(`Watching ${sourceFile} for changes...`);
 
@@ -101,9 +115,11 @@ async function startWatch(options = {}) {
     }
   }
 
-  // Watch the source file using stat polling (fs.watchFile).
-  // On change, debounce for 500ms then sync.
-  fs.watchFile(sourcePath, { interval: 500 }, (curr, prev) => {
+  // Watch the source file(s) using stat polling (fs.watchFile).
+  // On change, debounce for 500ms then sync. A namespace file ADDED after
+  // the watcher started is picked up by the next sync, but is not itself
+  // watched until watch mode restarts.
+  const onChange = (curr, prev) => {
     // Only react to actual content changes (mtime changed)
     if (curr.mtimeMs === prev.mtimeMs) return;
 
@@ -120,7 +136,10 @@ async function startWatch(options = {}) {
       // inside doSync() and logged, never thrown.
       doSync();
     }, 500);
-  });
+  };
+  for (const sourcePath of sourcePaths) {
+    fs.watchFile(sourcePath, { interval: 500 }, onChange);
+  }
 
   // Signal that the watcher is active and ready for file changes.
   // Tests use this line to know when it's safe to modify the source file.
@@ -135,7 +154,7 @@ async function startWatch(options = {}) {
     // if startWatch were ever called more than once in the same process.
     process.once('SIGINT', () => {
       if (debounceTimer) clearTimeout(debounceTimer);
-      fs.unwatchFile(sourcePath);
+      for (const sourcePath of sourcePaths) fs.unwatchFile(sourcePath);
       resolve(); // Allow the promise to resolve so cleanup can happen
       process.exit(0);
     });

package/lib/xliff.js

@@ -20,6 +20,15 @@
  *   third-party tools with extensions, inline markup, etc. If a user
  *   imports XLIFF from memoQ, we extract <target> text and ignore the rest.
  *
+ * UNIT IDS are locale keys. A gettext key with a context is
+ * `msgctxt + U+0004 + msgid` (lib/po.js), and XML 1.0 cannot carry U+0004
+ * at all — not even as a character reference — so the id writes it as "␄"
+ * (U+2404, the visible symbol for that character, the same form reports
+ * and `--force-keys` use) and import maps it back. Newlines, tabs and CRs
+ * in an id (multi-line msgids) are written as character references: a
+ * literal newline in an attribute is turned into a space by every XML
+ * parser, which would silently rename the key in a CAT tool round trip.
+ *
  * ZERO DEPENDENCIES. Uses regex-based XML parsing because:
  *   1. Our XLIFF output is predictable and well-formed
  *   2. We only need to extract source/target text from <trans-unit> elements
@@ -66,7 +75,7 @@ function exportXLIFF({ sourceLocale, targetLocale, sourceFlat, targetFlat, origi
     const body = exempt ? sourceValue : (hasTarget ? targetValue : '');
 
     units.push(
-      `      <trans-unit id="${escapeXML(key)}"${exempt ? ' translate="no"' : ''} xml:space="preserve">` +
+      `      <trans-unit id="${encodeUnitId(key)}"${exempt ? ' translate="no"' : ''} xml:space="preserve">` +
       `\n        <source>${escapeXML(sourceValue)}</source>` +
       `\n        <target state="${state}">${escapeXML(body)}</target>` +
       `\n      </trans-unit>`
@@ -125,7 +134,7 @@ function importXLIFF(xliffString) {
   let match;
 
   while ((match = unitPattern.exec(xliffString)) !== null) {
-    const unitId = unescapeXML(match[1]);
+    const unitId = decodeUnitId(match[1]);
     const unitContent = match[0];
 
     // Extract target content (if present and non-empty)
@@ -157,14 +166,44 @@ function escapeXML(str) {
     .replace(/'/g, '&apos;');
 }
 
+/** gettext's msgctxt separator, and how an XLIFF id writes it. */
+const CONTEXT_SEPARATOR = '\u0004';
+const CONTEXT_SEPARATOR_VISIBLE = '\u2404';
+
+/**
+ * A locale key as an XLIFF unit id (see "UNIT IDS" above).
+ *
+ * @param {string} key
+ * @returns {string} Attribute-safe id
+ */
+function encodeUnitId(key) {
+  return escapeXML(key.split(CONTEXT_SEPARATOR).join(CONTEXT_SEPARATOR_VISIBLE))
+    .replace(/\n/g, '&#10;')
+    .replace(/\r/g, '&#13;')
+    .replace(/\t/g, '&#9;');
+}
+
+/**
+ * The locale key an XLIFF unit id names.
+ *
+ * @param {string} raw - Attribute value as found in the file
+ * @returns {string}
+ */
+function decodeUnitId(raw) {
+  return unescapeXML(raw).split(CONTEXT_SEPARATOR_VISIBLE).join(CONTEXT_SEPARATOR);
+}
+
 /**
- * Unescape XML entities back to raw characters.
+ * Unescape XML entities back to raw characters (named entities and
+ * numeric character references).
  *
  * @param {string} str - XML-escaped string
  * @returns {string} Raw string
  */
 function unescapeXML(str) {
   return str
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
+    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec)))
     .replace(/&apos;/g, "'")
     .replace(/&quot;/g, '"')
     .replace(/&gt;/g, '>')
@@ -181,4 +220,6 @@ export {
   importXLIFF,
   escapeXML,
   unescapeXML,
+  encodeUnitId,
+  decodeUnitId,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "champollion",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "description": "Research-grade translation engine for i18n projects. Pluggable methods, per-pair quality tiers, and deterministic script converters. Supports JSON (next-intl, i18next), TOML, and YAML (Hugo).",
   "type": "module",
   "bin": {
@@ -88,6 +88,7 @@
     "language-subtag-registry": "^0.4.2"
   },
   "dependencies": {
-    "@translated/lara": "^1.12.1"
+    "@translated/lara": "^1.12.1",
+    "cree-sro-syllabics": "2021.6.8"
   }
 }

package/shared/CORPORA-CARDS.md

@@ -54,6 +54,8 @@ A pair-specific, community-curated dataset used for official Champollion evaluat
 
 The `eval-` prefix makes it instantly clear this is a benchmark, not just a dataset catalogue entry.
 
+The `{segment}` part is optional: the schema's id pattern only requires `eval-`/`ref-` plus lower-case kebab-case, and some cards carry none (`eval-in22-gen-v1`). Cards written by `champollion register-corpus` are `eval-{src}-{tgt}-{name}[-{role}]-v1`: `{name}` is the slugged `--name` (the publisher only when the name has no a–z/0–9 letters), and `{role}` (`test` / `dev` / `train`) appears only when the author passed `--role`. No role is ever guessed, and an id already registered is never re-derived.
+
 ---
 
 ## Official Champollion Evaluation

package/shared/DATA-SOVEREIGNTY.md

@@ -39,19 +39,19 @@ Each sovereignty-related field maps to one or more external standards. The table
 
 | Field / Concept | Primary Standard | Standard URL | Relationship |
 |---|---|---|---|
-| `sovereignty.frameworks` | OCAP®, CARE, Te Mana Raraunga, FAIR, IEEE 2890 | See individual URLs below | Records which frameworks have been **explicitly invoked** by the governing body |
-| `sovereignty.governanceOrg` | OCAP® Ownership | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Maps to the "O" in OCAP — who owns the data |
-| `sovereignty.custodian` | OCAP® Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Maps to the "P" in OCAP — who physically holds the data |
-| `sovereignty.consentModel` | OCAP® Control, CARE Authority | [https://www.gida-global.org/care](https://www.gida-global.org/care) | Maps to the "C" in OCAP (who controls access) and CARE's Authority to Control principle |
+| `sovereignty.frameworks` | CARE, Te Mana Raraunga, FAIR, IEEE 2890, community ownership-and-control assertions | See individual URLs below | Records which frameworks have been **explicitly invoked** by the governing body |
+| `sovereignty.governanceOrg` | Community ownership | — | Who owns the data |
+| `sovereignty.custodian` | Community possession | — | Who physically holds the data |
+| `sovereignty.consentModel` | Community control, CARE Authority | [https://www.gida-global.org/care](https://www.gida-global.org/care) | Who controls access — maps to CARE's Authority to Control principle |
 | `sovereignty.tkLabels[]` | Local Contexts TK/BC Labels | [https://localcontexts.org/](https://localcontexts.org/) | Community-applied labels from the Local Contexts Hub |
 | `usageRestrictions.training` | DUO (Data Use Ontology), ODRL | [DUO](https://github.com/EBISPOT/DUO), [ODRL](https://www.w3.org/TR/odrl-model/) | `prohibited-by-license` maps to DUO:0000004 (no general methods research) and ODRL `odrl:prohibition`. `prohibited-by-community` maps to a governance assertion (no DUO equivalent — DUO is license-derived) |
 | `usageRestrictions.commercialUse` | DUO:0000018 (not for profit), ODRL | [DUO](https://github.com/EBISPOT/DUO), [ODRL](https://www.w3.org/TR/odrl-model/) | `prohibited-by-license` maps to DUO:0000018 and ODRL `odrl:prohibition` with `odrl:commercialize` action |
 | `usageRestrictions.redistribution` | ODRL `odrl:distribute` | [ODRL](https://www.w3.org/TR/odrl-model/) | Maps to an ODRL permission/prohibition on the `odrl:distribute` action |
 | `usageRestrictions.communityNotes` | CARE Ethics | [https://www.gida-global.org/care](https://www.gida-global.org/care) | Maps to CARE's Ethics principle — ensuring legitimate concerns are surfaced |
 | `doNotTrain` | IEEE 2890 | No public URL (IEEE standard, paywall) | Methodological constraint for evaluation data; IEEE 2890 §7.4 recommends flagging datasets not intended for model training |
-| `stewardship.authorizationModel` | OCAP® Control | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | Community controls how evaluation access is granted |
-| `submission.transfer.*` | OCAP® Ownership + Possession, CARE Collective Benefit | [OCAP](https://fnigc.ca/ocap-training/), [CARE](https://www.gida-global.org/care) | Transfer provisions map to OCAP O+P (community owns and possesses the method). Revenue model maps to CARE Collective Benefit |
-| `submission.admissibility.selfHostable` | OCAP® Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) | If the community can't run it independently, they don't possess it |
+| `stewardship.authorizationModel` | Community control | — | Community controls how evaluation access is granted |
+| `submission.transfer.*` | Community ownership + possession, CARE Collective Benefit | [CARE](https://www.gida-global.org/care) | Transfer provisions mean the community owns and possesses the method. Revenue model maps to CARE Collective Benefit |
+| `submission.admissibility.selfHostable` | Community possession | — | If the community can't run it independently, they don't possess it |
 
 ### Standard References
 
@@ -61,7 +61,6 @@ Each sovereignty-related field maps to one or more external standards. The table
 | DUO | Data Use Ontology | [https://github.com/EBISPOT/DUO](https://github.com/EBISPOT/DUO) |
 | Local Contexts | TK and BC Labels | [https://localcontexts.org/](https://localcontexts.org/) |
 | CARE | Collective Benefit, Authority to Control, Responsibility, Ethics | [https://www.gida-global.org/care](https://www.gida-global.org/care) |
-| OCAP® | Ownership, Control, Access, Possession | [https://fnigc.ca/ocap-training/](https://fnigc.ca/ocap-training/) |
 | IEEE 2890 | IEEE Standard for Recommended Practice for Provenance of Datasets | IEEE (paywall; no stable public URL) |
 | Te Mana Raraunga | Māori Data Sovereignty Network | [https://www.temanararaunga.maori.nz/](https://www.temanararaunga.maori.nz/) |
 | FAIR | Findable, Accessible, Interoperable, Reusable | [https://www.go-fair.org/fair-principles/](https://www.go-fair.org/fair-principles/) |
@@ -108,7 +107,7 @@ The `sovereignty` object records governance structures that actually exist for t
 | Field | Type | Allowed Values | Description | When to use `null` |
 |---|---|---|---|---|
 | `sovereignty` | `object \| null` | — | Container for governance metadata. | No governance body has been identified. No frameworks have been invoked. No labels have been applied. |
-| `.frameworks` | `array` of `string` | `"OCAP"`, `"CARE"`, `"Te-Mana-Raraunga"`, `"FAIR"`, `"IEEE-2890"` | Data sovereignty frameworks the data creators or governing body have **explicitly invoked**. Only list frameworks where there is documented evidence of adoption. | *(Field is an array — use `[]` if sovereignty object exists but no frameworks have been invoked. But if no sovereignty object at all, the whole object is null.)* |
+| `.frameworks` | `array` of `string` | `"community-ownership-control"`, `"CARE"`, `"Te-Mana-Raraunga"`, `"FAIR"`, `"IEEE-2890"` | Data sovereignty frameworks the data creators or governing body have **explicitly invoked**. Only list frameworks where there is documented evidence of adoption. | *(Field is an array — use `[]` if sovereignty object exists but no frameworks have been invoked. But if no sovereignty object at all, the whole object is null.)* |
 | `.governanceOrg` | `string \| null` | Free text | Organization or body with governance authority. For Indigenous-governed corpora, this is the language trust, tribal council, or delegated body. | No governance body has been identified or established. |
 | `.custodian` | `string \| null` | Free text | Who physically holds the data. Distinct from `governanceOrg` — a university may be custodian while a community is the governance authority. | Custodian is the same as `source.publisher` (no need to repeat). |
 | `.consentModel` | `string \| null` | `"per-submission"`, `"blanket"`, `"open-access"` | How consent is structured for use of this data in method evaluation. | No consent model has been established. |
@@ -123,7 +122,7 @@ The `sovereignty` object records governance structures that actually exist for t
 ```json
 {
   "sovereignty": {
-    "frameworks": ["OCAP", "CARE"],
+    "frameworks": ["community-ownership-control", "CARE"],
     "governanceOrg": "Plains Cree Language Trust (hypothetical)",
     "custodian": "University of Alberta — ALTLab",
     "consentModel": "per-submission",
@@ -264,13 +263,13 @@ The `submission` field (§7) defines what happens when a method is accepted: wha
 
 ## 7. Admissibility Rules
 
-The `submission.admissibility` fields define what methods are eligible for prize evaluation. These constraints are technical, not arbitrary — they flow directly from OCAP® Possession requirements and the air-gapped sandbox architecture.
+The `submission.admissibility` fields define what methods are eligible for prize evaluation. These constraints are technical, not arbitrary — they flow directly from the community-possession requirement (the community must be able to hold and run the method itself) and the air-gapped sandbox architecture.
 
 ### Why Coached API Calls Are Inadmissible
 
 A "coached API call" is a method that wraps a third-party API (e.g., GPT-4, Google Translate) with prompt engineering. These are inadmissible for prize evaluation because:
 
-1. **OCAP® Possession violation.** The community cannot possess a method that depends on someone else's API. When the API key expires, the method dies. The community doesn't control the model, its weights, or its availability.
+1. **Community-possession violation.** The community cannot possess a method that depends on someone else's API. When the API key expires, the method dies. The community doesn't control the model, its weights, or its availability.
 
 2. **Non-reproducible.** API providers change models, pricing, and availability without notice. A score measured today may not be reproducible tomorrow.
 
@@ -286,9 +285,9 @@ A "coached API call" is a method that wraps a third-party API (e.g., GPT-4, Goog
 - No license-restricted runtime dependencies that the community cannot independently obtain
 - The community can inspect, modify, and deploy the method on their own hardware
 
-### OCAP® Possession Logic
+### Community-Possession Logic
 
-The OCAP® Possession principle states that the community must physically hold the data and tools. In the context of prize evaluation:
+The community-possession principle states that the community must physically hold the data and tools. In the context of prize evaluation:
 
 | What | Possession requirement |
 |---|---|
@@ -311,15 +310,15 @@ The evaluation sandbox operates with no network access:
 |---|---|---|
 | `selfHostable` | `boolean` | Method must run on community infrastructure without any third-party dependencies. |
 | `inadmissible` | `array` of `string` | Explicitly inadmissible method types. e.g., `"coached-api-calls"`, `"proprietary-api-wrappers"`, `"external-api-dependencies"`. |
-| `notes` | `string \| null` | Rationale for admissibility constraints. Should reference OCAP Possession and the air-gapped sandbox architecture. |
+| `notes` | `string \| null` | Rationale for admissibility constraints. Should reference community possession and the air-gapped sandbox architecture. |
 
 ### `submission.transfer` Fields — What the Community Gets
 
-| Field | Type | Description | OCAP/CARE mapping |
+| Field | Type | Description | Sovereignty/CARE mapping |
 |---|---|---|---|
-| `sourceCode` | `boolean` | Source code ownership transfers to governance org | OCAP Ownership + Possession |
-| `modelWeights` | `boolean` | Trained model weights transfer to governance org | OCAP Possession |
-| `deploymentRights` | `boolean` | Exclusive deployment rights transfer to governance org | OCAP Control |
+| `sourceCode` | `boolean` | Source code ownership transfers to governance org | Community ownership + possession |
+| `modelWeights` | `boolean` | Trained model weights transfer to governance org | Community possession |
+| `deploymentRights` | `boolean` | Exclusive deployment rights transfer to governance org | Community control |
 | `revenueModel` | `string \| null` | Community-set terms for any commercial deployment, held by the governance org (e.g., `"commercial use by written permission only"`) — Champollion is non-commercial and takes no share | CARE Collective Benefit |
 
 ### `submission.retained` Fields — What the Researcher Keeps
@@ -438,7 +437,7 @@ These are documented mistakes to avoid when populating sovereignty fields. Each
 
 ### Don't derive sovereignty from vitality
 
-**Wrong:** Seeing that a language is `"severely-endangered"` in `vitality.unescoStatus` and automatically populating OCAP/CARE frameworks, TK Labels, or consent models.
+**Wrong:** Seeing that a language is `"severely-endangered"` in `vitality.unescoStatus` and automatically populating sovereignty frameworks, TK Labels, or consent models.
 
 **Right:** Sovereignty fields are populated from **documented evidence of governance adoption** — a formal resolution, a published data governance policy, or a registered Local Contexts Hub project. Endangerment status and sovereignty are orthogonal. A safe, widely-spoken language may have strong data sovereignty frameworks (e.g., Māori → Te Mana Raraunga). A critically endangered language may have no formal governance body.
 

package/shared/LANGUAGE-CARD-FIELDS.md

@@ -7,7 +7,7 @@
 
 This document describes every field in the Champollion language card schema. For each field you will find the JSON key, type, data source(s), a plain-English description, and a real value from an existing card.
 
-For the enrichment philosophy, provenance rules, and merge semantics see [DATA-ENRICHMENT.md](../DATA-ENRICHMENT.md).
+For provenance rules and how disagreeing sources are shown, see the [language card specification](https://champollion.dev/docs/reference/language-card-spec).
 For license obligations see [ATTRIBUTION.md](./ATTRIBUTION.md).
 
 ---