@katerynakhar/i18n-keeper 0.16.0

package/src/formats/po-write.ts

@@ -0,0 +1,230 @@
+import { poKey, parsePo } from './po.js';
+import type { Edit, WriteOutcome } from './write.js';
+
+/**
+ * gettext catalogues, edited line by line.
+ *
+ * One deliberate choice beyond preserving the file: a translation written here
+ * is marked `#, fuzzy`. That flag is gettext's own word for "not reviewed by a
+ * person yet", which is exactly what this tool records as `reviewed: false` in
+ * the memory. Leaving it off would claim a human had approved the string.
+ * It does mean `check` then reports those entries as stale — correctly.
+ */
+
+const ESCAPES = new Map<number, string>([
+  [0x0a, '\\n'],
+  [0x0d, '\\r'],
+  [0x09, '\\t'],
+]);
+
+export function poString(value: string): string {
+  let out = '';
+  for (const ch of value) {
+    const cp = ch.codePointAt(0)!;
+    if (ch === '\\' || ch === '"') out += `\\${ch}`;
+    else if (cp < 0x20) out += ESCAPES.get(cp) ?? `\\x${cp.toString(16).padStart(2, '0')}`;
+    else out += ch;
+  }
+  return `"${out}"`;
+}
+
+interface Block {
+  /** Index of the `#,` flag line, if the entry has one. */
+  flagLine: number | null;
+  /** First line of the entry proper, where a flag line can go before it. */
+  headLine: number;
+  /** msgstr index -> the lines it occupies, [start, end). */
+  msgstr: Map<number, { start: number; end: number }>;
+  /** True for the catalogue header, which carries no translation. */
+  headerLike: boolean;
+}
+
+const KEYWORD = /^(msgctxt|msgid_plural|msgid|msgstr)(?:\[(\d+)\])?(.*)$/;
+const EMPTY_STRING = /^\s*""\s*$/;
+
+/**
+ * Lines up each entry in the text with the key the parser resolved for it.
+ *
+ * The parser drops the header entry, so the scanner has to drop it too or every
+ * key lands one entry early — which is exactly the bug this shape prevents.
+ */
+function scan(content: string, file: string): { lines: string[]; blocks: Map<string, Block> } {
+  const lines = content.split(/\r?\n/);
+  const keysInOrder = parsePo(content, file).entries.map(poKey);
+  const blocks = new Map<string, Block>();
+
+  let current: Block | null = null;
+  let msgidEmpty = true;
+  let inMsgid = false;
+  let lastMsgstr: number | null = null;
+  let obsolete = false;
+  let entryIndex = 0;
+
+  const close = (): void => {
+    if (current && current.msgstr.size > 0 && !obsolete) {
+      current.headerLike = msgidEmpty;
+      if (!msgidEmpty) {
+        const key = keysInOrder[entryIndex];
+        if (key !== undefined) blocks.set(key, current);
+        entryIndex++;
+      }
+    }
+    current = null;
+    msgidEmpty = true;
+    inMsgid = false;
+    lastMsgstr = null;
+    obsolete = false;
+  };
+
+  for (const [index, raw] of lines.entries()) {
+    const line = raw.trim();
+
+    if (line === '') {
+      close();
+      continue;
+    }
+    if (line.startsWith('#~')) {
+      obsolete = true;
+      continue;
+    }
+    if (line.startsWith('#')) {
+      if (line.startsWith('#,')) {
+        current ??= { flagLine: null, headLine: index, msgstr: new Map(), headerLike: false };
+        current.flagLine = index;
+      }
+      continue;
+    }
+
+    const match = KEYWORD.exec(line);
+    if (match) {
+      if (!current) current = { flagLine: null, headLine: index, msgstr: new Map(), headerLike: false };
+      else if (current.msgstr.size === 0 && current.flagLine === null) {
+        current.headLine = Math.min(current.headLine, index);
+      }
+
+      if (match[1] === 'msgstr') {
+        const at = match[2] === undefined ? 0 : Number.parseInt(match[2], 10);
+        current.msgstr.set(at, { start: index, end: index + 1 });
+        lastMsgstr = at;
+        inMsgid = false;
+      } else {
+        if (match[1] === 'msgid') {
+          inMsgid = true;
+          msgidEmpty = EMPTY_STRING.test(match[3] ?? '');
+        } else {
+          inMsgid = false;
+        }
+        lastMsgstr = null;
+      }
+      continue;
+    }
+
+    if (line.startsWith('"') && current) {
+      if (lastMsgstr !== null) {
+        current.msgstr.get(lastMsgstr)!.end = index + 1;
+      } else if (inMsgid && !EMPTY_STRING.test(line)) {
+        // A msgid spelled across continuation lines is not the empty header.
+        msgidEmpty = false;
+      }
+    }
+  }
+  close();
+
+  return { lines, blocks };
+}
+
+function splitForm(key: string): { base: string; form: number | null } {
+  const match = /^(.*)\.(\d+)$/.exec(key);
+  return match
+    ? { base: match[1]!, form: Number.parseInt(match[2]!, 10) }
+    : { base: key, form: null };
+}
+
+/**
+ * A new entry has to be spelled back out as msgctxt + msgid. The key format
+ * joins them with `|`, which a msgid could itself contain; splitting at the
+ * first one matches how the key was built and is wrong only for a contextless
+ * msgid that happens to contain a pipe.
+ */
+function appendEntry(key: string, value: string): string[] {
+  const pipe = key.indexOf('|');
+  const head =
+    pipe > 0
+      ? [`msgctxt ${poString(key.slice(0, pipe))}`, `msgid ${poString(key.slice(pipe + 1))}`]
+      : [`msgid ${poString(key)}`];
+  return ['', '#, fuzzy', ...head, `msgstr ${poString(value)}`];
+}
+
+export function writePoLocale(
+  content: string,
+  file: string,
+  _locale: string,
+  edits: Edit[],
+): WriteOutcome {
+  const { lines, blocks } = scan(content, file);
+  const skipped: Array<{ key: string; reason: string }> = [];
+
+  // line index -> replacement lines, or null to drop the line.
+  const replaced = new Map<number, string[] | null>();
+  const needFuzzy = new Set<Block>();
+  const appended: string[] = [];
+
+  for (const edit of edits) {
+    let block = blocks.get(edit.key);
+    let form = 0;
+
+    if (!block) {
+      const split = splitForm(edit.key);
+      if (split.form !== null) {
+        const candidate = blocks.get(split.base);
+        if (candidate) {
+          block = candidate;
+          form = split.form;
+        }
+      }
+    }
+
+    if (!block) {
+      appended.push(...appendEntry(edit.key, edit.value));
+      continue;
+    }
+
+    const span = block.msgstr.get(form);
+    if (!span) {
+      skipped.push({ key: edit.key, reason: `the entry has no msgstr[${form}]` });
+      continue;
+    }
+
+    const label = block.msgstr.size > 1 ? `msgstr[${form}]` : 'msgstr';
+    replaced.set(span.start, [`${label} ${poString(edit.value)}`]);
+    for (let line = span.start + 1; line < span.end; line++) replaced.set(line, null);
+    needFuzzy.add(block);
+  }
+
+  for (const block of needFuzzy) {
+    if (block.flagLine !== null) {
+      const flags = lines[block.flagLine]!;
+      if (!/\bfuzzy\b/.test(flags)) replaced.set(block.flagLine, [`${flags}, fuzzy`]);
+      continue;
+    }
+    const head: string[] = replaced.get(block.headLine) ?? [lines[block.headLine]!];
+    replaced.set(block.headLine, ['#, fuzzy', ...head]);
+  }
+
+  const out: string[] = [];
+  for (const [index, line] of lines.entries()) {
+    const replacement = replaced.get(index);
+    if (replacement === undefined) out.push(line);
+    else if (replacement !== null) out.push(...replacement);
+  }
+
+  if (appended.length > 0) {
+    while (out.length > 0 && out[out.length - 1] === '') out.pop();
+    out.push(...appended, '');
+  }
+
+  // A catalogue checked out on Windows may use CRLF; writing back with bare
+  // newlines would leave the file mixed.
+  const newline = content.includes('\r\n') ? '\r\n' : '\n';
+  return { content: out.join(newline), skipped };
+}

package/src/formats/po.ts

@@ -0,0 +1,233 @@
+import { readFileSync } from 'node:fs';
+import type { ReadTarget } from '../types.js';
+import { FormatError } from './error.js';
+
+/**
+ * gettext .po / .pot.
+ *
+ * Two things make this format different from the others: the msgid *is* the
+ * source text, so a .pot with empty translations is a usable source locale; and
+ * the format already has a notion of an outdated translation — the fuzzy flag —
+ * which maps onto the same rule the translation memory feeds.
+ */
+
+export class PoParseError extends FormatError {
+  constructor(file: string, line: number, message: string) {
+    super(file, message, line);
+  }
+}
+
+export interface PoEntry {
+  context: string | null;
+  msgid: string;
+  msgidPlural: string | null;
+  msgstr: string[];
+  fuzzy: boolean;
+  line: number;
+}
+
+export interface PoFile {
+  entries: PoEntry[];
+  /** From the header's Plural-Forms, when present. */
+  nplurals: number | null;
+}
+
+const SIMPLE_ESCAPES: Record<string, string> = {
+  n: '\n',
+  t: '\t',
+  r: '\r',
+  a: '\x07',
+  b: '\b',
+  f: '\f',
+  v: '\v',
+  '"': '"',
+  '\\': '\\',
+};
+
+function unescape(raw: string): string {
+  let out = '';
+  for (let i = 0; i < raw.length; i++) {
+    const ch = raw[i]!;
+    if (ch !== '\\') {
+      out += ch;
+      continue;
+    }
+    const next = raw[i + 1];
+    if (next === undefined) {
+      out += ch;
+      continue;
+    }
+    if (next in SIMPLE_ESCAPES) {
+      out += SIMPLE_ESCAPES[next]!;
+      i++;
+      continue;
+    }
+    if (next === 'x') {
+      const hex = /^[0-9A-Fa-f]{1,2}/.exec(raw.slice(i + 2));
+      if (hex) {
+        out += String.fromCharCode(Number.parseInt(hex[0], 16));
+        i += 1 + hex[0].length;
+        continue;
+      }
+    }
+    const octal = /^[0-7]{1,3}/.exec(raw.slice(i + 1));
+    if (octal) {
+      out += String.fromCharCode(Number.parseInt(octal[0], 8) & 0xff);
+      i += octal[0].length;
+      continue;
+    }
+    out += next;
+    i++;
+  }
+  return out;
+}
+
+/** A quoted chunk, as it appears after a keyword or on a continuation line. */
+function readQuoted(file: string, line: number, text: string): string {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith('"') || !trimmed.endsWith('"') || trimmed.length < 2) {
+    throw new PoParseError(file, line, 'Expected a quoted string');
+  }
+  return unescape(trimmed.slice(1, -1));
+}
+
+const KEYWORD = /^(msgctxt|msgid_plural|msgid|msgstr)(\[(\d+)\])?\s*(.*)$/;
+
+export function parsePo(text: string, file: string): PoFile {
+  const entries: PoEntry[] = [];
+  const lines = text.split(/\r?\n/);
+
+  // Assignments stay in the loop body rather than in closures, so the compiler
+  // can follow whether an entry is open.
+  let current: PoEntry | null = null;
+  let field: 'msgctxt' | 'msgid' | 'msgid_plural' | null = null;
+  let msgstrIndex: number | null = null;
+  let fuzzy = false;
+  let seenKeyword = false;
+
+  for (const [index, rawLine] of lines.entries()) {
+    const lineNo = index + 1;
+    const line = rawLine.trim();
+
+    // A blank line ends an entry; #~ marks one already removed from the
+    // catalogue, so its block is dropped rather than parsed.
+    if (line === '' || line.startsWith('#~')) {
+      if (current && seenKeyword) entries.push(current);
+      current = null;
+      field = null;
+      msgstrIndex = null;
+      fuzzy = false;
+      seenKeyword = false;
+      continue;
+    }
+
+    if (line.startsWith('#')) {
+      if (line.startsWith('#,')) {
+        const flags = line
+          .slice(2)
+          .split(',')
+          .map((f) => f.trim());
+        if (flags.includes('fuzzy')) {
+          fuzzy = true;
+          if (current) current.fuzzy = true;
+        }
+      }
+      continue;
+    }
+
+    const match = KEYWORD.exec(line);
+    if (match) {
+      if (!current) {
+        current = { context: null, msgid: '', msgidPlural: null, msgstr: [], fuzzy, line: lineNo };
+      }
+      const entry = current;
+      const keyword = match[1]!;
+      seenKeyword = true;
+      const value = readQuoted(file, lineNo, match[4] || '""');
+
+      if (keyword === 'msgstr') {
+        const at = match[3] === undefined ? 0 : Number.parseInt(match[3], 10);
+        entry.msgstr[at] = value;
+        msgstrIndex = at;
+        field = null;
+      } else if (keyword === 'msgctxt') {
+        entry.context = value;
+        field = 'msgctxt';
+        msgstrIndex = null;
+      } else if (keyword === 'msgid') {
+        entry.msgid = value;
+        field = 'msgid';
+        msgstrIndex = null;
+      } else {
+        entry.msgidPlural = value;
+        field = 'msgid_plural';
+        msgstrIndex = null;
+      }
+      continue;
+    }
+
+    if (line.startsWith('"')) {
+      if (!current) throw new PoParseError(file, lineNo, 'Continuation without a preceding keyword');
+      const entry = current;
+      const value = readQuoted(file, lineNo, line);
+      if (msgstrIndex !== null) entry.msgstr[msgstrIndex] = (entry.msgstr[msgstrIndex] ?? '') + value;
+      else if (field === 'msgctxt') entry.context = (entry.context ?? '') + value;
+      else if (field === 'msgid') entry.msgid += value;
+      else if (field === 'msgid_plural') entry.msgidPlural = (entry.msgidPlural ?? '') + value;
+      else throw new PoParseError(file, lineNo, 'Continuation without a preceding keyword');
+      continue;
+    }
+
+    throw new PoParseError(file, lineNo, `Unexpected line: ${line.slice(0, 40)}`);
+  }
+
+  if (current && seenKeyword) entries.push(current);
+
+  let nplurals: number | null = null;
+  const header = entries.find((e) => e.msgid === '' && e.context === null);
+  if (header) {
+    const declared = /nplurals\s*=\s*(\d+)/.exec(header.msgstr[0] ?? '');
+    if (declared) nplurals = Number.parseInt(declared[1]!, 10);
+  }
+
+  return { entries: entries.filter((e) => e.msgid !== '' || e.context !== null), nplurals };
+}
+
+/** msgctxt disambiguates an otherwise identical msgid. */
+export function poKey(entry: PoEntry): string {
+  return entry.context === null ? entry.msgid : `${entry.context}|${entry.msgid}`;
+}
+
+export function readPoLocale(
+  file: string,
+  namespace: string,
+  _locale: string,
+  isSource: boolean,
+  target: ReadTarget,
+): void {
+  const parsed = parsePo(readFileSync(file, 'utf8'), file);
+  if (parsed.nplurals !== null) target.nplurals = parsed.nplurals;
+
+  for (const entry of parsed.entries) {
+    const base = namespace ? `${namespace}.${poKey(entry)}` : poKey(entry);
+
+    if (entry.msgidPlural === null) {
+      // The msgid is the source text by definition, whatever this catalogue
+      // has translated it to.
+      const value = isSource ? entry.msgid : (entry.msgstr[0] ?? '');
+      target.leaves.set(base, { value, kind: 'string', file, fuzzy: entry.fuzzy });
+      continue;
+    }
+
+    const sourceForms = [entry.msgid, entry.msgidPlural];
+    const count = Math.max(entry.msgstr.length, isSource ? sourceForms.length : 1);
+    target.plurals.set(base, entry.msgstr.length);
+    target.containers.add(base);
+
+    for (let i = 0; i < count; i++) {
+      const translated = entry.msgstr[i] ?? '';
+      const value = isSource ? sourceForms[Math.min(i, 1)]! : translated;
+      target.leaves.set(`${base}.${i}`, { value, kind: 'string', file, fuzzy: entry.fuzzy });
+    }
+  }
+}

package/src/formats/write.ts

@@ -0,0 +1,41 @@
+/**
+ * Writing back into a locale file.
+ *
+ * Reading throws away everything that is not a key or a value: comments,
+ * quote styles, blank lines, anchors, translator notes. Writing must not, so
+ * none of these writers re-serialise a parsed tree. They edit the text in
+ * place — replacing the exact span of one value, or inserting one entry —
+ * and leave every byte they did not have a reason to touch.
+ */
+
+export interface Edit {
+  /** Key relative to this file, with any file-level namespace already removed. */
+  key: string;
+  value: string;
+}
+
+export interface WriteOutcome {
+  content: string;
+  /** Edits the format could not take, and why. */
+  skipped: Array<{ key: string; reason: string }>;
+}
+
+export type LocaleWriter = (
+  content: string,
+  file: string,
+  locale: string,
+  edits: Edit[],
+) => WriteOutcome;
+
+/** Applies text edits from the end backwards, so earlier offsets stay valid. */
+export function spliceAll(
+  content: string,
+  edits: Array<{ start: number; end: number; text: string }>,
+): string {
+  const ordered = [...edits].sort((a, b) => b.start - a.start);
+  let out = content;
+  for (const edit of ordered) {
+    out = out.slice(0, edit.start) + edit.text + out.slice(edit.end);
+  }
+  return out;
+}

package/src/formats/yaml-write.ts

@@ -0,0 +1,53 @@
+import { isAlias, isScalar, parseDocument } from 'yaml';
+import { FormatError } from './error.js';
+import { localeRootKey } from './yaml.js';
+import type { Edit, WriteOutcome } from './write.js';
+
+/**
+ * YAML is the one format edited through a document model rather than by
+ * splicing text, because the library keeps comments, anchors and node styles
+ * across a round trip and a hand-rolled splicer would not.
+ *
+ * Two things are still done by hand. An existing scalar is mutated in place
+ * instead of replaced, so a block scalar stays a block scalar and keeps its
+ * comments. And an alias is refused outright: writing through `*shared` would
+ * silently break every other key that shares the anchor.
+ */
+export function writeYamlLocale(
+  content: string,
+  file: string,
+  locale: string,
+  edits: Edit[],
+): WriteOutcome {
+  const doc = parseDocument(content);
+  if (doc.errors.length > 0) {
+    throw new FormatError(file, doc.errors[0]!.message);
+  }
+
+  const root = localeRootKey(doc.toJS(), locale);
+  const skipped: Array<{ key: string; reason: string }> = [];
+
+  for (const edit of edits) {
+    const path = (root ? [root] : []).concat(edit.key.split('.'));
+    const existing = doc.getIn(path, true);
+
+    if (isAlias(existing)) {
+      skipped.push({ key: edit.key, reason: 'the value is a YAML alias, shared with other keys' });
+      continue;
+    }
+    if (isScalar(existing)) {
+      // Keeps the node's style — block scalars stay block scalars.
+      existing.value = edit.value;
+      continue;
+    }
+    if (existing !== undefined && existing !== null) {
+      skipped.push({ key: edit.key, reason: 'the key holds a collection, not a string' });
+      continue;
+    }
+
+    doc.setIn(path, edit.value);
+  }
+
+  // lineWidth 0 disables re-wrapping, so untouched long lines stay as written.
+  return { content: doc.toString({ lineWidth: 0 }), skipped };
+}

package/src/formats/yaml.ts

@@ -0,0 +1,62 @@
+import { readFileSync } from 'node:fs';
+import { parse } from 'yaml';
+import type { ReadTarget } from '../types.js';
+import { FormatError } from './error.js';
+import { flattenValue } from './flatten.js';
+
+/**
+ * Rails and Symfony locale files.
+ *
+ * Unlike the PHP parser, this one delegates: reading YAML is not a security
+ * question, and the spec is deep enough — anchors, block scalars, implicit
+ * typing — that a hand-rolled subset would quietly misread real files.
+ */
+
+export class YamlParseError extends FormatError {}
+
+function normaliseLocale(value: string): string {
+  return value.toLowerCase().replace(/_/g, '-');
+}
+
+/**
+ * Rails nests the whole file under its locale code. Left in place, every key in
+ * en.yml would differ from every key in fr.yml.
+ */
+export function localeRootKey(parsed: unknown, locale: string): string | null {
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return null;
+
+  const keys = Object.keys(parsed as Record<string, unknown>);
+  if (keys.length !== 1) return null;
+
+  const root = keys[0]!;
+  return normaliseLocale(root) === normaliseLocale(locale) ? root : null;
+}
+
+export function stripLocaleRoot(parsed: unknown, locale: string): unknown {
+  const root = localeRootKey(parsed, locale);
+  return root === null ? parsed : (parsed as Record<string, unknown>)[root];
+}
+
+export function readYamlLocale(
+  file: string,
+  namespace: string,
+  locale: string,
+  _isSource: boolean,
+  target: ReadTarget,
+): void {
+  let parsed: unknown;
+  try {
+    parsed = parse(readFileSync(file, 'utf8'));
+  } catch (err) {
+    throw new YamlParseError(file, err instanceof Error ? err.message : String(err));
+  }
+
+  if (parsed === null || parsed === undefined) return; // an empty document
+
+  const inner = stripLocaleRoot(parsed, locale);
+  if (Array.isArray(inner)) {
+    target.skipped.push({ file, reason: 'the file holds a list, not keyed translations' });
+    return;
+  }
+  flattenValue(inner, namespace, file, target.leaves, target.containers);
+}