@lingo.dev/spec 1.0.0 → 1.0.2

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "@lingo.dev/spec",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "type": "module",
-  "main": "./dist/index.js",
   "exports": {
     ".": {
+      "typescript": {
+        "types": "./src/index.ts",
+        "default": "./src/index.ts"
+      },
       "import": {
         "types": "./dist/index.d.ts",
         "default": "./dist/index.js"
@@ -16,7 +19,8 @@
     }
   },
   "files": [
-    "dist"
+    "dist",
+    "src"
   ],
   "sideEffects": false,
   "dependencies": {

package/src/hash.test.ts

@@ -0,0 +1,96 @@
+import { describe, it, expect } from "vitest";
+import { computeKey } from "./hash";
+
+describe("computeKey", () => {
+  it("returns an 8-character string", () => {
+    expect(computeKey("Hello")).toHaveLength(8);
+  });
+
+  it("is deterministic - same input produces same output", () => {
+    expect(computeKey("Save")).toBe(computeKey("Save"));
+    expect(computeKey("Save", "Form button")).toBe(computeKey("Save", "Form button"));
+  });
+
+  it("produces different keys for different source text", () => {
+    expect(computeKey("Save")).not.toBe(computeKey("Cancel"));
+  });
+
+  it("produces different keys for same text with different context", () => {
+    const withoutContext = computeKey("Save");
+    const withContext = computeKey("Save", "Form button");
+    const withOtherContext = computeKey("Save", "Toolbar action");
+
+    expect(withoutContext).not.toBe(withContext);
+    expect(withContext).not.toBe(withOtherContext);
+    expect(withoutContext).not.toBe(withOtherContext);
+  });
+
+  it("uses only base62 characters", () => {
+    const key = computeKey("Hello, world!");
+    expect(key).toMatch(/^[0-9A-Za-z]+$/);
+  });
+
+  it("handles empty string", () => {
+    const key = computeKey("");
+    expect(key).toHaveLength(8);
+    expect(key).toMatch(/^[0-9A-Za-z]+$/);
+  });
+
+  it("handles unicode text", () => {
+    const key = computeKey("こんにちは");
+    expect(key).toHaveLength(8);
+    expect(key).toMatch(/^[0-9A-Za-z]+$/);
+  });
+
+  it("handles long strings", () => {
+    const key = computeKey("a".repeat(10_000));
+    expect(key).toHaveLength(8);
+    expect(key).toMatch(/^[0-9A-Za-z]+$/);
+  });
+
+  it("treats empty context same as no context", () => {
+    expect(computeKey("Save", "")).toBe(computeKey("Save"));
+    expect(computeKey("Save", undefined)).toBe(computeKey("Save"));
+  });
+
+  it("NFC-normalizes input - composed and decomposed forms produce the same key", () => {
+    // e + combining acute accent (U+0065 U+0301) vs precomposed e-acute (U+00E9)
+    const decomposed = "re\u0301sume\u0301"; // r + e + ́ + s + u + m + e + ́
+    const composed = "r\u00E9sum\u00E9"; // résumé (precomposed)
+    expect(computeKey(decomposed)).toBe(computeKey(composed));
+  });
+
+  it("NFC-normalizes context too", () => {
+    const decomposed = "cafe\u0301"; // cafe + combining acute
+    const composed = "caf\u00E9"; // café (precomposed)
+    expect(computeKey("Save", decomposed)).toBe(computeKey("Save", composed));
+  });
+
+  it("handles emoji and surrogate pairs", () => {
+    const key = computeKey("Hello 👋 World 🌍");
+    expect(key).toHaveLength(8);
+    expect(key).toMatch(/^[0-9A-Za-z]+$/);
+  });
+
+  it("handles strings with null bytes", () => {
+    const key = computeKey("hello\0world");
+    expect(key).toHaveLength(8);
+    expect(key).not.toBe(computeKey("hello"));
+    expect(key).not.toBe(computeKey("world"));
+  });
+
+  it("produces stable reference vectors for cross-platform verification", () => {
+    // These values anchor the algorithm. Any change to hashing, normalization,
+    // or encoding MUST update these vectors across all platform implementations.
+    const vectors: Record<string, string> = {
+      "": computeKey(""),
+      Hello: computeKey("Hello"),
+      "Hello, world!": computeKey("Hello, world!"),
+    };
+
+    // Snapshot: values must never change once published
+    expect(vectors[""]).toMatchInlineSnapshot(`"00000000"`);
+    expect(vectors["Hello"]).toMatchInlineSnapshot(`"TT6OR9Mc"`);
+    expect(vectors["Hello, world!"]).toMatchInlineSnapshot(`"D3IiJAcZ"`);
+  });
+});

package/src/hash.ts

@@ -0,0 +1,195 @@
+/**
+ * Deterministic key generation from (source text, context) pairs.
+ * Shared between runtime (message lookup) and build tools (extraction, JSONC generation).
+ *
+ * Uses MurmurHash3 x86_128 on NFC-normalized UTF-8 bytes.
+ * Outputs an 8-character base62 string (e.g., "aB3dEf9x").
+ *
+ * Cross-platform deterministic: JS, Rust, Go, Python produce identical keys
+ * when given the same (source, context) pair.
+ *
+ * 1% collision probability at ~2.4 million messages (48-bit effective entropy).
+ */
+
+const BASE62_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+const KEY_LENGTH = 8;
+const BASE62_SPACE = 62 ** KEY_LENGTH;
+
+const encoder = new TextEncoder();
+
+// --- MurmurHash3 x86_128 (reference: github.com/aappleby/smhasher) ---
+
+const C1 = 0x239b961b;
+const C2 = 0xab0e9789;
+const C3 = 0x38b34ae5;
+const C4 = 0xa1e38b93;
+
+function rotl32(x: number, r: number): number {
+  return ((x << r) | (x >>> (32 - r))) >>> 0;
+}
+
+function fmix32(h: number): number {
+  h ^= h >>> 16;
+  h = Math.imul(h, 0x85ebca6b) >>> 0;
+  h ^= h >>> 13;
+  h = Math.imul(h, 0xc2b2ae35) >>> 0;
+  h ^= h >>> 16;
+  return h >>> 0;
+}
+
+function readU32LE(bytes: Uint8Array, i: number): number {
+  return (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) >>> 0;
+}
+
+function murmurhash3_x86_128(bytes: Uint8Array): [number, number, number, number] {
+  const len = bytes.length;
+  const nblocks = len >>> 4;
+
+  let h1 = 0;
+  let h2 = 0;
+  let h3 = 0;
+  let h4 = 0;
+
+  for (let i = 0; i < nblocks; i++) {
+    const off = i << 4;
+    let k1 = readU32LE(bytes, off);
+    let k2 = readU32LE(bytes, off + 4);
+    let k3 = readU32LE(bytes, off + 8);
+    let k4 = readU32LE(bytes, off + 12);
+
+    k1 = Math.imul(k1, C1) >>> 0;
+    k1 = rotl32(k1, 15);
+    k1 = Math.imul(k1, C2) >>> 0;
+    h1 ^= k1;
+    h1 = rotl32(h1, 19);
+    h1 = (h1 + h2) >>> 0;
+    h1 = (Math.imul(h1, 5) + 0x561ccd1b) >>> 0;
+
+    k2 = Math.imul(k2, C2) >>> 0;
+    k2 = rotl32(k2, 16);
+    k2 = Math.imul(k2, C3) >>> 0;
+    h2 ^= k2;
+    h2 = rotl32(h2, 17);
+    h2 = (h2 + h3) >>> 0;
+    h2 = (Math.imul(h2, 5) + 0x0bcaa747) >>> 0;
+
+    k3 = Math.imul(k3, C3) >>> 0;
+    k3 = rotl32(k3, 17);
+    k3 = Math.imul(k3, C4) >>> 0;
+    h3 ^= k3;
+    h3 = rotl32(h3, 15);
+    h3 = (h3 + h4) >>> 0;
+    h3 = (Math.imul(h3, 5) + 0x96cd1c35) >>> 0;
+
+    k4 = Math.imul(k4, C4) >>> 0;
+    k4 = rotl32(k4, 18);
+    k4 = Math.imul(k4, C1) >>> 0;
+    h4 ^= k4;
+    h4 = rotl32(h4, 13);
+    h4 = (h4 + h1) >>> 0;
+    h4 = (Math.imul(h4, 5) + 0x32ac3b17) >>> 0;
+  }
+
+  const tail = nblocks << 4;
+  let k1 = 0;
+  let k2 = 0;
+  let k3 = 0;
+  let k4 = 0;
+  const rem = len & 15;
+
+  if (rem >= 15) k4 ^= bytes[tail + 14] << 16;
+  if (rem >= 14) k4 ^= bytes[tail + 13] << 8;
+  if (rem >= 13) {
+    k4 ^= bytes[tail + 12];
+    k4 = Math.imul(k4, C4) >>> 0;
+    k4 = rotl32(k4, 18);
+    k4 = Math.imul(k4, C1) >>> 0;
+    h4 ^= k4;
+  }
+  if (rem >= 12) k3 ^= bytes[tail + 11] << 24;
+  if (rem >= 11) k3 ^= bytes[tail + 10] << 16;
+  if (rem >= 10) k3 ^= bytes[tail + 9] << 8;
+  if (rem >= 9) {
+    k3 ^= bytes[tail + 8];
+    k3 = Math.imul(k3, C3) >>> 0;
+    k3 = rotl32(k3, 17);
+    k3 = Math.imul(k3, C4) >>> 0;
+    h3 ^= k3;
+  }
+  if (rem >= 8) k2 ^= bytes[tail + 7] << 24;
+  if (rem >= 7) k2 ^= bytes[tail + 6] << 16;
+  if (rem >= 6) k2 ^= bytes[tail + 5] << 8;
+  if (rem >= 5) {
+    k2 ^= bytes[tail + 4];
+    k2 = Math.imul(k2, C2) >>> 0;
+    k2 = rotl32(k2, 16);
+    k2 = Math.imul(k2, C3) >>> 0;
+    h2 ^= k2;
+  }
+  if (rem >= 4) k1 ^= bytes[tail + 3] << 24;
+  if (rem >= 3) k1 ^= bytes[tail + 2] << 16;
+  if (rem >= 2) k1 ^= bytes[tail + 1] << 8;
+  if (rem >= 1) {
+    k1 ^= bytes[tail];
+    k1 = Math.imul(k1, C1) >>> 0;
+    k1 = rotl32(k1, 15);
+    k1 = Math.imul(k1, C2) >>> 0;
+    h1 ^= k1;
+  }
+
+  h1 ^= len;
+  h2 ^= len;
+  h3 ^= len;
+  h4 ^= len;
+
+  h1 = (h1 + h2) >>> 0;
+  h1 = (h1 + h3) >>> 0;
+  h1 = (h1 + h4) >>> 0;
+  h2 = (h2 + h1) >>> 0;
+  h3 = (h3 + h1) >>> 0;
+  h4 = (h4 + h1) >>> 0;
+
+  h1 = fmix32(h1);
+  h2 = fmix32(h2);
+  h3 = fmix32(h3);
+  h4 = fmix32(h4);
+
+  h1 = (h1 + h2) >>> 0;
+  h1 = (h1 + h3) >>> 0;
+  h1 = (h1 + h4) >>> 0;
+  h2 = (h2 + h1) >>> 0;
+  h3 = (h3 + h1) >>> 0;
+  h4 = (h4 + h1) >>> 0;
+
+  return [h1, h2, h3, h4];
+}
+
+// --- Key generation ---
+
+function toBase62(num: number, length: number): string {
+  let result = "";
+  let remaining = num;
+  for (let i = 0; i < length; i++) {
+    result = BASE62_CHARS[remaining % 62] + result;
+    remaining = Math.floor(remaining / 62);
+  }
+  return result;
+}
+
+/**
+ * Computes a deterministic short key from source text and optional context.
+ *
+ * Same source + same context = same key (deterministic).
+ * Same source + different context = different key (disambiguation).
+ */
+export function computeKey(source: string, context?: string): string {
+  const input = context != null && context !== "" ? `${source}\0${context}` : source;
+  const bytes = encoder.encode(input.normalize("NFC"));
+  const [h1, h2] = murmurhash3_x86_128(bytes);
+
+  // 48-bit value: h1 (32 bits) + upper 16 bits of h2
+  // Safe for JS Number arithmetic (well within 2^53)
+  const value = (h1 >>> 0) + (h2 >>> 16) * 0x100000000;
+
+  return toBase62(value % BASE62_SPACE, KEY_LENGTH);
+}

package/src/icu.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, expect } from "vitest";
+import { buildIcuPlural, buildIcuSelect } from "./icu";
+
+describe("buildIcuPlural", () => {
+  it("builds ICU expression with one/other", () => {
+    expect(buildIcuPlural({ one: "# item", other: "# items" })).toBe("{count, plural, one {# item} other {# items}}");
+  });
+
+  it("orders categories by CLDR convention", () => {
+    // Even if passed out of order, output follows CLDR: zero, one, two, few, many, other
+    const result = buildIcuPlural({ other: "# things", zero: "no things", one: "# thing" });
+    expect(result).toBe("{count, plural, zero {no things} one {# thing} other {# things}}");
+  });
+
+  it("handles other-only", () => {
+    expect(buildIcuPlural({ other: "# items" })).toBe("{count, plural, other {# items}}");
+  });
+
+  it("handles all six categories", () => {
+    const forms = { zero: "0", one: "1", two: "2", few: "few", many: "many", other: "other" };
+    expect(buildIcuPlural(forms)).toBe("{count, plural, zero {0} one {1} two {2} few {few} many {many} other {other}}");
+  });
+});
+
+describe("buildIcuSelect", () => {
+  it("builds ICU expression sorted alphabetically", () => {
+    expect(buildIcuSelect({ admin: "Admin", other: "Home", user: "Dashboard" })).toBe(
+      "{value, select, admin {Admin} other {Home} user {Dashboard}}",
+    );
+  });
+
+  it("sorts keys regardless of insertion order", () => {
+    const a = buildIcuSelect({ user: "U", admin: "A", other: "O" });
+    const b = buildIcuSelect({ admin: "A", other: "O", user: "U" });
+    expect(a).toBe(b);
+  });
+
+  it("handles other-only", () => {
+    expect(buildIcuSelect({ other: "Default" })).toBe("{value, select, other {Default}}");
+  });
+});

package/src/icu.ts

@@ -0,0 +1,24 @@
+/**
+ * Canonical ICU MessageFormat expression builders.
+ * Shared between runtime (l.plural/l.select) and build tools (extraction).
+ *
+ * These must produce identical output everywhere so that computeKey()
+ * generates matching hash keys at build time and runtime.
+ */
+
+const CLDR_PLURAL_ORDER = ["zero", "one", "two", "few", "many", "other"];
+
+export function buildIcuPlural(forms: Record<string, string>): string {
+  const parts = CLDR_PLURAL_ORDER.filter((cat) => forms[cat] !== undefined)
+    .map((cat) => `${cat} {${forms[cat]!}}`)
+    .join(" ");
+  return `{count, plural, ${parts}}`;
+}
+
+export function buildIcuSelect(forms: Record<string, string>): string {
+  const parts = Object.keys(forms)
+    .sort()
+    .map((key) => `${key} {${forms[key]}}`)
+    .join(" ");
+  return `{value, select, ${parts}}`;
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export { computeKey } from "./hash";
+export { buildIcuPlural, buildIcuSelect } from "./icu";
+export { getActiveEntries, readLocaleFile, type EntryMetadata, type LocaleEntry, type LocaleFile } from "./jsonc";

package/src/jsonc.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from "vitest";
+import { readLocaleFile } from "./jsonc";
+
+describe("readLocaleFile", () => {
+  it("reads empty object", () => {
+    expect(readLocaleFile("{}")).toEqual({ entries: [] });
+  });
+
+  it("reads simple key-value pairs", () => {
+    const result = readLocaleFile('{ "abc": "Hello" }');
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries[0]).toEqual({ key: "abc", value: "Hello", metadata: {} });
+  });
+
+  it("extracts @context and @src from block comments", () => {
+    const jsonc = `{
+  /*
+   * @context Hero heading
+   * @src app/hero.tsx:12
+   */
+  "mK9xqZ": "Welcome"
+}`;
+    const result = readLocaleFile(jsonc);
+    expect(result.entries[0].metadata).toEqual({
+      context: "Hero heading",
+      src: "app/hero.tsx:12",
+    });
+  });
+
+  it("detects @orphan marker", () => {
+    const jsonc = `{
+  /*
+   * @orphan
+   */
+  "old": "Removed"
+}`;
+    const result = readLocaleFile(jsonc);
+    expect(result.entries[0].metadata.orphan).toBe(true);
+  });
+
+  it("handles trailing commas", () => {
+    const jsonc = '{ "a": "one", "b": "two", }';
+    const result = readLocaleFile(jsonc);
+    expect(result.entries).toHaveLength(2);
+  });
+
+  it("throws on invalid JSONC", () => {
+    expect(() => readLocaleFile("{ broken }")).toThrow(/Failed to parse JSONC/);
+  });
+
+  it("reads multiple entries with metadata", () => {
+    const jsonc = `{
+  /*
+   * @context Form button
+   * @src checkout.tsx:42
+   */
+  "aB3dEf": "Save",
+
+  /*
+   * @src hero.tsx:12
+   */
+  "mK9xqZ": "Welcome"
+}`;
+    const result = readLocaleFile(jsonc);
+    expect(result.entries).toHaveLength(2);
+    expect(result.entries[0].metadata.context).toBe("Form button");
+    expect(result.entries[1].metadata.src).toBe("hero.tsx:12");
+  });
+});

package/src/jsonc.ts

@@ -0,0 +1,125 @@
+/**
+ * JSONC locale file reader with structured metadata comments.
+ * Lives in @lingo.dev/spec so framework adapters can parse locale files
+ * without depending on @lingo.dev/cli.
+ *
+ * File format:
+ * ```jsonc
+ * {
+ *   /*
+ *    * @context Hero heading
+ *    * @src app/hero.tsx:12
+ *    *​/
+ *   "mK9xqZ": "Welcome to Acme"
+ * }
+ * ```
+ */
+
+import { parse as parseJsonc, type ParseError, printParseErrorCode } from "jsonc-parser";
+
+// --- Types (shared between reader in spec and writer in cli) ---
+
+export type EntryMetadata = {
+  context?: string;
+  src?: string;
+  orphan?: boolean;
+};
+
+export type LocaleEntry = {
+  key: string;
+  value: string;
+  metadata: EntryMetadata;
+};
+
+export type LocaleFile = {
+  entries: LocaleEntry[];
+};
+
+// --- Helpers ---
+
+/** Filters out orphaned entries, returning only active (non-orphaned) ones. */
+export function getActiveEntries(entries: LocaleEntry[]): LocaleEntry[] {
+  return entries.filter((e) => !e.metadata.orphan);
+}
+
+// --- Reader ---
+
+const METADATA_PATTERN = /@(\w+)(?:\s+(.*))?/;
+
+function parseMetadataBlock(comment: string): EntryMetadata {
+  const metadata: EntryMetadata = {};
+  for (const line of comment.split("\n")) {
+    const match = line.match(METADATA_PATTERN);
+    if (!match) continue;
+    const [, tag, value] = match;
+    if (tag === "context" && value) metadata.context = value.trim();
+    else if (tag === "src" && value) metadata.src = value.trim();
+    else if (tag === "orphan") metadata.orphan = true;
+  }
+  return metadata;
+}
+
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Scans JSONC source to associate block comments with the key that follows them.
+ */
+function extractCommentsForKeys(content: string, keys: string[]): Map<string, EntryMetadata> {
+  const result = new Map<string, EntryMetadata>();
+  const blockCommentPattern = /\/\*[\s\S]*?\*\//g;
+
+  const comments: { end: number; text: string }[] = [];
+  let match: RegExpExecArray | null;
+  while ((match = blockCommentPattern.exec(content)) !== null) {
+    comments.push({ end: match.index + match[0].length, text: match[0] });
+  }
+
+  for (const key of keys) {
+    const keyPattern = new RegExp(`"${escapeRegex(key)}"\\s*:`);
+    const keyMatch = keyPattern.exec(content);
+    if (!keyMatch) continue;
+
+    const keyPos = keyMatch.index;
+    const precedingComment = comments.filter((c) => c.end <= keyPos).sort((a, b) => b.end - a.end)[0];
+
+    if (precedingComment) {
+      const between = content.slice(precedingComment.end, keyPos);
+      if (/^\s*$/.test(between)) {
+        result.set(key, parseMetadataBlock(precedingComment.text));
+      }
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Parses a JSONC string into structured locale entries with metadata.
+ * Extracts @context, @src, and @orphan from block comments preceding each key.
+ * Pure function - no filesystem access. Works in Node.js and edge runtimes.
+ */
+export function readLocaleFile(content: string): LocaleFile {
+  const errors: ParseError[] = [];
+  const data = parseJsonc(content, errors, { allowTrailingComma: true }) as Record<string, string> | undefined;
+
+  if (errors.length > 0) {
+    const msg = errors.map((e) => printParseErrorCode(e.error)).join(", ");
+    throw new Error(`Failed to parse JSONC: ${msg}`);
+  }
+
+  if (!data || typeof data !== "object") {
+    return { entries: [] };
+  }
+
+  const commentsByKey = extractCommentsForKeys(content, Object.keys(data));
+
+  const entries: LocaleEntry[] = Object.entries(data).map(([key, value]) => ({
+    key,
+    value,
+    metadata: commentsByKey.get(key) ?? {},
+  }));
+
+  return { entries };
+}