anylang-dev 0.1.0

package/src/extract.js

@@ -0,0 +1,348 @@
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+
+const DEFAULT_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx", ".vue", ".html", ".svelte", ".astro"]);
+
+export async function extractProjectStrings(config) {
+  const files = await listCandidateFiles(process.cwd(), config);
+  const items = [];
+  const seen = new Set();
+
+  for (const file of files) {
+    const source = await readFile(file, "utf8");
+    for (const match of extractFromSource(source, config.functionName)) {
+      const key = `${match.value}\0${file}\0${match.index}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      items.push({
+        ...match,
+        file,
+        ...lineColumnForIndex(source, match.index)
+      });
+    }
+  }
+
+  return { files, items };
+}
+
+export function extractFromSource(source, functionName = "$tr") {
+  const matches = [];
+  let index = 0;
+  let state = "code";
+  let quote = "";
+
+  while (index < source.length) {
+    const char = source[index];
+    const next = source[index + 1];
+
+    if (state === "lineComment") {
+      if (char === "\n") state = "code";
+      index += 1;
+      continue;
+    }
+    if (state === "blockComment") {
+      if (char === "*" && next === "/") {
+        index += 2;
+        state = "code";
+      } else {
+        index += 1;
+      }
+      continue;
+    }
+    if (state === "string") {
+      if (char === "\\") {
+        index += 2;
+      } else if (char === quote) {
+        index += 1;
+        state = "code";
+      } else {
+        index += 1;
+      }
+      continue;
+    }
+    if (state === "template") {
+      if (char === "\\") {
+        index += 2;
+      } else if (char === "`") {
+        index += 1;
+        state = "code";
+      } else {
+        index += 1;
+      }
+      continue;
+    }
+
+    if (char === "/" && next === "/") {
+      state = "lineComment";
+      index += 2;
+      continue;
+    }
+    if (char === "/" && next === "*") {
+      state = "blockComment";
+      index += 2;
+      continue;
+    }
+    if (char === "'" || char === "\"") {
+      quote = char;
+      state = "string";
+      index += 1;
+      continue;
+    }
+    if (char === "`") {
+      state = "template";
+      index += 1;
+      continue;
+    }
+
+    if (source.startsWith(functionName, index) && hasIdentifierBoundary(source, index, functionName)) {
+      const call = parseTranslationCall(source, index + functionName.length);
+      if (call) {
+        matches.push({ key: call.key, value: call.value, variables: call.variables, index, raw: source.slice(index, call.endIndex) });
+        index = call.endIndex;
+        continue;
+      }
+    }
+
+    index += 1;
+  }
+
+  return matches;
+}
+
+async function listCandidateFiles(root, config) {
+  const allFiles = [];
+  const excluded = new Set(config.exclude || []);
+  const allowedExtensions = extensionsFromInclude(config.include);
+  const roots = includeRoots(root, config.include);
+
+  async function walk(dir) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      if (error && error.code === "ENOENT") return;
+      throw error;
+    }
+    for (const entry of entries) {
+      if (excluded.has(entry.name) || hasExcludedSegment(path.join(dir, entry.name), excluded)) continue;
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(fullPath);
+      } else if (entry.isFile() && allowedExtensions.has(path.extname(entry.name))) {
+        allFiles.push(fullPath);
+      }
+    }
+  }
+
+  for (const includeRoot of roots) {
+    await walk(includeRoot);
+  }
+  return allFiles.sort();
+}
+
+function includeRoots(root, include) {
+  const roots = new Set();
+  for (const pattern of include) {
+    const firstGlobIndex = pattern.search(/[*{?]/);
+    const staticPart = firstGlobIndex === -1 ? pattern : pattern.slice(0, firstGlobIndex);
+    const normalized = staticPart.replace(/[/\\][^/\\]*$/, "");
+    roots.add(path.resolve(root, normalized || "."));
+  }
+  return roots.size > 0 ? Array.from(roots) : [root];
+}
+
+function hasExcludedSegment(filePath, excluded) {
+  return filePath.split(path.sep).some((segment) => excluded.has(segment));
+}
+
+function extensionsFromInclude(include) {
+  const extensions = new Set();
+  for (const pattern of include) {
+    const braceMatch = pattern.match(/\.\{([^}]+)\}/);
+    if (braceMatch) {
+      for (const ext of braceMatch[1].split(",")) extensions.add(`.${ext.trim()}`);
+      continue;
+    }
+    const ext = path.extname(pattern.replace(/\*/g, "x"));
+    if (ext) extensions.add(ext);
+  }
+  return extensions.size > 0 ? extensions : DEFAULT_EXTENSIONS;
+}
+
+function hasIdentifierBoundary(source, start, functionName) {
+  const before = source[start - 1];
+  const after = source[start + functionName.length];
+  return !isIdentifierChar(before) && !isIdentifierChar(after);
+}
+
+function isIdentifierChar(char) {
+  return Boolean(char && /[A-Za-z0-9_$]/.test(char));
+}
+
+function parseTranslationCall(source, offset) {
+  let index = skipWhitespace(source, offset);
+  if (source[index] !== "(") return null;
+  index = skipWhitespace(source, index + 1);
+  const keyLiteral = parseLiteral(source, index);
+  if (!keyLiteral) return null;
+  index = skipWhitespace(source, keyLiteral.endIndex);
+
+  let text = keyLiteral.value;
+  if (source[index] === ",") {
+    const secondArgIndex = skipWhitespace(source, index + 1);
+    const textLiteral = parseLiteral(source, secondArgIndex);
+    if (textLiteral) {
+      text = textLiteral.value;
+      index = skipWhitespace(source, textLiteral.endIndex);
+    }
+  }
+
+  const endIndex = findCallEnd(source, index);
+  if (endIndex === -1) return null;
+  return {
+    key: keyLiteral.value,
+    value: text,
+    variables: extractVariables(text),
+    endIndex: endIndex + 1
+  };
+}
+
+function findCallEnd(source, index) {
+  let cursor = index;
+  let depth = 0;
+  let state = "code";
+  let quote = "";
+
+  while (cursor < source.length) {
+    const char = source[cursor];
+    const next = source[cursor + 1];
+
+    if (state === "lineComment") {
+      if (char === "\n") state = "code";
+      cursor += 1;
+      continue;
+    }
+    if (state === "blockComment") {
+      if (char === "*" && next === "/") {
+        cursor += 2;
+        state = "code";
+      } else {
+        cursor += 1;
+      }
+      continue;
+    }
+    if (state === "string" || state === "template") {
+      if (char === "\\") {
+        cursor += 2;
+      } else if (char === quote) {
+        cursor += 1;
+        state = "code";
+      } else {
+        cursor += 1;
+      }
+      continue;
+    }
+
+    if (char === "/" && next === "/") {
+      state = "lineComment";
+      cursor += 2;
+      continue;
+    }
+    if (char === "/" && next === "*") {
+      state = "blockComment";
+      cursor += 2;
+      continue;
+    }
+    if (char === "'" || char === "\"" || char === "`") {
+      quote = char;
+      state = char === "`" ? "template" : "string";
+      cursor += 1;
+      continue;
+    }
+    if (char === "(" || char === "[" || char === "{") {
+      depth += 1;
+      cursor += 1;
+      continue;
+    }
+    if (char === ")" && depth === 0) return cursor;
+    if (char === ")" || char === "]" || char === "}") {
+      depth -= 1;
+      cursor += 1;
+      continue;
+    }
+
+    cursor += 1;
+  }
+
+  return -1;
+}
+
+function parseLiteral(source, index) {
+  const quote = source[index];
+  if (quote !== "'" && quote !== "\"" && quote !== "`") return null;
+
+  let cursor = index + 1;
+  let value = "";
+  while (cursor < source.length) {
+    const char = source[cursor];
+    if (char === "\\") {
+      const escape = source[cursor + 1];
+      value += decodeEscape(escape);
+      cursor += 2;
+      continue;
+    }
+    if (quote === "`" && char === "$" && source[cursor + 1] === "{") {
+      return null;
+    }
+    if (char === quote) {
+      return { value, endIndex: cursor + 1 };
+    }
+    value += char;
+    cursor += 1;
+  }
+  return null;
+}
+
+function decodeEscape(char) {
+  switch (char) {
+    case "n":
+      return "\n";
+    case "r":
+      return "\r";
+    case "t":
+      return "\t";
+    case "\\":
+      return "\\";
+    case "\"":
+      return "\"";
+    case "'":
+      return "'";
+    case "`":
+      return "`";
+    default:
+      return char || "";
+  }
+}
+
+function skipWhitespace(source, index) {
+  let cursor = index;
+  while (/\s/.test(source[cursor] || "")) cursor += 1;
+  return cursor;
+}
+
+function lineColumnForIndex(source, index) {
+  const before = source.slice(0, index);
+  const lines = before.split("\n");
+  return {
+    line: lines.length,
+    column: lines[lines.length - 1].length + 1
+  };
+}
+
+function extractVariables(text) {
+  const variables = new Set();
+  for (const match of text.matchAll(/\{([A-Za-z_$][A-Za-z0-9_$]*)\}/g)) {
+    variables.add(match[1]);
+  }
+  return Array.from(variables).sort();
+}

package/src/pipeline.js

@@ -0,0 +1,214 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { extractProjectStrings } from "./extract.js";
+import { createTranslator } from "./providers.js";
+
+export async function runPipeline(config, options = {}) {
+  const outDir = path.resolve(config.outDir);
+  await mkdir(outDir, { recursive: true });
+
+  const extraction = await extractProjectStrings(config);
+  const sourceEntries = sourceEntriesFromItems(extraction.items);
+  const sourceCatalog = sortObject(Object.fromEntries(sourceEntries.map((entry) => [
+    entry.key,
+    {
+      text: entry.text,
+      variables: entry.variables
+    }
+  ])));
+  await writeJson(path.join(outDir, `${config.sourceLocale}.json`), sourceCatalog);
+
+  const lockPath = path.join(outDir, "anylang.lock.json");
+  const lock = await readJson(lockPath, { version: 1, entries: {} });
+  const translator = options.translator || (options.dryRun ? null : createTranslator(config.provider));
+  let translatedCount = 0;
+  let skippedTranslationCount = 0;
+
+  for (const locale of config.targetLocales) {
+    if (locale === config.sourceLocale) continue;
+    const localePath = path.join(outDir, `${locale}.json`);
+    const catalog = await readJson(localePath, {});
+
+    for (const entry of sourceEntries) {
+      const existing = normalizeTargetEntry(catalog[entry.key]);
+      const fingerprint = hashString(entry.text);
+      const lockKey = `${locale}:${entry.key}`;
+      const isFresh = existing && existing.source === entry.text && existing.text;
+      if (isFresh) continue;
+
+      if (options.dryRun || !translator) {
+        if (!existing) {
+          catalog[entry.key] = {
+            source: entry.text,
+            text: "",
+            variables: entry.variables
+          };
+        }
+        skippedTranslationCount += 1;
+      } else {
+        catalog[entry.key] = {
+          source: entry.text,
+          text: await translator.translate({ text: entry.text, sourceLocale: config.sourceLocale, targetLocale: locale }),
+          variables: entry.variables
+        };
+        translatedCount += 1;
+      }
+
+      lock.entries[lockKey] = {
+        fingerprint,
+        updatedAt: new Date().toISOString()
+      };
+    }
+
+    for (const key of Object.keys(catalog)) {
+      if (!sourceCatalog[key]) delete catalog[key];
+    }
+    await writeJson(localePath, sortObject(catalog));
+  }
+
+  await writeJson(lockPath, lock);
+  await writeGeneratedRuntime(config);
+
+  return {
+    sourceCount: sourceEntries.length,
+    localeCount: 1 + config.targetLocales.filter((locale) => locale !== config.sourceLocale).length,
+    translatedCount,
+    skippedTranslationCount,
+    outDir: path.relative(process.cwd(), outDir) || "."
+  };
+}
+
+async function readJson(file, fallback) {
+  try {
+    return JSON.parse(await readFile(file, "utf8"));
+  } catch (error) {
+    if (error && error.code === "ENOENT") return fallback;
+    throw error;
+  }
+}
+
+async function writeJson(file, value) {
+  await writeFile(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function hashString(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function sortObject(object) {
+  return Object.fromEntries(Object.entries(object).sort(([left], [right]) => left.localeCompare(right)));
+}
+
+function sourceEntriesFromItems(items) {
+  const entries = new Map();
+  for (const item of items) {
+    const existing = entries.get(item.key);
+    if (existing && existing.text !== item.value) {
+      throw new Error(`Translation key "${item.key}" has multiple source texts: "${existing.text}" and "${item.value}".`);
+    }
+    entries.set(item.key, {
+      key: item.key,
+      text: item.value,
+      variables: item.variables || []
+    });
+  }
+  return Array.from(entries.values()).sort((left, right) => left.key.localeCompare(right.key));
+}
+
+function normalizeTargetEntry(entry) {
+  if (!entry) return null;
+  if (typeof entry === "string") {
+    return { source: undefined, text: entry, variables: [] };
+  }
+  return {
+    source: entry.source,
+    text: typeof entry.text === "string" ? entry.text : "",
+    variables: Array.isArray(entry.variables) ? entry.variables : []
+  };
+}
+
+async function writeGeneratedRuntime(config) {
+  if (config.runtime === false) return;
+
+  const output = path.resolve(config.runtime?.output || "src/anylang.generated.ts");
+  const outDir = path.resolve(config.outDir);
+  const locales = [config.sourceLocale, ...config.targetLocales.filter((locale) => locale !== config.sourceLocale)];
+  const importFrom = config.runtime?.importFrom || "anylang-dev/runtime";
+  const runtimeDir = path.dirname(output);
+  const localeImports = locales.map((locale) => ({
+    locale,
+    identifier: localeIdentifier(locale),
+    importPath: toImportPath(path.relative(runtimeDir, path.join(outDir, `${locale}.json`)))
+  }));
+
+  await mkdir(runtimeDir, { recursive: true });
+  await writeFile(output, `${generatedRuntimeSource({ importFrom, localeImports, sourceLocale: config.sourceLocale })}\n`);
+}
+
+function generatedRuntimeSource({ importFrom, localeImports, sourceLocale }) {
+  const imports = localeImports
+    .map((item) => `import ${item.identifier} from '${item.importPath}'`)
+    .join("\n");
+  const languageUnion = localeImports.map((item) => `'${item.locale}'`).join(" | ");
+  const languageItems = localeImports
+    .map((item) => `  { code: '${item.locale}' as const, label: '${languageLabel(item.locale)}', nativeLabel: '${nativeLanguageLabel(item.locale)}' }`)
+    .join(",\n");
+  const catalogItems = localeImports
+    .map((item) => `  '${item.locale}': ${item.identifier}`)
+    .join(",\n");
+
+  return `/* This file is generated by anylang. Do not edit by hand. */
+import { useCallback } from 'react'
+import { $tr as translate, configureAnyLang, setAnyLangLocale } from '${importFrom}'
+${imports}
+
+export type LanguageCode = ${languageUnion}
+
+export const languages = [
+${languageItems}
+]
+
+configureAnyLang({
+  locale: '${sourceLocale}',
+  catalogs: {
+${catalogItems}
+  },
+})
+
+export function useAnyLang(locale: LanguageCode) {
+  return useCallback((key: string, source?: string) => {
+    return translate(key, source, locale)
+  }, [locale])
+}
+
+export function setLanguage(locale: LanguageCode) {
+  setAnyLangLocale(locale)
+}
+`;
+}
+
+function localeIdentifier(locale) {
+  return `catalog_${locale.replace(/[^A-Za-z0-9_$]/g, "_")}`;
+}
+
+function toImportPath(relativePath) {
+  const normalized = relativePath.split(path.sep).join("/");
+  return normalized.startsWith(".") ? normalized : `./${normalized}`;
+}
+
+function languageLabel(locale) {
+  try {
+    return new Intl.DisplayNames(["en"], { type: "language" }).of(locale) || locale;
+  } catch {
+    return locale;
+  }
+}
+
+function nativeLanguageLabel(locale) {
+  try {
+    return new Intl.DisplayNames([locale], { type: "language" }).of(locale) || languageLabel(locale);
+  } catch {
+    return languageLabel(locale);
+  }
+}