@dsi18n/airtable-sync 0.1.0

package/README.md

@@ -0,0 +1,94 @@
+# @dsi18n/airtable-sync
+
+**Build-time** tool: pull translation rows from [Airtable](https://airtable.com) and write **dsi18n** locale files (`manifest.json`, `en.json`, `es.json`, …).
+
+Use with **`@dsi18n/dsi18n`** at runtime (no Airtable calls in production).
+
+## Install
+
+```bash
+npm install -D @dsi18n/airtable-sync
+```
+
+Requires **Node.js 18+**.
+
+## CLI
+
+Create a `.env` in your project root:
+
+```env
+AIRTABLE_API_KEY=pat_...
+AIRTABLE_BASE_ID=app...
+AIRTABLE_TABLE=Translations
+I18N_OUT_DIR=./locales
+```
+
+Run:
+
+```bash
+npx dsi18n-sync
+```
+
+`dsi18n-sync` loads `.env` from the current directory and parent folders (nested overrides win), then runs the build.
+
+### Common env vars
+
+| Variable | Description |
+|----------|-------------|
+| `AIRTABLE_API_KEY` | Personal access token |
+| `AIRTABLE_BASE_ID` | Base id (`app…`) |
+| `AIRTABLE_TABLE` | Table name (or use `I18N_TABLES` comma-separated) |
+| `I18N_OUT_DIR` | Output directory (default: `./locales`) |
+| `I18N_KEY_FIELD` | Key column name (default: `key`) |
+| `I18N_DEFAULT_LOCALE` | Default locale tag (default: `en`) |
+| `I18N_LOCALES` | Force locale list, e.g. `en,es` |
+| `I18N_TYPES_OUT_FILE` | Optional path to generate `TranslationKey` TypeScript |
+| `I18N_VERSION` | Bundle version string in manifest |
+| `I18N_DEBUG` | `1` or `true` for verbose logs |
+
+## Programmatic API
+
+```js
+import {
+  loadBuildConfigFromEnv,
+  loadProjectEnv,
+  runBuild,
+} from "@dsi18n/airtable-sync";
+
+loadProjectEnv();
+process.env.I18N_OUT_DIR ??= "./locales";
+
+await runBuild(loadBuildConfigFromEnv());
+```
+
+## `package.json` script
+
+```json
+{
+  "scripts": {
+    "i18n:sync": "dsi18n-sync"
+  },
+  "devDependencies": {
+    "@dsi18n/airtable-sync": "^0.1.0"
+  }
+}
+```
+
+Commit the generated `locales/` folder (or generate in CI before deploy), then load them with `@dsi18n/dsi18n`.
+
+## Output layout
+
+```text
+locales/
+  manifest.json
+  en.json
+  es.json
+```
+
+## Related
+
+- Runtime: [`@dsi18n/dsi18n`](https://www.npmjs.com/package/@dsi18n/dsi18n) — `createI18n`, `t()`, `loadLocalesFromDirectory`.
+
+## License
+
+MIT

package/dist/cli.js

@@ -0,0 +1,537 @@
+#!/usr/bin/env node
+import { mkdir as mkdir$1, writeFile } from 'fs/promises';
+import path, { resolve, dirname as dirname$1, join } from 'path';
+import { existsSync } from 'fs';
+import { config } from 'dotenv';
+
+// src/config.ts
+function parseBooleanEnv(value) {
+  return value === "1" || value === "true";
+}
+function parseNumberEnv(value) {
+  if (!value) {
+    return void 0;
+  }
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : void 0;
+}
+function loadBuildConfigFromEnv() {
+  return {
+    airtableApiKey: process.env.AIRTABLE_API_KEY,
+    baseId: process.env.AIRTABLE_BASE_ID,
+    table: process.env.AIRTABLE_TABLE,
+    tablesCsv: process.env.I18N_TABLES,
+    outDir: process.env.I18N_OUT_DIR,
+    version: process.env.I18N_VERSION,
+    keyField: process.env.I18N_KEY_FIELD,
+    defaultLocale: process.env.I18N_DEFAULT_LOCALE,
+    localesCsv: process.env.I18N_LOCALES,
+    typesOutFile: process.env.I18N_TYPES_OUT_FILE,
+    debug: parseBooleanEnv(process.env.I18N_DEBUG),
+    rateLimitDelayMs: parseNumberEnv(process.env.I18N_RATE_LIMIT_DELAY_MS)
+  };
+}
+async function mkdir(pathValue) {
+  await mkdir$1(pathValue, { recursive: true });
+}
+async function writeUtf8File(pathValue, content) {
+  await writeFile(pathValue, content, "utf8");
+}
+function joinPath(...parts) {
+  return path.join(...parts);
+}
+function dirname(pathValue) {
+  return path.dirname(pathValue);
+}
+
+// src/core/fetch.ts
+var RETRY_ATTEMPTS = 3;
+var RETRY_BASE_DELAY_MS = 300;
+var DEFAULT_RATE_LIMIT_DELAY_MS = 200;
+function sleep(ms) {
+  return new Promise((resolve2) => {
+    setTimeout(resolve2, ms);
+  });
+}
+function isRetryableStatus(status) {
+  return status === 429 || status >= 500;
+}
+async function fetchWithRetry(url, init, fetchFn, debug) {
+  let lastError = null;
+  for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
+    try {
+      if (debug) {
+        console.log(`[fetch] attempt=${attempt + 1} url=${url.toString()}`);
+      }
+      const response = await fetchFn(url, init);
+      if (!isRetryableStatus(response.status)) {
+        return response;
+      }
+      if (debug) {
+        console.warn(
+          `[fetch] retryable_status=${response.status} attempt=${attempt + 1} url=${url.toString()}`
+        );
+      }
+      const body = await response.text();
+      lastError = new Error(
+        `Retryable Airtable response (${response.status}): ${response.statusText}
+${body}`
+      );
+    } catch (error) {
+      if (debug) {
+        console.warn(
+          `[fetch] network_error attempt=${attempt + 1} url=${url.toString()}`
+        );
+      }
+      lastError = error instanceof Error ? error : new Error("Unknown fetch error");
+    }
+    if (attempt < RETRY_ATTEMPTS - 1) {
+      await sleep(RETRY_BASE_DELAY_MS * (attempt + 1));
+    }
+  }
+  throw lastError ?? new Error("Fetch failed after retries");
+}
+async function fetchAirtablePage(params) {
+  const encodedTable = encodeURIComponent(params.table);
+  const url = new URL(
+    `https://api.airtable.com/v0/${params.baseId}/${encodedTable}`
+  );
+  if (params.offset) {
+    url.searchParams.set("offset", params.offset);
+  }
+  const headers = {};
+  if (params.apiKey) {
+    headers.Authorization = `Bearer ${params.apiKey}`;
+  }
+  const fetchFn = params.fetchFn ?? fetch;
+  if (params.debug) {
+    const offsetValue = params.offset ?? "none";
+    console.log(`[fetch] table=${params.table} offset=${offsetValue}`);
+  }
+  const response = await fetchWithRetry(url, { headers }, fetchFn, params.debug);
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(
+      `Airtable request failed (${response.status}): ${response.statusText}
+${body}`
+    );
+  }
+  return await response.json();
+}
+async function fetchAllRecords(params) {
+  const records = [];
+  let offset;
+  const rateLimitDelayMs = params.rateLimitDelayMs ?? DEFAULT_RATE_LIMIT_DELAY_MS;
+  do {
+    const page = await fetchAirtablePage({
+      apiKey: params.apiKey,
+      baseId: params.baseId,
+      table: params.table,
+      offset,
+      fetchFn: params.fetchFn,
+      debug: params.debug
+    });
+    records.push(...page.records);
+    offset = page.offset;
+    if (offset) {
+      await sleep(rateLimitDelayMs);
+    }
+  } while (offset);
+  return records;
+}
+async function listBaseTables(params) {
+  const url = new URL(`https://api.airtable.com/v0/meta/bases/${params.baseId}/tables`);
+  const fetchFn = params.fetchFn ?? fetch;
+  const response = await fetchWithRetry(
+    url,
+    {
+      headers: {
+        Authorization: `Bearer ${params.apiKey}`
+      }
+    },
+    fetchFn,
+    params.debug
+  );
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(
+      `Airtable metadata request failed (${response.status}): ${response.statusText}
+${body}`
+    );
+  }
+  const json = await response.json();
+  return json.tables.map((table) => table.name);
+}
+
+// src/core/normalize.ts
+function parseLocalesFromEnv(value) {
+  if (!value) {
+    return [];
+  }
+  return value.split(",").map((locale) => locale.trim()).filter(Boolean);
+}
+function inferLocalesFromRecords(records, keyField) {
+  const locales = /* @__PURE__ */ new Set();
+  for (const record of records) {
+    const fields = Object.keys(record.fields);
+    for (const fieldName of fields) {
+      if (fieldName === keyField) {
+        continue;
+      }
+      const value = record.fields[fieldName];
+      if (typeof value === "string") {
+        locales.add(fieldName);
+      }
+    }
+  }
+  return [...locales].sort();
+}
+function normalizeRecords(records, locales, keyField, options) {
+  const normalizedEntries = records.map((record) => {
+    const keyValue = record.fields[keyField];
+    if (typeof keyValue !== "string" || keyValue.trim().length === 0) {
+      if (options?.debug) {
+        console.warn(`Skipping record ${record.id}: invalid key`);
+      }
+      return null;
+    }
+    const key = keyValue.trim();
+    const translations = Object.fromEntries(
+      locales.map((locale) => {
+        const value = record.fields[locale];
+        if (typeof value !== "string") {
+          return null;
+        }
+        const trimmed = value.trim();
+        return trimmed.length > 0 ? [locale, trimmed] : null;
+      }).filter((item) => item !== null)
+    );
+    return { key, translations };
+  });
+  return normalizedEntries.filter(
+    (entry) => entry !== null
+  );
+}
+
+// src/core/validate.ts
+function extractPlaceholders(text) {
+  const matches = text.match(/\{([a-zA-Z0-9_]+)\}/g) ?? [];
+  const cleaned = matches.map((token) => token.replace(/[{}]/g, ""));
+  return [...new Set(cleaned)];
+}
+function detectDuplicateKeys(entries) {
+  const seen = /* @__PURE__ */ new Set();
+  const duplicates = /* @__PURE__ */ new Set();
+  for (const entry of entries) {
+    if (seen.has(entry.key)) {
+      duplicates.add(entry.key);
+      continue;
+    }
+    seen.add(entry.key);
+  }
+  return [...duplicates].sort();
+}
+function validateMissingTranslations(entries, locales) {
+  return entries.flatMap(
+    (entry) => locales.map((locale) => ({
+      key: entry.key,
+      locale,
+      value: entry.translations[locale]
+    }))
+  ).filter((item) => item.value == null || item.value === "").map((item) => ({
+    type: "missing_translation",
+    key: item.key,
+    locale: item.locale
+  }));
+}
+function validatePlaceholdersConsistency(entries, locales, baseLocale) {
+  const otherLocales = locales.filter((locale) => locale !== baseLocale);
+  return entries.flatMap((entry) => {
+    const baseTranslation = entry.translations[baseLocale];
+    if (!baseTranslation) {
+      return [];
+    }
+    const basePlaceholders = extractPlaceholders(baseTranslation);
+    return otherLocales.map((locale) => {
+      const translation = entry.translations[locale];
+      if (!translation) {
+        return null;
+      }
+      const placeholders = extractPlaceholders(translation);
+      const missingIssues = basePlaceholders.filter((placeholder) => !placeholders.includes(placeholder)).map((placeholder) => ({
+        type: "missing_placeholder",
+        key: entry.key,
+        locale,
+        baseLocale,
+        placeholder
+      }));
+      const extraIssues = placeholders.filter((placeholder) => !basePlaceholders.includes(placeholder)).map((placeholder) => ({
+        type: "extra_placeholder",
+        key: entry.key,
+        locale,
+        baseLocale,
+        placeholder
+      }));
+      return [...missingIssues, ...extraIssues];
+    }).flat().filter((item) => item !== null);
+  });
+}
+function validateDefaultLocale(locales, defaultLocale) {
+  if (!locales.includes(defaultLocale)) {
+    throw new Error(
+      `Default locale "${defaultLocale}" is not present in locales: ${locales.join(", ")}`
+    );
+  }
+}
+
+// src/core/run-build.ts
+var DICTIONARY_SCHEMA_REF = "../schema/dictionary.schema.json";
+var MANIFEST_SCHEMA_REF = "../schema/manifest.schema.json";
+var DEFAULT_OUT_DIR = "packages/locales";
+var DEFAULT_VERSION = "0.1.0";
+var DEFAULT_KEY_FIELD = "key";
+var DEFAULT_LOCALE = "en";
+var DEFAULT_TYPES_OUT_FILE = "packages/i18n-core/src/generated/translation-keys.ts";
+var DEFAULT_RATE_LIMIT_DELAY_MS2 = 200;
+function sleep2(ms) {
+  return new Promise((resolve2) => {
+    setTimeout(resolve2, ms);
+  });
+}
+function formatValidationIssue(issue) {
+  if (issue.type === "missing_translation") {
+    return `Missing ${issue.locale} for key: ${issue.key}`;
+  }
+  if (issue.type === "missing_placeholder") {
+    return `Missing placeholder "${issue.placeholder}" for key "${issue.key}" (${issue.locale}) using base locale "${issue.baseLocale}"`;
+  }
+  return `Extra placeholder "${issue.placeholder}" for key "${issue.key}" (${issue.locale}) using base locale "${issue.baseLocale}"`;
+}
+function parseTablesFromEnv(value) {
+  if (!value) {
+    return [];
+  }
+  return value.split(",").map((table) => table.trim()).filter(Boolean);
+}
+function extractPlaceholders2(text) {
+  const matches = text.match(/\{([a-zA-Z0-9_]+)\}/g) ?? [];
+  const cleaned = matches.map((token) => token.replace(/[{}]/g, ""));
+  return [...new Set(cleaned)];
+}
+function buildDictionaries(entries, locales, version) {
+  const dictionaries = Object.fromEntries(
+    locales.map((locale) => [
+      locale,
+      {
+        locale,
+        version,
+        entries: {}
+      }
+    ])
+  );
+  entries.forEach((entry) => {
+    locales.forEach((locale) => {
+      const value = entry.translations[locale];
+      if (!value) {
+        return;
+      }
+      dictionaries[locale].entries[entry.key] = {
+        value,
+        placeholders: extractPlaceholders2(value)
+      };
+    });
+  });
+  return dictionaries;
+}
+async function writeOutputs(params) {
+  await mkdir(params.outDir);
+  const dictionaryEntries = Object.entries(params.dictionaries);
+  await Promise.all(
+    dictionaryEntries.map(async ([locale, dictionary]) => {
+      const filePath = joinPath(params.outDir, `${locale}.json`);
+      const payload = {
+        $schema: DICTIONARY_SCHEMA_REF,
+        locale: dictionary.locale,
+        version: dictionary.version,
+        entries: dictionary.entries
+      };
+      await writeUtf8File(filePath, `${JSON.stringify(payload, null, 2)}
+`);
+    })
+  );
+  const manifestPath = joinPath(params.outDir, "manifest.json");
+  const manifestPayload = {
+    ...params.manifest,
+    $schema: MANIFEST_SCHEMA_REF
+  };
+  await writeUtf8File(
+    manifestPath,
+    `${JSON.stringify(manifestPayload, null, 2)}
+`
+  );
+}
+function buildTypeDefinitions(keys) {
+  const sortedKeys = [...keys].sort();
+  const keyUnion = sortedKeys.length > 0 ? sortedKeys.map((key) => `  | "${key}"`).join("\n") : '  | "__no_translation_keys__"';
+  return [
+    "// AUTO-GENERATED FILE. DO NOT EDIT.",
+    "// Generated by packages/airtable-sync/src/core/run-build.ts",
+    "",
+    "export type TranslationKey =",
+    keyUnion + ";",
+    "",
+    "export const translationKeys = [",
+    ...sortedKeys.map((key) => `  "${key}",`),
+    "] as const;",
+    ""
+  ].join("\n");
+}
+async function writeTypeDefinitions(typeOutputPath, entries) {
+  const keys = entries.map((entry) => entry.key);
+  const typeContent = buildTypeDefinitions(keys);
+  await mkdir(dirname(typeOutputPath));
+  await writeUtf8File(typeOutputPath, typeContent);
+}
+async function runBuild(config) {
+  const baseId = config.baseId;
+  const airtableApiKey = config.airtableApiKey;
+  if (!baseId) {
+    throw new Error("Missing required config: baseId");
+  }
+  if (!airtableApiKey) {
+    throw new Error("Missing airtableApiKey");
+  }
+  const outDir = config.outDir ?? DEFAULT_OUT_DIR;
+  const version = config.version ?? DEFAULT_VERSION;
+  const keyField = config.keyField ?? DEFAULT_KEY_FIELD;
+  const defaultLocale = config.defaultLocale ?? DEFAULT_LOCALE;
+  const typesOutFile = config.typesOutFile ?? DEFAULT_TYPES_OUT_FILE;
+  const rateLimitDelayMs = config.rateLimitDelayMs ?? DEFAULT_RATE_LIMIT_DELAY_MS2;
+  const tablesFromCsv = parseTablesFromEnv(config.tablesCsv);
+  const selectedTables = config.table && config.table.trim().length > 0 ? [config.table.trim()] : tablesFromCsv;
+  const tables = selectedTables.length > 0 ? selectedTables : await (async () => {
+    if (!config.airtableApiKey) {
+      throw new Error(
+        "AIRTABLE_TABLE or I18N_TABLES is required unless AIRTABLE_API_KEY can access Airtable metadata."
+      );
+    }
+    return listBaseTables({
+      apiKey: airtableApiKey,
+      baseId,
+      fetchFn: config.fetchFn,
+      debug: config.debug
+    });
+  })();
+  if (tables.length === 0) {
+    throw new Error("No tables found to read from Airtable base.");
+  }
+  const records = await tables.reduce(
+    async (accPromise, table, index) => {
+      const acc = await accPromise;
+      if (index > 0) {
+        await sleep2(rateLimitDelayMs);
+      }
+      const tableRecords = await fetchAllRecords({
+        apiKey: airtableApiKey,
+        baseId,
+        table,
+        fetchFn: config.fetchFn,
+        debug: config.debug,
+        rateLimitDelayMs
+      });
+      return [...acc, ...tableRecords];
+    },
+    Promise.resolve([])
+  );
+  const configuredLocales = parseLocalesFromEnv(config.localesCsv);
+  const inferredLocales = inferLocalesFromRecords(records, keyField);
+  const locales = configuredLocales.length > 0 ? configuredLocales : inferredLocales;
+  if (locales.length === 0) {
+    throw new Error(
+      "No locales found. Set I18N_LOCALES or provide language columns in Airtable."
+    );
+  }
+  const normalized = normalizeRecords(records, locales, keyField, {
+    debug: config.debug
+  });
+  const duplicateKeys = detectDuplicateKeys(normalized);
+  if (duplicateKeys.length > 0) {
+    throw new Error(
+      `Duplicate translation keys found: ${duplicateKeys.join(", ")}`
+    );
+  }
+  const missingTranslations = validateMissingTranslations(normalized, locales);
+  if (missingTranslations.length > 0) {
+    console.warn("Missing translations:");
+    missingTranslations.forEach(
+      (issue) => console.warn(` - ${formatValidationIssue(issue)}`)
+    );
+  }
+  validateDefaultLocale(locales, defaultLocale);
+  const placeholderMismatches = validatePlaceholdersConsistency(
+    normalized,
+    locales,
+    defaultLocale
+  );
+  if (placeholderMismatches.length > 0) {
+    console.warn("Placeholder consistency issues:");
+    placeholderMismatches.forEach(
+      (issue) => console.warn(` - ${formatValidationIssue(issue)}`)
+    );
+  }
+  const dictionaries = buildDictionaries(normalized, locales, version);
+  const manifest = {
+    version,
+    defaultLocale,
+    fallbackChain: [defaultLocale],
+    locales,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    source: {
+      type: "airtable",
+      baseId,
+      tables
+    }
+  };
+  await writeOutputs({
+    outDir,
+    dictionaries,
+    manifest
+  });
+  await writeTypeDefinitions(typesOutFile, normalized);
+  console.log(
+    `Generated locales: ${locales.join(", ")} | records: ${normalized.length} | out: ${outDir} | types: ${typesOutFile}`
+  );
+}
+function loadProjectEnv(startDir = process.cwd()) {
+  const chain = [];
+  let dir = resolve(startDir);
+  for (let i = 0; i < 6; i += 1) {
+    chain.unshift(dir);
+    const parent = dirname$1(dir);
+    if (parent === dir) {
+      break;
+    }
+    dir = parent;
+  }
+  for (const folder of chain) {
+    const path2 = join(folder, ".env");
+    if (existsSync(path2)) {
+      config({ path: path2 });
+    }
+  }
+}
+
+// src/index.ts
+async function main() {
+  await runBuild(loadBuildConfigFromEnv());
+}
+
+// src/cli.ts
+loadProjectEnv();
+main().catch((error) => {
+  const message = error instanceof Error ? error.message : String(error);
+  console.error(message);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map
+//# sourceMappingURL=cli.js.map