@omega.js/web 0.1.0

package/dist/vendor/devkit/translate/engine.js

@@ -0,0 +1,243 @@
+/**
+ * The translation engine — one provider-agnostic protocol for translating a
+ * batch of strings: JSON array in → same-length JSON array out, with a control
+ * sentinel appended to every batch to prove alignment, automatic batching,
+ * validation retries, and a HALVING re-ask for a batch the model keeps
+ * collapsing. Frameworks feed it strings (page text nodes, extension messages)
+ * and get positionally-aligned translations back.
+ */
+
+// Alignment sentinel — appended to every batch, must return unchanged
+const CONTROL = 'OMEGA-TRANSLATION-CONTROL';
+
+const BATCH_SIZE = 25;
+const MAX_RETRIES = 2;
+
+// How many batches are in flight at once (#604). A batch is one model call and
+// the model's latency dominates it, so a serial pass paid that latency once per
+// 25 strings — a language took minutes it never needed to. Raise it only as far
+// as the providers' rate limits allow.
+const CONCURRENCY = 5;
+
+const SYSTEM_PROMPT = `<role>
+Professional translator. Return ONLY a valid JSON array — no commentary, no markdown fences, no wrapping.
+</role>
+
+<task>
+Translate the input JSON array of strings into the target language.
+The output array MUST have the EXACT same length as the input array.
+</task>
+
+<rules>
+- Input: a JSON array of strings. Output: a JSON array of translated strings of the SAME length.
+- DO NOT add, remove, merge, or reorder elements.
+- Consider adjacent strings for context — they come from the same document.
+- Preserve leading and trailing whitespace in each string.
+- Keep HTML tags, attributes, URLs, and email addresses verbatim — never translate them.
+- Preserve placeholder tokens exactly (e.g. $1, $2, %s, {name}, {{ value }}).
+- Keep brand names, product names, and company names in their original language.
+- Never translate the brand name "{brand}".
+- The control string "${CONTROL}" must be returned unchanged at its exact position.
+{extraRules}
+</rules>
+
+<example lang="es">
+Input: ["Welcome to {brand}", "Get started today", "${CONTROL}"]
+Output: ["Bienvenido a {brand}", "Comienza hoy", "${CONTROL}"]
+</example>`;
+
+/**
+ * Parse the provider's response into a JSON array (strips accidental fences).
+ * @param {string} text - raw provider response
+ * @returns {Array} parsed array
+ */
+function parseArrayResponse(text) {
+  const cleaned = (text || '').trim().replace(/^```json?\n?|\n?```$/g, '');
+  const parsed = JSON.parse(cleaned);
+
+  if (!Array.isArray(parsed)) {
+    throw new Error(`translation result is not an array (got ${typeof parsed})`);
+  }
+
+  return parsed;
+}
+
+/**
+ * Re-apply the original string's leading/trailing whitespace to a translation
+ * (models trim despite instructions; surrounding markup relies on it).
+ * @param {string} original - source string
+ * @param {string} translated - translated string
+ * @returns {string}
+ */
+function preserveWhitespace(original, translated) {
+  const leading = original.match(/^\s*/)[0];
+  const trailing = original.match(/\s*$/)[0];
+
+  return `${leading}${String(translated).trim()}${trailing}`;
+}
+
+/**
+ * An alignment failure — the batch came back, but not one-for-one. Marked so
+ * the caller can tell it from a provider that never answered at all.
+ * @param {string} message
+ * @returns {Error}
+ */
+function alignmentError(message) {
+  const error = new Error(message);
+  error.alignment = true;
+  return error;
+}
+
+/**
+ * Translate one batch (with the control sentinel and validation retries).
+ * @param {object} ctx - { send, system, language, languageName }
+ * @param {string[]} batch - strings to translate
+ * @param {number} attempt - current attempt (0-based)
+ * @returns {Promise<{ result: string[], usage: object }>}
+ */
+async function translateBatch(ctx, batch, attempt) {
+  const payload = [...batch, CONTROL];
+  const user = `Target language: ${ctx.language} (${ctx.languageName})\nArray length: ${payload.length}\n\n${JSON.stringify(payload)}`;
+
+  const { text, usage } = await ctx.send({ system: ctx.system, user });
+
+  try {
+    const parsed = parseArrayResponse(text);
+
+    if (parsed.length !== payload.length) {
+      throw alignmentError(`length mismatch: expected ${payload.length}, got ${parsed.length}`);
+    }
+
+    if (parsed[parsed.length - 1] !== CONTROL) {
+      throw alignmentError('control sentinel missing or altered at the last position');
+    }
+
+    return { result: parsed.slice(0, -1), usage };
+  } catch (e) {
+    if (attempt < MAX_RETRIES) {
+      return translateBatch(ctx, batch, attempt + 1);
+    }
+
+    // A batch a model MERGES is deterministic (#523): the playground's
+    // ship-the-docs page came back 25-for-26 on all six attempts, in two
+    // languages — re-asking the same array can only fail the same way. Halving
+    // it separates whatever pair collapsed, and each half is asked in full,
+    // so nothing is guessed at and the page stops shipping untranslated.
+    if (e.alignment && batch.length > 1) {
+      return splitBatch(ctx, batch);
+    }
+
+    throw new Error(`translation failed after ${MAX_RETRIES + 1} attempts: ${e.message}`);
+  }
+}
+
+/**
+ * Re-ask a collapsing batch in halves, bottoming out at one string (a single
+ * string plus the sentinel is unambiguous — a provider that still drops it is
+ * broken, and the caller skips the page-language pair whole).
+ * @param {object} ctx - { send, system, language, languageName }
+ * @param {string[]} batch - the batch that would not come back aligned
+ * @returns {Promise<{ result: string[], usage: object }>}
+ */
+async function splitBatch(ctx, batch) {
+  const middle = Math.ceil(batch.length / 2);
+  const result = [];
+  const usage = { input: 0, output: 0 };
+
+  for (const half of [batch.slice(0, middle), batch.slice(middle)]) {
+    const outcome = await translateBatch(ctx, half, 0);
+    result.push(...outcome.result);
+    usage.input += outcome.usage?.input || 0;
+    usage.output += outcome.usage?.output || 0;
+  }
+
+  return { result, usage };
+}
+
+/**
+ * Run a worker over every item with at most CONCURRENCY of them in flight,
+ * returning results in the ITEMS' order — never completion order.
+ * @param {Array} items - the work items
+ * @param {Function} worker - (item) → Promise
+ * @returns {Promise<Array>} results, positionally aligned with items
+ */
+async function mapLimited(items, worker) {
+  const results = new Array(items.length);
+  let next = 0;
+
+  const drain = async () => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await worker(items[index]);
+    }
+  };
+
+  await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, drain));
+
+  return results;
+}
+
+/**
+ * Translate an array of strings, batching + validating along the way.
+ * @param {object} options
+ * @param {string[]} options.strings - source strings
+ * @param {string} options.language - target language code (e.g. 'es')
+ * @param {string} options.languageName - English name of the target language
+ * @param {Function} options.send - provider send ({ system, user }) → { text, usage }
+ * @param {string} [options.brand] - brand name to keep untranslated
+ * @param {string} [options.extraRules] - extra rule lines for the system prompt
+ * @returns {Promise<{ result: string[], usage: { input: number, output: number } }>}
+ *   result is positionally aligned with options.strings, original whitespace preserved
+ */
+async function translateStrings(options) {
+  const { strings, language, languageName, send, brand, extraRules } = options;
+
+  if (!strings.length) {
+    return { result: [], usage: { input: 0, output: 0 } };
+  }
+
+  const system = SYSTEM_PROMPT
+    .split('{brand}').join(brand || 'the brand')
+    .replace('{extraRules}', extraRules ? `${extraRules.trim()}\n` : '');
+
+  const ctx = { send, system, language, languageName };
+  const translated = [];
+  const usage = { input: 0, output: 0 };
+
+  // Dedupe before batching (#529): a page sends its title and description once
+  // per meta tag, so the same string rode a batch three times over — three
+  // times the AI spend, and adjacent twins are exactly what a model merges.
+  // Each unique string is translated ONCE and fanned back below.
+  const unique = [];
+  const position = new Map();
+  for (const string of strings) {
+    if (!position.has(string)) {
+      position.set(string, unique.length);
+      unique.push(string);
+    }
+  }
+
+  const batches = [];
+  for (let i = 0; i < unique.length; i += BATCH_SIZE) {
+    batches.push(unique.slice(i, i + BATCH_SIZE));
+  }
+
+  // Batches fly CONCURRENCY-wide (#604) and are stitched back in BATCH order:
+  // the sentinel, the alignment checks, and the #523 split backstop all stay
+  // per batch, so what a batch does on its own is exactly what it always did.
+  const outcomes = await mapLimited(batches, (batch) => translateBatch(ctx, batch, 0));
+
+  outcomes.forEach((outcome, i) => {
+    translated.push(...outcome.result.map((text, j) => preserveWhitespace(batches[i][j], text)));
+    usage.input += outcome.usage?.input || 0;
+    usage.output += outcome.usage?.output || 0;
+  });
+
+  // Occurrences are keyed on the exact source string, whitespace included, so
+  // every one of them gets back the translation its own text asked for.
+  const result = strings.map((string) => translated[position.get(string)]);
+
+  return { result, usage };
+}
+
+module.exports = { translateStrings, preserveWhitespace, CONTROL, BATCH_SIZE, CONCURRENCY };

package/dist/vendor/devkit/translate/index.js

@@ -0,0 +1,49 @@
+/**
+ * @omega.js/devkit/translate — the shared OMEGA translation system: language
+ * SSOT, provider abstraction (claude via local Claude Code / chatgpt via
+ * OpenAI), the batch translation engine, and the committed per-string cache.
+ * Config contract (shared `translation` section of omega.json5):
+ *   { enabled, default, languages, providers: { claude|chatgpt: {} }, model, exclude }
+ */
+const { chosenProvider } = require('../../config/index.js');
+const { LANGUAGE_NAMES, LANGUAGE_LOCALES, RTL_LANGUAGES, isRTL, languageName, ogLocale, assertKnownLanguages } = require('./languages.js');
+const { PROVIDERS, DEFAULT_MODELS, resolveProvider } = require('./providers.js');
+const { translateStrings, preserveWhitespace, CONTROL, BATCH_SIZE, CONCURRENCY } = require('./engine.js');
+const { hashKey, cachePath, loadCache, saveCache } = require('./cache.js');
+
+/**
+ * Read + validate the `translation` section of a resolved config.
+ * @param {object} config - resolved omega.json5 config
+ * @returns {{ enabled: boolean, default: string, languages: string[], provider: string, model: string|null, exclude: string[] }}
+ *   enabled is false when the section is off or lists no languages
+ */
+function resolveTranslationSettings(config) {
+  const section = config?.translation || {};
+  const languages = section.languages || [];
+
+  assertKnownLanguages(languages);
+
+  return {
+    enabled: section.enabled !== false && languages.length > 0,
+    default: section.default || 'en',
+    languages,
+    // The engine is a KEY under translation.providers (#425); no block at all
+    // means claude, the no-API-key default.
+    provider: chosenProvider(section.providers) || 'claude',
+    model: section.model || null,
+    exclude: section.exclude || [],
+  };
+}
+
+module.exports = {
+  // languages
+  LANGUAGE_NAMES, LANGUAGE_LOCALES, RTL_LANGUAGES, isRTL, languageName, ogLocale, assertKnownLanguages,
+  // providers
+  PROVIDERS, DEFAULT_MODELS, resolveProvider,
+  // engine
+  translateStrings, preserveWhitespace, CONTROL, BATCH_SIZE, CONCURRENCY,
+  // cache
+  hashKey, cachePath, loadCache, saveCache,
+  // config
+  resolveTranslationSettings,
+};

package/dist/vendor/devkit/translate/languages.js

@@ -0,0 +1,134 @@
+/**
+ * Language SSOT for the OMEGA translation system — the code → English-name
+ * map every framework validates against, plus the RTL set. Brand configs pick
+ * target codes in `translation.languages`; frameworks never carry their own
+ * language lists.
+ */
+
+// Supported target languages (ISO 639-1 codes → English names). The names are
+// sent to the AI provider so the model knows exactly which variant is meant.
+const LANGUAGE_NAMES = {
+  ar: 'Arabic',
+  bn: 'Bengali',
+  cs: 'Czech',
+  da: 'Danish',
+  de: 'German',
+  el: 'Greek',
+  es: 'Spanish',
+  fa: 'Persian (Farsi)',
+  fi: 'Finnish',
+  fr: 'French',
+  he: 'Hebrew',
+  hi: 'Hindi',
+  hu: 'Hungarian',
+  id: 'Indonesian',
+  it: 'Italian',
+  ja: 'Japanese',
+  ko: 'Korean',
+  ms: 'Malay',
+  nl: 'Dutch',
+  no: 'Norwegian',
+  pl: 'Polish',
+  pt: 'Portuguese',
+  ro: 'Romanian',
+  ru: 'Russian',
+  sv: 'Swedish',
+  th: 'Thai',
+  tl: 'Tagalog/Filipino',
+  tr: 'Turkish',
+  uk: 'Ukrainian',
+  ur: 'Urdu',
+  vi: 'Vietnamese',
+  zh: 'Chinese (Simplified)',
+};
+
+// Open Graph locales (`language_TERRITORY` — the og:locale convention) for the
+// supported set plus the common source languages. One territory per language:
+// the most widely used one, matching Facebook's locale list (Norwegian is
+// `nb_NO` there, Portuguese `pt_PT`).
+const LANGUAGE_LOCALES = {
+  ar: 'ar_AR',
+  bn: 'bn_IN',
+  cs: 'cs_CZ',
+  da: 'da_DK',
+  de: 'de_DE',
+  el: 'el_GR',
+  en: 'en_US',
+  es: 'es_ES',
+  fa: 'fa_IR',
+  fi: 'fi_FI',
+  fr: 'fr_FR',
+  he: 'he_IL',
+  hi: 'hi_IN',
+  hu: 'hu_HU',
+  id: 'id_ID',
+  it: 'it_IT',
+  ja: 'ja_JP',
+  ko: 'ko_KR',
+  ms: 'ms_MY',
+  nl: 'nl_NL',
+  no: 'nb_NO',
+  pl: 'pl_PL',
+  pt: 'pt_PT',
+  ro: 'ro_RO',
+  ru: 'ru_RU',
+  sv: 'sv_SE',
+  th: 'th_TH',
+  tl: 'tl_PH',
+  tr: 'tr_TR',
+  uk: 'uk_UA',
+  ur: 'ur_PK',
+  vi: 'vi_VN',
+  zh: 'zh_CN',
+};
+
+// Right-to-left languages (drives <html dir="rtl"> on translated pages)
+const RTL_LANGUAGES = ['ar', 'he', 'fa', 'ur', 'ps', 'sd', 'ku', 'yi', 'ckb', 'dv'];
+
+/**
+ * Whether a language renders right-to-left.
+ * @param {string} code - language code
+ * @returns {boolean}
+ */
+function isRTL(code) {
+  return RTL_LANGUAGES.includes(code);
+}
+
+/**
+ * English name for a language code.
+ * @param {string} code - language code
+ * @returns {string} the name, or the code itself if unknown
+ */
+function languageName(code) {
+  return LANGUAGE_NAMES[code] || code;
+}
+
+/**
+ * Open Graph locale for a language code — `es` → `es_ES`. Target codes are
+ * validated by assertKnownLanguages, but the SOURCE language is free-form
+ * config, so an unmapped code falls back to the code itself (a bare code is
+ * what og:locale carried before this map existed).
+ * @param {string} code - language code
+ * @returns {string} the language_TERRITORY locale, or the code if unmapped
+ */
+function ogLocale(code) {
+  return LANGUAGE_LOCALES[code] || code;
+}
+
+/**
+ * Assert every code is a supported target language.
+ * @param {string[]} codes - language codes from config
+ * @throws {Error} naming the unknown codes and the supported set
+ */
+function assertKnownLanguages(codes) {
+  const unknown = (codes || []).filter((code) => !LANGUAGE_NAMES[code]);
+
+  if (unknown.length) {
+    throw new Error(
+      `Unknown translation language(s): ${unknown.join(', ')}. `
+      + `Supported codes: ${Object.keys(LANGUAGE_NAMES).join(', ')}`
+    );
+  }
+}
+
+module.exports = { LANGUAGE_NAMES, LANGUAGE_LOCALES, RTL_LANGUAGES, isRTL, languageName, ogLocale, assertKnownLanguages };

package/dist/vendor/devkit/translate/providers.js

@@ -0,0 +1,188 @@
+/**
+ * AI providers for the translation engine — `claude` (default; rides the
+ * locally-installed Claude Code via @anthropic-ai/claude-agent-sdk, so no API
+ * key is needed) and `chatgpt` (OpenAI Responses API via native fetch;
+ * requires OPENAI_API_KEY). Both expose the same contract:
+ *   provider.send({ system, user }) → Promise<{ text, usage }>
+ * where usage is { input, output } token counts (zeros when unreported).
+ *
+ * The Agent SDK is lazy-required, so it is only touched when a claude call
+ * actually happens, and devkit itself stays dependency-free here. It resolves
+ * from whoever installed it: @omega.js/extension declares it, @omega.js/web
+ * does NOT (#37: translation is opt-in and the SDK is heavy, so a web brand
+ * that enables translation installs it in the target). When it is absent, the
+ * call fails loud with the install command instead of skipping translation.
+ */
+
+const PROVIDERS = ['claude', 'chatgpt'];
+
+// Default model per provider. 'sonnet' is the Claude Code alias that tracks
+// the current Sonnet — no stale date pins in framework code.
+const DEFAULT_MODELS = {
+  claude: 'sonnet',
+  chatgpt: 'gpt-5.4-nano',
+};
+
+/**
+ * Build a provider handle from config values.
+ * @param {object} options
+ * @param {string} [options.provider] - 'claude' (default) or 'chatgpt'
+ * @param {string} [options.model] - model override (default per provider)
+ * @param {Function} [options.fetchFn] - fetch override (tests)
+ * @returns {{ name: string, model: string, send: Function }}
+ */
+function resolveProvider(options) {
+  options = options || {};
+  const name = options.provider || 'claude';
+
+  if (!PROVIDERS.includes(name)) {
+    throw new Error(`Unknown translation provider "${name}" — use ${PROVIDERS.map((p) => `'${p}'`).join(' or ')}`);
+  }
+
+  const model = options.model || DEFAULT_MODELS[name];
+  const send = name === 'claude'
+    ? (message) => sendClaude(model, message)
+    : (message) => sendChatgpt(model, message, options.fetchFn);
+
+  return { name, model, send };
+}
+
+/**
+ * Send one prompt through the local Claude Code install (Agent SDK).
+ * @param {string} model - model name/alias
+ * @param {{ system: string, user: string }} message
+ * @returns {Promise<{ text: string, usage: { input: number, output: number } }>}
+ */
+async function sendClaude(model, message) {
+  let query;
+  try {
+    ({ query } = require('@anthropic-ai/claude-agent-sdk'));
+  } catch (e) {
+    throw new Error('The claude translation provider needs @anthropic-ai/claude-agent-sdk, which is not installed here. Translation is opt-in and the SDK is heavy, so the framework does not bundle it: install it in this target with `npm install @anthropic-ai/claude-agent-sdk` (it also needs a local Claude Code install/auth), or set translation.providers to { chatgpt: {} }.');
+  }
+
+  let text = '';
+  const usage = { input: 0, output: 0 };
+
+  // Hermetic single-shot: settingSources [] keeps the user's CLAUDE.md/hooks
+  // out of the session (they pollute mechanical output), the system prompt
+  // replaces the CLI persona, and no maxTurns cap — a plain reply already
+  // counts as the final turn and a cap of 1 reports error_max_turns.
+  //
+  // Subscription-only by contract: this provider rides the local Claude
+  // Code install's own login. A stray ANTHROPIC_API_KEY anywhere in the
+  // env cascade would silently flip the SDK to API-credit billing — the
+  // child env hides it so the subscription is the ONLY auth path.
+  const env = { ...process.env };
+  delete env.ANTHROPIC_API_KEY;
+
+  const consume = async () => {
+    for await (const event of query({
+      prompt: message.user,
+      options: {
+        model,
+        allowedTools: [],
+        settingSources: [],
+        systemPrompt: message.system,
+        env,
+      },
+    })) {
+      if (event.type === 'assistant' && event.message?.content) {
+        for (const block of event.message.content) {
+          if (block.type === 'text') {
+            text += block.text;
+          }
+        }
+      }
+
+      if (event.type === 'result') {
+        // A failed call must throw NOW — a session-limited install can
+        // otherwise stall the whole build (the 28-minute cp119 hang)
+        if (event.subtype && event.subtype !== 'success') {
+          throw new Error(`Claude translate call failed (${event.subtype})${event.result ? `: ${String(event.result).slice(0, 200)}` : ''} — if this is the session limit, wait for the 5-hour window to reset or build with --cached-only.`);
+        }
+        if (event.usage) {
+          usage.input += event.usage.input_tokens || 0;
+          usage.output += event.usage.output_tokens || 0;
+        }
+      }
+    }
+  };
+
+  // Hard per-call ceiling — a stalled SDK call fails loud and fast instead
+  // of hanging the caller (OMEGA_TRANSLATE_TIMEOUT_MS overrides; tests use it)
+  const timeoutMs = Number(process.env.OMEGA_TRANSLATE_TIMEOUT_MS || 240000);
+  let timer = null;
+  try {
+    await Promise.race([
+      consume(),
+      new Promise((resolve, reject) => {
+        // NOT unref'd on purpose: the finally-clearTimeout already stops a
+        // settled call from holding the process, and an unref'd watchdog
+        // lets a stalled SDK call with no other live handles drain the
+        // event loop and exit SILENTLY before the timer can fire — the
+        // exact opposite of failing loud (and a test-run flake: the
+        // stalled-call test was cancelled whenever nothing else kept the
+        // loop alive for the 80ms window).
+        timer = setTimeout(() => reject(new Error(`Claude translate call exceeded ${Math.round(timeoutMs / 1000)}s — a stalled SDK call (often the session limit); wait for the window to reset or build with --cached-only.`)), timeoutMs);
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+
+  return { text, usage };
+}
+
+/**
+ * Send one prompt through the OpenAI Responses API.
+ * @param {string} model - model name
+ * @param {{ system: string, user: string }} message
+ * @param {Function} [fetchFn] - fetch override (tests)
+ * @returns {Promise<{ text: string, usage: { input: number, output: number } }>}
+ */
+async function sendChatgpt(model, message, fetchFn) {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error('The chatgpt translation provider requires OPENAI_API_KEY in the environment (.env cascade) — or use the default "claude" provider (local Claude Code, no key).');
+  }
+
+  const doFetch = fetchFn || fetch;
+  const response = await doFetch('https://api.openai.com/v1/responses', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    signal: AbortSignal.timeout(240000),
+    body: JSON.stringify({
+      model,
+      input: [
+        { role: 'developer', content: message.system },
+        { role: 'user', content: message.user },
+      ],
+      reasoning: { effort: 'low' },
+    }),
+  });
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    throw new Error(`OpenAI API error ${response.status}: ${body.slice(0, 300)}`);
+  }
+
+  const data = await response.json();
+
+  // Reasoning models emit a reasoning item first — find the message item
+  const item = (data.output || []).find((o) => o.type === 'message');
+  const text = item?.content?.[0]?.text || '';
+
+  return {
+    text,
+    usage: {
+      input: data.usage?.input_tokens || 0,
+      output: data.usage?.output_tokens || 0,
+    },
+  };
+}
+
+module.exports = { PROVIDERS, DEFAULT_MODELS, resolveProvider };