glotfile 0.5.0 → 0.5.1

package/dist/server/cli.js

@@ -2806,7 +2806,7 @@ function attachScreenshotsForProvider(reqs, state, projectRoot, supportsVision)
   const keys = new Set(reqs.filter((r) => state.keys[r.key]?.screenshot).map((r) => r.key));
   return { skipped: keys.size };
 }
-async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAULT_LOCALE_CONCURRENCY, signal) {
+async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAULT_LOCALE_CONCURRENCY, signal, batchSize = Infinity) {
   if (!reqs.length) return [];
   const byLocale = /* @__PURE__ */ new Map();
   for (const req of reqs) {
@@ -2817,26 +2817,42 @@ async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAU
     }
     group.push(req);
   }
+  const localeBatches = [...byLocale.entries()].map(([locale, group]) => ({
+    locale,
+    batches: chunk(group, Math.max(1, batchSize))
+  }));
+  const jobs = [];
+  const maxBatches = Math.max(...localeBatches.map((g) => g.batches.length));
+  for (let i = 0; i < maxBatches; i++) {
+    for (const g of localeBatches) {
+      if (i < g.batches.length) jobs.push({ locale: g.locale, batch: g.batches[i] });
+    }
+  }
+  const remaining = new Map(localeBatches.map((g) => [g.locale, g.batches.length]));
+  const started = /* @__PURE__ */ new Set();
   const total = reqs.length;
   let done = 0;
   const allResults = [];
-  const groups = [...byLocale.values()];
   let next = 0;
   async function worker() {
-    while (next < groups.length) {
+    while (next < jobs.length) {
       if (signal?.aborted) break;
-      const group = groups[next++];
-      const locale = group[0].targetLocale;
-      hooks.onLocaleStart?.(locale);
-      const localeResults = await provider.translate(group, (_localeDone, _localeTotal, batchResults) => {
-        done += batchResults.length;
-        hooks.onBatchComplete?.(done, total, batchResults, locale);
-      }, signal, (raw, batchSize) => hooks.onMalformedReply?.(raw, batchSize, locale));
-      allResults.push(...localeResults);
-      if (!signal?.aborted) hooks.onLocaleDone?.(locale);
+      const { locale, batch } = jobs[next++];
+      if (!started.has(locale)) {
+        started.add(locale);
+        hooks.onLocaleStart?.(locale);
+      }
+      const batchResults = await provider.translate(batch, (_localeDone, _localeTotal, results) => {
+        done += results.length;
+        hooks.onBatchComplete?.(done, total, results, locale);
+      }, signal, (raw, size) => hooks.onMalformedReply?.(raw, size, locale));
+      allResults.push(...batchResults);
+      const left = remaining.get(locale) - 1;
+      remaining.set(locale, left);
+      if (left === 0 && !signal?.aborted) hooks.onLocaleDone?.(locale);
     }
   }
-  const workers = Array.from({ length: Math.min(concurrency, groups.length) }, worker);
+  const workers = Array.from({ length: Math.min(concurrency, jobs.length) }, worker);
   await Promise.all(workers);
   return allResults;
 }
@@ -2872,6 +2888,7 @@ var init_run = __esm({
     init_plurals();
     init_state();
     init_glob();
+    init_batch();
     MEDIA_TYPES = {
       ".png": "image/png",
       ".jpg": "image/jpeg",
@@ -5453,7 +5470,7 @@ function createApi(deps) {
             raw
           });
         }
-      }, aiCfg.concurrency, signal);
+      }, aiCfg.concurrency, signal, aiCfg.batchSize);
       if (!signal?.aborted) {
         console.log(`[translate] done \u2014 wrote ${totalWritten}, ${allErrors.length} error(s)`);
         await stream.writeSSE({ event: "done", data: JSON.stringify({ written: totalWritten, errors: allErrors }) });
@@ -5496,7 +5513,7 @@ function createApi(deps) {
             raw
           });
         }
-      }, aiCfg.concurrency);
+      }, aiCfg.concurrency, void 0, aiCfg.batchSize);
       const latest = load();
       ({ written, errors } = applyResults(latest, toTranslate, results, void 0, force));
       const entry = {
@@ -6123,7 +6140,7 @@ async function runTranslate(args) {
           raw
         });
       }
-    });
+    }, ai.concurrency, void 0, ai.batchSize);
     process.stdout.write("\n");
     if (!batchCallbackFired) {
       ({ written, errors } = applyResults(state, toTranslate, results));

package/dist/server/server.js

@@ -1493,6 +1493,88 @@ function globToRegExp2(glob) {
   return new RegExp(`^${escaped}$`);
 }
 
+// src/server/ai/batch.ts
+var MalformedReplyError = class extends Error {
+  constructor(raw) {
+    super("Model reply was not valid translation JSON.");
+    this.raw = raw;
+    this.name = "MalformedReplyError";
+  }
+  raw;
+};
+function parseReplyItems(text) {
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new MalformedReplyError(text);
+  }
+  if (!Array.isArray(parsed.items)) throw new MalformedReplyError(text);
+  return parsed.items;
+}
+function chunk(items, size) {
+  const out = [];
+  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
+  return out;
+}
+function validateTranslation(req, translation) {
+  if (translation === void 0) return { id: req.id, error: "No translation returned." };
+  if (!placeholdersMatch(req.source, translation)) {
+    return { id: req.id, error: "Placeholder mismatch between source and translation." };
+  }
+  if (req.maxLength !== void 0 && translation.length > req.maxLength) {
+    return { id: req.id, translation, error: `Exceeds maxLength (${translation.length} > ${req.maxLength}).` };
+  }
+  return { id: req.id, translation };
+}
+function validatePlural(req, forms) {
+  if (!forms) return { id: req.id, error: "No translation returned." };
+  const plural = req.plural;
+  if (!plural) return { id: req.id, error: "validatePlural called on a non-plural request." };
+  const cats = plural.categories;
+  const missing = cats.filter((c) => typeof forms[c] !== "string");
+  if (missing.length) return { id: req.id, error: `Missing plural categories: ${missing.join(", ")}.` };
+  const badPh = cats.find((c) => !pluralFormPlaceholdersMatch(c, req.source, forms[c]));
+  if (badPh) return { id: req.id, error: `Placeholder mismatch in plural form "${badPh}".` };
+  if (req.maxLength !== void 0) {
+    const over = cats.find((c) => forms[c].length > req.maxLength);
+    if (over) return { id: req.id, error: `Plural form "${over}" exceeds maxLength (${forms[over].length} > ${req.maxLength}).` };
+  }
+  const out = {};
+  for (const c of cats) out[c] = forms[c];
+  return { id: req.id, forms: out };
+}
+function validateReply(req, item) {
+  return req.plural ? validatePlural(req, item?.forms) : validateTranslation(req, item?.translation);
+}
+async function runBatched(reqs, batchSize, callBatch, onBatchComplete, signal, onMalformedReply) {
+  const failBatch = (batch) => batch.map((req) => ({ id: req.id, error: "Model returned malformed JSON for this string." }));
+  async function resolveBatch(batch, isRetry = false) {
+    let reply;
+    try {
+      reply = await callBatch(batch, signal);
+    } catch (err) {
+      if (!(err instanceof MalformedReplyError)) throw err;
+      onMalformedReply?.(err.raw, batch.length);
+      if (signal?.aborted) return failBatch(batch);
+      if (batch.length === 1) return isRetry ? failBatch(batch) : resolveBatch(batch, true);
+      const mid = Math.ceil(batch.length / 2);
+      return [...await resolveBatch(batch.slice(0, mid)), ...await resolveBatch(batch.slice(mid))];
+    }
+    const byId = new Map(reply.map((r) => [r.id, r]));
+    return batch.map((req) => validateReply(req, byId.get(req.id)));
+  }
+  const results = [];
+  const total = reqs.length;
+  for (const batch of chunk(reqs, Math.max(1, batchSize))) {
+    if (signal?.aborted) break;
+    const batchResults = await resolveBatch(batch);
+    results.push(...batchResults);
+    onBatchComplete?.(results.length, total, batchResults);
+  }
+  return results;
+}
+
 // src/server/ai/run.ts
 function selectRequests(state, opts) {
   const targets = (opts.locales ?? state.config.locales).filter((l) => l !== state.config.sourceLocale);
@@ -1603,7 +1685,7 @@ function attachScreenshotsForProvider(reqs, state, projectRoot, supportsVision)
   return { skipped: keys.size };
 }
 var DEFAULT_LOCALE_CONCURRENCY = 3;
-async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAULT_LOCALE_CONCURRENCY, signal) {
+async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAULT_LOCALE_CONCURRENCY, signal, batchSize = Infinity) {
   if (!reqs.length) return [];
   const byLocale = /* @__PURE__ */ new Map();
   for (const req of reqs) {
@@ -1614,26 +1696,42 @@ async function runLocaleParallel(reqs, provider, hooks = {}, concurrency = DEFAU
     }
     group.push(req);
   }
+  const localeBatches = [...byLocale.entries()].map(([locale, group]) => ({
+    locale,
+    batches: chunk(group, Math.max(1, batchSize))
+  }));
+  const jobs = [];
+  const maxBatches = Math.max(...localeBatches.map((g) => g.batches.length));
+  for (let i = 0; i < maxBatches; i++) {
+    for (const g of localeBatches) {
+      if (i < g.batches.length) jobs.push({ locale: g.locale, batch: g.batches[i] });
+    }
+  }
+  const remaining = new Map(localeBatches.map((g) => [g.locale, g.batches.length]));
+  const started = /* @__PURE__ */ new Set();
   const total = reqs.length;
   let done = 0;
   const allResults = [];
-  const groups = [...byLocale.values()];
   let next = 0;
   async function worker() {
-    while (next < groups.length) {
+    while (next < jobs.length) {
       if (signal?.aborted) break;
-      const group = groups[next++];
-      const locale = group[0].targetLocale;
-      hooks.onLocaleStart?.(locale);
-      const localeResults = await provider.translate(group, (_localeDone, _localeTotal, batchResults) => {
-        done += batchResults.length;
-        hooks.onBatchComplete?.(done, total, batchResults, locale);
-      }, signal, (raw, batchSize) => hooks.onMalformedReply?.(raw, batchSize, locale));
-      allResults.push(...localeResults);
-      if (!signal?.aborted) hooks.onLocaleDone?.(locale);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, groups.length) }, worker);
+      const { locale, batch } = jobs[next++];
+      if (!started.has(locale)) {
+        started.add(locale);
+        hooks.onLocaleStart?.(locale);
+      }
+      const batchResults = await provider.translate(batch, (_localeDone, _localeTotal, results) => {
+        done += results.length;
+        hooks.onBatchComplete?.(done, total, results, locale);
+      }, signal, (raw, size) => hooks.onMalformedReply?.(raw, size, locale));
+      allResults.push(...batchResults);
+      const left = remaining.get(locale) - 1;
+      remaining.set(locale, left);
+      if (left === 0 && !signal?.aborted) hooks.onLocaleDone?.(locale);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, jobs.length) }, worker);
   await Promise.all(workers);
   return allResults;
 }
@@ -3042,88 +3140,6 @@ var BATCH_SCHEMA = {
   additionalProperties: false
 };
 
-// src/server/ai/batch.ts
-var MalformedReplyError = class extends Error {
-  constructor(raw) {
-    super("Model reply was not valid translation JSON.");
-    this.raw = raw;
-    this.name = "MalformedReplyError";
-  }
-  raw;
-};
-function parseReplyItems(text) {
-  let parsed;
-  try {
-    parsed = JSON.parse(text);
-  } catch {
-    throw new MalformedReplyError(text);
-  }
-  if (!Array.isArray(parsed.items)) throw new MalformedReplyError(text);
-  return parsed.items;
-}
-function chunk(items, size) {
-  const out = [];
-  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
-  return out;
-}
-function validateTranslation(req, translation) {
-  if (translation === void 0) return { id: req.id, error: "No translation returned." };
-  if (!placeholdersMatch(req.source, translation)) {
-    return { id: req.id, error: "Placeholder mismatch between source and translation." };
-  }
-  if (req.maxLength !== void 0 && translation.length > req.maxLength) {
-    return { id: req.id, translation, error: `Exceeds maxLength (${translation.length} > ${req.maxLength}).` };
-  }
-  return { id: req.id, translation };
-}
-function validatePlural(req, forms) {
-  if (!forms) return { id: req.id, error: "No translation returned." };
-  const plural = req.plural;
-  if (!plural) return { id: req.id, error: "validatePlural called on a non-plural request." };
-  const cats = plural.categories;
-  const missing = cats.filter((c) => typeof forms[c] !== "string");
-  if (missing.length) return { id: req.id, error: `Missing plural categories: ${missing.join(", ")}.` };
-  const badPh = cats.find((c) => !pluralFormPlaceholdersMatch(c, req.source, forms[c]));
-  if (badPh) return { id: req.id, error: `Placeholder mismatch in plural form "${badPh}".` };
-  if (req.maxLength !== void 0) {
-    const over = cats.find((c) => forms[c].length > req.maxLength);
-    if (over) return { id: req.id, error: `Plural form "${over}" exceeds maxLength (${forms[over].length} > ${req.maxLength}).` };
-  }
-  const out = {};
-  for (const c of cats) out[c] = forms[c];
-  return { id: req.id, forms: out };
-}
-function validateReply(req, item) {
-  return req.plural ? validatePlural(req, item?.forms) : validateTranslation(req, item?.translation);
-}
-async function runBatched(reqs, batchSize, callBatch, onBatchComplete, signal, onMalformedReply) {
-  const failBatch = (batch) => batch.map((req) => ({ id: req.id, error: "Model returned malformed JSON for this string." }));
-  async function resolveBatch(batch, isRetry = false) {
-    let reply;
-    try {
-      reply = await callBatch(batch, signal);
-    } catch (err) {
-      if (!(err instanceof MalformedReplyError)) throw err;
-      onMalformedReply?.(err.raw, batch.length);
-      if (signal?.aborted) return failBatch(batch);
-      if (batch.length === 1) return isRetry ? failBatch(batch) : resolveBatch(batch, true);
-      const mid = Math.ceil(batch.length / 2);
-      return [...await resolveBatch(batch.slice(0, mid)), ...await resolveBatch(batch.slice(mid))];
-    }
-    const byId = new Map(reply.map((r) => [r.id, r]));
-    return batch.map((req) => validateReply(req, byId.get(req.id)));
-  }
-  const results = [];
-  const total = reqs.length;
-  for (const batch of chunk(reqs, Math.max(1, batchSize))) {
-    if (signal?.aborted) break;
-    const batchResults = await resolveBatch(batch);
-    results.push(...batchResults);
-    onBatchComplete?.(results.length, total, batchResults);
-  }
-  return results;
-}
-
 // src/server/ai/anthropic.ts
 var AnthropicProvider = class {
   constructor(config, client) {
@@ -5002,7 +5018,7 @@ function createApi(deps) {
             raw
           });
         }
-      }, aiCfg.concurrency, signal);
+      }, aiCfg.concurrency, signal, aiCfg.batchSize);
       if (!signal?.aborted) {
         console.log(`[translate] done \u2014 wrote ${totalWritten}, ${allErrors.length} error(s)`);
         await stream.writeSSE({ event: "done", data: JSON.stringify({ written: totalWritten, errors: allErrors }) });
@@ -5045,7 +5061,7 @@ function createApi(deps) {
             raw
           });
         }
-      }, aiCfg.concurrency);
+      }, aiCfg.concurrency, void 0, aiCfg.batchSize);
       const latest = load();
       ({ written, errors } = applyResults(latest, toTranslate, results, void 0, force));
       const entry = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "glotfile",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Local-first, git-native translation management.",
   "type": "module",
   "bin": {