glotto 2.9.0 → 3.0.0

package/esm/src/translator.js

@@ -0,0 +1,200 @@
+import { logger } from './logger.js';
+import { delay, formatBytes } from './utilites.js';
+import { BASE_RETRY_DELAY_MS, INTER_BATCH_DELAY_MS, MAX_RETRIES } from './contants.js';
+const encoder = new TextEncoder();
+const byteSize = (s) => encoder.encode(s).byteLength;
+export const extractLeaves = (data) => {
+    const out = [];
+    let nextId = 0;
+    const walk = (node, path) => {
+        if (node === null || typeof node !== 'object') {
+            const isString = typeof node === 'string';
+            out.push({
+                id: ++nextId,
+                path,
+                value: node,
+                translatable: isString && node.trim().length > 0,
+            });
+            return;
+        }
+        if (Array.isArray(node)) {
+            if (node.length === 0) {
+                out.push({ id: ++nextId, path, value: node, translatable: false });
+                return;
+            }
+            node.forEach((item, i) => walk(item, [...path, i]));
+            return;
+        }
+        const entries = Object.entries(node);
+        if (entries.length === 0) {
+            out.push({ id: ++nextId, path, value: node, translatable: false });
+            return;
+        }
+        for (const [k, v] of entries)
+            walk(v, [...path, k]);
+    };
+    walk(data, []);
+    return out;
+};
+const TAG_OPEN = (id) => `≪${id}≫`;
+const TAG_CLOSE = (id) => `≪/${id}≫`;
+const TAG_OVERHEAD_CHARS = 8;
+const wrapEntry = (leaf) => `${TAG_OPEN(leaf.id)}${String(leaf.value)}${TAG_CLOSE(leaf.id)}`;
+export const groupIntoBatches = (leaves, maxBytes) => {
+    const translatable = leaves.filter((l) => l.translatable);
+    const batches = [];
+    let current = [];
+    let currentSize = 0;
+    for (const leaf of translatable) {
+        const valueBytes = byteSize(String(leaf.value));
+        const idChars = String(leaf.id).length;
+        const overhead = TAG_OVERHEAD_CHARS + idChars * 2 + 1;
+        const itemSize = valueBytes + overhead;
+        if (current.length > 0 && currentSize + itemSize > maxBytes) {
+            batches.push({ index: batches.length, leaves: current, byteSize: currentSize });
+            current = [];
+            currentSize = 0;
+        }
+        current.push(leaf);
+        currentSize += itemSize;
+    }
+    if (current.length > 0) {
+        batches.push({ index: batches.length, leaves: current, byteSize: currentSize });
+    }
+    return batches;
+};
+export const buildBatchPrompt = (from, to, leaves) => {
+    const entries = leaves.map(wrapEntry).join('\n');
+    return `Translate each tagged entry below from ${from} to ${to}.
+
+Tag format:
+- Every entry is wrapped exactly as ≪N≫value≪/N≫, where N is the entry id (a positive integer).
+- The opening marker is the character ≪ (U+226A), then the id, then ≫ (U+226B). The closing marker is ≪, /, the same id, then ≫.
+- Copy these markers verbatim. Do not put spaces inside them. Do not change them to <<>>, «», <tag>, [n], or any other notation.
+- The id must match the input id of the same entry — never renumber, merge, split, or reorder entries.
+
+What to write back:
+- One ≪N≫translated_value≪/N≫ block per input entry, in the same order, separated by single newlines.
+- Translate only the text that sits between ≪N≫ and ≪/N≫.
+- Preserve every variable ({{name}}, {name}, __VAR__, $t(...), %s, %d), HTML tag, markdown token, escape sequence, and special character inside the value.
+- Output nothing else: no headings, no prose, no commentary, no code fences, no language labels.
+
+Now translate the following entries:
+${entries}`;
+};
+const TAG_PATTERNS = [
+    /≪\s*(\d+)\s*≫([\s\S]*?)≪\s*\/\s*\1\s*≫/g,
+    /<<\s*(\d+)\s*>>([\s\S]*?)<<\s*\/\s*\1\s*>>/g,
+    /«\s*(\d+)\s*»([\s\S]*?)«\s*\/\s*\1\s*»/g,
+];
+export const decodeResponse = (text) => {
+    const map = new Map();
+    for (const pattern of TAG_PATTERNS) {
+        for (const match of text.matchAll(pattern)) {
+            const id = parseInt(match[1], 10);
+            if (!map.has(id))
+                map.set(id, match[2].trim());
+        }
+    }
+    return map;
+};
+const setPath = (root, path, value) => {
+    let node = root;
+    for (let i = 0; i < path.length - 1; i++) {
+        const key = path[i];
+        const nextKey = path[i + 1];
+        const container = node;
+        if (container[key] === undefined) {
+            container[key] = typeof nextKey === 'number' ? [] : {};
+        }
+        node = container[key];
+    }
+    node[path[path.length - 1]] = value;
+};
+export const reconstruct = (allLeaves, translations) => {
+    if (allLeaves.length === 0)
+        return {};
+    const firstStep = allLeaves[0].path[0];
+    if (firstStep === undefined) {
+        const only = allLeaves[0];
+        return only.translatable ? translations.get(only.id) ?? only.value : only.value;
+    }
+    const root = typeof firstStep === 'number' ? [] : {};
+    for (const leaf of allLeaves) {
+        const value = leaf.translatable && translations.has(leaf.id) ? translations.get(leaf.id) : leaf.value;
+        setPath(root, leaf.path, value);
+    }
+    return root;
+};
+export const runBatches = async (batches, translator, from, to, options) => {
+    const translations = new Map();
+    const failedIds = [];
+    for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        const label = `[Batch ${batch.index + 1}/${batches.length}]`;
+        if (i > 0 && !options.noLimit) {
+            logger.info(`Waiting ${INTER_BATCH_DELAY_MS}ms before next batch (rate limit protection)...`);
+            await delay(INTER_BATCH_DELAY_MS);
+        }
+        let remaining = [...batch.leaves];
+        for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+            if (attempt > 0) {
+                const backoffMs = BASE_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
+                logger.warn(`${label} Retry ${attempt}/${MAX_RETRIES} after ${backoffMs}ms backoff (${remaining.length} entries)...`);
+                await delay(backoffMs);
+            }
+            try {
+                const prompt = buildBatchPrompt(from, to, remaining);
+                const response = await translator.translate(prompt);
+                if (!response || response.trim().length === 0) {
+                    if (attempt === MAX_RETRIES) {
+                        logger.error(`${label} Failed after ${MAX_RETRIES + 1} attempts: empty response`);
+                        for (const leaf of remaining)
+                            failedIds.push(leaf.id);
+                        break;
+                    }
+                    logger.warn(`${label} Attempt ${attempt + 1} returned empty response`);
+                    continue;
+                }
+                const decoded = decodeResponse(response);
+                const stillMissing = [];
+                for (const leaf of remaining) {
+                    const value = decoded.get(leaf.id);
+                    if (!value || value.trim().length === 0) {
+                        stillMissing.push(leaf);
+                        continue;
+                    }
+                    translations.set(leaf.id, value);
+                }
+                if (stillMissing.length === 0) {
+                    logger.info(`${label} Translated successfully (${formatBytes(batch.byteSize)}, ${batch.leaves.length} entries)`);
+                    break;
+                }
+                if (attempt === MAX_RETRIES) {
+                    logger.error(`${label} Failed after ${MAX_RETRIES + 1} attempts: ${stillMissing.length} entries missing`);
+                    for (const leaf of stillMissing)
+                        failedIds.push(leaf.id);
+                    break;
+                }
+                logger.warn(`${label} ${stillMissing.length}/${remaining.length} entries missing, will retry`);
+                remaining = stillMissing;
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                if (attempt === MAX_RETRIES) {
+                    logger.error(`${label} Failed after ${MAX_RETRIES + 1} attempts: ${message}`);
+                    for (const leaf of remaining)
+                        failedIds.push(leaf.id);
+                    break;
+                }
+                logger.warn(`${label} Attempt ${attempt + 1} failed: ${message}`);
+            }
+        }
+    }
+    if (failedIds.length > 0) {
+        const preview = failedIds.slice(0, 20).join(', ');
+        const more = failedIds.length > 20 ? `, …(+${failedIds.length - 20})` : '';
+        throw new Error(`Translation failed for ${failedIds.length} entry/entries (ids: ${preview}${more})`);
+    }
+    return translations;
+};

package/esm/src/types.d.ts

@@ -1,4 +1,8 @@
 export type Provider = 'gemini' | 'openai' | 'anthropic';
+export type TranslateOptions = {
+    noLimit: boolean;
+    noTimeout: boolean;
+};
 export type TranslateArgs = {
     key?: string;
     provider?: string;
@@ -8,6 +12,9 @@ export type TranslateArgs = {
     from?: string;
     to?: string;
     url?: string;
+    'no-limit'?: boolean;
+    'no-timeout'?: boolean;
+    'max-batch-size'?: string;
 };
 export type ValidatedTranslateArgs = {
     key: string;
@@ -18,19 +25,29 @@ export type ValidatedTranslateArgs = {
     from: string;
     to: string;
     url?: string;
+    noLimit: boolean;
+    noTimeout: boolean;
+    maxBatchBytes: number;
 };
-export interface Translatable {
-    chunks: ChunkInfo[];
-    from: string;
-    to: string;
-    translate(): Promise<string>;
+export type JsonValue = string | number | boolean | null | JsonObject | JsonArray;
+export interface JsonObject {
+    [key: string]: JsonValue;
 }
-export type JsonObject = Record<string, any>;
-export type Chunk = Uint8Array;
-export type ChunkInfo = {
-    data: Uint8Array;
-    keyCount: number;
-    byteSize: number;
+export type JsonArray = JsonValue[];
+export type PathSegment = string | number;
+export type Path = PathSegment[];
+export type Leaf = {
+    id: number;
+    path: Path;
+    value: JsonValue;
+    translatable: boolean;
+};
+export type Batch = {
     index: number;
+    leaves: Leaf[];
+    byteSize: number;
 };
+export interface TextTranslator {
+    translate(prompt: string): Promise<string>;
+}
 //# sourceMappingURL=types.d.ts.map

package/esm/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEzD,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE7C,MAAM,MAAM,KAAK,GAAG,UAAU,CAAC;AAE/B,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,UAAU,CAAC;IAEjB,QAAQ,EAAE,MAAM,CAAC;IAEjB,QAAQ,EAAE,MAAM,CAAC;IAEjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEzD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,UAAU,GAAG,SAAS,CAAC;AAClF,MAAM,WAAW,UAAU;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B;AACD,MAAM,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;AAEpC,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;AAEjC,MAAM,MAAM,IAAI,GAAG;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,SAAS,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,IAAI,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC5C"}

package/esm/src/utilites.d.ts

@@ -1,12 +1,6 @@
 import type { TranslateArgs, ValidatedTranslateArgs } from './types.js';
 import type { Args } from '../deps/jsr.io/@std/cli/1.0.29/mod.js';
 export declare const validateArgs: (args: Args<TranslateArgs>) => ValidatedTranslateArgs;
-export declare const generatePrompts: (from: string, to: string) => {
-    systemPrompt: string;
-    userPrompt: string;
-};
-export declare const stripJsonMarkdown: (text: string) => string;
-export declare const isValidJson: (value: string) => boolean;
 export declare const formatBytes: (bytes: number) => string;
 export declare const delay: (ms: number) => Promise<void>;
 //# sourceMappingURL=utilites.d.ts.map

package/esm/src/utilites.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utilites.d.ts","sourceRoot":"","sources":["../../src/src/utilites.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAClF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,uCAAuC,CAAC;AAElE,eAAO,MAAM,YAAY,GAAI,MAAM,IAAI,CAAC,aAAa,CAAC,KAAG,sBAmCxD,CAAC;AAEF,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,IAAI,MAAM;;;CA2DvD,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,KAAG,MAyDhD,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,MAAM,KAAG,OAO3C,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,MAAM,KAAG,MAI3C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,IAAI,MAAM,KAAG,OAAO,CAAC,IAAI,CAE9C,CAAC"}
+{"version":3,"file":"utilites.d.ts","sourceRoot":"","sources":["../../src/src/utilites.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAClF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,uCAAuC,CAAC;AAGlE,eAAO,MAAM,YAAY,GAAI,MAAM,IAAI,CAAC,aAAa,CAAC,KAAG,sBA+BxD,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,MAAM,KAAG,MAI3C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,IAAI,MAAM,KAAG,OAAO,CAAC,IAAI,CAE9C,CAAC"}

package/esm/src/utilites.js

@@ -1,21 +1,25 @@
+import { DEFAULT_MAX_BATCH_BYTES } from './contants.js';
 export const validateArgs = (args) => {
-    if (!args.key) {
+    if (!args.key)
         throw new Error('AI Key is required');
-    }
-    if (!args.provider) {
+    if (!args.provider)
         throw new Error('Provider parameter is required');
-    }
-    if (!args.input) {
+    if (!args.input)
         throw new Error('Input parameter is required');
-    }
-    if (!args.output) {
+    if (!args.output)
         throw new Error('Output parameter is required');
-    }
-    if (!args.from) {
+    if (!args.from)
         throw new Error('Source language (from) parameter is required');
-    }
-    if (!args.to) {
+    if (!args.to)
         throw new Error('Target language (to) parameter is required');
+    let maxBatchBytes = DEFAULT_MAX_BATCH_BYTES;
+    const rawMaxBatchSize = args['max-batch-size'];
+    if (rawMaxBatchSize !== undefined) {
+        const parsed = parseInt(rawMaxBatchSize, 10);
+        if (isNaN(parsed) || parsed <= 0) {
+            throw new Error('--max-batch-size must be a positive integer (value in KB, e.g. --max-batch-size 8)');
+        }
+        maxBatchBytes = parsed * 1024;
     }
     return {
         key: args.key,
@@ -26,129 +30,11 @@ export const validateArgs = (args) => {
         from: args.from,
         to: args.to,
         url: args.url,
+        noLimit: args['no-limit'] ?? false,
+        noTimeout: args['no-timeout'] ?? false,
+        maxBatchBytes,
     };
 };
-export const generatePrompts = (from, to) => {
-    const systemPrompt = `You are a specialized i18next JSON translation expert. Your role is to translate content from ${from} to ${to} with these strict requirements:
-  
-  1. COMPLETE TRANSLATION:
-     - Translate ALL text values comprehensively
-     - Double-check to ensure no text is left untranslated
-     - Pay special attention to arrays and nested objects to ensure everything is translated
-     - If unsure about any translation, provide the most accurate and natural translation possible
-  
-  2. TRANSLATION QUALITY:
-     - Use natural, context-appropriate language
-     - Maintain consistent terminology throughout the translation
-     - Use formal language unless the source is clearly casual
-     - Preserve the exact meaning and tone of the original text
-     - For UI elements, use standard localized terms common in ${to} applications
-  
-  3. STRUCTURAL PRESERVATION:
-     - Keep all JSON structure and keys exactly as they are
-     - Maintain all variables and interpolation patterns ({{name}}, __VARIABLE__, $t(), etc.)
-     - Preserve all HTML tags and markdown formatting
-     - Keep all whitespace, nesting, and formatting intact
-  
-  4. VALIDATION:
-     - Return only valid JSON
-     - Verify that all text is translated
-     - Ensure no source language text remains
-     - Confirm all arrays and nested objects are fully translated`;
-    const userPrompt = `Please translate this i18next JSON file with these specific requirements:
-  
-  1. THOROUGH TRANSLATION:
-     - Translate every single text value from ${from} to ${to}
-     - Pay special attention to arrays and nested objects
-     - Verify no text is left in ${from}
-     - Double-check all translations for completeness
-  
-  2. PRESERVE STRUCTURE:
-     - Keep all keys unchanged (e.g. "button.submit")
-     - Maintain all variables: {{name}}, __VAR__, $t()
-     - Preserve HTML tags and markdown
-     - Keep all special characters
-     - Maintain exact JSON structure
-  
-  3. QUALITY REQUIREMENTS:
-     - Use natural ${to} language
-     - Maintain consistent terminology
-     - Use formal language unless source is casual
-     - Ensure translations match the context
-     - Use standard ${to} UI terminology for interface elements
-  
-  4. OUTPUT:
-     - Return only the translated JSON
-     - No explanations or comments
-     - No additional text
-     - No formatting changes
-     - Must be valid JSON`;
-    return { systemPrompt, userPrompt };
-};
-export const stripJsonMarkdown = (text) => {
-    const trimmed = text.trim();
-    // Extract from code fence if present (handles surrounding text too)
-    const fenceMatch = trimmed.match(/```(?:\w+)?\s*\n?([\s\S]*?)\n?```/);
-    if (fenceMatch) {
-        const candidate = fenceMatch[1].trim();
-        if (candidate.startsWith('{') || candidate.startsWith('[')) {
-            return candidate;
-        }
-    }
-    // Find outermost JSON structure by tracking bracket depth,
-    // correctly ignoring brackets inside strings and escape sequences
-    const firstBrace = trimmed.indexOf('{');
-    const firstBracket = trimmed.indexOf('[');
-    if (firstBrace === -1 && firstBracket === -1)
-        return trimmed;
-    let startIndex;
-    let openChar;
-    let closeChar;
-    if (firstBrace === -1 || (firstBracket !== -1 && firstBracket < firstBrace)) {
-        startIndex = firstBracket;
-        openChar = '[';
-        closeChar = ']';
-    }
-    else {
-        startIndex = firstBrace;
-        openChar = '{';
-        closeChar = '}';
-    }
-    let depth = 0;
-    let inString = false;
-    let escaped = false;
-    for (let i = startIndex; i < trimmed.length; i++) {
-        const char = trimmed[i];
-        if (escaped) {
-            escaped = false;
-            continue;
-        }
-        if (char === '\\' && inString) {
-            escaped = true;
-            continue;
-        }
-        if (char === '"') {
-            inString = !inString;
-            continue;
-        }
-        if (inString)
-            continue;
-        if (char === openChar)
-            depth++;
-        else if (char === closeChar && --depth === 0)
-            return trimmed.slice(startIndex, i + 1);
-    }
-    return trimmed;
-};
-export const isValidJson = (value) => {
-    try {
-        JSON.parse(value);
-        return true;
-    }
-    catch {
-        return false;
-    }
-};
 export const formatBytes = (bytes) => {
     if (bytes < 1024)
         return `${bytes} B`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "glotto",
-  "version": "2.9.0",
+  "version": "3.0.0",
   "description": "A tool for translating i18n JSON files using AI services.",
   "author": "Ibrahim Odev <me@ibrahimo.dev>",
   "repository": {

package/script/cli.js

@@ -42,6 +42,7 @@ const dntShim = __importStar(require("./_dnt.shims.js"));
 const unstable_spinner_js_1 = require("./deps/jsr.io/@std/cli/1.0.29/unstable_spinner.js");
 const mod_js_1 = require("./deps/jsr.io/@std/cli/1.0.29/mod.js");
 const file_js_1 = require("./src/file.js");
+const translator_js_1 = require("./src/translator.js");
 const contants_js_1 = require("./src/contants.js");
 const utilites_js_1 = require("./src/utilites.js");
 const logger_js_1 = require("./src/logger.js");
@@ -50,11 +51,23 @@ const gemini_js_1 = __importDefault(require("./src/providers/gemini.js"));
 const openai_js_1 = __importDefault(require("./src/providers/openai.js"));
 const anthropic_js_1 = __importDefault(require("./src/providers/anthropic.js"));
 const spinner = new unstable_spinner_js_1.Spinner({ message: 'AI Thinks...', color: 'cyan' });
+const createTranslator = (args, options) => {
+    switch (args.provider) {
+        case 'gemini':
+            return new gemini_js_1.default(args.key, args.model, options);
+        case 'openai':
+            return new openai_js_1.default(args.key, args.url, args.model, options);
+        case 'anthropic':
+            return new anthropic_js_1.default(args.key, args.url, args.model, options);
+        default:
+            throw new Error(`Unknown provider: ${args.provider}`);
+    }
+};
 async function main() {
     try {
         const args = (0, mod_js_1.parseArgs)(dntShim.Deno.args, {
-            string: ['key', 'provider', 'model', 'input', 'output', 'from', 'to', 'url'],
-            boolean: ['help', 'version'],
+            string: ['key', 'provider', 'model', 'input', 'output', 'from', 'to', 'url', 'max-batch-size'],
+            boolean: ['help', 'version', 'no-limit', 'no-timeout'],
             alias: {
                 provider: 'p',
                 model: 'm',
@@ -71,8 +84,7 @@ async function main() {
         const help = args.help || dntShim.Deno.args.length === 0;
         const version = args.version;
         if (version) {
-            const VERSION = deno_js_1.default.version;
-            logger_js_1.logger.info('Glotto version: ' + VERSION);
+            logger_js_1.logger.info('Glotto version: ' + deno_js_1.default.version);
             dntShim.Deno.exit(0);
         }
         if (help) {
@@ -81,49 +93,45 @@ async function main() {
         }
         const validatedArgs = (0, utilites_js_1.validateArgs)(args);
         const fileContent = await (0, file_js_1.getImportJson)(validatedArgs.input);
-        const chunks = (0, file_js_1.splitJson)(fileContent);
-        const totalBytes = chunks.reduce((sum, c) => sum + c.byteSize, 0);
-        const totalKeys = chunks.reduce((sum, c) => sum + c.keyCount, 0);
-        const outputPath = (0, file_js_1.resolvePath)(validatedArgs.output);
+        const allLeaves = (0, translator_js_1.extractLeaves)(fileContent);
+        const translatableLeaves = allLeaves.filter((leaf) => leaf.translatable);
+        const batches = (0, translator_js_1.groupIntoBatches)(allLeaves, validatedArgs.maxBatchBytes);
+        const totalBytes = batches.reduce((sum, b) => sum + b.byteSize, 0);
         logger_js_1.logger.info('Provider: ', validatedArgs.provider);
         logger_js_1.logger.info('Input: ', validatedArgs.input);
         logger_js_1.logger.info('Output: ', validatedArgs.output);
         logger_js_1.logger.info('From: ', validatedArgs.from);
         logger_js_1.logger.info('To: ', validatedArgs.to);
-        if (validatedArgs.model) {
+        if (validatedArgs.model)
             logger_js_1.logger.info('Model: ', validatedArgs.model);
-        }
-        if (validatedArgs.url) {
+        if (validatedArgs.url)
             logger_js_1.logger.info('URL: ', validatedArgs.url);
+        if (validatedArgs.noLimit)
+            logger_js_1.logger.info('Rate limit protection: disabled (--no-limit)');
+        if (validatedArgs.noTimeout)
+            logger_js_1.logger.info('Request timeout: disabled (--no-timeout)');
+        if (validatedArgs.maxBatchBytes !== contants_js_1.DEFAULT_MAX_BATCH_BYTES) {
+            logger_js_1.logger.info(`Max batch size: ${(0, utilites_js_1.formatBytes)(validatedArgs.maxBatchBytes)}`);
         }
-        logger_js_1.logger.info(`Total: ${totalKeys} keys, ${(0, utilites_js_1.formatBytes)(totalBytes)}, split into ${chunks.length} chunk(s)`);
-        for (const chunk of chunks) {
-            logger_js_1.logger.info(`  Chunk ${chunk.index + 1}: ${chunk.keyCount} keys, ${(0, utilites_js_1.formatBytes)(chunk.byteSize)}`);
+        logger_js_1.logger.info(`Total: ${translatableLeaves.length} translatable entries (of ${allLeaves.length} leaves), ${(0, utilites_js_1.formatBytes)(totalBytes)}, split into ${batches.length} batch(es)`);
+        for (const batch of batches) {
+            logger_js_1.logger.info(`  Batch ${batch.index + 1}: ${batch.leaves.length} entries, ${(0, utilites_js_1.formatBytes)(batch.byteSize)}`);
         }
-        spinner.start();
-        let result = '';
-        switch (validatedArgs.provider) {
-            case 'gemini': {
-                const gemini = new gemini_js_1.default(validatedArgs.key, chunks, validatedArgs.from, validatedArgs.to, validatedArgs.model);
-                result = await gemini.translate();
-                break;
-            }
-            case 'openai': {
-                const openai = new openai_js_1.default(validatedArgs.key, chunks, validatedArgs.from, validatedArgs.to, validatedArgs.url, validatedArgs.model);
-                result = await openai.translate();
-                break;
-            }
-            case 'anthropic': {
-                const anthropic = new anthropic_js_1.default(validatedArgs.key, chunks, validatedArgs.from, validatedArgs.to, validatedArgs.url, validatedArgs.model);
-                result = await anthropic.translate();
-                break;
-            }
-            default: {
-                logger_js_1.logger.warn('Provider not found');
-                break;
-            }
+        if (batches.length === 0) {
+            logger_js_1.logger.warn('No translatable entries found, copying input to output');
+            await (0, file_js_1.writeOutput)((0, file_js_1.resolvePath)(validatedArgs.output), JSON.stringify(fileContent, null, 2));
+            dntShim.Deno.exit(0);
         }
-        await (0, file_js_1.writeOutput)(outputPath, result);
+        const translateOptions = {
+            noLimit: validatedArgs.noLimit,
+            noTimeout: validatedArgs.noTimeout,
+        };
+        const translator = createTranslator(validatedArgs, translateOptions);
+        spinner.start();
+        const translations = await (0, translator_js_1.runBatches)(batches, translator, validatedArgs.from, validatedArgs.to, translateOptions);
+        const result = (0, translator_js_1.reconstruct)(allLeaves, translations);
+        const outputPath = (0, file_js_1.resolvePath)(validatedArgs.output);
+        await (0, file_js_1.writeOutput)(outputPath, JSON.stringify(result, null, 2));
         spinner.stop();
         logger_js_1.logger.success('Translation completed');
     }

package/script/deno.d.ts

@@ -23,7 +23,6 @@ declare namespace _default {
         "@google/genai": string;
         "@openai/openai": string;
         "@std/cli": string;
-        "@std/encoding": string;
         "@std/path": string;
         consola: string;
     };

package/script/deno.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = {
     "name": "@ibodev/glotto",
-    "version": "2.9.0",
+    "version": "3.0.0",
     "exports": "./cli.ts",
     "lock": false,
     "nodeModulesDir": "auto",
@@ -25,7 +25,6 @@ exports.default = {
         "@google/genai": "npm:@google/genai@^1.52.0",
         "@openai/openai": "npm:openai@^6.36.0",
         "@std/cli": "jsr:@std/cli@^1.0.29",
-        "@std/encoding": "jsr:@std/encoding@^1.0.10",
         "@std/path": "jsr:@std/path@^1.1.4",
         "consola": "npm:consola@^3.4.2"
     },

package/script/src/contants.d.ts

@@ -1,9 +1,9 @@
 import type { Provider } from './types.js';
 export declare const DEFAULT_PROVIDER: Provider;
 export declare const DEFAULT_MODELS: Record<Provider, string>;
-export declare const DEFAULT_MAX_CHUNK_BYTES: 30000;
+export declare const DEFAULT_MAX_BATCH_BYTES: 12000;
 export declare const MAX_RETRIES: 3;
 export declare const BASE_RETRY_DELAY_MS: 2000;
-export declare const INTER_CHUNK_DELAY_MS: 1500;
+export declare const INTER_BATCH_DELAY_MS: 1500;
 export declare const HELP_TEXT: string;
 //# sourceMappingURL=contants.d.ts.map

package/script/src/contants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"contants.d.ts","sourceRoot":"","sources":["../../src/src/contants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,eAAO,MAAM,gBAAgB,EAAE,QAAmB,CAAC;AAEnD,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAInD,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAG,KAAe,CAAC;AAEvD,eAAO,MAAM,WAAW,EAAG,CAAU,CAAC;AAEtC,eAAO,MAAM,mBAAmB,EAAG,IAAc,CAAC;AAElD,eAAO,MAAM,oBAAoB,EAAG,IAAc,CAAC;AAEnD,eAAO,MAAM,SAAS,QAsBrB,CAAC"}
+{"version":3,"file":"contants.d.ts","sourceRoot":"","sources":["../../src/src/contants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,eAAO,MAAM,gBAAgB,EAAE,QAAmB,CAAC;AAEnD,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAInD,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAG,KAAe,CAAC;AAEvD,eAAO,MAAM,WAAW,EAAG,CAAU,CAAC;AAEtC,eAAO,MAAM,mBAAmB,EAAG,IAAc,CAAC;AAElD,eAAO,MAAM,oBAAoB,EAAG,IAAc,CAAC;AAEnD,eAAO,MAAM,SAAS,QAgCrB,CAAC"}