@lyxa.ai/core 1.4.318 → 1.4.319
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -48,9 +48,9 @@ class OpenAICompletionProvider {
|
|
|
48
48
|
}
|
|
49
49
|
async generateSearchVariantsChunksInBatch(items, model, instructions, options) {
|
|
50
50
|
const finalModel = model ?? this.model;
|
|
51
|
-
const chunkSize = options?.chunkSize ??
|
|
51
|
+
const chunkSize = options?.chunkSize ?? 25;
|
|
52
52
|
const maxArabic = options?.maxArabic ?? 1;
|
|
53
|
-
const maxArabizi = options?.maxArabizi ??
|
|
53
|
+
const maxArabizi = options?.maxArabizi ?? 1;
|
|
54
54
|
const maxTransliteration = options?.maxTransliteration ?? 1;
|
|
55
55
|
const itemsToSend = items.filter(i => !i.skipAutoTranslation);
|
|
56
56
|
const skippedItems = items.filter(i => i.skipAutoTranslation);
|
|
@@ -109,6 +109,7 @@ class OpenAICompletionProvider {
|
|
|
109
109
|
const outputByChunk = await this.readJsonlFile(batch.output_file_id);
|
|
110
110
|
const errorsByChunk = await this.readJsonlFile(batch.error_file_id);
|
|
111
111
|
const resultsById = new Map();
|
|
112
|
+
let chunkParseFailures = 0;
|
|
112
113
|
for (const [chunkId, record] of outputByChunk.entries()) {
|
|
113
114
|
const body = record?.response?.body;
|
|
114
115
|
const outputText = body?.output_text ??
|
|
@@ -140,10 +141,19 @@ class OpenAICompletionProvider {
|
|
|
140
141
|
});
|
|
141
142
|
}
|
|
142
143
|
}
|
|
143
|
-
catch {
|
|
144
|
-
|
|
144
|
+
catch (parseErr) {
|
|
145
|
+
chunkParseFailures++;
|
|
146
|
+
console.error(`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`, {
|
|
147
|
+
error: parseErr,
|
|
148
|
+
outputTextLength: outputText.length,
|
|
149
|
+
outputTextPreview: outputText.slice(0, 2000),
|
|
150
|
+
outputTextTail: outputText.slice(-500),
|
|
151
|
+
});
|
|
145
152
|
}
|
|
146
153
|
}
|
|
154
|
+
if (chunkParseFailures) {
|
|
155
|
+
console.error(`generateSearchVariantsChunksInBatch: ${chunkParseFailures}/${chunks.length} chunks failed to parse entirely — every item in those chunks will show as missing from LLM output below.`);
|
|
156
|
+
}
|
|
147
157
|
const missingIds = itemsToSend.filter(i => !resultsById.has(i.id)).map(i => i.id);
|
|
148
158
|
if (missingIds.length) {
|
|
149
159
|
console.error(`generateSearchVariantsChunksInBatch: ${missingIds.length}/${itemsToSend.length} items missing from LLM output`, missingIds);
|
|
@@ -152,7 +162,7 @@ class OpenAICompletionProvider {
|
|
|
152
162
|
const r = resultsById.get(id);
|
|
153
163
|
if (!r) {
|
|
154
164
|
console.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);
|
|
155
|
-
return { id, original: text,
|
|
165
|
+
return { id, original: text, isProcessingFailed: true, isEmpty: true };
|
|
156
166
|
}
|
|
157
167
|
return { id, original: text, ...r };
|
|
158
168
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-provider.js","sourceRoot":"/","sources":["libraries/llm/providers/openai-provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwC;AAGxC,MAAa,wBAAwB;IACnB,MAAM,CAAS;IACf,KAAK,CAAS;IACd,iBAAiB,GAAG,iBAAiB,CAAC;IAEvD,YAAY,MAA0B,EAAE,QAAgB,YAAY;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;IAcD,KAAK,CAAC,mCAAmC,CACxC,KAAsB,EACtB,KAAc,EACd,YAAqB,EACrB,OAOC;QAED,MAAM,UAAU,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACvC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;QAC5C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,kBAAkB,IAAI,CAAC,CAAC;QAG5D,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAE9D,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACV,iDAAiD,YAAY,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,wCAAwC,CAC5H,CAAC;QACH,CAAC;QAGD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAEzB,OAAO,EAAE,CAAC;QACX,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,iBAAiB,GACtB,YAAY,IAAI,IAAI,CAAC,0BAA0B,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAEhG,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAElG,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACd,oBAAoB,MAAM,CAAC,MAAM,0FAA0F,CAC3H,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CACV,8CAA8C,UAAU,UAAU,WAAW,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,uBAAuB,SAAS,WAAW,MAAM,CAAC,MAAM,EAAE,CACtK,CAAC;YAGF,MAAM,KAAK,GAAG,MAAM;iBAClB,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CACtB,IAAI,CAAC,SAAS,CAAC;gBACd,SAAS,EAAE,SAAS,CAAC,EAAE;gBACvB,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,eAAe;gBACpB,IAAI,EAAE;oBACL,KAAK,EAAE,UAAU;oBACjB,YAAY,EAAE,iBAAiB;oBAC/B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC3E,IAAI,EAAE;wBACL,MAAM,EAAE;4BACP,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,WAAW;4BACnB,MAAM,EAAE,IAAI;yBACZ;qBACD;iBACD;aACD,CAAC,CACF;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YAGb,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBAChD,IAAI,EAAE,MAAM,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,6BAA6B,CAAC;gBAC9E,OAAO,EAAE,OAAO;aAChB,CAAC,CAAC;YAGH,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC5C,aAAa,EAAE,SAAS,CAAC,EAAE;gBAC3B,QAAQ,EAAE,eAAe;gBACzB,iBAAiB,EAAE,KAAK;aACxB,CAAC,CAAC;YAGH,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAEpF,OAAO,CAAC,GAAG,CAAC;gBACX,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACd,SAAS,KAAK,CAAC,EAAE,uBAAuB,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,UAAU,CACrG,CAAC;YACH,CAAC;YAGD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAGpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsC,CAAC;YAElE,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC;gBACpC,MAAM,UAAU,GACf,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,MAAM;wBACX,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;wBACxC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC;gBAE9D,IAAI,CAAC,UAAU,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACvC,OAAO,CAAC,KAAK,CACZ,sDAAsD,OAAO,EAAE,EAC/D,GAAG,IAAI,MAAM,EAAE,KAAK,CACpB,CAAC;oBACF,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAEnC,CAAC;oBACF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAChC,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAEzC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC9F,MAAM,kBAAkB,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;wBAEpE,IAAI,kBAAkB,EAAE,CAAC;4BACxB,OAAO,CAAC,IAAI,CACX,mDAAmD,CAAC,CAAC,EAAE,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAChK,CAAC;wBACH,CAAC;wBAED,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;4BACrB,MAAM,EAAE,EAAE;4BACV,OAAO,EAAE,EAAE;4BACX,qBAAqB,EAAE,EAAE;4BACzB,kBAAkB;4BAClB,OAAO;yBACP,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,CAAC,KAAK,CACZ,mEAAmE,OAAO,EAAE,EAC5E,UAAU,CACV,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClF,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CACZ,wCAAwC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,gCAAgC,EAC/G,UAAU,CACV,CAAC;YACH,CAAC;YAID,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBACjC,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,6DAA6D,EAAE,EAAE,CAAC,CAAC;oBACjF,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBACtE,CAAC;gBACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAuB,CAAC;YAC1D,CAAC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC3D,KAAK;gBACL,SAAS,EAAE,WAAW,CAAC,MAAM;gBAC7B,KAAK,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAMO,mBAAmB,CAAC,MAA4B;QACvD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,CAAC;IAIO,kBAAkB,CAAC,MAA4B;QACtD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAMO,0BAA0B,CAAC,IAIlC;QACA,OAAO;YACN,8EAA8E;YAC9E,oEAAoE;YACpE,uCAAuC;YACvC,yCAAyC;YACzC,qDAAqD,IAAI,CAAC,SAAS,GAAG;YACtE,0EAA0E,IAAI,CAAC,UAAU,GAAG;YAC5F,qFAAqF,IAAI,CAAC,kBAAkB,GAAG;YAC/G,2EAA2E;YAC3E,mFAAmF;YACnF,+EAA+E;SAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAIO,yBAAyB,CAAC,IAIjC;QACA,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,OAAO,EAAE;oBACR,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACX,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BACtB,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE;4BACvF,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;4BACxF,EAAE,EAAE;gCACH,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,QAAQ,EAAE,CAAC;gCACX,QAAQ,EAAE,IAAI,CAAC,kBAAkB;6BACjC;yBACD;wBACD,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;wBAClC,oBAAoB,EAAE,KAAK;qBAC3B;iBACD;aACD;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;YACrB,oBAAoB,EAAE,KAAK;SAC3B,CAAC;IACH,CAAC;IAEO,UAAU,CAAI,KAAU,EAAE,IAAY;QAC7C,MAAM,GAAG,GAAU,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,SAAS,CACtB,OAAe,EACf,UAAU,GAAG,IAAI,EACjB,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QAE/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CACV,cAAc,OAAO,WAAW,KAAK,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAClG,CAAC;YACF,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7C,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,0BAA0B,SAAS,eAAe,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACpG,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAC/D,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAiC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAKO,YAAY,CAAC,MAAmC;QACvD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;CACD;AAtVD,4DAsVC","sourcesContent":["import OpenAI, { toFile } from 'openai';\nimport { IAICompletionProvider, ItemTextInput, TextVariantResult } from '../interfaces/common-llm-interfaces';\n\nexport class OpenAICompletionProvider implements IAICompletionProvider {\n\tprivate readonly client: OpenAI;\n\tprivate readonly model: string;\n\tprivate readonly ARABIC_CHAR_REGEX = /[\\u0600-\\u06FF]/;\n\n\tconstructor(apiKey: string | undefined, model: string = 'gpt-5-nano') {\n\t\tif (!apiKey) {\n\t\t\tthrow new Error('OPENAI_API_KEY is required to construct OpenAICompletionProvider');\n\t\t}\n\t\tthis.client = new OpenAI({ apiKey });\n\t\tthis.model = model;\n\t}\n\n\t// ── Cost-optimized batch path ────────────────────────────────────\n\t// Implements everything discussed:\n\t// 1. Chunking — groups items (default 200/chunk) into one line per\n\t// chunk, so `instructions` is billed once per chunk instead of\n\t// once per item (biggest single cost lever).\n\t// 2. Short output keys — model emits {id, ar, az, tr} instead of\n\t// {id, arabic, arabizi, arabicTransliteration}; mapped back\n\t// to full field names locally, at zero token cost.\n\t// 3. Capped array lengths — schema enforces maxItems (not just a\n\t// prose suggestion), so the model can't drift to longer arrays.\n\t// 4. No echoed fields — model never re-emits id/original text;\n\t// both are re-attached locally from the input you already have.\n\tasync generateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number;\n\t\t\tmaxArabic?: number;\n\t\t\tmaxArabizi?: number;\n\t\t\tmaxTransliteration?: number;\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]> {\n\t\tconst finalModel = model ?? this.model;\n\t\tconst chunkSize = options?.chunkSize ?? 200;\n\t\tconst maxArabic = options?.maxArabic ?? 1;\n\t\tconst maxArabizi = options?.maxArabizi ?? 2;\n\t\tconst maxTransliteration = options?.maxTransliteration ?? 1;\n\n\t\t// Items explicitly opted out — never sent to the model, just passed through.\n\t\tconst itemsToSend = items.filter(i => !i.skipAutoTranslation);\n\t\tconst skippedItems = items.filter(i => i.skipAutoTranslation);\n\n\t\tif (skippedItems.length) {\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: skipping ${skippedItems.length}/${items.length} items due to skipAutoTranslation=true`\n\t\t\t);\n\t\t}\n\n\t\t// Nothing left to actually send — short-circuit before touching the OpenAI API at all.\n\t\tif (!itemsToSend.length) {\n\t\t\t// return skippedItems.map(({ id, text }) => ({ id, original: text, skipped: true }));\n\t\t\treturn [];\n\t\t}\n\n\t\ttry {\n\t\t\tconst finalInstructions =\n\t\t\t\tinstructions ?? this.buildOptimizedInstructions({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunkSchema = this.buildOptimizedChunkSchema({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunks = this.chunkItems(itemsToSend, chunkSize);\n\n\t\t\tif (chunks.length > 50_000) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Too many chunks (${chunks.length}) for a single batch file — increase chunkSize or split into multiple batch submissions.`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: model=${finalModel} items=${itemsToSend.length} (${skippedItems.length} skipped) chunkSize=${chunkSize} chunks=${chunks.length}`\n\t\t\t);\n\n\t\t\t// 1. Build JSONL — ONE line per chunk, not per item\n\t\t\tconst jsonl = chunks\n\t\t\t\t.map((chunkItems, i) =>\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\tcustom_id: `chunk-${i}`,\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\turl: '/v1/responses',\n\t\t\t\t\t\tbody: {\n\t\t\t\t\t\t\tmodel: finalModel,\n\t\t\t\t\t\t\tinstructions: finalInstructions,\n\t\t\t\t\t\t\tinput: JSON.stringify(chunkItems.map(({ id, text }) => ({ id: id, text }))),\n\t\t\t\t\t\t\ttext: {\n\t\t\t\t\t\t\t\tformat: {\n\t\t\t\t\t\t\t\t\ttype: 'json_schema',\n\t\t\t\t\t\t\t\t\tname: 'text_variants_batch',\n\t\t\t\t\t\t\t\t\tschema: chunkSchema,\n\t\t\t\t\t\t\t\t\tstrict: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t})\n\t\t\t\t)\n\t\t\t\t.join('\\n');\n\n\t\t\t// 2. Upload input file\n\t\t\tconst inputFile = await this.client.files.create({\n\t\t\t\tfile: await toFile(Buffer.from(jsonl, 'utf-8'), 'batch-input-optimized.jsonl'),\n\t\t\t\tpurpose: 'batch',\n\t\t\t});\n\n\t\t\t// 3. Create batch job\n\t\t\tlet batch = await this.client.batches.create({\n\t\t\t\tinput_file_id: inputFile.id,\n\t\t\t\tendpoint: '/v1/responses',\n\t\t\t\tcompletion_window: '24h',\n\t\t\t});\n\n\t\t\t// 4. Poll until done\n\t\t\tbatch = await this.pollBatch(batch.id, options?.pollIntervalMs, options?.maxWaitMs);\n\n\t\t\tconsole.log({\n\t\t\t\tstatus: batch.status,\n\t\t\t\trequest_counts: batch.request_counts,\n\t\t\t\terrors: batch.errors,\n\t\t\t});\n\n\t\t\tif (batch.status !== 'completed') {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Batch ${batch.id} ended with status \"${batch.status}\" (${batch.request_counts?.failed ?? 0} failed)`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 5. Parse output\n\t\t\tconst outputByChunk = await this.readJsonlFile(batch.output_file_id);\n\t\t\tconst errorsByChunk = await this.readJsonlFile(batch.error_file_id);\n\n\t\t\t// 6. Flatten chunk results into a per-itemId map\n\t\t\tconst resultsById = new Map<string, Partial<TextVariantResult>>();\n\n\t\t\tfor (const [chunkId, record] of outputByChunk.entries()) {\n\t\t\t\tconst body = record?.response?.body;\n\t\t\t\tconst outputText =\n\t\t\t\t\tbody?.output_text ??\n\t\t\t\t\tbody?.output\n\t\t\t\t\t\t?.find((o: any) => o.type === 'message')\n\t\t\t\t\t\t?.content?.find((c: any) => c.type === 'output_text')?.text;\n\n\t\t\t\tif (!outputText) {\n\t\t\t\t\tconst err = errorsByChunk.get(chunkId);\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: no output for ${chunkId}`,\n\t\t\t\t\t\terr ?? record?.error\n\t\t\t\t\t);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(outputText) as {\n\t\t\t\t\t\tresults: Array<{ id: string; ar: string[]; az: string[]; tr: string[] }>;\n\t\t\t\t\t};\n\t\t\t\t\tfor (const r of parsed.results) {\n\t\t\t\t\t\tconst ar = this.sanitizeArabicArray(r.ar);\n\t\t\t\t\t\tconst tr = this.sanitizeArabicArray(r.tr);\n\t\t\t\t\t\tconst az = this.sanitizeLatinArray(r.az);\n\n\t\t\t\t\t\tconst isEmpty = this.isEmptyValue(r.ar) || this.isEmptyValue(r.az) || this.isEmptyValue(r.tr);\n\t\t\t\t\t\tconst isProcessingFailed = (!ar.length && !tr.length) || !az.length;\n\n\t\t\t\t\t\tif (isProcessingFailed) {\n\t\t\t\t\t\t\tconsole.warn(\n\t\t\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: marking id=${r.id} as processingFailed — ar=${JSON.stringify(r.ar)} az=${JSON.stringify(r.az)} tr=${JSON.stringify(r.tr)}`\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tresultsById.set(r.id, {\n\t\t\t\t\t\t\tarabic: ar,\n\t\t\t\t\t\t\tarabizi: az,\n\t\t\t\t\t\t\tarabicTransliteration: tr,\n\t\t\t\t\t\t\tisProcessingFailed,\n\t\t\t\t\t\t\tisEmpty,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t} catch {\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`,\n\t\t\t\t\t\toutputText\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// 7. Surface items the model silently dropped (only checked against what we actually sent)\n\t\t\tconst missingIds = itemsToSend.filter(i => !resultsById.has(i.id)).map(i => i.id);\n\t\t\tif (missingIds.length) {\n\t\t\t\tconsole.error(\n\t\t\t\t\t`generateSearchVariantsChunksInBatch: ${missingIds.length}/${itemsToSend.length} items missing from LLM output`,\n\t\t\t\t\tmissingIds\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 8. Merge back with the FULL original item list — skipped items pass through untouched,\n\t\t\t// everything else is looked up in resultsById.\n\t\t\treturn items.map(({ id, text }) => {\n\t\t\t\tconst r = resultsById.get(id);\n\t\t\t\tif (!r) {\n\t\t\t\t\tconsole.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);\n\t\t\t\t\treturn { id, original: text, processingFailed: true, isEmpty: true };\n\t\t\t\t}\n\t\t\t\treturn { id, original: text, ...r } as TextVariantResult;\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tconsole.error('generateSearchVariantsChunksInBatch failed', {\n\t\t\t\terror,\n\t\t\t\titemCount: itemsToSend.length,\n\t\t\t\tmodel: finalModel,\n\t\t\t});\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t// ── Helpers ──────────────────────────────────────────────────────\n\n\t// Keeps only entries that actually contain Arabic-script characters — used for `ar`/`tr`,\n\t// which must never contain the raw English/Latin text the model was given.\n\tprivate sanitizeArabicArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Keeps only entries that do NOT contain Arabic-script characters — used for `az`\n\t// (Latin/Arabizi variants), which must never contain Arabic script.\n\tprivate sanitizeLatinArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && !this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Compact instructions for the optimized path: short-key output format,\n\t// explicit array caps stated in prose (schema also enforces them),\n\t// and an explicit \"no commentary inside values\" rule to stop the model\n\t// padding fields with parenthetical notes.\n\tprivate buildOptimizedInstructions(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}): string {\n\t\treturn [\n\t\t\t'You generate search-name variants for a food delivery app product/shop name.',\n\t\t\t'You will receive a JSON array of items, each with \"id\" and \"text\".',\n\t\t\t'For EACH item, return an object with:',\n\t\t\t`- id: the exact id provided (unchanged)`,\n\t\t\t`- ar: Arabic translation(s) in Arabic script, max ${caps.maxArabic}.`,\n\t\t\t`- az: Arabizi spelling(s) (Latin script/numerals, e.g. \"zaytoon\"), max ${caps.maxArabizi}.`,\n\t\t\t`- tr: Arabic-script transliteration(s) of the original word (not translated), max ${caps.maxTransliteration}.`,\n\t\t\t'Return one result per input item, same order, no items skipped or merged.',\n\t\t\t'Values must contain only the term itself — no parenthetical notes, no commentary.',\n\t\t\t'Return strict JSON matching the schema. No extra commentary, no extra fields.',\n\t\t].join('\\n');\n\t}\n\n\t// Schema for the OPTIMIZED chunk path: wraps a `results` array (one\n\t// entry per item in the chunk), short keys, and hard maxItems caps.\n\tprivate buildOptimizedChunkSchema(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}) {\n\t\treturn {\n\t\t\ttype: 'object',\n\t\t\tproperties: {\n\t\t\t\tresults: {\n\t\t\t\t\ttype: 'array',\n\t\t\t\t\titems: {\n\t\t\t\t\t\ttype: 'object',\n\t\t\t\t\t\tproperties: {\n\t\t\t\t\t\t\tid: { type: 'string' },\n\t\t\t\t\t\t\tar: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabic },\n\t\t\t\t\t\t\taz: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabizi },\n\t\t\t\t\t\t\ttr: {\n\t\t\t\t\t\t\t\ttype: 'array',\n\t\t\t\t\t\t\t\titems: { type: 'string' },\n\t\t\t\t\t\t\t\tminItems: 1,\n\t\t\t\t\t\t\t\tmaxItems: caps.maxTransliteration,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\trequired: ['id', 'ar', 'az', 'tr'],\n\t\t\t\t\t\tadditionalProperties: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trequired: ['results'],\n\t\t\tadditionalProperties: false,\n\t\t};\n\t}\n\n\tprivate chunkItems<T>(items: T[], size: number): T[][] {\n\t\tconst out: T[][] = [];\n\t\tfor (let i = 0; i < items.length; i += size) {\n\t\t\tout.push(items.slice(i, i + size));\n\t\t}\n\t\treturn out;\n\t}\n\n\tprivate async pollBatch(\n\t\tbatchId: string,\n\t\tintervalMs = 5000,\n\t\tmaxWaitMs = 24 * 60 * 60 * 1000\n\t): Promise<OpenAI.Batches.Batch> {\n\t\tconst terminal = new Set(['completed', 'failed', 'expired', 'cancelled']);\n\t\tconst start = Date.now();\n\n\t\twhile (true) {\n\t\t\tconst batch = await this.client.batches.retrieve(batchId);\n\t\t\tconsole.log(\n\t\t\t\t`pollBatch: ${batchId} status=${batch.status} elapsed=${Math.round((Date.now() - start) / 1000)}s`\n\t\t\t);\n\t\t\tif (terminal.has(batch.status)) return batch;\n\n\t\t\tif (Date.now() - start > maxWaitMs) {\n\t\t\t\tthrow new Error(`Batch ${batchId} did not finish within ${maxWaitMs}ms (status: ${batch.status})`);\n\t\t\t}\n\n\t\t\tawait new Promise(resolve => setTimeout(resolve, intervalMs));\n\t\t}\n\t}\n\n\tprivate async readJsonlFile(fileId: string | null | undefined): Promise<Map<string, any>> {\n\t\tconst result = new Map<string, any>();\n\t\tif (!fileId) return result;\n\n\t\tconst fileResponse = await this.client.files.content(fileId);\n\t\tconst text = await fileResponse.text();\n\n\t\tfor (const line of text.split('\\n').filter(Boolean)) {\n\t\t\tconst record = JSON.parse(line);\n\t\t\tresult.set(record.custom_id, record);\n\t\t}\n\n\t\treturn result;\n\t}\n\n\t// Checks whether ar/az/tr came back empty in any sense the model might produce:\n\t// missing entirely (null/undefined), an empty array, or an array whose entries\n\t// are just empty/whitespace strings.\n\tprivate isEmptyValue(values: string[] | null | undefined): boolean {\n\t\tif (!values || values.length === 0) return true;\n\t\treturn values.every(v => typeof v !== 'string' || !v.trim());\n\t}\n}\n"]}
|
|
1
|
+
{"version":3,"file":"openai-provider.js","sourceRoot":"/","sources":["libraries/llm/providers/openai-provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwC;AAGxC,MAAa,wBAAwB;IACnB,MAAM,CAAS;IACf,KAAK,CAAS;IACd,iBAAiB,GAAG,iBAAiB,CAAC;IAEvD,YAAY,MAA0B,EAAE,QAAgB,YAAY;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;IAeD,KAAK,CAAC,mCAAmC,CACxC,KAAsB,EACtB,KAAc,EACd,YAAqB,EACrB,OAOC;QAED,MAAM,UAAU,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QAGvC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;QAG1C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,kBAAkB,IAAI,CAAC,CAAC;QAG5D,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAE9D,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACV,iDAAiD,YAAY,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,wCAAwC,CAC5H,CAAC;QACH,CAAC;QAGD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,EAAE,CAAC;QACX,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,iBAAiB,GACtB,YAAY,IAAI,IAAI,CAAC,0BAA0B,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAEhG,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAElG,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACd,oBAAoB,MAAM,CAAC,MAAM,0FAA0F,CAC3H,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CACV,8CAA8C,UAAU,UAAU,WAAW,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,uBAAuB,SAAS,WAAW,MAAM,CAAC,MAAM,EAAE,CACtK,CAAC;YAGF,MAAM,KAAK,GAAG,MAAM;iBAClB,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CACtB,IAAI,CAAC,SAAS,CAAC;gBACd,SAAS,EAAE,SAAS,CAAC,EAAE;gBACvB,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,eAAe;gBACpB,IAAI,EAAE;oBACL,KAAK,EAAE,UAAU;oBACjB,YAAY,EAAE,iBAAiB;oBAC/B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC3E,IAAI,EAAE;wBACL,MAAM,EAAE;4BACP,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,WAAW;4BACnB,MAAM,EAAE,IAAI;yBACZ;qBACD;iBACD;aACD,CAAC,CACF;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YAGb,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBAChD,IAAI,EAAE,MAAM,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,6BAA6B,CAAC;gBAC9E,OAAO,EAAE,OAAO;aAChB,CAAC,CAAC;YAGH,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC5C,aAAa,EAAE,SAAS,CAAC,EAAE;gBAC3B,QAAQ,EAAE,eAAe;gBACzB,iBAAiB,EAAE,KAAK;aACxB,CAAC,CAAC;YAGH,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAEpF,OAAO,CAAC,GAAG,CAAC;gBACX,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACd,SAAS,KAAK,CAAC,EAAE,uBAAuB,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,UAAU,CACrG,CAAC;YACH,CAAC;YAGD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAGpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsC,CAAC;YAClE,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAE3B,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC;gBACpC,MAAM,UAAU,GACf,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,MAAM;wBACX,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;wBACxC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC;gBAE9D,IAAI,CAAC,UAAU,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACvC,OAAO,CAAC,KAAK,CACZ,sDAAsD,OAAO,EAAE,EAC/D,GAAG,IAAI,MAAM,EAAE,KAAK,CACpB,CAAC;oBACF,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAEnC,CAAC;oBACF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAChC,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAEzC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC9F,MAAM,kBAAkB,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;wBAEpE,IAAI,kBAAkB,EAAE,CAAC;4BACxB,OAAO,CAAC,IAAI,CACX,mDAAmD,CAAC,CAAC,EAAE,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAChK,CAAC;wBACH,CAAC;wBAED,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;4BACrB,MAAM,EAAE,EAAE;4BACV,OAAO,EAAE,EAAE;4BACX,qBAAqB,EAAE,EAAE;4BACzB,kBAAkB;4BAClB,OAAO;yBACP,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,OAAO,QAAQ,EAAE,CAAC;oBACnB,kBAAkB,EAAE,CAAC;oBAGrB,OAAO,CAAC,KAAK,CAAC,mEAAmE,OAAO,EAAE,EAAE;wBAC3F,KAAK,EAAE,QAAQ;wBACf,gBAAgB,EAAE,UAAU,CAAC,MAAM;wBACnC,iBAAiB,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;wBAC5C,cAAc,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;qBACtC,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;YAED,IAAI,kBAAkB,EAAE,CAAC;gBACxB,OAAO,CAAC,KAAK,CACZ,wCAAwC,kBAAkB,IAAI,MAAM,CAAC,MAAM,2GAA2G,CACtL,CAAC;YACH,CAAC;YAGD,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClF,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CACZ,wCAAwC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,gCAAgC,EAC/G,UAAU,CACV,CAAC;YACH,CAAC;YAID,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBACjC,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,6DAA6D,EAAE,EAAE,CAAC,CAAC;oBAKjF,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBACxE,CAAC;gBACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAuB,CAAC;YAC1D,CAAC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC3D,KAAK;gBACL,SAAS,EAAE,WAAW,CAAC,MAAM;gBAC7B,KAAK,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAMO,mBAAmB,CAAC,MAA4B;QACvD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,CAAC;IAIO,kBAAkB,CAAC,MAA4B;QACtD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAMO,0BAA0B,CAAC,IAIlC;QACA,OAAO;YACN,8EAA8E;YAC9E,oEAAoE;YACpE,uCAAuC;YACvC,yCAAyC;YACzC,qDAAqD,IAAI,CAAC,SAAS,GAAG;YACtE,0EAA0E,IAAI,CAAC,UAAU,GAAG;YAC5F,qFAAqF,IAAI,CAAC,kBAAkB,GAAG;YAC/G,2EAA2E;YAC3E,mFAAmF;YACnF,+EAA+E;SAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAIO,yBAAyB,CAAC,IAIjC;QACA,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,OAAO,EAAE;oBACR,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACX,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BACtB,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE;4BACvF,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;4BACxF,EAAE,EAAE;gCACH,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,QAAQ,EAAE,CAAC;gCACX,QAAQ,EAAE,IAAI,CAAC,kBAAkB;6BACjC;yBACD;wBACD,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;wBAClC,oBAAoB,EAAE,KAAK;qBAC3B;iBACD;aACD;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;YACrB,oBAAoB,EAAE,KAAK;SAC3B,CAAC;IACH,CAAC;IAEO,UAAU,CAAI,KAAU,EAAE,IAAY;QAC7C,MAAM,GAAG,GAAU,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,SAAS,CACtB,OAAe,EACf,UAAU,GAAG,IAAI,EACjB,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QAE/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CACV,cAAc,OAAO,WAAW,KAAK,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAClG,CAAC;YACF,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7C,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,0BAA0B,SAAS,eAAe,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACpG,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAC/D,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAiC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAKO,YAAY,CAAC,MAAmC;QACvD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;CACD;AA1WD,4DA0WC","sourcesContent":["import OpenAI, { toFile } from 'openai';\nimport { IAICompletionProvider, ItemTextInput, TextVariantResult } from '../interfaces/common-llm-interfaces';\n\nexport class OpenAICompletionProvider implements IAICompletionProvider {\n\tprivate readonly client: OpenAI;\n\tprivate readonly model: string;\n\tprivate readonly ARABIC_CHAR_REGEX = /[\\u0600-\\u06FF]/;\n\n\tconstructor(apiKey: string | undefined, model: string = 'gpt-5-nano') {\n\t\tif (!apiKey) {\n\t\t\tthrow new Error('OPENAI_API_KEY is required to construct OpenAICompletionProvider');\n\t\t}\n\t\tthis.client = new OpenAI({ apiKey });\n\t\tthis.model = model;\n\t}\n\n\t// ── Cost-optimized batch path ────────────────────────────────────\n\t// Implements everything discussed:\n\t// 1. Chunking — groups items (default 25/chunk — kept small so a single\n\t// malformed/truncated model response only takes out a small slice of\n\t// items instead of ~100+ at once) into one line per chunk, so\n\t// `instructions` is billed once per chunk instead of once per item.\n\t// 2. Short output keys — model emits {id, ar, az, tr} instead of\n\t// {id, arabic, arabizi, arabicTransliteration}; mapped back\n\t// to full field names locally, at zero token cost.\n\t// 3. Capped array lengths — schema enforces maxItems (not just a\n\t// prose suggestion), so the model can't drift to longer arrays.\n\t// 4. No echoed fields — model never re-emits id/original text;\n\t// both are re-attached locally from the input you already have.\n\tasync generateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number;\n\t\t\tmaxArabic?: number;\n\t\t\tmaxArabizi?: number;\n\t\t\tmaxTransliteration?: number;\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]> {\n\t\tconst finalModel = model ?? this.model;\n\t\t// Kept intentionally small — large chunks (100+) risk the model truncating or\n\t\t// malforming output for the WHOLE chunk, which drops every item in it at once.\n\t\tconst chunkSize = options?.chunkSize ?? 25;\n\t\tconst maxArabic = options?.maxArabic ?? 1;\n\t\t// Reduced from 2 → 1 by default: fewer required strings per item means less total\n\t\t// output per chunk, which lowers the chance of hitting a truncation/formatting issue.\n\t\tconst maxArabizi = options?.maxArabizi ?? 1;\n\t\tconst maxTransliteration = options?.maxTransliteration ?? 1;\n\n\t\t// Items explicitly opted out — never sent to the model, just passed through.\n\t\tconst itemsToSend = items.filter(i => !i.skipAutoTranslation);\n\t\tconst skippedItems = items.filter(i => i.skipAutoTranslation);\n\n\t\tif (skippedItems.length) {\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: skipping ${skippedItems.length}/${items.length} items due to skipAutoTranslation=true`\n\t\t\t);\n\t\t}\n\n\t\t// Nothing left to actually send — short-circuit before touching the OpenAI API at all.\n\t\tif (!itemsToSend.length) {\n\t\t\treturn [];\n\t\t}\n\n\t\ttry {\n\t\t\tconst finalInstructions =\n\t\t\t\tinstructions ?? this.buildOptimizedInstructions({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunkSchema = this.buildOptimizedChunkSchema({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunks = this.chunkItems(itemsToSend, chunkSize);\n\n\t\t\tif (chunks.length > 50_000) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Too many chunks (${chunks.length}) for a single batch file — increase chunkSize or split into multiple batch submissions.`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: model=${finalModel} items=${itemsToSend.length} (${skippedItems.length} skipped) chunkSize=${chunkSize} chunks=${chunks.length}`\n\t\t\t);\n\n\t\t\t// 1. Build JSONL — ONE line per chunk, not per item\n\t\t\tconst jsonl = chunks\n\t\t\t\t.map((chunkItems, i) =>\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\tcustom_id: `chunk-${i}`,\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\turl: '/v1/responses',\n\t\t\t\t\t\tbody: {\n\t\t\t\t\t\t\tmodel: finalModel,\n\t\t\t\t\t\t\tinstructions: finalInstructions,\n\t\t\t\t\t\t\tinput: JSON.stringify(chunkItems.map(({ id, text }) => ({ id: id, text }))),\n\t\t\t\t\t\t\ttext: {\n\t\t\t\t\t\t\t\tformat: {\n\t\t\t\t\t\t\t\t\ttype: 'json_schema',\n\t\t\t\t\t\t\t\t\tname: 'text_variants_batch',\n\t\t\t\t\t\t\t\t\tschema: chunkSchema,\n\t\t\t\t\t\t\t\t\tstrict: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t})\n\t\t\t\t)\n\t\t\t\t.join('\\n');\n\n\t\t\t// 2. Upload input file\n\t\t\tconst inputFile = await this.client.files.create({\n\t\t\t\tfile: await toFile(Buffer.from(jsonl, 'utf-8'), 'batch-input-optimized.jsonl'),\n\t\t\t\tpurpose: 'batch',\n\t\t\t});\n\n\t\t\t// 3. Create batch job\n\t\t\tlet batch = await this.client.batches.create({\n\t\t\t\tinput_file_id: inputFile.id,\n\t\t\t\tendpoint: '/v1/responses',\n\t\t\t\tcompletion_window: '24h',\n\t\t\t});\n\n\t\t\t// 4. Poll until done\n\t\t\tbatch = await this.pollBatch(batch.id, options?.pollIntervalMs, options?.maxWaitMs);\n\n\t\t\tconsole.log({\n\t\t\t\tstatus: batch.status,\n\t\t\t\trequest_counts: batch.request_counts,\n\t\t\t\terrors: batch.errors,\n\t\t\t});\n\n\t\t\tif (batch.status !== 'completed') {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Batch ${batch.id} ended with status \"${batch.status}\" (${batch.request_counts?.failed ?? 0} failed)`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 5. Parse output\n\t\t\tconst outputByChunk = await this.readJsonlFile(batch.output_file_id);\n\t\t\tconst errorsByChunk = await this.readJsonlFile(batch.error_file_id);\n\n\t\t\t// 6. Flatten chunk results into a per-itemId map\n\t\t\tconst resultsById = new Map<string, Partial<TextVariantResult>>();\n\t\t\tlet chunkParseFailures = 0;\n\n\t\t\tfor (const [chunkId, record] of outputByChunk.entries()) {\n\t\t\t\tconst body = record?.response?.body;\n\t\t\t\tconst outputText =\n\t\t\t\t\tbody?.output_text ??\n\t\t\t\t\tbody?.output\n\t\t\t\t\t\t?.find((o: any) => o.type === 'message')\n\t\t\t\t\t\t?.content?.find((c: any) => c.type === 'output_text')?.text;\n\n\t\t\t\tif (!outputText) {\n\t\t\t\t\tconst err = errorsByChunk.get(chunkId);\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: no output for ${chunkId}`,\n\t\t\t\t\t\terr ?? record?.error\n\t\t\t\t\t);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(outputText) as {\n\t\t\t\t\t\tresults: Array<{ id: string; ar: string[]; az: string[]; tr: string[] }>;\n\t\t\t\t\t};\n\t\t\t\t\tfor (const r of parsed.results) {\n\t\t\t\t\t\tconst ar = this.sanitizeArabicArray(r.ar);\n\t\t\t\t\t\tconst tr = this.sanitizeArabicArray(r.tr);\n\t\t\t\t\t\tconst az = this.sanitizeLatinArray(r.az);\n\n\t\t\t\t\t\tconst isEmpty = this.isEmptyValue(r.ar) || this.isEmptyValue(r.az) || this.isEmptyValue(r.tr);\n\t\t\t\t\t\tconst isProcessingFailed = (!ar.length && !tr.length) || !az.length;\n\n\t\t\t\t\t\tif (isProcessingFailed) {\n\t\t\t\t\t\t\tconsole.warn(\n\t\t\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: marking id=${r.id} as processingFailed — ar=${JSON.stringify(r.ar)} az=${JSON.stringify(r.az)} tr=${JSON.stringify(r.tr)}`\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tresultsById.set(r.id, {\n\t\t\t\t\t\t\tarabic: ar,\n\t\t\t\t\t\t\tarabizi: az,\n\t\t\t\t\t\t\tarabicTransliteration: tr,\n\t\t\t\t\t\t\tisProcessingFailed,\n\t\t\t\t\t\t\tisEmpty,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t} catch (parseErr) {\n\t\t\t\t\tchunkParseFailures++;\n\t\t\t\t\t// Log a preview + length of the raw output so you can tell WHY it failed —\n\t\t\t\t\t// truncated mid-JSON (hit output length) vs. malformed/extra text vs. something else.\n\t\t\t\t\tconsole.error(`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`, {\n\t\t\t\t\t\terror: parseErr,\n\t\t\t\t\t\toutputTextLength: outputText.length,\n\t\t\t\t\t\toutputTextPreview: outputText.slice(0, 2000),\n\t\t\t\t\t\toutputTextTail: outputText.slice(-500),\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (chunkParseFailures) {\n\t\t\t\tconsole.error(\n\t\t\t\t\t`generateSearchVariantsChunksInBatch: ${chunkParseFailures}/${chunks.length} chunks failed to parse entirely — every item in those chunks will show as missing from LLM output below.`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 7. Surface items the model silently dropped (only checked against what we actually sent)\n\t\t\tconst missingIds = itemsToSend.filter(i => !resultsById.has(i.id)).map(i => i.id);\n\t\t\tif (missingIds.length) {\n\t\t\t\tconsole.error(\n\t\t\t\t\t`generateSearchVariantsChunksInBatch: ${missingIds.length}/${itemsToSend.length} items missing from LLM output`,\n\t\t\t\t\tmissingIds\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 8. Merge back with the FULL original item list — skipped items pass through untouched,\n\t\t\t// everything else is looked up in resultsById.\n\t\t\treturn items.map(({ id, text }) => {\n\t\t\t\tconst r = resultsById.get(id);\n\t\t\t\tif (!r) {\n\t\t\t\t\tconsole.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);\n\t\t\t\t\t// NOTE: key must be isProcessingFailed (matches the shape set above and what\n\t\t\t\t\t// hasAnyFailedField/isFieldUsable check downstream) — a prior version of this\n\t\t\t\t\t// fallback used `processingFailed` (wrong key), which meant callers saw\n\t\t\t\t\t// isProcessingFailed=undefined for every dropped item instead of true.\n\t\t\t\t\treturn { id, original: text, isProcessingFailed: true, isEmpty: true };\n\t\t\t\t}\n\t\t\t\treturn { id, original: text, ...r } as TextVariantResult;\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tconsole.error('generateSearchVariantsChunksInBatch failed', {\n\t\t\t\terror,\n\t\t\t\titemCount: itemsToSend.length,\n\t\t\t\tmodel: finalModel,\n\t\t\t});\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t// ── Helpers ──────────────────────────────────────────────────────\n\n\t// Keeps only entries that actually contain Arabic-script characters — used for `ar`/`tr`,\n\t// which must never contain the raw English/Latin text the model was given.\n\tprivate sanitizeArabicArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Keeps only entries that do NOT contain Arabic-script characters — used for `az`\n\t// (Latin/Arabizi variants), which must never contain Arabic script.\n\tprivate sanitizeLatinArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && !this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Compact instructions for the optimized path: short-key output format,\n\t// explicit array caps stated in prose (schema also enforces them),\n\t// and an explicit \"no commentary inside values\" rule to stop the model\n\t// padding fields with parenthetical notes.\n\tprivate buildOptimizedInstructions(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}): string {\n\t\treturn [\n\t\t\t'You generate search-name variants for a food delivery app product/shop name.',\n\t\t\t'You will receive a JSON array of items, each with \"id\" and \"text\".',\n\t\t\t'For EACH item, return an object with:',\n\t\t\t`- id: the exact id provided (unchanged)`,\n\t\t\t`- ar: Arabic translation(s) in Arabic script, max ${caps.maxArabic}.`,\n\t\t\t`- az: Arabizi spelling(s) (Latin script/numerals, e.g. \"zaytoon\"), max ${caps.maxArabizi}.`,\n\t\t\t`- tr: Arabic-script transliteration(s) of the original word (not translated), max ${caps.maxTransliteration}.`,\n\t\t\t'Return one result per input item, same order, no items skipped or merged.',\n\t\t\t'Values must contain only the term itself — no parenthetical notes, no commentary.',\n\t\t\t'Return strict JSON matching the schema. No extra commentary, no extra fields.',\n\t\t].join('\\n');\n\t}\n\n\t// Schema for the OPTIMIZED chunk path: wraps a `results` array (one\n\t// entry per item in the chunk), short keys, and hard maxItems caps.\n\tprivate buildOptimizedChunkSchema(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}) {\n\t\treturn {\n\t\t\ttype: 'object',\n\t\t\tproperties: {\n\t\t\t\tresults: {\n\t\t\t\t\ttype: 'array',\n\t\t\t\t\titems: {\n\t\t\t\t\t\ttype: 'object',\n\t\t\t\t\t\tproperties: {\n\t\t\t\t\t\t\tid: { type: 'string' },\n\t\t\t\t\t\t\tar: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabic },\n\t\t\t\t\t\t\taz: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabizi },\n\t\t\t\t\t\t\ttr: {\n\t\t\t\t\t\t\t\ttype: 'array',\n\t\t\t\t\t\t\t\titems: { type: 'string' },\n\t\t\t\t\t\t\t\tminItems: 1,\n\t\t\t\t\t\t\t\tmaxItems: caps.maxTransliteration,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\trequired: ['id', 'ar', 'az', 'tr'],\n\t\t\t\t\t\tadditionalProperties: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trequired: ['results'],\n\t\t\tadditionalProperties: false,\n\t\t};\n\t}\n\n\tprivate chunkItems<T>(items: T[], size: number): T[][] {\n\t\tconst out: T[][] = [];\n\t\tfor (let i = 0; i < items.length; i += size) {\n\t\t\tout.push(items.slice(i, i + size));\n\t\t}\n\t\treturn out;\n\t}\n\n\tprivate async pollBatch(\n\t\tbatchId: string,\n\t\tintervalMs = 5000,\n\t\tmaxWaitMs = 24 * 60 * 60 * 1000\n\t): Promise<OpenAI.Batches.Batch> {\n\t\tconst terminal = new Set(['completed', 'failed', 'expired', 'cancelled']);\n\t\tconst start = Date.now();\n\n\t\twhile (true) {\n\t\t\tconst batch = await this.client.batches.retrieve(batchId);\n\t\t\tconsole.log(\n\t\t\t\t`pollBatch: ${batchId} status=${batch.status} elapsed=${Math.round((Date.now() - start) / 1000)}s`\n\t\t\t);\n\t\t\tif (terminal.has(batch.status)) return batch;\n\n\t\t\tif (Date.now() - start > maxWaitMs) {\n\t\t\t\tthrow new Error(`Batch ${batchId} did not finish within ${maxWaitMs}ms (status: ${batch.status})`);\n\t\t\t}\n\n\t\t\tawait new Promise(resolve => setTimeout(resolve, intervalMs));\n\t\t}\n\t}\n\n\tprivate async readJsonlFile(fileId: string | null | undefined): Promise<Map<string, any>> {\n\t\tconst result = new Map<string, any>();\n\t\tif (!fileId) return result;\n\n\t\tconst fileResponse = await this.client.files.content(fileId);\n\t\tconst text = await fileResponse.text();\n\n\t\tfor (const line of text.split('\\n').filter(Boolean)) {\n\t\t\tconst record = JSON.parse(line);\n\t\t\tresult.set(record.custom_id, record);\n\t\t}\n\n\t\treturn result;\n\t}\n\n\t// Checks whether ar/az/tr came back empty in any sense the model might produce:\n\t// missing entirely (null/undefined), an empty array, or an array whose entries\n\t// are just empty/whitespace strings.\n\tprivate isEmptyValue(values: string[] | null | undefined): boolean {\n\t\tif (!values || values.length === 0) return true;\n\t\treturn values.every(v => typeof v !== 'string' || !v.trim());\n\t}\n}\n"]}
|
package/dist/types/README.md
CHANGED
package/dist/types/package.json
CHANGED