@lyxa.ai/core 1.4.311 → 1.4.312

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/dist/libraries/llm/interfaces/common-llm-interfaces.d.ts +3 -0
  2. package/dist/libraries/llm/interfaces/common-llm-interfaces.js.map +1 -1
  3. package/dist/libraries/llm/providers/openai-provider.d.ts +4 -0
  4. package/dist/libraries/llm/providers/openai-provider.js +42 -13
  5. package/dist/libraries/llm/providers/openai-provider.js.map +1 -1
  6. package/dist/types/README.md +1 -1
  7. package/dist/types/package.json +1 -1
  8. package/package.json +1 -1
package/dist/libraries/llm/interfaces/common-llm-interfaces.d.ts CHANGED
@@ -15,6 +15,8 @@ export interface TextVariantResult {
15
15
  arabic?: string[];
16
16
  arabizi?: string[];
17
17
  arabicTransliteration?: string[];
18
+ isProcessingFailed?: boolean;
19
+ isEmpty?: boolean;
18
20
  }
19
21
  export interface AiConfiguration {
20
22
  provider: LLMProvider;
@@ -26,4 +28,5 @@ export interface AiConfiguration {
26
28
  export interface ItemTextInput {
27
29
  id: string;
28
30
  text: string;
31
+ skipAutoTranslation?: boolean;
29
32
  }
package/dist/libraries/llm/interfaces/common-llm-interfaces.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"common-llm-interfaces.js","sourceRoot":"/","sources":["libraries/llm/interfaces/common-llm-interfaces.ts"],"names":[],"mappings":"","sourcesContent":["import { LLMProvider } from '../llm-provider-service';\n\nexport interface IAICompletionProvider {\n\tgenerateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number;\n\t\t\tmaxArabic?: number;\n\t\t\tmaxArabizi?: number;\n\t\t\tmaxTransliteration?: number;\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]>;\n}\n\nexport interface TextVariantResult {\n\tid: string;\n\toriginal: string;\n\tarabic?: string[];\n\tarabizi?: string[];\n\tarabicTransliteration?: string[];\n}\n\nexport interface AiConfiguration {\n\tprovider: LLMProvider;\n\tmodel?: string;\n\tinstructions?: string;\n\tprompt?: string;\n\tchunkSize?: number\n}\n\nexport interface ItemTextInput {\n\tid: string;\n\ttext: string;\n}\n"]}
1
+ {"version":3,"file":"common-llm-interfaces.js","sourceRoot":"/","sources":["libraries/llm/interfaces/common-llm-interfaces.ts"],"names":[],"mappings":"","sourcesContent":["import { LLMProvider } from '../llm-provider-service';\n\nexport interface IAICompletionProvider {\n\tgenerateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number;\n\t\t\tmaxArabic?: number;\n\t\t\tmaxArabizi?: number;\n\t\t\tmaxTransliteration?: number;\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]>;\n}\n\nexport interface TextVariantResult {\n\tid: string;\n\toriginal: string;\n\tarabic?: string[];\n\tarabizi?: string[];\n\tarabicTransliteration?: string[];\n\tisProcessingFailed?: boolean;\n\tisEmpty?: boolean;\n}\n\nexport interface AiConfiguration {\n\tprovider: LLMProvider;\n\tmodel?: string;\n\tinstructions?: string;\n\tprompt?: string;\n\tchunkSize?: number;\n}\n\nexport interface ItemTextInput {\n\tid: string;\n\ttext: string;\n\tskipAutoTranslation?: boolean;\n}\n"]}
package/dist/libraries/llm/providers/openai-provider.d.ts CHANGED
@@ -2,6 +2,7 @@ import { IAICompletionProvider, ItemTextInput, TextVariantResult } from '../inte
2
2
  export declare class OpenAICompletionProvider implements IAICompletionProvider {
3
3
  private readonly client;
4
4
  private readonly model;
5
+ private readonly ARABIC_CHAR_REGEX;
5
6
  constructor(apiKey: string | undefined, model?: string);
6
7
  generateSearchVariantsChunksInBatch(items: ItemTextInput[], model?: string, instructions?: string, options?: {
7
8
  chunkSize?: number;
@@ -11,9 +12,12 @@ export declare class OpenAICompletionProvider implements IAICompletionProvider {
11
12
  pollIntervalMs?: number;
12
13
  maxWaitMs?: number;
13
14
  }): Promise<TextVariantResult[]>;
15
+ private sanitizeArabicArray;
16
+ private sanitizeLatinArray;
14
17
  private buildOptimizedInstructions;
15
18
  private buildOptimizedChunkSchema;
16
19
  private chunkItems;
17
20
  private pollBatch;
18
21
  private readJsonlFile;
22
+ private isEmptyValue;
19
23
  }
package/dist/libraries/llm/providers/openai-provider.js CHANGED
@@ -38,6 +38,7 @@ const openai_1 = __importStar(require("openai"));
38
38
  class OpenAICompletionProvider {
39
39
  client;
40
40
  model;
41
+ ARABIC_CHAR_REGEX = /[\u0600-\u06FF]/;
41
42
  constructor(apiKey, model = 'gpt-5-nano') {
42
43
  if (!apiKey) {
43
44
  throw new Error('OPENAI_API_KEY is required to construct OpenAICompletionProvider');
@@ -51,19 +52,22 @@ class OpenAICompletionProvider {
51
52
  const maxArabic = options?.maxArabic ?? 1;
52
53
  const maxArabizi = options?.maxArabizi ?? 2;
53
54
  const maxTransliteration = options?.maxTransliteration ?? 1;
55
+ const itemsToSend = items.filter(i => !i.skipAutoTranslation);
56
+ const skippedItems = items.filter(i => i.skipAutoTranslation);
57
+ if (skippedItems.length) {
58
+ console.log(`generateSearchVariantsChunksInBatch: skipping ${skippedItems.length}/${items.length} items due to skipAutoTranslation=true`);
59
+ }
60
+ if (!itemsToSend.length) {
61
+ return [];
62
+ }
54
63
  try {
55
- const finalInstructions = instructions ??
56
- this.buildOptimizedInstructions({
57
- maxArabic,
58
- maxArabizi,
59
- maxTransliteration,
60
- });
64
+ const finalInstructions = instructions ?? this.buildOptimizedInstructions({ maxArabic, maxArabizi, maxTransliteration });
61
65
  const chunkSchema = this.buildOptimizedChunkSchema({ maxArabic, maxArabizi, maxTransliteration });
62
- const chunks = this.chunkItems(items, chunkSize);
66
+ const chunks = this.chunkItems(itemsToSend, chunkSize);
63
67
  if (chunks.length > 50_000) {
64
68
  throw new Error(`Too many chunks (${chunks.length}) for a single batch file — increase chunkSize or split into multiple batch submissions.`);
65
69
  }
66
- console.log(`generateSearchVariantsChunksInBatch: model=${finalModel} items=${items.length} chunkSize=${chunkSize} chunks=${chunks.length}`);
70
+ console.log(`generateSearchVariantsChunksInBatch: model=${finalModel} items=${itemsToSend.length} (${skippedItems.length} skipped) chunkSize=${chunkSize} chunks=${chunks.length}`);
67
71
  const jsonl = chunks
68
72
  .map((chunkItems, i) => JSON.stringify({
69
73
  custom_id: `chunk-${i}`,
@@ -114,10 +118,20 @@ class OpenAICompletionProvider {
114
118
  try {
115
119
  const parsed = JSON.parse(outputText);
116
120
  for (const r of parsed.results) {
121
+ const ar = this.sanitizeArabicArray(r.ar);
122
+ const tr = this.sanitizeArabicArray(r.tr);
123
+ const az = this.sanitizeLatinArray(r.az);
124
+ const isEmpty = this.isEmptyValue(r.ar) || this.isEmptyValue(r.az) || this.isEmptyValue(r.tr);
125
+ const isProcessingFailed = (!ar.length && !tr.length) || !az.length;
126
+ if (isProcessingFailed) {
127
+ console.warn(`generateSearchVariantsChunksInBatch: marking id=${r.id} as processingFailed — ar=${JSON.stringify(r.ar)} az=${JSON.stringify(r.az)} tr=${JSON.stringify(r.tr)}`);
128
+ }
117
129
  resultsById.set(r.id, {
118
- arabic: r.ar,
119
- arabizi: r.az,
120
- arabicTransliteration: r.tr,
130
+ arabic: ar,
131
+ arabizi: az,
132
+ arabicTransliteration: tr,
133
+ isProcessingFailed,
134
+ isEmpty,
121
135
  });
122
136
  }
123
137
  }
@@ -125,11 +139,15 @@ class OpenAICompletionProvider {
125
139
  console.error(`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`, outputText);
126
140
  }
127
141
  }
142
+ const missingIds = itemsToSend.filter(i => !resultsById.has(i.id)).map(i => i.id);
143
+ if (missingIds.length) {
144
+ console.error(`generateSearchVariantsChunksInBatch: ${missingIds.length}/${itemsToSend.length} items missing from LLM output`, missingIds);
145
+ }
128
146
  return items.map(({ id, text }) => {
129
147
  const r = resultsById.get(id);
130
148
  if (!r) {
131
149
  console.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);
132
- return { id, original: text };
150
+ return { id, original: text, processingFailed: true, isEmpty: true };
133
151
  }
134
152
  return { id, original: text, ...r };
135
153
  });
@@ -137,12 +155,18 @@ class OpenAICompletionProvider {
137
155
  catch (error) {
138
156
  console.error('generateSearchVariantsChunksInBatch failed', {
139
157
  error,
140
- itemCount: items.length,
158
+ itemCount: itemsToSend.length,
141
159
  model: finalModel,
142
160
  });
143
161
  throw error;
144
162
  }
145
163
  }
164
+ sanitizeArabicArray(values) {
165
+ return (values ?? []).filter(v => typeof v === 'string' && v.trim() && this.ARABIC_CHAR_REGEX.test(v));
166
+ }
167
+ sanitizeLatinArray(values) {
168
+ return (values ?? []).filter(v => typeof v === 'string' && v.trim() && !this.ARABIC_CHAR_REGEX.test(v));
169
+ }
146
170
  buildOptimizedInstructions(caps) {
147
171
  return [
148
172
  'You generate search-name variants for a food delivery app product/shop name.',
@@ -218,6 +242,11 @@ class OpenAICompletionProvider {
218
242
  }
219
243
  return result;
220
244
  }
245
+ isEmptyValue(values) {
246
+ if (!values || values.length === 0)
247
+ return true;
248
+ return values.every(v => typeof v !== 'string' || !v.trim());
249
+ }
221
250
  }
222
251
  exports.OpenAICompletionProvider = OpenAICompletionProvider;
223
252
  //# sourceMappingURL=openai-provider.js.map
package/dist/libraries/llm/providers/openai-provider.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"openai-provider.js","sourceRoot":"/","sources":["libraries/llm/providers/openai-provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwC;AAGxC,MAAa,wBAAwB;IACnB,MAAM,CAAS;IACf,KAAK,CAAS;IAE/B,YAAY,MAA0B,EAAE,QAAgB,YAAY;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;IAcD,KAAK,CAAC,mCAAmC,CACxC,KAAsB,EACtB,KAAc,EACd,YAAqB,EACrB,OAOC;QAED,MAAM,UAAU,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACvC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;QAC5C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,kBAAkB,IAAI,CAAC,CAAC;QAE5D,IAAI,CAAC;YACJ,MAAM,iBAAiB,GACtB,YAAY;gBACZ,IAAI,CAAC,0BAA0B,CAAC;oBAC/B,SAAS;oBACT,UAAU;oBACV,kBAAkB;iBAClB,CAAC,CAAC;YAEJ,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAElG,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAEjD,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACd,oBAAoB,MAAM,CAAC,MAAM,0FAA0F,CAC3H,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CACV,8CAA8C,UAAU,UAAU,KAAK,CAAC,MAAM,cAAc,SAAS,WAAW,MAAM,CAAC,MAAM,EAAE,CAC/H,CAAC;YAGF,MAAM,KAAK,GAAG,MAAM;iBAClB,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CACtB,IAAI,CAAC,SAAS,CAAC;gBACd,SAAS,EAAE,SAAS,CAAC,EAAE;gBACvB,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,eAAe;gBACpB,IAAI,EAAE;oBACL,KAAK,EAAE,UAAU;oBACjB,YAAY,EAAE,iBAAiB;oBAG/B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC3E,IAAI,EAAE;wBACL,MAAM,EAAE;4BACP,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,WAAW;4BACnB,MAAM,EAAE,IAAI;yBACZ;qBACD;iBACD;aACD,CAAC,CACF;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YAGb,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBAChD,IAAI,EAAE,MAAM,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,6BAA6B,CAAC;gBAC9E,OAAO,EAAE,OAAO;aAChB,CAAC,CAAC;YAGH,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC5C,aAAa,EAAE,SAAS,CAAC,EAAE;gBAC3B,QAAQ,EAAE,eAAe;gBACzB,iBAAiB,EAAE,KAAK;aACxB,CAAC,CAAC;YAGH,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAEpF,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACd,SAAS,KAAK,CAAC,EAAE,uBAAuB,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,UAAU,CACrG,CAAC;YACH,CAAC;YAGD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAGpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsC,CAAC;YAElE,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC;gBACpC,MAAM,UAAU,GACf,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,MAAM;wBACX,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;wBACxC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC;gBAE9D,IAAI,CAAC,UAAU,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACvC,OAAO,CAAC,KAAK,CACZ,sDAAsD,OAAO,EAAE,EAC/D,GAAG,IAAI,MAAM,EAAE,KAAK,CACpB,CAAC;oBACF,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAEnC,CAAC;oBACF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAChC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;4BACrB,MAAM,EAAE,CAAC,CAAC,EAAE;4BACZ,OAAO,EAAE,CAAC,CAAC,EAAE;4BACb,qBAAqB,EAAE,CAAC,CAAC,EAAE;yBAC3B,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,CAAC,KAAK,CACZ,mEAAmE,OAAO,EAAE,EAC5E,UAAU,CACV,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBACjC,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,6DAA6D,EAAE,EAAE,CAAC,CAAC;oBACjF,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;gBAC/B,CAAC;gBACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAuB,CAAC;YAC1D,CAAC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC3D,KAAK;gBACL,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,KAAK,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAQO,0BAA0B,CAAC,IAIlC;QACA,OAAO;YACN,8EAA8E;YAC9E,oEAAoE;YACpE,uCAAuC;YACvC,yCAAyC;YACzC,qDAAqD,IAAI,CAAC,SAAS,GAAG;YACtE,0EAA0E,IAAI,CAAC,UAAU,GAAG;YAC5F,qFAAqF,IAAI,CAAC,kBAAkB,GAAG;YAC/G,2EAA2E;YAC3E,mFAAmF;YACnF,+EAA+E;SAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAIO,yBAAyB,CAAC,IAIjC;QACA,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,OAAO,EAAE;oBACR,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACX,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BACtB,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE;4BACvF,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;4BACxF,EAAE,EAAE;gCACH,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,QAAQ,EAAE,CAAC;gCACX,QAAQ,EAAE,IAAI,CAAC,kBAAkB;6BACjC;yBACD;wBACD,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;wBAClC,oBAAoB,EAAE,KAAK;qBAC3B;iBACD;aACD;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;YACrB,oBAAoB,EAAE,KAAK;SAC3B,CAAC;IACH,CAAC;IAEO,UAAU,CAAI,KAAU,EAAE,IAAY;QAC7C,MAAM,GAAG,GAAU,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,SAAS,CACtB,OAAe,EACf,UAAU,GAAG,IAAI,EACjB,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QAE/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CACV,cAAc,OAAO,WAAW,KAAK,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAClG,CAAC;YACF,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7C,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,0BAA0B,SAAS,eAAe,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACpG,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAC/D,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAiC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;CACD;AAzRD,4DAyRC","sourcesContent":["import OpenAI, { toFile } from 'openai';\nimport { IAICompletionProvider, ItemTextInput, TextVariantResult } from '../interfaces/common-llm-interfaces';\n\nexport class OpenAICompletionProvider implements IAICompletionProvider {\n\tprivate readonly client: OpenAI;\n\tprivate readonly model: string;\n\n\tconstructor(apiKey: string | undefined, model: string = 'gpt-5-nano') {\n\t\tif (!apiKey) {\n\t\t\tthrow new Error('OPENAI_API_KEY is required to construct OpenAICompletionProvider');\n\t\t}\n\t\tthis.client = new OpenAI({ apiKey });\n\t\tthis.model = model;\n\t}\n\n\t// ── Cost-optimized batch path ────────────────────────────────────\n\t// Implements everything discussed:\n\t// 1. Chunking — groups items (default 200/chunk) into one line per\n\t// chunk, so `instructions` is billed once per chunk instead of\n\t// once per item (biggest single cost lever).\n\t// 2. Short output keys — model emits {id, ar, az, tr} instead of\n\t// {id, arabic, arabizi, arabicTransliteration}; mapped back\n\t// to full field names locally, at zero token cost.\n\t// 3. Capped array lengths — schema enforces maxItems (not just a\n\t// prose suggestion), so the model can't drift to longer arrays.\n\t// 4. No echoed fields — model never re-emits id/original text;\n\t// both are re-attached locally from the input you already have.\n\tasync generateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number; // items per chunk/line — tune after testing token usage\n\t\t\tmaxArabic?: number; // cap on `arabic` variants (set to 1 for a single string-like result)\n\t\t\tmaxArabizi?: number; // cap on `arabizi` variants\n\t\t\tmaxTransliteration?: number; // cap on `arabicTransliteration` variants\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]> {\n\t\tconst finalModel = model ?? this.model;\n\t\tconst chunkSize = options?.chunkSize ?? 200;\n\t\tconst maxArabic = options?.maxArabic ?? 1;\n\t\tconst maxArabizi = options?.maxArabizi ?? 2;\n\t\tconst maxTransliteration = options?.maxTransliteration ?? 1;\n\n\t\ttry {\n\t\t\tconst finalInstructions =\n\t\t\t\tinstructions ??\n\t\t\t\tthis.buildOptimizedInstructions({\n\t\t\t\t\tmaxArabic,\n\t\t\t\t\tmaxArabizi,\n\t\t\t\t\tmaxTransliteration,\n\t\t\t\t});\n\n\t\t\tconst chunkSchema = this.buildOptimizedChunkSchema({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunks = this.chunkItems(items, chunkSize);\n\n\t\t\tif (chunks.length > 50_000) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Too many chunks (${chunks.length}) for a single batch file — increase chunkSize or split into multiple batch submissions.`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: model=${finalModel} items=${items.length} chunkSize=${chunkSize} chunks=${chunks.length}`\n\t\t\t);\n\n\t\t\t// 1. Build JSONL — ONE line per chunk, not per item\n\t\t\tconst jsonl = chunks\n\t\t\t\t.map((chunkItems, i) =>\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\tcustom_id: `chunk-${i}`,\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\turl: '/v1/responses',\n\t\t\t\t\t\tbody: {\n\t\t\t\t\t\t\tmodel: finalModel,\n\t\t\t\t\t\t\tinstructions: finalInstructions,\n\t\t\t\t\t\t\t// Only send what the model needs to generate from: id + text.\n\t\t\t\t\t\t\t// No other fields, no restating of instructions per item.\n\t\t\t\t\t\t\tinput: JSON.stringify(chunkItems.map(({ id, text }) => ({ id: id, text }))),\n\t\t\t\t\t\t\ttext: {\n\t\t\t\t\t\t\t\tformat: {\n\t\t\t\t\t\t\t\t\ttype: 'json_schema',\n\t\t\t\t\t\t\t\t\tname: 'text_variants_batch',\n\t\t\t\t\t\t\t\t\tschema: chunkSchema,\n\t\t\t\t\t\t\t\t\tstrict: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t})\n\t\t\t\t)\n\t\t\t\t.join('\\n');\n\n\t\t\t// 2. Upload input file\n\t\t\tconst inputFile = await this.client.files.create({\n\t\t\t\tfile: await toFile(Buffer.from(jsonl, 'utf-8'), 'batch-input-optimized.jsonl'),\n\t\t\t\tpurpose: 'batch',\n\t\t\t});\n\n\t\t\t// 3. Create batch job\n\t\t\tlet batch = await this.client.batches.create({\n\t\t\t\tinput_file_id: inputFile.id,\n\t\t\t\tendpoint: '/v1/responses',\n\t\t\t\tcompletion_window: '24h',\n\t\t\t});\n\n\t\t\t// 4. Poll until done\n\t\t\tbatch = await this.pollBatch(batch.id, options?.pollIntervalMs, options?.maxWaitMs);\n\n\t\t\tif (batch.status !== 'completed') {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Batch ${batch.id} ended with status \"${batch.status}\" (${batch.request_counts?.failed ?? 0} failed)`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 5. Parse output — keyed by chunk custom_id, each containing an array of per-item results\n\t\t\tconst outputByChunk = await this.readJsonlFile(batch.output_file_id);\n\t\t\tconst errorsByChunk = await this.readJsonlFile(batch.error_file_id);\n\n\t\t\t// 6. Flatten chunk results into a per-itemId map, expanding short keys back to full field names\n\t\t\tconst resultsById = new Map<string, Partial<TextVariantResult>>();\n\n\t\t\tfor (const [chunkId, record] of outputByChunk.entries()) {\n\t\t\t\tconst body = record?.response?.body;\n\t\t\t\tconst outputText =\n\t\t\t\t\tbody?.output_text ??\n\t\t\t\t\tbody?.output\n\t\t\t\t\t\t?.find((o: any) => o.type === 'message')\n\t\t\t\t\t\t?.content?.find((c: any) => c.type === 'output_text')?.text;\n\n\t\t\t\tif (!outputText) {\n\t\t\t\t\tconst err = errorsByChunk.get(chunkId);\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: no output for ${chunkId}`,\n\t\t\t\t\t\terr ?? record?.error\n\t\t\t\t\t);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(outputText) as {\n\t\t\t\t\t\tresults: Array<{ id: string; ar: string[]; az: string[]; tr: string[] }>;\n\t\t\t\t\t};\n\t\t\t\t\tfor (const r of parsed.results) {\n\t\t\t\t\t\tresultsById.set(r.id, {\n\t\t\t\t\t\t\tarabic: r.ar,\n\t\t\t\t\t\t\tarabizi: r.az,\n\t\t\t\t\t\t\tarabicTransliteration: r.tr,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t} catch {\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`,\n\t\t\t\t\t\toutputText\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// 7. Merge back with original items, preserving order and filling gaps\n\t\t\treturn items.map(({ id, text }) => {\n\t\t\t\tconst r = resultsById.get(id);\n\t\t\t\tif (!r) {\n\t\t\t\t\tconsole.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);\n\t\t\t\t\treturn { id, original: text };\n\t\t\t\t}\n\t\t\t\treturn { id, original: text, ...r } as TextVariantResult;\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tconsole.error('generateSearchVariantsChunksInBatch failed', {\n\t\t\t\terror,\n\t\t\t\titemCount: items.length,\n\t\t\t\tmodel: finalModel,\n\t\t\t});\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t// ── Helpers ──────────────────────────────────────────────────────\n\n\t// Compact instructions for the optimized path: short-key output format,\n\t// explicit array caps stated in prose (schema also enforces them),\n\t// and an explicit \"no commentary inside values\" rule to stop the model\n\t// padding fields with parenthetical notes.\n\tprivate buildOptimizedInstructions(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}): string {\n\t\treturn [\n\t\t\t'You generate search-name variants for a food delivery app product/shop name.',\n\t\t\t'You will receive a JSON array of items, each with \"id\" and \"text\".',\n\t\t\t'For EACH item, return an object with:',\n\t\t\t`- id: the exact id provided (unchanged)`,\n\t\t\t`- ar: Arabic translation(s) in Arabic script, max ${caps.maxArabic}.`,\n\t\t\t`- az: Arabizi spelling(s) (Latin script/numerals, e.g. \"zaytoon\"), max ${caps.maxArabizi}.`,\n\t\t\t`- tr: Arabic-script transliteration(s) of the original word (not translated), max ${caps.maxTransliteration}.`,\n\t\t\t'Return one result per input item, same order, no items skipped or merged.',\n\t\t\t'Values must contain only the term itself — no parenthetical notes, no commentary.',\n\t\t\t'Return strict JSON matching the schema. No extra commentary, no extra fields.',\n\t\t].join('\\n');\n\t}\n\n\t// Schema for the OPTIMIZED chunk path: wraps a `results` array (one\n\t// entry per item in the chunk), short keys, and hard maxItems caps.\n\tprivate buildOptimizedChunkSchema(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}) {\n\t\treturn {\n\t\t\ttype: 'object',\n\t\t\tproperties: {\n\t\t\t\tresults: {\n\t\t\t\t\ttype: 'array',\n\t\t\t\t\titems: {\n\t\t\t\t\t\ttype: 'object',\n\t\t\t\t\t\tproperties: {\n\t\t\t\t\t\t\tid: { type: 'string' },\n\t\t\t\t\t\t\tar: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabic },\n\t\t\t\t\t\t\taz: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabizi },\n\t\t\t\t\t\t\ttr: {\n\t\t\t\t\t\t\t\ttype: 'array',\n\t\t\t\t\t\t\t\titems: { type: 'string' },\n\t\t\t\t\t\t\t\tminItems: 1,\n\t\t\t\t\t\t\t\tmaxItems: caps.maxTransliteration,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\trequired: ['id', 'ar', 'az', 'tr'],\n\t\t\t\t\t\tadditionalProperties: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trequired: ['results'],\n\t\t\tadditionalProperties: false,\n\t\t};\n\t}\n\n\tprivate chunkItems<T>(items: T[], size: number): T[][] {\n\t\tconst out: T[][] = [];\n\t\tfor (let i = 0; i < items.length; i += size) {\n\t\t\tout.push(items.slice(i, i + size));\n\t\t}\n\t\treturn out;\n\t}\n\n\tprivate async pollBatch(\n\t\tbatchId: string,\n\t\tintervalMs = 5000,\n\t\tmaxWaitMs = 24 * 60 * 60 * 1000\n\t): Promise<OpenAI.Batches.Batch> {\n\t\tconst terminal = new Set(['completed', 'failed', 'expired', 'cancelled']);\n\t\tconst start = Date.now();\n\n\t\twhile (true) {\n\t\t\tconst batch = await this.client.batches.retrieve(batchId);\n\t\t\tconsole.log(\n\t\t\t\t`pollBatch: ${batchId} status=${batch.status} elapsed=${Math.round((Date.now() - start) / 1000)}s`\n\t\t\t);\n\t\t\tif (terminal.has(batch.status)) return batch;\n\n\t\t\tif (Date.now() - start > maxWaitMs) {\n\t\t\t\tthrow new Error(`Batch ${batchId} did not finish within ${maxWaitMs}ms (status: ${batch.status})`);\n\t\t\t}\n\n\t\t\tawait new Promise(resolve => setTimeout(resolve, intervalMs));\n\t\t}\n\t}\n\n\tprivate async readJsonlFile(fileId: string | null | undefined): Promise<Map<string, any>> {\n\t\tconst result = new Map<string, any>();\n\t\tif (!fileId) return result;\n\n\t\tconst fileResponse = await this.client.files.content(fileId);\n\t\tconst text = await fileResponse.text();\n\n\t\tfor (const line of text.split('\\n').filter(Boolean)) {\n\t\t\tconst record = JSON.parse(line);\n\t\t\tresult.set(record.custom_id, record);\n\t\t}\n\n\t\treturn result;\n\t}\n}\n"]}
1
+ {"version":3,"file":"openai-provider.js","sourceRoot":"/","sources":["libraries/llm/providers/openai-provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwC;AAGxC,MAAa,wBAAwB;IACnB,MAAM,CAAS;IACf,KAAK,CAAS;IACd,iBAAiB,GAAG,iBAAiB,CAAC;IAEvD,YAAY,MAA0B,EAAE,QAAgB,YAAY;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;IAcD,KAAK,CAAC,mCAAmC,CACxC,KAAsB,EACtB,KAAc,EACd,YAAqB,EACrB,OAOC;QAED,MAAM,UAAU,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACvC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;QAC5C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,kBAAkB,IAAI,CAAC,CAAC;QAG5D,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAE9D,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACV,iDAAiD,YAAY,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,wCAAwC,CAC5H,CAAC;QACH,CAAC;QAGD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAEzB,OAAO,EAAE,CAAC;QACX,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,iBAAiB,GACtB,YAAY,IAAI,IAAI,CAAC,0BAA0B,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAEhG,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAElG,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACd,oBAAoB,MAAM,CAAC,MAAM,0FAA0F,CAC3H,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CACV,8CAA8C,UAAU,UAAU,WAAW,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,uBAAuB,SAAS,WAAW,MAAM,CAAC,MAAM,EAAE,CACtK,CAAC;YAGF,MAAM,KAAK,GAAG,MAAM;iBAClB,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CACtB,IAAI,CAAC,SAAS,CAAC;gBACd,SAAS,EAAE,SAAS,CAAC,EAAE;gBACvB,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,eAAe;gBACpB,IAAI,EAAE;oBACL,KAAK,EAAE,UAAU;oBACjB,YAAY,EAAE,iBAAiB;oBAC/B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC3E,IAAI,EAAE;wBACL,MAAM,EAAE;4BACP,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,WAAW;4BACnB,MAAM,EAAE,IAAI;yBACZ;qBACD;iBACD;aACD,CAAC,CACF;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YAGb,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBAChD,IAAI,EAAE,MAAM,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,6BAA6B,CAAC;gBAC9E,OAAO,EAAE,OAAO;aAChB,CAAC,CAAC;YAGH,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC5C,aAAa,EAAE,SAAS,CAAC,EAAE;gBAC3B,QAAQ,EAAE,eAAe;gBACzB,iBAAiB,EAAE,KAAK;aACxB,CAAC,CAAC;YAGH,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAEpF,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACd,SAAS,KAAK,CAAC,EAAE,uBAAuB,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,UAAU,CACrG,CAAC;YACH,CAAC;YAGD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAGpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsC,CAAC;YAElE,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC;gBACpC,MAAM,UAAU,GACf,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,MAAM;wBACX,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;wBACxC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,IAAI,CAAC;gBAE9D,IAAI,CAAC,UAAU,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACvC,OAAO,CAAC,KAAK,CACZ,sDAAsD,OAAO,EAAE,EAC/D,GAAG,IAAI,MAAM,EAAE,KAAK,CACpB,CAAC;oBACF,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAEnC,CAAC;oBACF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAChC,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAEzC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC9F,MAAM,kBAAkB,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;wBAEpE,IAAI,kBAAkB,EAAE,CAAC;4BACxB,OAAO,CAAC,IAAI,CACX,mDAAmD,CAAC,CAAC,EAAE,6BAA6B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAChK,CAAC;wBACH,CAAC;wBAED,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;4BACrB,MAAM,EAAE,EAAE;4BACV,OAAO,EAAE,EAAE;4BACX,qBAAqB,EAAE,EAAE;4BACzB,kBAAkB;4BAClB,OAAO;yBACP,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBAAC,MAAM,CAAC;oBACR,OAAO,CAAC,KAAK,CACZ,mEAAmE,OAAO,EAAE,EAC5E,UAAU,CACV,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClF,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CACZ,wCAAwC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,gCAAgC,EAC/G,UAAU,CACV,CAAC;YACH,CAAC;YAID,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBACjC,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,6DAA6D,EAAE,EAAE,CAAC,CAAC;oBACjF,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBACtE,CAAC;gBACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAuB,CAAC;YAC1D,CAAC,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC3D,KAAK;gBACL,SAAS,EAAE,WAAW,CAAC,MAAM;gBAC7B,KAAK,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAMO,mBAAmB,CAAC,MAA4B;QACvD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,CAAC;IAIO,kBAAkB,CAAC,MAA4B;QACtD,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAMO,0BAA0B,CAAC,IAIlC;QACA,OAAO;YACN,8EAA8E;YAC9E,oEAAoE;YACpE,uCAAuC;YACvC,yCAAyC;YACzC,qDAAqD,IAAI,CAAC,SAAS,GAAG;YACtE,0EAA0E,IAAI,CAAC,UAAU,GAAG;YAC5F,qFAAqF,IAAI,CAAC,kBAAkB,GAAG;YAC/G,2EAA2E;YAC3E,mFAAmF;YACnF,+EAA+E;SAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAIO,yBAAyB,CAAC,IAIjC;QACA,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACX,OAAO,EAAE;oBACR,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACX,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;4BACtB,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE;4BACvF,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;4BACxF,EAAE,EAAE;gCACH,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,QAAQ,EAAE,CAAC;gCACX,QAAQ,EAAE,IAAI,CAAC,kBAAkB;6BACjC;yBACD;wBACD,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;wBAClC,oBAAoB,EAAE,KAAK;qBAC3B;iBACD;aACD;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;YACrB,oBAAoB,EAAE,KAAK;SAC3B,CAAC;IACH,CAAC;IAEO,UAAU,CAAI,KAAU,EAAE,IAAY;QAC7C,MAAM,GAAG,GAAU,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,SAAS,CACtB,OAAe,EACf,UAAU,GAAG,IAAI,EACjB,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QAE/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CACV,cAAc,OAAO,WAAW,KAAK,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAClG,CAAC;YACF,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7C,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,0BAA0B,SAAS,eAAe,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACpG,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAC/D,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAiC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAKO,YAAY,CAAC,MAAmC;QACvD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;CACD;AAhVD,4DAgVC","sourcesContent":["import OpenAI, { toFile } from 'openai';\nimport { IAICompletionProvider, ItemTextInput, TextVariantResult } from '../interfaces/common-llm-interfaces';\n\nexport class OpenAICompletionProvider implements IAICompletionProvider {\n\tprivate readonly client: OpenAI;\n\tprivate readonly model: string;\n\tprivate readonly ARABIC_CHAR_REGEX = /[\\u0600-\\u06FF]/;\n\n\tconstructor(apiKey: string | undefined, model: string = 'gpt-5-nano') {\n\t\tif (!apiKey) {\n\t\t\tthrow new Error('OPENAI_API_KEY is required to construct OpenAICompletionProvider');\n\t\t}\n\t\tthis.client = new OpenAI({ apiKey });\n\t\tthis.model = model;\n\t}\n\n\t// ── Cost-optimized batch path ────────────────────────────────────\n\t// Implements everything discussed:\n\t// 1. Chunking — groups items (default 200/chunk) into one line per\n\t// chunk, so `instructions` is billed once per chunk instead of\n\t// once per item (biggest single cost lever).\n\t// 2. Short output keys — model emits {id, ar, az, tr} instead of\n\t// {id, arabic, arabizi, arabicTransliteration}; mapped back\n\t// to full field names locally, at zero token cost.\n\t// 3. Capped array lengths — schema enforces maxItems (not just a\n\t// prose suggestion), so the model can't drift to longer arrays.\n\t// 4. No echoed fields — model never re-emits id/original text;\n\t// both are re-attached locally from the input you already have.\n\tasync generateSearchVariantsChunksInBatch(\n\t\titems: ItemTextInput[],\n\t\tmodel?: string,\n\t\tinstructions?: string,\n\t\toptions?: {\n\t\t\tchunkSize?: number;\n\t\t\tmaxArabic?: number;\n\t\t\tmaxArabizi?: number;\n\t\t\tmaxTransliteration?: number;\n\t\t\tpollIntervalMs?: number;\n\t\t\tmaxWaitMs?: number;\n\t\t}\n\t): Promise<TextVariantResult[]> {\n\t\tconst finalModel = model ?? this.model;\n\t\tconst chunkSize = options?.chunkSize ?? 200;\n\t\tconst maxArabic = options?.maxArabic ?? 1;\n\t\tconst maxArabizi = options?.maxArabizi ?? 2;\n\t\tconst maxTransliteration = options?.maxTransliteration ?? 1;\n\n\t\t// Items explicitly opted out — never sent to the model, just passed through.\n\t\tconst itemsToSend = items.filter(i => !i.skipAutoTranslation);\n\t\tconst skippedItems = items.filter(i => i.skipAutoTranslation);\n\n\t\tif (skippedItems.length) {\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: skipping ${skippedItems.length}/${items.length} items due to skipAutoTranslation=true`\n\t\t\t);\n\t\t}\n\n\t\t// Nothing left to actually send — short-circuit before touching the OpenAI API at all.\n\t\tif (!itemsToSend.length) {\n\t\t\t// return skippedItems.map(({ id, text }) => ({ id, original: text, skipped: true }));\n\t\t\treturn [];\n\t\t}\n\n\t\ttry {\n\t\t\tconst finalInstructions =\n\t\t\t\tinstructions ?? this.buildOptimizedInstructions({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunkSchema = this.buildOptimizedChunkSchema({ maxArabic, maxArabizi, maxTransliteration });\n\n\t\t\tconst chunks = this.chunkItems(itemsToSend, chunkSize);\n\n\t\t\tif (chunks.length > 50_000) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Too many chunks (${chunks.length}) for a single batch file — increase chunkSize or split into multiple batch submissions.`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tconsole.log(\n\t\t\t\t`generateSearchVariantsChunksInBatch: model=${finalModel} items=${itemsToSend.length} (${skippedItems.length} skipped) chunkSize=${chunkSize} chunks=${chunks.length}`\n\t\t\t);\n\n\t\t\t// 1. Build JSONL — ONE line per chunk, not per item\n\t\t\tconst jsonl = chunks\n\t\t\t\t.map((chunkItems, i) =>\n\t\t\t\t\tJSON.stringify({\n\t\t\t\t\t\tcustom_id: `chunk-${i}`,\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\turl: '/v1/responses',\n\t\t\t\t\t\tbody: {\n\t\t\t\t\t\t\tmodel: finalModel,\n\t\t\t\t\t\t\tinstructions: finalInstructions,\n\t\t\t\t\t\t\tinput: JSON.stringify(chunkItems.map(({ id, text }) => ({ id: id, text }))),\n\t\t\t\t\t\t\ttext: {\n\t\t\t\t\t\t\t\tformat: {\n\t\t\t\t\t\t\t\t\ttype: 'json_schema',\n\t\t\t\t\t\t\t\t\tname: 'text_variants_batch',\n\t\t\t\t\t\t\t\t\tschema: chunkSchema,\n\t\t\t\t\t\t\t\t\tstrict: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t})\n\t\t\t\t)\n\t\t\t\t.join('\\n');\n\n\t\t\t// 2. Upload input file\n\t\t\tconst inputFile = await this.client.files.create({\n\t\t\t\tfile: await toFile(Buffer.from(jsonl, 'utf-8'), 'batch-input-optimized.jsonl'),\n\t\t\t\tpurpose: 'batch',\n\t\t\t});\n\n\t\t\t// 3. Create batch job\n\t\t\tlet batch = await this.client.batches.create({\n\t\t\t\tinput_file_id: inputFile.id,\n\t\t\t\tendpoint: '/v1/responses',\n\t\t\t\tcompletion_window: '24h',\n\t\t\t});\n\n\t\t\t// 4. Poll until done\n\t\t\tbatch = await this.pollBatch(batch.id, options?.pollIntervalMs, options?.maxWaitMs);\n\n\t\t\tif (batch.status !== 'completed') {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Batch ${batch.id} ended with status \"${batch.status}\" (${batch.request_counts?.failed ?? 0} failed)`\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 5. Parse output\n\t\t\tconst outputByChunk = await this.readJsonlFile(batch.output_file_id);\n\t\t\tconst errorsByChunk = await this.readJsonlFile(batch.error_file_id);\n\n\t\t\t// 6. Flatten chunk results into a per-itemId map\n\t\t\tconst resultsById = new Map<string, Partial<TextVariantResult>>();\n\n\t\t\tfor (const [chunkId, record] of outputByChunk.entries()) {\n\t\t\t\tconst body = record?.response?.body;\n\t\t\t\tconst outputText =\n\t\t\t\t\tbody?.output_text ??\n\t\t\t\t\tbody?.output\n\t\t\t\t\t\t?.find((o: any) => o.type === 'message')\n\t\t\t\t\t\t?.content?.find((c: any) => c.type === 'output_text')?.text;\n\n\t\t\t\tif (!outputText) {\n\t\t\t\t\tconst err = errorsByChunk.get(chunkId);\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: no output for ${chunkId}`,\n\t\t\t\t\t\terr ?? record?.error\n\t\t\t\t\t);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\ttry {\n\t\t\t\t\tconst parsed = JSON.parse(outputText) as {\n\t\t\t\t\t\tresults: Array<{ id: string; ar: string[]; az: string[]; tr: string[] }>;\n\t\t\t\t\t};\n\t\t\t\t\tfor (const r of parsed.results) {\n\t\t\t\t\t\tconst ar = this.sanitizeArabicArray(r.ar);\n\t\t\t\t\t\tconst tr = this.sanitizeArabicArray(r.tr);\n\t\t\t\t\t\tconst az = this.sanitizeLatinArray(r.az);\n\n\t\t\t\t\t\tconst isEmpty = this.isEmptyValue(r.ar) || this.isEmptyValue(r.az) || this.isEmptyValue(r.tr);\n\t\t\t\t\t\tconst isProcessingFailed = (!ar.length && !tr.length) || !az.length;\n\n\t\t\t\t\t\tif (isProcessingFailed) {\n\t\t\t\t\t\t\tconsole.warn(\n\t\t\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: marking id=${r.id} as processingFailed — ar=${JSON.stringify(r.ar)} az=${JSON.stringify(r.az)} tr=${JSON.stringify(r.tr)}`\n\t\t\t\t\t\t\t);\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tresultsById.set(r.id, {\n\t\t\t\t\t\t\tarabic: ar,\n\t\t\t\t\t\t\tarabizi: az,\n\t\t\t\t\t\t\tarabicTransliteration: tr,\n\t\t\t\t\t\t\tisProcessingFailed,\n\t\t\t\t\t\t\tisEmpty,\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\t\t\t\t} catch {\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`generateSearchVariantsChunksInBatch: failed to parse output for ${chunkId}`,\n\t\t\t\t\t\toutputText\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// 7. Surface items the model silently dropped (only checked against what we actually sent)\n\t\t\tconst missingIds = itemsToSend.filter(i => !resultsById.has(i.id)).map(i => i.id);\n\t\t\tif (missingIds.length) {\n\t\t\t\tconsole.error(\n\t\t\t\t\t`generateSearchVariantsChunksInBatch: ${missingIds.length}/${itemsToSend.length} items missing from LLM output`,\n\t\t\t\t\tmissingIds\n\t\t\t\t);\n\t\t\t}\n\n\t\t\t// 8. Merge back with the FULL original item list — skipped items pass through untouched,\n\t\t\t// everything else is looked up in resultsById.\n\t\t\treturn items.map(({ id, text }) => {\n\t\t\t\tconst r = resultsById.get(id);\n\t\t\t\tif (!r) {\n\t\t\t\t\tconsole.error(`generateSearchVariantsChunksInBatch: no result for itemId=${id}`);\n\t\t\t\t\treturn { id, original: text, processingFailed: true, isEmpty: true };\n\t\t\t\t}\n\t\t\t\treturn { id, original: text, ...r } as TextVariantResult;\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tconsole.error('generateSearchVariantsChunksInBatch failed', {\n\t\t\t\terror,\n\t\t\t\titemCount: itemsToSend.length,\n\t\t\t\tmodel: finalModel,\n\t\t\t});\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t// ── Helpers ──────────────────────────────────────────────────────\n\n\t// Keeps only entries that actually contain Arabic-script characters — used for `ar`/`tr`,\n\t// which must never contain the raw English/Latin text the model was given.\n\tprivate sanitizeArabicArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Keeps only entries that do NOT contain Arabic-script characters — used for `az`\n\t// (Latin/Arabizi variants), which must never contain Arabic script.\n\tprivate sanitizeLatinArray(values: string[] | undefined): string[] {\n\t\treturn (values ?? []).filter(v => typeof v === 'string' && v.trim() && !this.ARABIC_CHAR_REGEX.test(v));\n\t}\n\n\t// Compact instructions for the optimized path: short-key output format,\n\t// explicit array caps stated in prose (schema also enforces them),\n\t// and an explicit \"no commentary inside values\" rule to stop the model\n\t// padding fields with parenthetical notes.\n\tprivate buildOptimizedInstructions(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}): string {\n\t\treturn [\n\t\t\t'You generate search-name variants for a food delivery app product/shop name.',\n\t\t\t'You will receive a JSON array of items, each with \"id\" and \"text\".',\n\t\t\t'For EACH item, return an object with:',\n\t\t\t`- id: the exact id provided (unchanged)`,\n\t\t\t`- ar: Arabic translation(s) in Arabic script, max ${caps.maxArabic}.`,\n\t\t\t`- az: Arabizi spelling(s) (Latin script/numerals, e.g. \"zaytoon\"), max ${caps.maxArabizi}.`,\n\t\t\t`- tr: Arabic-script transliteration(s) of the original word (not translated), max ${caps.maxTransliteration}.`,\n\t\t\t'Return one result per input item, same order, no items skipped or merged.',\n\t\t\t'Values must contain only the term itself — no parenthetical notes, no commentary.',\n\t\t\t'Return strict JSON matching the schema. No extra commentary, no extra fields.',\n\t\t].join('\\n');\n\t}\n\n\t// Schema for the OPTIMIZED chunk path: wraps a `results` array (one\n\t// entry per item in the chunk), short keys, and hard maxItems caps.\n\tprivate buildOptimizedChunkSchema(caps: {\n\t\tmaxArabic: number;\n\t\tmaxArabizi: number;\n\t\tmaxTransliteration: number;\n\t}) {\n\t\treturn {\n\t\t\ttype: 'object',\n\t\t\tproperties: {\n\t\t\t\tresults: {\n\t\t\t\t\ttype: 'array',\n\t\t\t\t\titems: {\n\t\t\t\t\t\ttype: 'object',\n\t\t\t\t\t\tproperties: {\n\t\t\t\t\t\t\tid: { type: 'string' },\n\t\t\t\t\t\t\tar: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabic },\n\t\t\t\t\t\t\taz: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: caps.maxArabizi },\n\t\t\t\t\t\t\ttr: {\n\t\t\t\t\t\t\t\ttype: 'array',\n\t\t\t\t\t\t\t\titems: { type: 'string' },\n\t\t\t\t\t\t\t\tminItems: 1,\n\t\t\t\t\t\t\t\tmaxItems: caps.maxTransliteration,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\trequired: ['id', 'ar', 'az', 'tr'],\n\t\t\t\t\t\tadditionalProperties: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trequired: ['results'],\n\t\t\tadditionalProperties: false,\n\t\t};\n\t}\n\n\tprivate chunkItems<T>(items: T[], size: number): T[][] {\n\t\tconst out: T[][] = [];\n\t\tfor (let i = 0; i < items.length; i += size) {\n\t\t\tout.push(items.slice(i, i + size));\n\t\t}\n\t\treturn out;\n\t}\n\n\tprivate async pollBatch(\n\t\tbatchId: string,\n\t\tintervalMs = 5000,\n\t\tmaxWaitMs = 24 * 60 * 60 * 1000\n\t): Promise<OpenAI.Batches.Batch> {\n\t\tconst terminal = new Set(['completed', 'failed', 'expired', 'cancelled']);\n\t\tconst start = Date.now();\n\n\t\twhile (true) {\n\t\t\tconst batch = await this.client.batches.retrieve(batchId);\n\t\t\tconsole.log(\n\t\t\t\t`pollBatch: ${batchId} status=${batch.status} elapsed=${Math.round((Date.now() - start) / 1000)}s`\n\t\t\t);\n\t\t\tif (terminal.has(batch.status)) return batch;\n\n\t\t\tif (Date.now() - start > maxWaitMs) {\n\t\t\t\tthrow new Error(`Batch ${batchId} did not finish within ${maxWaitMs}ms (status: ${batch.status})`);\n\t\t\t}\n\n\t\t\tawait new Promise(resolve => setTimeout(resolve, intervalMs));\n\t\t}\n\t}\n\n\tprivate async readJsonlFile(fileId: string | null | undefined): Promise<Map<string, any>> {\n\t\tconst result = new Map<string, any>();\n\t\tif (!fileId) return result;\n\n\t\tconst fileResponse = await this.client.files.content(fileId);\n\t\tconst text = await fileResponse.text();\n\n\t\tfor (const line of text.split('\\n').filter(Boolean)) {\n\t\t\tconst record = JSON.parse(line);\n\t\t\tresult.set(record.custom_id, record);\n\t\t}\n\n\t\treturn result;\n\t}\n\n\t// Checks whether ar/az/tr came back empty in any sense the model might produce:\n\t// missing entirely (null/undefined), an empty array, or an array whose entries\n\t// are just empty/whitespace strings.\n\tprivate isEmptyValue(values: string[] | null | undefined): boolean {\n\t\tif (!values || values.length === 0) return true;\n\t\treturn values.every(v => typeof v !== 'string' || !v.trim());\n\t}\n}\n"]}
package/dist/types/README.md CHANGED
@@ -22,7 +22,7 @@ Perfect for sharing types between frontend and backend applications.
22
22
 
23
23
  ## Version
24
24
 
25
- Version: 1.4.311
25
+ Version: 1.4.312
26
26
 
27
27
  ## Dependencies
28
28
 
package/dist/types/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lyxa.ai/types",
3
- "version": "1.4.311",
3
+ "version": "1.4.312",
4
4
  "description": "Lyxa type definitions and validation schemas for both frontend and backend",
5
5
  "author": "elie <42282499+Internalizable@users.noreply.github.com>",
6
6
  "license": "MIT",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lyxa.ai/core",
3
- "version": "1.4.311",
3
+ "version": "1.4.312",
4
4
  "description": "The Core system of the Lyxa services.",
5
5
  "author": "elie <42282499+Internalizable@users.noreply.github.com>",
6
6
  "license": "MIT",