@alauda/doom 2.11.2 → 2.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -138,6 +138,8 @@ export interface TranslateDocumentOptions {
|
|
|
138
138
|
maxAssemblyRounds?: number;
|
|
139
139
|
contextTail?: number;
|
|
140
140
|
repairer?: SegmentRepairer;
|
|
141
|
+
/** Maximum number of uncached segments worked on at once. */
|
|
142
|
+
segmentConcurrency?: number;
|
|
141
143
|
/**
|
|
142
144
|
* The translation already on disk, for reusing the segments that did not
|
|
143
145
|
* change. Absent on a first translation, and when `--force` is given.
|
|
@@ -159,6 +161,14 @@ export interface TranslateDocumentOptions {
|
|
|
159
161
|
*/
|
|
160
162
|
export declare const maskFindingToTranslationFinding: (finding: MaskIntegrityError["findings"][number]) => TranslationFinding;
|
|
161
163
|
export declare const translateDocument: (options: TranslateDocumentOptions) => Promise<TranslateDocumentResult>;
|
|
164
|
+
/**
|
|
165
|
+
* Run independent segment jobs with a small worker pool.
|
|
166
|
+
*
|
|
167
|
+
* A rejected job stops new work from being scheduled but lets already-started
|
|
168
|
+
* jobs settle. This preserves the document-level error boundary while avoiding
|
|
169
|
+
* dangling promises when one model call fails.
|
|
170
|
+
*/
|
|
171
|
+
export declare const runWithConcurrency: <T>(items: readonly T[], concurrency: number, worker: (item: T, index: number) => Promise<unknown>) => Promise<void>;
|
|
162
172
|
/**
|
|
163
173
|
* Whether a segment has any text a reviewer could judge.
|
|
164
174
|
*
|
|
@@ -55,7 +55,7 @@ export const maskFindingToTranslationFinding = (finding) => ({
|
|
|
55
55
|
placeholder: finding.placeholder,
|
|
56
56
|
});
|
|
57
57
|
export const translateDocument = async (options) => {
|
|
58
|
-
const { tree, maskedSource, maskEntries, processor, compose, targetPath, sourceLabel, sourceLanguage, targetLanguage, translator, checker, segmentJudge, documentJudge, terms, segmentCap, segmentHardCap, segmentFloor, maxSegmentAttempts = DEFAULT_MAX_SEGMENT_ATTEMPTS, maxAssemblyRounds = DEFAULT_MAX_ASSEMBLY_ROUNDS, contextTail, repairer, previous, onProgress, } = options;
|
|
58
|
+
const { tree, maskedSource, maskEntries, processor, compose, targetPath, sourceLabel, sourceLanguage, targetLanguage, translator, checker, segmentJudge, documentJudge, terms, segmentCap, segmentHardCap, segmentFloor, maxSegmentAttempts = DEFAULT_MAX_SEGMENT_ATTEMPTS, maxAssemblyRounds = DEFAULT_MAX_ASSEMBLY_ROUNDS, contextTail, repairer, segmentConcurrency = 1, previous, onProgress, } = options;
|
|
59
59
|
let plan;
|
|
60
60
|
try {
|
|
61
61
|
plan = planSegments({
|
|
@@ -262,7 +262,8 @@ export const translateDocument = async (options) => {
|
|
|
262
262
|
}
|
|
263
263
|
return true;
|
|
264
264
|
};
|
|
265
|
-
// ------------------------------------------------
|
|
265
|
+
// ------------------------------------------------ cached segments first
|
|
266
|
+
const pending = [];
|
|
266
267
|
for (const segment of plan.segments) {
|
|
267
268
|
const reused = reuse.get(segment.index);
|
|
268
269
|
if (reused != null) {
|
|
@@ -270,8 +271,10 @@ export const translateDocument = async (options) => {
|
|
|
270
271
|
outcomes[segment.index].status = 'cached';
|
|
271
272
|
continue;
|
|
272
273
|
}
|
|
273
|
-
|
|
274
|
+
pending.push(segment);
|
|
274
275
|
}
|
|
276
|
+
// ------------------------------------------------ uncached segments, bounded
|
|
277
|
+
await runWithConcurrency(pending, segmentConcurrency, (segment) => workOn(segment));
|
|
275
278
|
if (reuse.size > 0) {
|
|
276
279
|
onProgress?.(`${sourceLabel}: reused ${reuse.size} of ${plan.segments.length} segment(s) from the previous translation`);
|
|
277
280
|
}
|
|
@@ -369,11 +372,51 @@ export const translateDocument = async (options) => {
|
|
|
369
372
|
};
|
|
370
373
|
}
|
|
371
374
|
assemblyRounds++;
|
|
375
|
+
const rerouted = [];
|
|
372
376
|
for (const [index, sentBack] of routed) {
|
|
373
377
|
const segment = plan.segments[index];
|
|
374
378
|
onProgress?.(`${sourceLabel} [segment ${index + 1}/${plan.segments.length}${describe(segment)}] sent back by the assembled document: ${summarise(sentBack)}${reasons(sentBack)}`);
|
|
375
|
-
|
|
379
|
+
rerouted.push({ segment, findings: sentBack });
|
|
376
380
|
}
|
|
381
|
+
await runWithConcurrency(rerouted, segmentConcurrency, ({ segment, findings }) => workOn(segment, findings));
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
/**
|
|
385
|
+
* Run independent segment jobs with a small worker pool.
|
|
386
|
+
*
|
|
387
|
+
* A rejected job stops new work from being scheduled but lets already-started
|
|
388
|
+
* jobs settle. This preserves the document-level error boundary while avoiding
|
|
389
|
+
* dangling promises when one model call fails.
|
|
390
|
+
*/
|
|
391
|
+
export const runWithConcurrency = async (items, concurrency, worker) => {
|
|
392
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
393
|
+
throw new Error(`segment concurrency must be a positive integer, got ${concurrency}`);
|
|
394
|
+
}
|
|
395
|
+
let next = 0;
|
|
396
|
+
let stopped = false;
|
|
397
|
+
const errors = [];
|
|
398
|
+
const runWorker = async () => {
|
|
399
|
+
while (!stopped) {
|
|
400
|
+
const index = next++;
|
|
401
|
+
const item = items[index];
|
|
402
|
+
if (item === undefined) {
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
try {
|
|
406
|
+
await worker(item, index);
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
if (errors.length === 0) {
|
|
410
|
+
errors.push(error);
|
|
411
|
+
stopped = true;
|
|
412
|
+
}
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => runWorker()));
|
|
418
|
+
if (errors.length > 0) {
|
|
419
|
+
throw errors[0];
|
|
377
420
|
}
|
|
378
421
|
};
|
|
379
422
|
const describe = (segment) => segment.label.heading ? ` ‹${segment.label.heading}›` : '';
|
package/lib/cli/translate.js
CHANGED