@lingjingai/scriptctl 0.15.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.js +7 -0
- package/dist/cli.js.map +1 -1
- package/dist/common.d.ts +10 -3
- package/dist/common.js +82 -60
- package/dist/common.js.map +1 -1
- package/dist/domain/direct-core.d.ts +39 -4
- package/dist/domain/direct-core.js +317 -56
- package/dist/domain/direct-core.js.map +1 -1
- package/dist/help-text.js +15 -1
- package/dist/help-text.js.map +1 -1
- package/dist/infra/providers.d.ts +12 -2
- package/dist/infra/providers.js +156 -108
- package/dist/infra/providers.js.map +1 -1
- package/dist/usecases/direct.js +352 -74
- package/dist/usecases/direct.js.map +1 -1
- package/dist/usecases/parse.js +104 -14
- package/dist/usecases/parse.js.map +1 -1
- package/dist/usecases/script.d.ts +2 -2
- package/dist/usecases/script.js +15 -11
- package/dist/usecases/script.js.map +1 -1
- package/package.json +1 -1
package/dist/usecases/direct.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, writeJson, } from "../common.js";
|
|
4
|
-
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResults, normalizeEpisodeResult, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
3
|
+
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
|
|
4
|
+
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResults, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
5
5
|
import { validateScript } from "../domain/script-core.js";
|
|
6
6
|
import { makeProvider } from "../infra/providers.js";
|
|
7
7
|
import { makeSourceManifest, prepareSource, } from "../infra/converters.js";
|
|
@@ -29,6 +29,32 @@ function parseIntList(v) {
|
|
|
29
29
|
function parseStrList(v) {
|
|
30
30
|
return strOf(v).split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
31
31
|
}
|
|
32
|
+
// Parse a positive-integer CLI option (default when unset) with a uniform USAGE
|
|
33
|
+
// error. Shared by --concurrency / --summary-threshold-chars / the batch opts.
|
|
34
|
+
function parsePositiveIntOption(raw, flag, title, fallback) {
|
|
35
|
+
const value = parseInt(strOf(raw || fallback), 10);
|
|
36
|
+
if (Number.isNaN(value) || value < 1) {
|
|
37
|
+
throw new CliError(title, `${title}.`, {
|
|
38
|
+
exitCode: EXIT_USAGE,
|
|
39
|
+
required: [`${flag}: positive integer`],
|
|
40
|
+
received: [`${flag}: ${raw}`],
|
|
41
|
+
nextSteps: ["Use a positive integer and rerun init."],
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
// Status/report line for the synopsis stage: skipped, degraded (with episode
|
|
47
|
+
// list when known), or null when the overview is complete. One formatter so
|
|
48
|
+
// `direct init`'s report and `direct status` never word it differently.
|
|
49
|
+
function formatSummaryStatusLine(skipped, degraded, degradedEpisodes) {
|
|
50
|
+
if (skipped)
|
|
51
|
+
return "summary: skipped (--skip-summary)";
|
|
52
|
+
if (!degraded)
|
|
53
|
+
return null;
|
|
54
|
+
return degradedEpisodes.length > 0
|
|
55
|
+
? `summary: degraded (episodes: ${degradedEpisodes.join(", ")})`
|
|
56
|
+
: "summary: degraded (overview may be partial)";
|
|
57
|
+
}
|
|
32
58
|
// ---------------------------------------------------------------------------
|
|
33
59
|
// run_state.json
|
|
34
60
|
// ---------------------------------------------------------------------------
|
|
@@ -78,6 +104,19 @@ function episodeErrorPath(dir, ep) {
|
|
|
78
104
|
function episodeResultKey(ep) {
|
|
79
105
|
return `ep_${pad3(Number(ep["episode"]))}`;
|
|
80
106
|
}
|
|
107
|
+
// Read the synopsis carried on a persisted episode result (compact key `syn`),
|
|
108
|
+
// for episode-synopsis reuse. Returns "" when absent/unreadable/legacy (no syn).
|
|
109
|
+
function readEpisodeSynopsis(dir, episode) {
|
|
110
|
+
const p = episodeResultPath(dir, episode);
|
|
111
|
+
if (!exists(p))
|
|
112
|
+
return "";
|
|
113
|
+
try {
|
|
114
|
+
return readCarriedSynopsis(readJson(p));
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return "";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
81
120
|
function batchResultKey(batch) {
|
|
82
121
|
const bid = strOf(batch["batch_id"]).trim();
|
|
83
122
|
if (bid)
|
|
@@ -205,8 +244,15 @@ export function mergeBatchResultsForEpisode(episode, batchResults) {
|
|
|
205
244
|
const props = [];
|
|
206
245
|
const speakers = [];
|
|
207
246
|
const stateDefinitions = [];
|
|
247
|
+
// Ordered, non-empty per-fragment synopses for the episode-level reduce. The
|
|
248
|
+
// reduce/passthrough decision (and LLM call) happens in the orchestrator, NOT
|
|
249
|
+
// here — this pure function only collects the beats in _batch_part order.
|
|
250
|
+
const batchSynopses = [];
|
|
208
251
|
const sorted = [...batchResults].sort((a, b) => Number(a["_batch_part"] ?? 0) - Number(b["_batch_part"] ?? 0));
|
|
209
252
|
for (const item of sorted) {
|
|
253
|
+
const synopsis = strOf(item["synopsis"]).trim();
|
|
254
|
+
if (synopsis)
|
|
255
|
+
batchSynopses.push(synopsis);
|
|
210
256
|
const itemScenes = asList(item["scenes"]);
|
|
211
257
|
if (itemScenes.length === 0)
|
|
212
258
|
continue;
|
|
@@ -249,14 +295,12 @@ export function mergeBatchResultsForEpisode(episode, batchResults) {
|
|
|
249
295
|
speakers,
|
|
250
296
|
state_definitions: stateDefinitions,
|
|
251
297
|
};
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
259
|
-
return {};
|
|
298
|
+
const normalized = normalizeEpisodeResult(result, episode);
|
|
299
|
+
// Carry the ordered fragment synopses on the merged result so the orchestrator
|
|
300
|
+
// can run the episode-level reduce (or passthrough) on them. Not part of the
|
|
301
|
+
// episode schema — stripped before the episode entry is built downstream.
|
|
302
|
+
normalized["batch_synopses"] = batchSynopses;
|
|
303
|
+
return normalized;
|
|
260
304
|
}
|
|
261
305
|
// Record a batch failure: write `<key>.error.json` (which marks the batch as
|
|
262
306
|
// "tried and failed" → sticky on plain resume) and drop any stale result file.
|
|
@@ -298,17 +342,21 @@ function writeBatchFailure(dir, batch, exc) {
|
|
|
298
342
|
return error;
|
|
299
343
|
}
|
|
300
344
|
function initFailedReport(workspace, opts) {
|
|
345
|
+
const lastError = { title: opts.title, received: opts.received, failed_at: checkpointTimestamp() };
|
|
346
|
+
if (opts.errorCode)
|
|
347
|
+
lastError["error_code"] = opts.errorCode;
|
|
301
348
|
const payload = {
|
|
302
349
|
status: "init_failed",
|
|
303
350
|
command: "direct init",
|
|
304
351
|
init_stage: opts.stage,
|
|
305
|
-
last_error:
|
|
352
|
+
last_error: lastError,
|
|
306
353
|
};
|
|
307
354
|
if (opts.updates)
|
|
308
355
|
Object.assign(payload, opts.updates);
|
|
309
356
|
updateRunState(workspace, payload);
|
|
310
357
|
return new CliError(opts.title, opts.title, {
|
|
311
358
|
exitCode: opts.exitCode ?? EXIT_RUNTIME,
|
|
359
|
+
errorCode: opts.errorCode,
|
|
312
360
|
required: opts.required,
|
|
313
361
|
received: opts.received,
|
|
314
362
|
nextSteps: opts.nextSteps,
|
|
@@ -396,6 +444,218 @@ async function enrichEpisodeTitlesChunked(workspace, sourceText, plan, provider,
|
|
|
396
444
|
return degraded;
|
|
397
445
|
}
|
|
398
446
|
// ---------------------------------------------------------------------------
|
|
447
|
+
// Synopsis reduce helpers (map-reduce overview)
|
|
448
|
+
// ---------------------------------------------------------------------------
|
|
449
|
+
// Stitch an ordered list of synopses into one. ≥2 beats → LLM stitch; falls
|
|
450
|
+
// back to a plain join when the provider has no synopsis method or returns
|
|
451
|
+
// empty. Throws if the provider call throws — callers catch for soft-degrade.
|
|
452
|
+
async function reduceSynopses(provider, synopses, ctx) {
|
|
453
|
+
if (provider.extractEpisodeSynopsis) {
|
|
454
|
+
const out = await provider.extractEpisodeSynopsis(synopses, ctx);
|
|
455
|
+
const stitched = isDict(out) ? strOf(out["synopsis"]).trim() : "";
|
|
456
|
+
if (stitched)
|
|
457
|
+
return stitched;
|
|
458
|
+
}
|
|
459
|
+
return synopses.join(" ");
|
|
460
|
+
}
|
|
461
|
+
// Reduce-1: resolve each merged episode's `synopsis` IN PLACE. ≤1 fragment beat
|
|
462
|
+
// → passthrough. ≥2 beats: if the episode is unchanged this round and has a
|
|
463
|
+
// persisted `syn`, REUSE it verbatim (never overwrite a good prior synopsis —
|
|
464
|
+
// including under --skip-summary); else --skip-summary → join the fresh beats
|
|
465
|
+
// (no LLM); else defer the LLM stitch. The deferred stitches run concurrently;
|
|
466
|
+
// a failed stitch degrades to the FRESH joined beats (never a stale prior, which
|
|
467
|
+
// for a changed episode would describe old content). Returns degraded episode
|
|
468
|
+
// numbers. Reads prior `syn` before the caller overwrites ep_NNN.json.
|
|
469
|
+
async function resolveEpisodeSynopses(provider, results, opts) {
|
|
470
|
+
const stitchPending = [];
|
|
471
|
+
for (const merged of results) {
|
|
472
|
+
const episodeNum = Number(merged["episode"]);
|
|
473
|
+
const beats = cleanSynopsisList(merged["batch_synopses"]);
|
|
474
|
+
delete merged["batch_synopses"];
|
|
475
|
+
if (beats.length <= 1) {
|
|
476
|
+
if (beats[0])
|
|
477
|
+
merged["synopsis"] = beats[0];
|
|
478
|
+
}
|
|
479
|
+
else {
|
|
480
|
+
const unchanged = !opts.resummarize && !opts.changedEpisodes.has(episodeNum);
|
|
481
|
+
const priorSyn = unchanged ? readEpisodeSynopsis(opts.resultsDir, merged) : "";
|
|
482
|
+
if (unchanged && priorSyn)
|
|
483
|
+
merged["synopsis"] = priorSyn;
|
|
484
|
+
else if (opts.skipSummary)
|
|
485
|
+
merged["synopsis"] = beats.join(" ");
|
|
486
|
+
else
|
|
487
|
+
stitchPending.push({ merged, beats });
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
const degraded = [];
|
|
491
|
+
await pMapWithConcurrency(stitchPending, opts.concurrency, async (it) => {
|
|
492
|
+
const episodeNum = Number(it.merged["episode"]);
|
|
493
|
+
try {
|
|
494
|
+
const syn = await reduceSynopses(provider, it.beats, { episode: episodeNum, title: it.merged["title"] });
|
|
495
|
+
if (syn)
|
|
496
|
+
it.merged["synopsis"] = syn;
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
degraded.push(episodeNum);
|
|
500
|
+
it.merged["synopsis"] = it.beats.join(" ");
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
return degraded;
|
|
504
|
+
}
|
|
505
|
+
// Single source of truth for the overview field-shaping rules (trim strings,
|
|
506
|
+
// keep only well-formed main_characters). Both the persisted synopsis.json and
|
|
507
|
+
// the in-memory script go through this, so the two can never normalize differently.
|
|
508
|
+
function normalizeOverviewFields(fields) {
|
|
509
|
+
return {
|
|
510
|
+
synopsis: strOf(fields["synopsis"]).trim(),
|
|
511
|
+
theme: strOf(fields["theme"]).trim(),
|
|
512
|
+
logline: strOf(fields["logline"]).trim(),
|
|
513
|
+
style: strOf(fields["style"]).trim(),
|
|
514
|
+
main_characters: asList(fields["main_characters"]).filter(isDict),
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
// Assign ALREADY-normalized overview fields onto the script. Callers with raw
|
|
518
|
+
// provider/disk fields must go through applyScriptSynopsisFields instead.
|
|
519
|
+
function assignOverviewFields(script, o) {
|
|
520
|
+
script["synopsis"] = o["synopsis"];
|
|
521
|
+
script["theme"] = o["theme"];
|
|
522
|
+
script["logline"] = o["logline"];
|
|
523
|
+
script["main_characters"] = o["main_characters"];
|
|
524
|
+
// style is produced by the summary layer (a whole-show judgement); only
|
|
525
|
+
// override the merge-stage placeholder when the reduce actually produced one.
|
|
526
|
+
if (o["style"])
|
|
527
|
+
script["style"] = o["style"];
|
|
528
|
+
}
|
|
529
|
+
// Normalize untrusted fields (provider output / a reused synopsis.json) and
|
|
530
|
+
// assign them onto the script.
|
|
531
|
+
function applyScriptSynopsisFields(script, fields) {
|
|
532
|
+
assignOverviewFields(script, normalizeOverviewFields(fields));
|
|
533
|
+
}
|
|
534
|
+
// Whole-script reduce-2: collapse the episode synopses down to a set that fits
|
|
535
|
+
// one final reduce, then produce the top-level overview. Reuses a prior
|
|
536
|
+
// synopsis.json when the episode synopses are unchanged (signature match). Soft
|
|
537
|
+
// per-group: a failed intermediate stitch degrades to its joined beats (the rest
|
|
538
|
+
// keep stitching) rather than wiping the whole overview; only a failed FINAL
|
|
539
|
+
// synthesis falls the overview back to joined beats. Never blocks init; the
|
|
540
|
+
// degrade reason is surfaced for diagnostics.
|
|
541
|
+
async function buildScriptSynopsis(provider, script, opts) {
|
|
542
|
+
const synopsisPath = path.join(opts.dd, "synopsis.json");
|
|
543
|
+
const episodeSynopses = cleanSynopsisList(asList(script["episodes"]).map((ep) => ep["synopsis"]));
|
|
544
|
+
// Nothing to summarize → leave the placeholders empty (not a degrade).
|
|
545
|
+
if (episodeSynopses.length === 0)
|
|
546
|
+
return { degraded: false, reused: false };
|
|
547
|
+
// Signature gates reuse: include every input that changes the overview —
|
|
548
|
+
// the episode synopses AND the title (it is injected into the reduce prompt)
|
|
549
|
+
// AND the reduce shape (threshold/groupSize). Otherwise a rename or a
|
|
550
|
+
// different threshold would silently reuse a stale overview. sortDeep keeps
|
|
551
|
+
// the digest stable under any future key reordering (repo cache-key convention).
|
|
552
|
+
const signature = sha256Text(JSON.stringify(sortDeep({
|
|
553
|
+
title: strOf(script["title"]),
|
|
554
|
+
thresholdChars: opts.thresholdChars,
|
|
555
|
+
groupSize: opts.groupSize,
|
|
556
|
+
episodes: episodeSynopses,
|
|
557
|
+
})));
|
|
558
|
+
// Reuse: a prior synopsis.json with a matching signature and no --resummarize.
|
|
559
|
+
// This runs before the --skip-summary check ON PURPOSE: --skip-summary means
|
|
560
|
+
// "don't spend LLM on summaries", not "discard a summary we already have".
|
|
561
|
+
if (!opts.resummarize && exists(synopsisPath)) {
|
|
562
|
+
try {
|
|
563
|
+
const prior = readJson(synopsisPath);
|
|
564
|
+
if (isDict(prior) && strOf(prior["source_episode_signature"]) === signature) {
|
|
565
|
+
applyScriptSynopsisFields(script, prior);
|
|
566
|
+
return { degraded: false, reused: true };
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
catch {
|
|
570
|
+
// fall through to recompute
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
// --skip-summary: no reusable overview above → skip all reduce LLM and leave
|
|
574
|
+
// the top-level overview empty (the placeholders from the merge stage stand).
|
|
575
|
+
if (opts.skipSummary)
|
|
576
|
+
return { degraded: false, reused: false };
|
|
577
|
+
let degraded = false;
|
|
578
|
+
let reason = "";
|
|
579
|
+
const noteDegrade = (err) => {
|
|
580
|
+
degraded = true;
|
|
581
|
+
if (!reason)
|
|
582
|
+
reason = providerErrorSummary(err);
|
|
583
|
+
};
|
|
584
|
+
try {
|
|
585
|
+
// Reduce one level at a time until the surviving set fits a single final
|
|
586
|
+
// reduce. planSynopsisReduction guarantees progress (it returns fitsInOnePass
|
|
587
|
+
// whenever it can't form fewer groups than items), so `current` strictly
|
|
588
|
+
// shrinks each non-terminal pass and this terminates without a counter.
|
|
589
|
+
let current = episodeSynopses;
|
|
590
|
+
while (true) {
|
|
591
|
+
const plan = planSynopsisReduction(current, opts.thresholdChars, opts.groupSize);
|
|
592
|
+
if (plan.fitsInOnePass)
|
|
593
|
+
break;
|
|
594
|
+
// Groups within a level are independent → stitch them concurrently. A
|
|
595
|
+
// singleton group passes through verbatim (no LLM rewrite), mirroring the
|
|
596
|
+
// reduce-1 passthrough. A group whose stitch FAILS degrades to its joined
|
|
597
|
+
// beats (soft, per-group) — one bad group never wipes the whole overview.
|
|
598
|
+
const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group) => {
|
|
599
|
+
const groupSynopses = group.map((i) => current[i]).filter((s) => s.trim());
|
|
600
|
+
if (groupSynopses.length <= 1)
|
|
601
|
+
return { value: groupSynopses[0] ?? "" };
|
|
602
|
+
try {
|
|
603
|
+
return { value: await reduceSynopses(provider, groupSynopses, { title: script["title"] }) };
|
|
604
|
+
}
|
|
605
|
+
catch (err) {
|
|
606
|
+
return { value: groupSynopses.join(" "), error: err };
|
|
607
|
+
}
|
|
608
|
+
});
|
|
609
|
+
const next = [];
|
|
610
|
+
for (const o of outcomes) {
|
|
611
|
+
if (!o.ok) {
|
|
612
|
+
noteDegrade(o.error);
|
|
613
|
+
continue;
|
|
614
|
+
} // unexpected worker throw (worker already soft-handles reduce)
|
|
615
|
+
if (o.value.error)
|
|
616
|
+
noteDegrade(o.value.error);
|
|
617
|
+
if (o.value.value.trim())
|
|
618
|
+
next.push(o.value.value);
|
|
619
|
+
}
|
|
620
|
+
current = next;
|
|
621
|
+
if (current.length === 0)
|
|
622
|
+
break;
|
|
623
|
+
}
|
|
624
|
+
// Final synthesis. On failure, fall the overview synopsis back to the joined
|
|
625
|
+
// surviving beats rather than wiping everything (theme/logline/etc. stay empty).
|
|
626
|
+
let fields = {};
|
|
627
|
+
if (provider.extractScriptSynopsis) {
|
|
628
|
+
try {
|
|
629
|
+
const out = await provider.extractScriptSynopsis(current, { title: script["title"] });
|
|
630
|
+
if (isDict(out))
|
|
631
|
+
fields = out;
|
|
632
|
+
}
|
|
633
|
+
catch (err) {
|
|
634
|
+
noteDegrade(err);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
const overview = {
|
|
638
|
+
...normalizeOverviewFields(fields),
|
|
639
|
+
source_episode_signature: signature,
|
|
640
|
+
};
|
|
641
|
+
if (!strOf(overview["synopsis"]))
|
|
642
|
+
overview["synopsis"] = current.join(" "); // fall back to joined beats
|
|
643
|
+
if (!strOf(overview["synopsis"])) {
|
|
644
|
+
degraded = true;
|
|
645
|
+
if (!reason)
|
|
646
|
+
reason = "empty reduce output";
|
|
647
|
+
}
|
|
648
|
+
assignOverviewFields(script, overview); // overview is already normalized — assign directly (no re-normalize)
|
|
649
|
+
writeJson(synopsisPath, overview);
|
|
650
|
+
return { degraded, reused: false, reason: reason || undefined };
|
|
651
|
+
}
|
|
652
|
+
catch (err) {
|
|
653
|
+
// Last-resort soft-degrade (e.g. writeJson failure): never block init.
|
|
654
|
+
noteDegrade(err);
|
|
655
|
+
return { degraded: true, reused: false, reason: reason || undefined };
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
// ---------------------------------------------------------------------------
|
|
399
659
|
// command_init
|
|
400
660
|
// ---------------------------------------------------------------------------
|
|
401
661
|
export async function commandInit(opts) {
|
|
@@ -414,28 +674,14 @@ export async function commandInit(opts) {
|
|
|
414
674
|
const selBatches = new Set(parseStrList(opts["batches"]));
|
|
415
675
|
const retryErrors = Boolean(opts["retry_errors"]);
|
|
416
676
|
const runAll = Boolean(opts["all"]);
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
exitCode: EXIT_USAGE,
|
|
426
|
-
required: ["--concurrency: positive integer"],
|
|
427
|
-
received: [`--concurrency: ${opts["concurrency"]}`],
|
|
428
|
-
nextSteps: ["Use a positive integer and rerun init."],
|
|
429
|
-
});
|
|
430
|
-
}
|
|
431
|
-
if (concurrency < 1) {
|
|
432
|
-
throw new CliError("USAGE ERROR: Invalid concurrency", "Invalid concurrency.", {
|
|
433
|
-
exitCode: EXIT_USAGE,
|
|
434
|
-
required: ["--concurrency: positive integer"],
|
|
435
|
-
received: [`--concurrency: ${concurrency}`],
|
|
436
|
-
nextSteps: ["Use a positive integer and rerun init."],
|
|
437
|
-
});
|
|
438
|
-
}
|
|
677
|
+
// Synopsis stage controls. --skip-summary skips all summary LLM (episode
|
|
678
|
+
// synopses degrade to joined beats, top-level overview stays empty);
|
|
679
|
+
// --resummarize forces the summary LLM to re-run even when structure is
|
|
680
|
+
// reused (e.g. after a prompt change).
|
|
681
|
+
const skipSummary = Boolean(opts["skip_summary"]);
|
|
682
|
+
const resummarize = Boolean(opts["resummarize"]);
|
|
683
|
+
const summaryThresholdChars = parsePositiveIntOption(opts["summary_threshold_chars"], "--summary-threshold-chars", "USAGE ERROR: Invalid summary threshold", DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS);
|
|
684
|
+
const concurrency = parsePositiveIntOption(opts["concurrency"], "--concurrency", "USAGE ERROR: Invalid concurrency", DEFAULT_CONCURRENCY);
|
|
439
685
|
const batchMode = strOf(opts["batch_mode"] || DEFAULT_BATCH_MODE).trim();
|
|
440
686
|
if (batchMode !== "episode") {
|
|
441
687
|
throw new CliError("USAGE ERROR: Invalid batch mode", "Invalid batch mode.", {
|
|
@@ -451,29 +697,7 @@ export async function commandInit(opts) {
|
|
|
451
697
|
["batch_max_chars", DEFAULT_BATCH_MAX_CHARS],
|
|
452
698
|
["batch_min_lines", DEFAULT_BATCH_MIN_LINES],
|
|
453
699
|
]) {
|
|
454
|
-
|
|
455
|
-
try {
|
|
456
|
-
value = parseInt(strOf(opts[optName] || defaultValue), 10);
|
|
457
|
-
if (Number.isNaN(value))
|
|
458
|
-
throw new Error("nan");
|
|
459
|
-
}
|
|
460
|
-
catch {
|
|
461
|
-
throw new CliError("USAGE ERROR: Invalid batch option", "Invalid batch option.", {
|
|
462
|
-
exitCode: EXIT_USAGE,
|
|
463
|
-
required: [`--${optName.replace(/_/g, "-")}: positive integer`],
|
|
464
|
-
received: [`--${optName.replace(/_/g, "-")}: ${opts[optName]}`],
|
|
465
|
-
nextSteps: ["Use positive integers for batch options and rerun init."],
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
if (value < 1) {
|
|
469
|
-
throw new CliError("USAGE ERROR: Invalid batch option", "Invalid batch option.", {
|
|
470
|
-
exitCode: EXIT_USAGE,
|
|
471
|
-
required: [`--${optName.replace(/_/g, "-")}: positive integer`],
|
|
472
|
-
received: [`--${optName.replace(/_/g, "-")}: ${value}`],
|
|
473
|
-
nextSteps: ["Use positive integers for batch options and rerun init."],
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
batchValues[optName] = value;
|
|
700
|
+
batchValues[optName] = parsePositiveIntOption(opts[optName], `--${optName.replace(/_/g, "-")}`, "USAGE ERROR: Invalid batch option", defaultValue);
|
|
477
701
|
}
|
|
478
702
|
const batchTargetLines = batchValues["batch_target_lines"];
|
|
479
703
|
const batchMaxChars = batchValues["batch_max_chars"];
|
|
@@ -642,6 +866,11 @@ export async function commandInit(opts) {
|
|
|
642
866
|
default: pending.push(batch); // never attempted (or corrupt → self-heal)
|
|
643
867
|
}
|
|
644
868
|
}
|
|
869
|
+
// Episodes whose batches were (re)run this round — their episode synopsis is
|
|
870
|
+
// recomputed; untouched episodes reuse their persisted `syn`. Summary failures
|
|
871
|
+
// accumulate here and are reported without ever blocking init.
|
|
872
|
+
const changedEpisodes = new Set(pending.map((b) => Number(b["episode"])));
|
|
873
|
+
const summaryDegradedEpisodes = [];
|
|
645
874
|
updateRunState(workspace, {
|
|
646
875
|
status: "init_running",
|
|
647
876
|
init_stage: "batch_extract",
|
|
@@ -680,6 +909,7 @@ export async function commandInit(opts) {
|
|
|
680
909
|
const results = [];
|
|
681
910
|
const incompleteEpisodes = [];
|
|
682
911
|
try {
|
|
912
|
+
// Assemble every complete episode (incomplete ones are reported and skipped).
|
|
683
913
|
for (const episode of asList(plan["episodes"])) {
|
|
684
914
|
const episodeNum = Number(episode["episode"]);
|
|
685
915
|
const epBatches = batchesByEpisode.get(episodeNum) ?? [];
|
|
@@ -695,9 +925,23 @@ export async function commandInit(opts) {
|
|
|
695
925
|
}
|
|
696
926
|
const merged = mergeBatchResultsForEpisode(episode, batchResults);
|
|
697
927
|
validateEpisodeExtractionQuality(sourceText, episode, merged);
|
|
698
|
-
results.push(merged);
|
|
699
|
-
|
|
700
|
-
|
|
928
|
+
results.push(merged); // complete episodes only; carries `episode` for the write below
|
|
929
|
+
}
|
|
930
|
+
// Episode-level synopsis (reduce-1): resolve each result's synopsis in place
|
|
931
|
+
// (passthrough / reuse / skip / concurrent LLM stitch). Reads prior `syn`
|
|
932
|
+
// BEFORE the persist loop below overwrites ep_NNN.json.
|
|
933
|
+
summaryDegradedEpisodes.push(...await resolveEpisodeSynopses(provider, results, {
|
|
934
|
+
resultsDir: episodeResultsDir,
|
|
935
|
+
resummarize,
|
|
936
|
+
skipSummary,
|
|
937
|
+
changedEpisodes,
|
|
938
|
+
concurrency,
|
|
939
|
+
}));
|
|
940
|
+
// Persist every assembled episode (synopsis now resolved for all of them).
|
|
941
|
+
// merged carries `episode`, so it doubles as the path key.
|
|
942
|
+
for (const merged of results) {
|
|
943
|
+
writeJson(episodeResultPath(episodeResultsDir, merged), compactEpisodeResult(merged));
|
|
944
|
+
const errPath = episodeErrorPath(episodeResultsDir, merged);
|
|
701
945
|
if (exists(errPath))
|
|
702
946
|
deletePath(errPath);
|
|
703
947
|
}
|
|
@@ -780,29 +1024,23 @@ export async function commandInit(opts) {
|
|
|
780
1024
|
}
|
|
781
1025
|
try {
|
|
782
1026
|
updateRunState(workspace, { status: "init_running", init_stage: "asset_curation" });
|
|
783
|
-
|
|
784
|
-
|
|
1027
|
+
// Location merge is deterministic-only: actor/prop pruning runs from scene
|
|
1028
|
+
// counts and every location is kept. The previous provider location-merge
|
|
1029
|
+
// call was removed (single large streaming request kept dropping the gateway
|
|
1030
|
+
// connection); duplicate locations can be merged later if needed.
|
|
1031
|
+
const curation = curateScriptAssets(script);
|
|
785
1032
|
writeJson(path.join(dd, "asset_curation.json"), curation);
|
|
786
1033
|
}
|
|
787
1034
|
catch (exc) {
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
title: exc.title,
|
|
791
|
-
stage: "asset_curation",
|
|
792
|
-
exitCode: exc.exitCode,
|
|
793
|
-
required: exc.required.length > 0 ? exc.required : ["asset curation JSON matching final script contract"],
|
|
794
|
-
received: exc.received.length > 0 ? exc.received : [String(exc.message).slice(0, 160)],
|
|
795
|
-
nextSteps: exc.nextSteps.length > 0 ? exc.nextSteps : ["Rerun init; extraction checkpoints will be reused and asset curation will retry."],
|
|
796
|
-
updates: { episode_completed: results.length },
|
|
797
|
-
});
|
|
798
|
-
}
|
|
1035
|
+
// curateScriptAssets is deterministic (no provider call) — a failure here is
|
|
1036
|
+
// a malformed assembled script, not a gateway/provider problem.
|
|
799
1037
|
const e = exc;
|
|
800
1038
|
throw initFailedReport(workspace, {
|
|
801
1039
|
title: "INIT FAILED: Asset curation failed",
|
|
802
1040
|
stage: "asset_curation",
|
|
803
|
-
required: ["
|
|
1041
|
+
required: ["a well-formed assembled script for deterministic asset curation"],
|
|
804
1042
|
received: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`],
|
|
805
|
-
nextSteps: ["Rerun init; extraction checkpoints will be reused
|
|
1043
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused."],
|
|
806
1044
|
updates: { episode_completed: results.length },
|
|
807
1045
|
});
|
|
808
1046
|
}
|
|
@@ -820,6 +1058,7 @@ export async function commandInit(opts) {
|
|
|
820
1058
|
title: exc.title,
|
|
821
1059
|
stage: "metadata_extract",
|
|
822
1060
|
exitCode: exc.exitCode,
|
|
1061
|
+
errorCode: exc.errorCode,
|
|
823
1062
|
required: exc.required.length > 0 ? exc.required : ["metadata JSON matching final script contract"],
|
|
824
1063
|
received: exc.received.length > 0 ? exc.received : [String(exc.message).slice(0, 160)],
|
|
825
1064
|
nextSteps: exc.nextSteps.length > 0 ? exc.nextSteps : ["Rerun init; extraction checkpoints will be reused and metadata will retry."],
|
|
@@ -836,6 +1075,30 @@ export async function commandInit(opts) {
|
|
|
836
1075
|
updates: { episode_completed: results.length },
|
|
837
1076
|
});
|
|
838
1077
|
}
|
|
1078
|
+
// Whole-script overview (reduce-2). Soft: a non-essential concept enhancement,
|
|
1079
|
+
// the opposite of batch extraction's hard-fail policy — failures degrade the
|
|
1080
|
+
// overview (recorded as summary_degraded + a sanitized reason) and NEVER block init.
|
|
1081
|
+
let summaryFullDegraded = false;
|
|
1082
|
+
let summaryDegradeReason = "";
|
|
1083
|
+
try {
|
|
1084
|
+
updateRunState(workspace, { status: "init_running", init_stage: "summary" });
|
|
1085
|
+
const summary = await buildScriptSynopsis(provider, script, {
|
|
1086
|
+
dd,
|
|
1087
|
+
thresholdChars: summaryThresholdChars,
|
|
1088
|
+
groupSize: DEFAULT_SUMMARY_GROUP_SIZE,
|
|
1089
|
+
resummarize,
|
|
1090
|
+
skipSummary,
|
|
1091
|
+
concurrency,
|
|
1092
|
+
});
|
|
1093
|
+
summaryFullDegraded = summary.degraded;
|
|
1094
|
+
summaryDegradeReason = summary.reason ?? "";
|
|
1095
|
+
}
|
|
1096
|
+
catch (exc) {
|
|
1097
|
+
summaryFullDegraded = true;
|
|
1098
|
+
summaryDegradeReason = providerErrorSummary(exc);
|
|
1099
|
+
}
|
|
1100
|
+
summaryDegradedEpisodes.sort((a, b) => a - b); // concurrent stitch pushes out of order
|
|
1101
|
+
const summaryDegraded = summaryFullDegraded || summaryDegradedEpisodes.length > 0;
|
|
839
1102
|
const scriptPath = path.join(dd, "script.initial.json");
|
|
840
1103
|
writeJson(scriptPath, script);
|
|
841
1104
|
updateRunState(workspace, { status: "init_running", init_stage: "validate" });
|
|
@@ -877,12 +1140,18 @@ export async function commandInit(opts) {
|
|
|
877
1140
|
batch_completed: doneBatches,
|
|
878
1141
|
batch_reused: reusedBatchCount,
|
|
879
1142
|
batch_failed: erroredBatches,
|
|
1143
|
+
summary_threshold_chars: summaryThresholdChars,
|
|
1144
|
+
summary_skipped: skipSummary,
|
|
1145
|
+
summary_degraded: summaryDegraded,
|
|
1146
|
+
summary_degraded_episodes: summaryDegradedEpisodes,
|
|
1147
|
+
summary_degraded_reason: summaryDegradeReason || null,
|
|
880
1148
|
last_error: null,
|
|
881
1149
|
});
|
|
882
1150
|
const title = passed
|
|
883
1151
|
? "INIT COMPLETE: Initial script ready"
|
|
884
1152
|
: "INIT NEEDS AGENT: Initial script written with repair issues";
|
|
885
1153
|
const stats = validation["stats"] ?? {};
|
|
1154
|
+
const summaryLine = formatSummaryStatusLine(skipSummary, summaryDegraded, summaryDegradedEpisodes);
|
|
886
1155
|
const report = {
|
|
887
1156
|
title,
|
|
888
1157
|
result: [
|
|
@@ -894,6 +1163,7 @@ export async function commandInit(opts) {
|
|
|
894
1163
|
`episodes: ${results.length}/${totalEpisodes} complete`,
|
|
895
1164
|
`batches: ${doneBatches}/${totalBatches} done (reused ${reusedBatchCount})`,
|
|
896
1165
|
...(titleDegraded.length > 0 ? [`titles degraded (deterministic): ${titleDegraded.join(", ")}`] : []),
|
|
1166
|
+
...(summaryLine ? [summaryLine] : []),
|
|
897
1167
|
],
|
|
898
1168
|
artifacts: [
|
|
899
1169
|
path.join(workspace, "source.txt"),
|
|
@@ -904,6 +1174,7 @@ export async function commandInit(opts) {
|
|
|
904
1174
|
episodeResultsDir,
|
|
905
1175
|
path.join(dd, "asset_curation.json"),
|
|
906
1176
|
path.join(dd, "asset_metadata.json"),
|
|
1177
|
+
path.join(dd, "synopsis.json"),
|
|
907
1178
|
scriptPath,
|
|
908
1179
|
path.join(dd, "validation.json"),
|
|
909
1180
|
path.join(dd, "run_state.json"),
|
|
@@ -1003,6 +1274,13 @@ export function commandStatus(opts) {
|
|
|
1003
1274
|
if (titleDeg && Number(titleDeg["count"] ?? 0) > 0) {
|
|
1004
1275
|
result.push(`titles degraded (deterministic): ${asList(titleDeg["episodes"]).join(", ")}`);
|
|
1005
1276
|
}
|
|
1277
|
+
// Surface synopsis state recorded by init: skipped or degraded summaries mean
|
|
1278
|
+
// the overview / some episode synopses are missing or fell back to joined
|
|
1279
|
+
// beats — an agent gating on status should know it isn't a complete summary.
|
|
1280
|
+
// Same formatter as init's report so the two never word it differently.
|
|
1281
|
+
const summaryLine = formatSummaryStatusLine(Boolean(state["summary_skipped"]), Boolean(state["summary_degraded"]), asList(state["summary_degraded_episodes"]).map((n) => Number(n)));
|
|
1282
|
+
if (summaryLine)
|
|
1283
|
+
result.push(summaryLine);
|
|
1006
1284
|
result.push(...lines);
|
|
1007
1285
|
const allDone = completeEpisodes === episodes.length;
|
|
1008
1286
|
const report = {
|