@lingjingai/scriptctl 0.15.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.js +7 -0
- package/dist/cli.js.map +1 -1
- package/dist/common.d.ts +10 -3
- package/dist/common.js +82 -60
- package/dist/common.js.map +1 -1
- package/dist/domain/direct-core.d.ts +27 -4
- package/dist/domain/direct-core.js +281 -55
- package/dist/domain/direct-core.js.map +1 -1
- package/dist/help-text.js +15 -1
- package/dist/help-text.js.map +1 -1
- package/dist/infra/providers.d.ts +12 -2
- package/dist/infra/providers.js +147 -98
- package/dist/infra/providers.js.map +1 -1
- package/dist/usecases/direct.js +342 -60
- package/dist/usecases/direct.js.map +1 -1
- package/dist/usecases/parse.js +104 -14
- package/dist/usecases/parse.js.map +1 -1
- package/dist/usecases/script.d.ts +2 -2
- package/dist/usecases/script.js +15 -11
- package/dist/usecases/script.js.map +1 -1
- package/package.json +1 -1
package/dist/usecases/direct.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, writeJson, } from "../common.js";
|
|
4
|
-
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResults, normalizeEpisodeResult, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
3
|
+
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
|
|
4
|
+
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResults, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
5
5
|
import { validateScript } from "../domain/script-core.js";
|
|
6
6
|
import { makeProvider } from "../infra/providers.js";
|
|
7
7
|
import { makeSourceManifest, prepareSource, } from "../infra/converters.js";
|
|
@@ -29,6 +29,32 @@ function parseIntList(v) {
|
|
|
29
29
|
function parseStrList(v) {
|
|
30
30
|
return strOf(v).split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
31
31
|
}
|
|
32
|
+
// Parse a positive-integer CLI option (default when unset) with a uniform USAGE
|
|
33
|
+
// error. Shared by --concurrency / --summary-threshold-chars / the batch opts.
|
|
34
|
+
function parsePositiveIntOption(raw, flag, title, fallback) {
|
|
35
|
+
const value = parseInt(strOf(raw || fallback), 10);
|
|
36
|
+
if (Number.isNaN(value) || value < 1) {
|
|
37
|
+
throw new CliError(title, `${title}.`, {
|
|
38
|
+
exitCode: EXIT_USAGE,
|
|
39
|
+
required: [`${flag}: positive integer`],
|
|
40
|
+
received: [`${flag}: ${raw}`],
|
|
41
|
+
nextSteps: ["Use a positive integer and rerun init."],
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
// Status/report line for the synopsis stage: skipped, degraded (with episode
|
|
47
|
+
// list when known), or null when the overview is complete. One formatter so
|
|
48
|
+
// `direct init`'s report and `direct status` never word it differently.
|
|
49
|
+
function formatSummaryStatusLine(skipped, degraded, degradedEpisodes) {
|
|
50
|
+
if (skipped)
|
|
51
|
+
return "summary: skipped (--skip-summary)";
|
|
52
|
+
if (!degraded)
|
|
53
|
+
return null;
|
|
54
|
+
return degradedEpisodes.length > 0
|
|
55
|
+
? `summary: degraded (episodes: ${degradedEpisodes.join(", ")})`
|
|
56
|
+
: "summary: degraded (overview may be partial)";
|
|
57
|
+
}
|
|
32
58
|
// ---------------------------------------------------------------------------
|
|
33
59
|
// run_state.json
|
|
34
60
|
// ---------------------------------------------------------------------------
|
|
@@ -78,6 +104,19 @@ function episodeErrorPath(dir, ep) {
|
|
|
78
104
|
function episodeResultKey(ep) {
|
|
79
105
|
return `ep_${pad3(Number(ep["episode"]))}`;
|
|
80
106
|
}
|
|
107
|
+
// Read the synopsis carried on a persisted episode result (compact key `syn`),
|
|
108
|
+
// for episode-synopsis reuse. Returns "" when absent/unreadable/legacy (no syn).
|
|
109
|
+
function readEpisodeSynopsis(dir, episode) {
|
|
110
|
+
const p = episodeResultPath(dir, episode);
|
|
111
|
+
if (!exists(p))
|
|
112
|
+
return "";
|
|
113
|
+
try {
|
|
114
|
+
return readCarriedSynopsis(readJson(p));
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return "";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
81
120
|
function batchResultKey(batch) {
|
|
82
121
|
const bid = strOf(batch["batch_id"]).trim();
|
|
83
122
|
if (bid)
|
|
@@ -205,8 +244,15 @@ export function mergeBatchResultsForEpisode(episode, batchResults) {
|
|
|
205
244
|
const props = [];
|
|
206
245
|
const speakers = [];
|
|
207
246
|
const stateDefinitions = [];
|
|
247
|
+
// Ordered, non-empty per-fragment synopses for the episode-level reduce. The
|
|
248
|
+
// reduce/passthrough decision (and LLM call) happens in the orchestrator, NOT
|
|
249
|
+
// here — this pure function only collects the beats in _batch_part order.
|
|
250
|
+
const batchSynopses = [];
|
|
208
251
|
const sorted = [...batchResults].sort((a, b) => Number(a["_batch_part"] ?? 0) - Number(b["_batch_part"] ?? 0));
|
|
209
252
|
for (const item of sorted) {
|
|
253
|
+
const synopsis = strOf(item["synopsis"]).trim();
|
|
254
|
+
if (synopsis)
|
|
255
|
+
batchSynopses.push(synopsis);
|
|
210
256
|
const itemScenes = asList(item["scenes"]);
|
|
211
257
|
if (itemScenes.length === 0)
|
|
212
258
|
continue;
|
|
@@ -249,14 +295,12 @@ export function mergeBatchResultsForEpisode(episode, batchResults) {
|
|
|
249
295
|
speakers,
|
|
250
296
|
state_definitions: stateDefinitions,
|
|
251
297
|
};
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
259
|
-
return {};
|
|
298
|
+
const normalized = normalizeEpisodeResult(result, episode);
|
|
299
|
+
// Carry the ordered fragment synopses on the merged result so the orchestrator
|
|
300
|
+
// can run the episode-level reduce (or passthrough) on them. Not part of the
|
|
301
|
+
// episode schema — stripped before the episode entry is built downstream.
|
|
302
|
+
normalized["batch_synopses"] = batchSynopses;
|
|
303
|
+
return normalized;
|
|
260
304
|
}
|
|
261
305
|
// Record a batch failure: write `<key>.error.json` (which marks the batch as
|
|
262
306
|
// "tried and failed" → sticky on plain resume) and drop any stale result file.
|
|
@@ -396,6 +440,218 @@ async function enrichEpisodeTitlesChunked(workspace, sourceText, plan, provider,
|
|
|
396
440
|
return degraded;
|
|
397
441
|
}
|
|
398
442
|
// ---------------------------------------------------------------------------
|
|
443
|
+
// Synopsis reduce helpers (map-reduce overview)
|
|
444
|
+
// ---------------------------------------------------------------------------
|
|
445
|
+
// Stitch an ordered list of synopses into one. ≥2 beats → LLM stitch; falls
|
|
446
|
+
// back to a plain join when the provider has no synopsis method or returns
|
|
447
|
+
// empty. Throws if the provider call throws — callers catch for soft-degrade.
|
|
448
|
+
async function reduceSynopses(provider, synopses, ctx) {
|
|
449
|
+
if (provider.extractEpisodeSynopsis) {
|
|
450
|
+
const out = await provider.extractEpisodeSynopsis(synopses, ctx);
|
|
451
|
+
const stitched = isDict(out) ? strOf(out["synopsis"]).trim() : "";
|
|
452
|
+
if (stitched)
|
|
453
|
+
return stitched;
|
|
454
|
+
}
|
|
455
|
+
return synopses.join(" ");
|
|
456
|
+
}
|
|
457
|
+
// Reduce-1: resolve each merged episode's `synopsis` IN PLACE. ≤1 fragment beat
|
|
458
|
+
// → passthrough. ≥2 beats: if the episode is unchanged this round and has a
|
|
459
|
+
// persisted `syn`, REUSE it verbatim (never overwrite a good prior synopsis —
|
|
460
|
+
// including under --skip-summary); else --skip-summary → join the fresh beats
|
|
461
|
+
// (no LLM); else defer the LLM stitch. The deferred stitches run concurrently;
|
|
462
|
+
// a failed stitch degrades to the FRESH joined beats (never a stale prior, which
|
|
463
|
+
// for a changed episode would describe old content). Returns degraded episode
|
|
464
|
+
// numbers. Reads prior `syn` before the caller overwrites ep_NNN.json.
|
|
465
|
+
async function resolveEpisodeSynopses(provider, results, opts) {
|
|
466
|
+
const stitchPending = [];
|
|
467
|
+
for (const merged of results) {
|
|
468
|
+
const episodeNum = Number(merged["episode"]);
|
|
469
|
+
const beats = cleanSynopsisList(merged["batch_synopses"]);
|
|
470
|
+
delete merged["batch_synopses"];
|
|
471
|
+
if (beats.length <= 1) {
|
|
472
|
+
if (beats[0])
|
|
473
|
+
merged["synopsis"] = beats[0];
|
|
474
|
+
}
|
|
475
|
+
else {
|
|
476
|
+
const unchanged = !opts.resummarize && !opts.changedEpisodes.has(episodeNum);
|
|
477
|
+
const priorSyn = unchanged ? readEpisodeSynopsis(opts.resultsDir, merged) : "";
|
|
478
|
+
if (unchanged && priorSyn)
|
|
479
|
+
merged["synopsis"] = priorSyn;
|
|
480
|
+
else if (opts.skipSummary)
|
|
481
|
+
merged["synopsis"] = beats.join(" ");
|
|
482
|
+
else
|
|
483
|
+
stitchPending.push({ merged, beats });
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
const degraded = [];
|
|
487
|
+
await pMapWithConcurrency(stitchPending, opts.concurrency, async (it) => {
|
|
488
|
+
const episodeNum = Number(it.merged["episode"]);
|
|
489
|
+
try {
|
|
490
|
+
const syn = await reduceSynopses(provider, it.beats, { episode: episodeNum, title: it.merged["title"] });
|
|
491
|
+
if (syn)
|
|
492
|
+
it.merged["synopsis"] = syn;
|
|
493
|
+
}
|
|
494
|
+
catch {
|
|
495
|
+
degraded.push(episodeNum);
|
|
496
|
+
it.merged["synopsis"] = it.beats.join(" ");
|
|
497
|
+
}
|
|
498
|
+
});
|
|
499
|
+
return degraded;
|
|
500
|
+
}
|
|
501
|
+
// Single source of truth for the overview field-shaping rules (trim strings,
|
|
502
|
+
// keep only well-formed main_characters). Both the persisted synopsis.json and
|
|
503
|
+
// the in-memory script go through this, so the two can never normalize differently.
|
|
504
|
+
function normalizeOverviewFields(fields) {
|
|
505
|
+
return {
|
|
506
|
+
synopsis: strOf(fields["synopsis"]).trim(),
|
|
507
|
+
theme: strOf(fields["theme"]).trim(),
|
|
508
|
+
logline: strOf(fields["logline"]).trim(),
|
|
509
|
+
style: strOf(fields["style"]).trim(),
|
|
510
|
+
main_characters: asList(fields["main_characters"]).filter(isDict),
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
// Assign ALREADY-normalized overview fields onto the script. Callers with raw
|
|
514
|
+
// provider/disk fields must go through applyScriptSynopsisFields instead.
|
|
515
|
+
function assignOverviewFields(script, o) {
|
|
516
|
+
script["synopsis"] = o["synopsis"];
|
|
517
|
+
script["theme"] = o["theme"];
|
|
518
|
+
script["logline"] = o["logline"];
|
|
519
|
+
script["main_characters"] = o["main_characters"];
|
|
520
|
+
// style is produced by the summary layer (a whole-show judgement); only
|
|
521
|
+
// override the merge-stage placeholder when the reduce actually produced one.
|
|
522
|
+
if (o["style"])
|
|
523
|
+
script["style"] = o["style"];
|
|
524
|
+
}
|
|
525
|
+
// Normalize untrusted fields (provider output / a reused synopsis.json) and
|
|
526
|
+
// assign them onto the script.
|
|
527
|
+
function applyScriptSynopsisFields(script, fields) {
|
|
528
|
+
assignOverviewFields(script, normalizeOverviewFields(fields));
|
|
529
|
+
}
|
|
530
|
+
// Whole-script reduce-2: collapse the episode synopses down to a set that fits
|
|
531
|
+
// one final reduce, then produce the top-level overview. Reuses a prior
|
|
532
|
+
// synopsis.json when the episode synopses are unchanged (signature match). Soft
|
|
533
|
+
// per-group: a failed intermediate stitch degrades to its joined beats (the rest
|
|
534
|
+
// keep stitching) rather than wiping the whole overview; only a failed FINAL
|
|
535
|
+
// synthesis falls the overview back to joined beats. Never blocks init; the
|
|
536
|
+
// degrade reason is surfaced for diagnostics.
|
|
537
|
+
async function buildScriptSynopsis(provider, script, opts) {
|
|
538
|
+
const synopsisPath = path.join(opts.dd, "synopsis.json");
|
|
539
|
+
const episodeSynopses = cleanSynopsisList(asList(script["episodes"]).map((ep) => ep["synopsis"]));
|
|
540
|
+
// Nothing to summarize → leave the placeholders empty (not a degrade).
|
|
541
|
+
if (episodeSynopses.length === 0)
|
|
542
|
+
return { degraded: false, reused: false };
|
|
543
|
+
// Signature gates reuse: include every input that changes the overview —
|
|
544
|
+
// the episode synopses AND the title (it is injected into the reduce prompt)
|
|
545
|
+
// AND the reduce shape (threshold/groupSize). Otherwise a rename or a
|
|
546
|
+
// different threshold would silently reuse a stale overview. sortDeep keeps
|
|
547
|
+
// the digest stable under any future key reordering (repo cache-key convention).
|
|
548
|
+
const signature = sha256Text(JSON.stringify(sortDeep({
|
|
549
|
+
title: strOf(script["title"]),
|
|
550
|
+
thresholdChars: opts.thresholdChars,
|
|
551
|
+
groupSize: opts.groupSize,
|
|
552
|
+
episodes: episodeSynopses,
|
|
553
|
+
})));
|
|
554
|
+
// Reuse: a prior synopsis.json with a matching signature and no --resummarize.
|
|
555
|
+
// This runs before the --skip-summary check ON PURPOSE: --skip-summary means
|
|
556
|
+
// "don't spend LLM on summaries", not "discard a summary we already have".
|
|
557
|
+
if (!opts.resummarize && exists(synopsisPath)) {
|
|
558
|
+
try {
|
|
559
|
+
const prior = readJson(synopsisPath);
|
|
560
|
+
if (isDict(prior) && strOf(prior["source_episode_signature"]) === signature) {
|
|
561
|
+
applyScriptSynopsisFields(script, prior);
|
|
562
|
+
return { degraded: false, reused: true };
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
// fall through to recompute
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
// --skip-summary: no reusable overview above → skip all reduce LLM and leave
|
|
570
|
+
// the top-level overview empty (the placeholders from the merge stage stand).
|
|
571
|
+
if (opts.skipSummary)
|
|
572
|
+
return { degraded: false, reused: false };
|
|
573
|
+
let degraded = false;
|
|
574
|
+
let reason = "";
|
|
575
|
+
const noteDegrade = (err) => {
|
|
576
|
+
degraded = true;
|
|
577
|
+
if (!reason)
|
|
578
|
+
reason = providerErrorSummary(err);
|
|
579
|
+
};
|
|
580
|
+
try {
|
|
581
|
+
// Reduce one level at a time until the surviving set fits a single final
|
|
582
|
+
// reduce. planSynopsisReduction guarantees progress (it returns fitsInOnePass
|
|
583
|
+
// whenever it can't form fewer groups than items), so `current` strictly
|
|
584
|
+
// shrinks each non-terminal pass and this terminates without a counter.
|
|
585
|
+
let current = episodeSynopses;
|
|
586
|
+
while (true) {
|
|
587
|
+
const plan = planSynopsisReduction(current, opts.thresholdChars, opts.groupSize);
|
|
588
|
+
if (plan.fitsInOnePass)
|
|
589
|
+
break;
|
|
590
|
+
// Groups within a level are independent → stitch them concurrently. A
|
|
591
|
+
// singleton group passes through verbatim (no LLM rewrite), mirroring the
|
|
592
|
+
// reduce-1 passthrough. A group whose stitch FAILS degrades to its joined
|
|
593
|
+
// beats (soft, per-group) — one bad group never wipes the whole overview.
|
|
594
|
+
const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group) => {
|
|
595
|
+
const groupSynopses = group.map((i) => current[i]).filter((s) => s.trim());
|
|
596
|
+
if (groupSynopses.length <= 1)
|
|
597
|
+
return { value: groupSynopses[0] ?? "" };
|
|
598
|
+
try {
|
|
599
|
+
return { value: await reduceSynopses(provider, groupSynopses, { title: script["title"] }) };
|
|
600
|
+
}
|
|
601
|
+
catch (err) {
|
|
602
|
+
return { value: groupSynopses.join(" "), error: err };
|
|
603
|
+
}
|
|
604
|
+
});
|
|
605
|
+
const next = [];
|
|
606
|
+
for (const o of outcomes) {
|
|
607
|
+
if (!o.ok) {
|
|
608
|
+
noteDegrade(o.error);
|
|
609
|
+
continue;
|
|
610
|
+
} // unexpected worker throw (worker already soft-handles reduce)
|
|
611
|
+
if (o.value.error)
|
|
612
|
+
noteDegrade(o.value.error);
|
|
613
|
+
if (o.value.value.trim())
|
|
614
|
+
next.push(o.value.value);
|
|
615
|
+
}
|
|
616
|
+
current = next;
|
|
617
|
+
if (current.length === 0)
|
|
618
|
+
break;
|
|
619
|
+
}
|
|
620
|
+
// Final synthesis. On failure, fall the overview synopsis back to the joined
|
|
621
|
+
// surviving beats rather than wiping everything (theme/logline/etc. stay empty).
|
|
622
|
+
let fields = {};
|
|
623
|
+
if (provider.extractScriptSynopsis) {
|
|
624
|
+
try {
|
|
625
|
+
const out = await provider.extractScriptSynopsis(current, { title: script["title"] });
|
|
626
|
+
if (isDict(out))
|
|
627
|
+
fields = out;
|
|
628
|
+
}
|
|
629
|
+
catch (err) {
|
|
630
|
+
noteDegrade(err);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
const overview = {
|
|
634
|
+
...normalizeOverviewFields(fields),
|
|
635
|
+
source_episode_signature: signature,
|
|
636
|
+
};
|
|
637
|
+
if (!strOf(overview["synopsis"]))
|
|
638
|
+
overview["synopsis"] = current.join(" "); // fall back to joined beats
|
|
639
|
+
if (!strOf(overview["synopsis"])) {
|
|
640
|
+
degraded = true;
|
|
641
|
+
if (!reason)
|
|
642
|
+
reason = "empty reduce output";
|
|
643
|
+
}
|
|
644
|
+
assignOverviewFields(script, overview); // overview is already normalized — assign directly (no re-normalize)
|
|
645
|
+
writeJson(synopsisPath, overview);
|
|
646
|
+
return { degraded, reused: false, reason: reason || undefined };
|
|
647
|
+
}
|
|
648
|
+
catch (err) {
|
|
649
|
+
// Last-resort soft-degrade (e.g. writeJson failure): never block init.
|
|
650
|
+
noteDegrade(err);
|
|
651
|
+
return { degraded: true, reused: false, reason: reason || undefined };
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
// ---------------------------------------------------------------------------
|
|
399
655
|
// command_init
|
|
400
656
|
// ---------------------------------------------------------------------------
|
|
401
657
|
export async function commandInit(opts) {
|
|
@@ -414,28 +670,14 @@ export async function commandInit(opts) {
|
|
|
414
670
|
const selBatches = new Set(parseStrList(opts["batches"]));
|
|
415
671
|
const retryErrors = Boolean(opts["retry_errors"]);
|
|
416
672
|
const runAll = Boolean(opts["all"]);
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
exitCode: EXIT_USAGE,
|
|
426
|
-
required: ["--concurrency: positive integer"],
|
|
427
|
-
received: [`--concurrency: ${opts["concurrency"]}`],
|
|
428
|
-
nextSteps: ["Use a positive integer and rerun init."],
|
|
429
|
-
});
|
|
430
|
-
}
|
|
431
|
-
if (concurrency < 1) {
|
|
432
|
-
throw new CliError("USAGE ERROR: Invalid concurrency", "Invalid concurrency.", {
|
|
433
|
-
exitCode: EXIT_USAGE,
|
|
434
|
-
required: ["--concurrency: positive integer"],
|
|
435
|
-
received: [`--concurrency: ${concurrency}`],
|
|
436
|
-
nextSteps: ["Use a positive integer and rerun init."],
|
|
437
|
-
});
|
|
438
|
-
}
|
|
673
|
+
// Synopsis stage controls. --skip-summary skips all summary LLM (episode
|
|
674
|
+
// synopses degrade to joined beats, top-level overview stays empty);
|
|
675
|
+
// --resummarize forces the summary LLM to re-run even when structure is
|
|
676
|
+
// reused (e.g. after a prompt change).
|
|
677
|
+
const skipSummary = Boolean(opts["skip_summary"]);
|
|
678
|
+
const resummarize = Boolean(opts["resummarize"]);
|
|
679
|
+
const summaryThresholdChars = parsePositiveIntOption(opts["summary_threshold_chars"], "--summary-threshold-chars", "USAGE ERROR: Invalid summary threshold", DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS);
|
|
680
|
+
const concurrency = parsePositiveIntOption(opts["concurrency"], "--concurrency", "USAGE ERROR: Invalid concurrency", DEFAULT_CONCURRENCY);
|
|
439
681
|
const batchMode = strOf(opts["batch_mode"] || DEFAULT_BATCH_MODE).trim();
|
|
440
682
|
if (batchMode !== "episode") {
|
|
441
683
|
throw new CliError("USAGE ERROR: Invalid batch mode", "Invalid batch mode.", {
|
|
@@ -451,29 +693,7 @@ export async function commandInit(opts) {
|
|
|
451
693
|
["batch_max_chars", DEFAULT_BATCH_MAX_CHARS],
|
|
452
694
|
["batch_min_lines", DEFAULT_BATCH_MIN_LINES],
|
|
453
695
|
]) {
|
|
454
|
-
|
|
455
|
-
try {
|
|
456
|
-
value = parseInt(strOf(opts[optName] || defaultValue), 10);
|
|
457
|
-
if (Number.isNaN(value))
|
|
458
|
-
throw new Error("nan");
|
|
459
|
-
}
|
|
460
|
-
catch {
|
|
461
|
-
throw new CliError("USAGE ERROR: Invalid batch option", "Invalid batch option.", {
|
|
462
|
-
exitCode: EXIT_USAGE,
|
|
463
|
-
required: [`--${optName.replace(/_/g, "-")}: positive integer`],
|
|
464
|
-
received: [`--${optName.replace(/_/g, "-")}: ${opts[optName]}`],
|
|
465
|
-
nextSteps: ["Use positive integers for batch options and rerun init."],
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
if (value < 1) {
|
|
469
|
-
throw new CliError("USAGE ERROR: Invalid batch option", "Invalid batch option.", {
|
|
470
|
-
exitCode: EXIT_USAGE,
|
|
471
|
-
required: [`--${optName.replace(/_/g, "-")}: positive integer`],
|
|
472
|
-
received: [`--${optName.replace(/_/g, "-")}: ${value}`],
|
|
473
|
-
nextSteps: ["Use positive integers for batch options and rerun init."],
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
batchValues[optName] = value;
|
|
696
|
+
batchValues[optName] = parsePositiveIntOption(opts[optName], `--${optName.replace(/_/g, "-")}`, "USAGE ERROR: Invalid batch option", defaultValue);
|
|
477
697
|
}
|
|
478
698
|
const batchTargetLines = batchValues["batch_target_lines"];
|
|
479
699
|
const batchMaxChars = batchValues["batch_max_chars"];
|
|
@@ -642,6 +862,11 @@ export async function commandInit(opts) {
|
|
|
642
862
|
default: pending.push(batch); // never attempted (or corrupt → self-heal)
|
|
643
863
|
}
|
|
644
864
|
}
|
|
865
|
+
// Episodes whose batches were (re)run this round — their episode synopsis is
|
|
866
|
+
// recomputed; untouched episodes reuse their persisted `syn`. Summary failures
|
|
867
|
+
// accumulate here and are reported without ever blocking init.
|
|
868
|
+
const changedEpisodes = new Set(pending.map((b) => Number(b["episode"])));
|
|
869
|
+
const summaryDegradedEpisodes = [];
|
|
645
870
|
updateRunState(workspace, {
|
|
646
871
|
status: "init_running",
|
|
647
872
|
init_stage: "batch_extract",
|
|
@@ -680,6 +905,7 @@ export async function commandInit(opts) {
|
|
|
680
905
|
const results = [];
|
|
681
906
|
const incompleteEpisodes = [];
|
|
682
907
|
try {
|
|
908
|
+
// Assemble every complete episode (incomplete ones are reported and skipped).
|
|
683
909
|
for (const episode of asList(plan["episodes"])) {
|
|
684
910
|
const episodeNum = Number(episode["episode"]);
|
|
685
911
|
const epBatches = batchesByEpisode.get(episodeNum) ?? [];
|
|
@@ -695,9 +921,23 @@ export async function commandInit(opts) {
|
|
|
695
921
|
}
|
|
696
922
|
const merged = mergeBatchResultsForEpisode(episode, batchResults);
|
|
697
923
|
validateEpisodeExtractionQuality(sourceText, episode, merged);
|
|
698
|
-
results.push(merged);
|
|
699
|
-
|
|
700
|
-
|
|
924
|
+
results.push(merged); // complete episodes only; carries `episode` for the write below
|
|
925
|
+
}
|
|
926
|
+
// Episode-level synopsis (reduce-1): resolve each result's synopsis in place
|
|
927
|
+
// (passthrough / reuse / skip / concurrent LLM stitch). Reads prior `syn`
|
|
928
|
+
// BEFORE the persist loop below overwrites ep_NNN.json.
|
|
929
|
+
summaryDegradedEpisodes.push(...await resolveEpisodeSynopses(provider, results, {
|
|
930
|
+
resultsDir: episodeResultsDir,
|
|
931
|
+
resummarize,
|
|
932
|
+
skipSummary,
|
|
933
|
+
changedEpisodes,
|
|
934
|
+
concurrency,
|
|
935
|
+
}));
|
|
936
|
+
// Persist every assembled episode (synopsis now resolved for all of them).
|
|
937
|
+
// merged carries `episode`, so it doubles as the path key.
|
|
938
|
+
for (const merged of results) {
|
|
939
|
+
writeJson(episodeResultPath(episodeResultsDir, merged), compactEpisodeResult(merged));
|
|
940
|
+
const errPath = episodeErrorPath(episodeResultsDir, merged);
|
|
701
941
|
if (exists(errPath))
|
|
702
942
|
deletePath(errPath);
|
|
703
943
|
}
|
|
@@ -780,8 +1020,11 @@ export async function commandInit(opts) {
|
|
|
780
1020
|
}
|
|
781
1021
|
try {
|
|
782
1022
|
updateRunState(workspace, { status: "init_running", init_stage: "asset_curation" });
|
|
783
|
-
|
|
784
|
-
|
|
1023
|
+
// Location merge is deterministic-only: actor/prop pruning runs from scene
|
|
1024
|
+
// counts and every location is kept. The previous provider location-merge
|
|
1025
|
+
// call was removed (single large streaming request kept dropping the gateway
|
|
1026
|
+
// connection); duplicate locations can be merged later if needed.
|
|
1027
|
+
const curation = curateScriptAssets(script);
|
|
785
1028
|
writeJson(path.join(dd, "asset_curation.json"), curation);
|
|
786
1029
|
}
|
|
787
1030
|
catch (exc) {
|
|
@@ -836,6 +1079,30 @@ export async function commandInit(opts) {
|
|
|
836
1079
|
updates: { episode_completed: results.length },
|
|
837
1080
|
});
|
|
838
1081
|
}
|
|
1082
|
+
// Whole-script overview (reduce-2). Soft: a non-essential concept enhancement,
|
|
1083
|
+
// the opposite of batch extraction's hard-fail policy — failures degrade the
|
|
1084
|
+
// overview (recorded as summary_degraded + a sanitized reason) and NEVER block init.
|
|
1085
|
+
let summaryFullDegraded = false;
|
|
1086
|
+
let summaryDegradeReason = "";
|
|
1087
|
+
try {
|
|
1088
|
+
updateRunState(workspace, { status: "init_running", init_stage: "summary" });
|
|
1089
|
+
const summary = await buildScriptSynopsis(provider, script, {
|
|
1090
|
+
dd,
|
|
1091
|
+
thresholdChars: summaryThresholdChars,
|
|
1092
|
+
groupSize: DEFAULT_SUMMARY_GROUP_SIZE,
|
|
1093
|
+
resummarize,
|
|
1094
|
+
skipSummary,
|
|
1095
|
+
concurrency,
|
|
1096
|
+
});
|
|
1097
|
+
summaryFullDegraded = summary.degraded;
|
|
1098
|
+
summaryDegradeReason = summary.reason ?? "";
|
|
1099
|
+
}
|
|
1100
|
+
catch (exc) {
|
|
1101
|
+
summaryFullDegraded = true;
|
|
1102
|
+
summaryDegradeReason = providerErrorSummary(exc);
|
|
1103
|
+
}
|
|
1104
|
+
summaryDegradedEpisodes.sort((a, b) => a - b); // concurrent stitch pushes out of order
|
|
1105
|
+
const summaryDegraded = summaryFullDegraded || summaryDegradedEpisodes.length > 0;
|
|
839
1106
|
const scriptPath = path.join(dd, "script.initial.json");
|
|
840
1107
|
writeJson(scriptPath, script);
|
|
841
1108
|
updateRunState(workspace, { status: "init_running", init_stage: "validate" });
|
|
@@ -877,12 +1144,18 @@ export async function commandInit(opts) {
|
|
|
877
1144
|
batch_completed: doneBatches,
|
|
878
1145
|
batch_reused: reusedBatchCount,
|
|
879
1146
|
batch_failed: erroredBatches,
|
|
1147
|
+
summary_threshold_chars: summaryThresholdChars,
|
|
1148
|
+
summary_skipped: skipSummary,
|
|
1149
|
+
summary_degraded: summaryDegraded,
|
|
1150
|
+
summary_degraded_episodes: summaryDegradedEpisodes,
|
|
1151
|
+
summary_degraded_reason: summaryDegradeReason || null,
|
|
880
1152
|
last_error: null,
|
|
881
1153
|
});
|
|
882
1154
|
const title = passed
|
|
883
1155
|
? "INIT COMPLETE: Initial script ready"
|
|
884
1156
|
: "INIT NEEDS AGENT: Initial script written with repair issues";
|
|
885
1157
|
const stats = validation["stats"] ?? {};
|
|
1158
|
+
const summaryLine = formatSummaryStatusLine(skipSummary, summaryDegraded, summaryDegradedEpisodes);
|
|
886
1159
|
const report = {
|
|
887
1160
|
title,
|
|
888
1161
|
result: [
|
|
@@ -894,6 +1167,7 @@ export async function commandInit(opts) {
|
|
|
894
1167
|
`episodes: ${results.length}/${totalEpisodes} complete`,
|
|
895
1168
|
`batches: ${doneBatches}/${totalBatches} done (reused ${reusedBatchCount})`,
|
|
896
1169
|
...(titleDegraded.length > 0 ? [`titles degraded (deterministic): ${titleDegraded.join(", ")}`] : []),
|
|
1170
|
+
...(summaryLine ? [summaryLine] : []),
|
|
897
1171
|
],
|
|
898
1172
|
artifacts: [
|
|
899
1173
|
path.join(workspace, "source.txt"),
|
|
@@ -904,6 +1178,7 @@ export async function commandInit(opts) {
|
|
|
904
1178
|
episodeResultsDir,
|
|
905
1179
|
path.join(dd, "asset_curation.json"),
|
|
906
1180
|
path.join(dd, "asset_metadata.json"),
|
|
1181
|
+
path.join(dd, "synopsis.json"),
|
|
907
1182
|
scriptPath,
|
|
908
1183
|
path.join(dd, "validation.json"),
|
|
909
1184
|
path.join(dd, "run_state.json"),
|
|
@@ -1003,6 +1278,13 @@ export function commandStatus(opts) {
|
|
|
1003
1278
|
if (titleDeg && Number(titleDeg["count"] ?? 0) > 0) {
|
|
1004
1279
|
result.push(`titles degraded (deterministic): ${asList(titleDeg["episodes"]).join(", ")}`);
|
|
1005
1280
|
}
|
|
1281
|
+
// Surface synopsis state recorded by init: skipped or degraded summaries mean
|
|
1282
|
+
// the overview / some episode synopses are missing or fell back to joined
|
|
1283
|
+
// beats — an agent gating on status should know it isn't a complete summary.
|
|
1284
|
+
// Same formatter as init's report so the two never word it differently.
|
|
1285
|
+
const summaryLine = formatSummaryStatusLine(Boolean(state["summary_skipped"]), Boolean(state["summary_degraded"]), asList(state["summary_degraded_episodes"]).map((n) => Number(n)));
|
|
1286
|
+
if (summaryLine)
|
|
1287
|
+
result.push(summaryLine);
|
|
1006
1288
|
result.push(...lines);
|
|
1007
1289
|
const allDone = completeEpisodes === episodes.length;
|
|
1008
1290
|
const report = {
|