@lingjingai/scriptctl 0.31.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_DIRECT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
4
- import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
4
+ import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, applyAssetGroupingDuplicateMerges, applyStateBindingPlan, bindDefaultStateRefsInScript, buildAssetCurationAuditContextText, buildAssetCurationContextText, buildAssetCurationRepairContextText, buildAssetGroupingContextText, buildLocationCanonicalizationAuditContextText, buildPrimaryAssetCurationChunks, groupedAssetCurationChunks, combineLocationCanonicalizationPlan, buildStateBindingContexts, assetCurationAuditSourceKeys, combineAssetCurationExpressionPlans, combinePrimaryAssetCurationPlans, combineAssetCurationPlans, curateScriptAssets, ensureDefaultStates, assetCurationAuditCandidateSourceKeys, locationCanonicalizationCandidateSourceKeys, missingAssetCurationRequiredSourceKeys, normalizeStateBindingPlanShape, parseAssetCurationExpressions, parseAssetCurationExpressionsTolerant, parseAssetGroupingExpressions, assertLocationCanonicalizationPlan, applyMetadataToScript, validateAssetGroupingPlan, } from "../domain/direct-core.js";
5
5
  import { validateScript } from "../domain/script/validate.js";
6
6
  import { makeProvider, providerCapabilities } from "../infra/providers.js";
7
7
  import { remoteScriptOutputStoreFromEnv } from "../infra/script-output-api.js";
@@ -118,6 +118,152 @@ export function readRunState(workspace) {
118
118
  return {};
119
119
  }
120
120
  }
121
+ function isoFromMs(ms) {
122
+ return new Date(ms).toISOString();
123
+ }
124
+ function durationMs(startedAtMs, finishedAtMs) {
125
+ return Math.max(0, Math.round(finishedAtMs - startedAtMs));
126
+ }
127
+ function timingErrorSummary(error) {
128
+ const e = error;
129
+ return `${e?.name ?? "Error"}: ${e?.message ?? ""}`.slice(0, 240);
130
+ }
131
+ function timingSummary(events) {
132
+ const stages = {};
133
+ const branches = {};
134
+ const slowChunks = events
135
+ .filter((event) => event.category === "chunk")
136
+ .sort((left, right) => right.duration_ms - left.duration_ms)
137
+ .slice(0, 20)
138
+ .map((event) => ({
139
+ name: event.name,
140
+ parent: event.parent ?? "",
141
+ duration_ms: event.duration_ms,
142
+ status: event.status,
143
+ }));
144
+ for (const event of events) {
145
+ if (event.category === "stage")
146
+ stages[event.name] = event.duration_ms;
147
+ if (event.category === "branch")
148
+ branches[event.name] = event.duration_ms;
149
+ }
150
+ return { stages, branches, slow_chunks: slowChunks };
151
+ }
152
+ class JsonTimingRecorder {
153
+ ws;
154
+ workspace;
155
+ command = "";
156
+ events = [];
157
+ activeSpans = [];
158
+ counter = 0;
159
+ constructor(ws, workspace) {
160
+ this.ws = ws;
161
+ this.workspace = workspace;
162
+ }
163
+ reset(command) {
164
+ this.command = command;
165
+ this.events = [];
166
+ this.activeSpans = [];
167
+ this.counter = 0;
168
+ this.flush();
169
+ updateRunState(this.workspace, {
170
+ timing_current: null,
171
+ timing_current_stage: null,
172
+ timing_current_branch: null,
173
+ timing_summary: { stages: {}, branches: {}, slow_chunks: [] },
174
+ });
175
+ }
176
+ async span(options, run) {
177
+ const startedAtMs = Date.now();
178
+ const active = {
179
+ name: options.name,
180
+ category: options.category,
181
+ ...(options.parent ? { parent: options.parent } : {}),
182
+ started_at: isoFromMs(startedAtMs),
183
+ };
184
+ this.activeSpans.push(active);
185
+ this.updateTimingCurrent();
186
+ try {
187
+ const value = await run();
188
+ this.removeActiveSpan(active);
189
+ this.record({
190
+ ...options,
191
+ status: "success",
192
+ startedAtMs,
193
+ finishedAtMs: Date.now(),
194
+ });
195
+ return value;
196
+ }
197
+ catch (error) {
198
+ this.removeActiveSpan(active);
199
+ this.record({
200
+ ...options,
201
+ status: "error",
202
+ startedAtMs,
203
+ finishedAtMs: Date.now(),
204
+ error: timingErrorSummary(error),
205
+ });
206
+ throw error;
207
+ }
208
+ }
209
+ removeActiveSpan(active) {
210
+ const index = this.activeSpans.lastIndexOf(active);
211
+ if (index >= 0)
212
+ this.activeSpans.splice(index, 1);
213
+ }
214
+ activeSpanByCategory(category) {
215
+ for (let index = this.activeSpans.length - 1; index >= 0; index -= 1) {
216
+ const span = this.activeSpans[index];
217
+ if (span.category === category)
218
+ return span;
219
+ }
220
+ return null;
221
+ }
222
+ currentSpanPayload(span) {
223
+ if (!span)
224
+ return null;
225
+ return {
226
+ name: span.name,
227
+ category: span.category,
228
+ parent: span.parent ?? null,
229
+ started_at: span.started_at,
230
+ };
231
+ }
232
+ updateTimingCurrent() {
233
+ updateRunState(this.workspace, {
234
+ timing_current: this.currentSpanPayload(this.activeSpans[this.activeSpans.length - 1] ?? null),
235
+ timing_current_stage: this.currentSpanPayload(this.activeSpanByCategory("stage")),
236
+ timing_current_branch: this.currentSpanPayload(this.activeSpanByCategory("branch")),
237
+ timing_summary: timingSummary(this.events),
238
+ });
239
+ }
240
+ record(options) {
241
+ const event = {
242
+ id: `tim_${String(++this.counter).padStart(5, "0")}`,
243
+ name: options.name,
244
+ category: options.category,
245
+ ...(options.parent ? { parent: options.parent } : {}),
246
+ status: options.status,
247
+ started_at: isoFromMs(options.startedAtMs),
248
+ finished_at: isoFromMs(options.finishedAtMs),
249
+ duration_ms: durationMs(options.startedAtMs, options.finishedAtMs),
250
+ ...(options.metadata ? { metadata: options.metadata } : {}),
251
+ ...(options.error ? { error: options.error } : {}),
252
+ };
253
+ this.events.push(event);
254
+ this.flush();
255
+ this.updateTimingCurrent();
256
+ }
257
+ flush() {
258
+ this.ws.writeJson("timings.json", {
259
+ version: 1,
260
+ command: this.command,
261
+ updated_at: checkpointTimestamp(),
262
+ events: this.events,
263
+ summary: timingSummary(this.events),
264
+ });
265
+ }
266
+ }
121
267
  // ---------------------------------------------------------------------------
122
268
  // Paths for episode/batch results
123
269
  // ---------------------------------------------------------------------------
@@ -214,6 +360,7 @@ function isDirectStageName(value) {
214
360
  case "episode_synopsis":
215
361
  case "script_merge":
216
362
  case "asset_curation":
363
+ case "state_binding":
217
364
  case "metadata_extract":
218
365
  case "script_synopsis":
219
366
  case "validate":
@@ -254,6 +401,56 @@ function stageSnapshot(name, status, completed, total, message) {
254
401
  message: message ?? "",
255
402
  };
256
403
  }
404
+ function formatDuration(ms) {
405
+ const value = Math.max(0, Math.round(ms));
406
+ if (value < 1000)
407
+ return `${value}ms`;
408
+ const seconds = value / 1000;
409
+ if (seconds < 60)
410
+ return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;
411
+ const minutes = Math.floor(seconds / 60);
412
+ const rest = Math.round(seconds % 60);
413
+ return `${minutes}m${String(rest).padStart(2, "0")}s`;
414
+ }
415
+ function stageTimingDurations(state) {
416
+ const timingSummaryRaw = state["timing_summary"];
417
+ const timingSummary = isDict(timingSummaryRaw) ? timingSummaryRaw : {};
418
+ const stagesRaw = timingSummary["stages"];
419
+ const stages = isDict(stagesRaw) ? stagesRaw : {};
420
+ const out = {};
421
+ for (const [key, value] of Object.entries(stages)) {
422
+ const n = Number(value);
423
+ if (!Number.isNaN(n))
424
+ out[key] = n;
425
+ }
426
+ return out;
427
+ }
428
+ function runningTimingElapsed(state, stageName) {
429
+ const stageRaw = state["timing_current_stage"];
430
+ const currentRaw = isDict(stageRaw) ? stageRaw : state["timing_current"];
431
+ const current = isDict(currentRaw) ? currentRaw : null;
432
+ if (!current || strOf(current["category"]) !== "stage" || strOf(current["name"]) !== stageName)
433
+ return null;
434
+ const startedAt = Date.parse(strOf(current["started_at"]));
435
+ if (Number.isNaN(startedAt))
436
+ return null;
437
+ return Date.now() - startedAt;
438
+ }
439
+ function applyStageTiming(stage, state, durations) {
440
+ const duration = durations[stage.name];
441
+ const elapsed = stage.status === "running" ? runningTimingElapsed(state, stage.name) : null;
442
+ const messageParts = [stage.message].filter((item) => item);
443
+ if (typeof duration === "number")
444
+ messageParts.push(`耗时 ${formatDuration(duration)}`);
445
+ else if (elapsed !== null)
446
+ messageParts.push(`已运行 ${formatDuration(elapsed)}`);
447
+ return {
448
+ ...stage,
449
+ ...(typeof duration === "number" ? { duration_ms: duration } : {}),
450
+ ...(elapsed !== null ? { elapsed_ms: elapsed } : {}),
451
+ message: messageParts.join("; "),
452
+ };
453
+ }
257
454
  function fileStage(name, filePath, runningStage) {
258
455
  if (exists(filePath))
259
456
  return stageSnapshot(name, "success", 1, 1);
@@ -304,6 +501,7 @@ function stageLabel(name) {
304
501
  case "episode_synopsis": return "补充信息";
305
502
  case "script_merge": return "生成剧本";
306
503
  case "asset_curation": return "补充信息";
504
+ case "state_binding": return "补充信息";
307
505
  case "metadata_extract": return "补充信息";
308
506
  case "script_synopsis": return "补充信息";
309
507
  case "validate": return "检查结果";
@@ -364,9 +562,10 @@ function stageOrder(name) {
364
562
  case "episode_synopsis": return 6;
365
563
  case "script_merge": return 7;
366
564
  case "asset_curation": return 8;
367
- case "metadata_extract": return 9;
368
- case "script_synopsis": return 10;
369
- case "validate": return 11;
565
+ case "state_binding": return 9;
566
+ case "metadata_extract": return 10;
567
+ case "script_synopsis": return 11;
568
+ case "validate": return 12;
370
569
  }
371
570
  }
372
571
  function stageRatio(stage) {
@@ -602,6 +801,7 @@ function buildDirectStatusSnapshot(workspace) {
602
801
  const episodeResultsDir = path.join(dd, "episode_results");
603
802
  const trustEpisodeMergeArtifacts = canTrustStageArtifacts("episode_merge", runningStage, runStatus);
604
803
  const trustEpisodeSynopsisArtifacts = canTrustStageArtifacts("episode_synopsis", runningStage, runStatus);
804
+ const trustStateBindingArtifacts = canTrustStageArtifacts("state_binding", runningStage, runStatus);
605
805
  const trustMetadataArtifacts = canTrustStageArtifacts("metadata_extract", runningStage, runStatus);
606
806
  const trustScriptSynopsisArtifacts = canTrustStageArtifacts("script_synopsis", runningStage, runStatus);
607
807
  const batchesByEpisode = new Map();
@@ -741,6 +941,9 @@ function buildDirectStatusSnapshot(workspace) {
741
941
  };
742
942
  const scriptStage = orderedFileStage("script_merge", path.join(dd, "script.initial.json"), runningStage, runStatus);
743
943
  const curationStage = orderedFileStage("asset_curation", path.join(dd, "asset_curation.json"), runningStage, runStatus);
944
+ const stateBindingStage = trustStateBindingArtifacts
945
+ ? orderedFileStage("state_binding", path.join(dd, "state_binding.json"), runningStage, runStatus)
946
+ : stageSnapshot("state_binding", runningStage === "state_binding" ? "running" : "pending", 0, 1);
744
947
  const metadataStage = Boolean(state["metadata_skipped"]) && trustMetadataArtifacts
745
948
  ? stageSnapshot("metadata_extract", "skipped", 0, 1, "metadata: skipped (--skip-metadata)")
746
949
  : orderedFileStage("metadata_extract", path.join(dd, "asset_metadata.json"), runningStage, runStatus);
@@ -761,6 +964,7 @@ function buildDirectStatusSnapshot(workspace) {
761
964
  const validation = readJson(validationPath);
762
965
  validationStage = stageSnapshot("validate", Boolean(validation["passed"]) ? "success" : "warning", 1, 1);
763
966
  }
967
+ const timingDurations = stageTimingDurations(state);
764
968
  const stages = [
765
969
  sourceStage,
766
970
  episodePlanStage,
@@ -771,10 +975,13 @@ function buildDirectStatusSnapshot(workspace) {
771
975
  episodeSynopsisStage,
772
976
  scriptStage,
773
977
  curationStage,
978
+ stateBindingStage,
774
979
  metadataStage,
775
980
  scriptSynopsisStage,
776
981
  validationStage,
777
- ].map((stage) => failedStage === stage.name ? { ...stage, status: "error", error: Math.max(1, stage.error) } : stage);
982
+ ]
983
+ .map((stage) => failedStage === stage.name ? { ...stage, status: "error", error: Math.max(1, stage.error) } : stage)
984
+ .map((stage) => applyStageTiming(stage, state, timingDurations));
778
985
  let overallStatus = "pending";
779
986
  const maxStage = stages.reduce((max, stage) => statusPriority(stage.status) > statusPriority(max) ? stage.status : max, "success");
780
987
  if (runStatus === "init_failed" || runStatus === "init_incomplete" || maxStage === "error")
@@ -1010,11 +1217,16 @@ function initFailedReport(workspace, opts) {
1010
1217
  // into every stage's catch block.
1011
1218
  function stageFailure(workspace, exc, opts) {
1012
1219
  const e = exc;
1220
+ const received = exc instanceof CliError && exc.received.length > 0
1221
+ ? exc.received
1222
+ : [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`];
1013
1223
  throw initFailedReport(workspace, {
1014
1224
  title: opts.title,
1015
1225
  stage: opts.stage,
1226
+ exitCode: exc instanceof CliError ? exc.exitCode : undefined,
1227
+ errorCode: exc instanceof CliError ? exc.errorCode : undefined,
1016
1228
  required: opts.required,
1017
- received: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`],
1229
+ received,
1018
1230
  nextSteps: opts.nextSteps,
1019
1231
  ...(opts.updates ? { updates: opts.updates } : {}),
1020
1232
  });
@@ -1042,6 +1254,12 @@ async function pMapWithConcurrency(items, concurrency, worker) {
1042
1254
  }));
1043
1255
  return out;
1044
1256
  }
1257
+ function unwrapConcurrentOutcomes(outcomes) {
1258
+ const failed = outcomes.find((outcome) => !outcome.ok);
1259
+ if (failed)
1260
+ throw failed.error;
1261
+ return outcomes.map((outcome) => outcome.value);
1262
+ }
1045
1263
  function providerErrorSummary(error) {
1046
1264
  if (error instanceof CliError && error.errorCode)
1047
1265
  return error.errorCode;
@@ -1051,6 +1269,9 @@ function providerErrorSummary(error) {
1051
1269
  return "PROVIDER_CONTENT_FILTERED";
1052
1270
  return `${e?.name ?? "Error"}: ${msg}`.slice(0, 160);
1053
1271
  }
1272
+ function cloneDict(value) {
1273
+ return structuredClone(value);
1274
+ }
1054
1275
  // Generate the plan's missing episode titles in bounded, concurrent chunks.
1055
1276
  // Title generation is non-essential (every episode has a deterministic
1056
1277
  // fallback), so a chunk that the provider filters or fails does NOT abort
@@ -1188,6 +1409,56 @@ function assignOverviewFields(script, o) {
1188
1409
  function applyScriptSynopsisFields(script, fields) {
1189
1410
  assignOverviewFields(script, normalizeOverviewFields(fields));
1190
1411
  }
1412
+ function synopsisGroupSignature(opts) {
1413
+ return sha256Text(JSON.stringify(sortDeep({
1414
+ title: strOf(opts.title),
1415
+ thresholdChars: opts.thresholdChars,
1416
+ groupSize: opts.groupSize,
1417
+ level: opts.level,
1418
+ groupIndex: opts.groupIndex,
1419
+ synopses: opts.synopses,
1420
+ })));
1421
+ }
1422
+ function synopsisGroupPartPath(dd, signature) {
1423
+ return path.join(dd, "synopsis_parts", `${signature}.json`);
1424
+ }
1425
+ async function loadOrRunSynopsisGroup(provider, groupSynopses, opts) {
1426
+ if (groupSynopses.length <= 1)
1427
+ return { value: groupSynopses[0] ?? "", reused: false };
1428
+ const signature = synopsisGroupSignature({
1429
+ title: opts.title,
1430
+ thresholdChars: opts.thresholdChars,
1431
+ groupSize: opts.groupSize,
1432
+ level: opts.level,
1433
+ groupIndex: opts.groupIndex,
1434
+ synopses: groupSynopses,
1435
+ });
1436
+ const partPath = synopsisGroupPartPath(opts.dd, signature);
1437
+ if (!opts.resummarize && exists(partPath)) {
1438
+ const prior = readJson(partPath);
1439
+ if (isDict(prior) && strOf(prior["source_group_signature"]) === signature) {
1440
+ const synopsis = strOf(prior["synopsis"]).trim();
1441
+ if (synopsis)
1442
+ return { value: synopsis, reused: true };
1443
+ }
1444
+ }
1445
+ try {
1446
+ const synopsis = await reduceSynopses(provider, [...groupSynopses], { title: opts.title });
1447
+ writeJson(partPath, {
1448
+ version: 1,
1449
+ source_group_signature: signature,
1450
+ level: opts.level,
1451
+ group_index: opts.groupIndex,
1452
+ source_count: groupSynopses.length,
1453
+ synopsis,
1454
+ updated_at: checkpointTimestamp(),
1455
+ });
1456
+ return { value: synopsis, reused: false };
1457
+ }
1458
+ catch (error) {
1459
+ return { value: groupSynopses.join(" "), error, reused: false };
1460
+ }
1461
+ }
1191
1462
  // Whole-script reduce-2: collapse the episode synopses down to a set that fits
1192
1463
  // one final reduce, then produce the top-level overview. Reuses a prior
1193
1464
  // synopsis.json when the episode synopses are unchanged (signature match). Soft
@@ -1244,6 +1515,7 @@ async function buildScriptSynopsis(provider, script, opts) {
1244
1515
  // whenever it can't form fewer groups than items), so `current` strictly
1245
1516
  // shrinks each non-terminal pass and this terminates without a counter.
1246
1517
  let current = episodeSynopses;
1518
+ let level = 1;
1247
1519
  while (true) {
1248
1520
  const plan = planSynopsisReduction(current, opts.thresholdChars, opts.groupSize);
1249
1521
  if (plan.fitsInOnePass)
@@ -1252,16 +1524,10 @@ async function buildScriptSynopsis(provider, script, opts) {
1252
1524
  // singleton group passes through verbatim (no LLM rewrite), mirroring the
1253
1525
  // reduce-1 passthrough. A group whose stitch FAILS degrades to its joined
1254
1526
  // beats (soft, per-group) — one bad group never wipes the whole overview.
1255
- const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group) => {
1527
+ const currentLevel = level;
1528
+ const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group, groupIndex) => {
1256
1529
  const groupSynopses = group.map((i) => current[i]).filter((s) => s.trim());
1257
- if (groupSynopses.length <= 1)
1258
- return { value: groupSynopses[0] ?? "" };
1259
- try {
1260
- return { value: await reduceSynopses(provider, groupSynopses, { title: script["title"] }) };
1261
- }
1262
- catch (err) {
1263
- return { value: groupSynopses.join(" "), error: err };
1264
- }
1530
+ return loadOrRunSynopsisGroup(provider, groupSynopses, { ...opts, title: script["title"], level: currentLevel, groupIndex });
1265
1531
  });
1266
1532
  const next = [];
1267
1533
  for (const o of outcomes) {
@@ -1275,6 +1541,7 @@ async function buildScriptSynopsis(provider, script, opts) {
1275
1541
  next.push(o.value.value);
1276
1542
  }
1277
1543
  current = next;
1544
+ level += 1;
1278
1545
  if (current.length === 0)
1279
1546
  break;
1280
1547
  }
@@ -1416,10 +1683,12 @@ function parseDirectRunOptions(opts) {
1416
1683
  function buildStageContext(options) {
1417
1684
  const ws = new Workspace(options.workspace);
1418
1685
  let provider = null;
1686
+ const timing = new JsonTimingRecorder(ws, options.workspace);
1419
1687
  const ctx = {
1420
1688
  ws,
1421
1689
  options,
1422
1690
  state: emptyRunState(),
1691
+ timing,
1423
1692
  provider() {
1424
1693
  if (provider)
1425
1694
  return provider;
@@ -1545,11 +1814,98 @@ function buildIncompleteReport(ctx) {
1545
1814
  };
1546
1815
  return [report, EXIT_RUNTIME];
1547
1816
  }
1817
+ function initPreFanoutStages() {
1818
+ const end = STAGES.findIndex((stage) => stage.name === "asset-curation");
1819
+ return STAGES.slice(0, end + 1);
1820
+ }
1821
+ function validateOnlyStages() {
1822
+ return STAGES.filter((stage) => stage.name === "validate");
1823
+ }
1824
+ function updatePostCurationBranch(ctx, name, status) {
1825
+ const current = readRunState(ctx.options.workspace);
1826
+ const rawBranches = current["post_curation_branches"];
1827
+ const branches = isDict(rawBranches) ? { ...rawBranches } : {};
1828
+ branches[name] = { status, updated_at: checkpointTimestamp() };
1829
+ ctx.updateRunState({ post_curation_branches: branches });
1830
+ }
1831
+ async function runPostCurationBranch(ctx, name, run) {
1832
+ updatePostCurationBranch(ctx, name, "running");
1833
+ try {
1834
+ const value = await ctx.timing.span({ name, category: "branch", parent: "post_curation" }, run);
1835
+ updatePostCurationBranch(ctx, name, "success");
1836
+ return value;
1837
+ }
1838
+ catch (error) {
1839
+ updatePostCurationBranch(ctx, name, "error");
1840
+ throw error;
1841
+ }
1842
+ }
1843
+ async function runPostCurationFanout(ctx) {
1844
+ const { ws, state, options } = ctx;
1845
+ const baseScript = ensureScript(ctx);
1846
+ ctx.updateRunState({
1847
+ status: "init_running",
1848
+ init_stage: "state_binding",
1849
+ post_curation_parallel: true,
1850
+ post_curation_branches: {
1851
+ state_binding: { status: "running", updated_at: checkpointTimestamp() },
1852
+ metadata_extract: { status: options.skipMetadata ? "skipped" : "running", updated_at: checkpointTimestamp() },
1853
+ script_synopsis: { status: options.skipSummary ? "skipped" : "running", updated_at: checkpointTimestamp() },
1854
+ },
1855
+ });
1856
+ ctx.emitProgress("state-binding");
1857
+ ctx.emitProgress("metadata", options.skipMetadata ? "skipped" : "running");
1858
+ ctx.emitProgress("script-synopsis", options.skipSummary ? "skipped" : "running");
1859
+ const stateBindingPromise = runPostCurationBranch(ctx, "state_binding", async () => {
1860
+ const script = cloneDict(baseScript);
1861
+ const result = await runStateBindingForScript(ctx, script, false);
1862
+ return { ...result, script };
1863
+ });
1864
+ const metadataPromise = runPostCurationBranch(ctx, "metadata_extract", async () => {
1865
+ const script = cloneDict(baseScript);
1866
+ return runMetadataForScript(ctx, script, false, false);
1867
+ });
1868
+ const synopsisPromise = runPostCurationBranch(ctx, "script_synopsis", async () => {
1869
+ const script = cloneDict(baseScript);
1870
+ return runScriptSynopsisForScript(ctx, script, false);
1871
+ });
1872
+ const settled = await Promise.allSettled([stateBindingPromise, metadataPromise, synopsisPromise]);
1873
+ const failed = settled.find((item) => item.status === "rejected");
1874
+ if (failed && failed.status === "rejected")
1875
+ throw failed.reason;
1876
+ const stateBinding = await stateBindingPromise;
1877
+ const metadata = await metadataPromise;
1878
+ const synopsis = await synopsisPromise;
1879
+ await ctx.timing.span({ name: "post_curation.merge", category: "substage", parent: "post_curation" }, () => {
1880
+ const finalScript = stateBinding.script;
1881
+ if (!metadata.skipped)
1882
+ applyMetadataToScript(finalScript, metadata.metadata);
1883
+ applyScriptSynopsisFields(finalScript, synopsis.fields);
1884
+ ws.writeJson("script.initial.json", finalScript);
1885
+ state.script = finalScript;
1886
+ state.summaryFullDegraded = synopsis.degraded;
1887
+ state.summaryDegradeReason = synopsis.reason;
1888
+ });
1889
+ state.summaryDegradedEpisodes.sort((a, b) => a - b);
1890
+ const summaryDegraded = state.summaryFullDegraded || state.summaryDegradedEpisodes.length > 0;
1891
+ ctx.updateRunState({
1892
+ post_curation_parallel: false,
1893
+ summary_skipped: options.skipSummary,
1894
+ summary_degraded: summaryDegraded,
1895
+ summary_degraded_episodes: state.summaryDegradedEpisodes,
1896
+ summary_degraded_reason: state.summaryDegradeReason || null,
1897
+ metadata_skipped: options.skipMetadata,
1898
+ });
1899
+ ctx.emitProgress("state-binding", "success");
1900
+ ctx.emitProgress("metadata", metadata.skipped ? "skipped" : "success");
1901
+ ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
1902
+ }
1548
1903
  async function commandInitImpl(opts) {
1549
1904
  const options = parseDirectRunOptions(opts);
1550
1905
  assertInitSource(options.source);
1551
1906
  const ctx = buildStageContext(options);
1552
1907
  ctx.ws.ensure();
1908
+ ctx.timing.reset("direct init");
1553
1909
  // Seed run_state with the init header before the first stage runs (provider /
1554
1910
  // model / source_path / metadata flag) — the god-command set these up front.
1555
1911
  updateRunState(options.workspace, {
@@ -1563,13 +1919,9 @@ async function commandInitImpl(opts) {
1563
1919
  metadata_skipped: options.skipMetadata,
1564
1920
  });
1565
1921
  try {
1566
- // `direct init` runs EVERY stage (force) exactly like the former god
1567
- // command, which had no whole-stage skip. The reuse model lives INSIDE the
1568
- // stages (per-batch file presence in batch-extract; per-episode `syn` reuse
1569
- // in the synopsis stages), so a deterministic stage must still re-derive its
1570
- // output from the (possibly reused/edited) upstream artifacts. Whole-stage
1571
- // skip-on-present is a `direct run <stage>` convenience only.
1572
- await runStages(ctx, STAGES, { force: true });
1922
+ await runStages(ctx, initPreFanoutStages(), { force: true, persistScriptAfterStages: false });
1923
+ await ctx.timing.span({ name: "post_curation", category: "stage" }, () => runPostCurationFanout(ctx));
1924
+ await runStages(ctx, validateOnlyStages(), { force: true, persistScriptAfterStages: false });
1573
1925
  }
1574
1926
  catch (exc) {
1575
1927
  // The resumable "incomplete" stop is not a failure — surface the INCOMPLETE
@@ -1645,6 +1997,8 @@ export async function commandRun(opts) {
1645
1997
  }
1646
1998
  const options = parseDirectRunOptions(opts);
1647
1999
  const ctx = buildStageContext(options);
2000
+ ctx.ws.ensure();
2001
+ ctx.timing.reset(`direct run ${stageName}`);
1648
2002
  const force = Boolean(opts["force"]);
1649
2003
  try {
1650
2004
  if (force) {
@@ -2131,16 +2485,661 @@ export async function stageScriptMerge(ctx) {
2131
2485
  }
2132
2486
  }
2133
2487
  // --- asset-curation -------------------------------------------------------
2488
+ const ASSET_GROUPING_CACHE_VERSION = 2;
2489
+ const ASSET_CURATION_CACHE_VERSION = 2;
2490
+ function assetCacheIdentity(options) {
2491
+ return {
2492
+ provider: options.providerName,
2493
+ model: options.model,
2494
+ };
2495
+ }
2496
+ function assetCacheArtifactFields(ctx, cacheVersion) {
2497
+ return {
2498
+ cache_version: cacheVersion,
2499
+ ...assetCacheIdentity(ctx.options),
2500
+ };
2501
+ }
2502
+ function assetCurationScriptFingerprint(script, options) {
2503
+ return sha256Text(JSON.stringify(sortDeep({
2504
+ cache_version: ASSET_CURATION_CACHE_VERSION,
2505
+ ...assetCacheIdentity(options),
2506
+ script: {
2507
+ actors: script["actors"],
2508
+ locations: script["locations"],
2509
+ props: script["props"],
2510
+ speakers: script["speakers"],
2511
+ episodes: script["episodes"],
2512
+ },
2513
+ })));
2514
+ }
2515
+ function assetCurationPlanFingerprint(scriptFingerprint, plan) {
2516
+ return sha256Text(JSON.stringify(sortDeep({
2517
+ script_fingerprint: scriptFingerprint,
2518
+ decisions: plan["decisions"],
2519
+ })));
2520
+ }
2521
+ function chunkStrings(values, chunkSize) {
2522
+ const chunks = [];
2523
+ for (let index = 0; index < values.length; index += chunkSize) {
2524
+ chunks.push(values.slice(index, index + chunkSize));
2525
+ }
2526
+ return chunks;
2527
+ }
2528
+ function curationSourceKey(kind, id) {
2529
+ return `${kind}:${id}`;
2530
+ }
2531
+ function curationDecisionSourceKey(decision) {
2532
+ return curationSourceKey(strOf(decision["kind"]), strOf(decision["source_id"]));
2533
+ }
2534
+ function curationSourceKeysForIds(kind, ids) {
2535
+ return ids.map((id) => curationSourceKey(kind, id));
2536
+ }
2537
+ function curationPartSelection(kind, ids) {
2538
+ if (kind === "actor")
2539
+ return { actor: ids };
2540
+ if (kind === "location")
2541
+ return { location: ids };
2542
+ return { prop: ids };
2543
+ }
2544
+ function curationIdsFromSourceKeys(kind, ids, sourceKeys) {
2545
+ const allowed = new Set(sourceKeys);
2546
+ return ids.filter((id) => allowed.has(curationSourceKey(kind, id)));
2547
+ }
2548
+ function invalidCurationPart(title, received) {
2549
+ return new CliError(title, title, {
2550
+ exitCode: EXIT_RUNTIME,
2551
+ required: ["repairable asset curation expressions"],
2552
+ received,
2553
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and only invalid curation parts will retry."],
2554
+ });
2555
+ }
2556
+ const MAX_CURATION_PART_REPAIR_ATTEMPTS = 2;
2557
+ function assetGroupingKindListKey(kind) {
2558
+ if (kind === "actor")
2559
+ return "actors";
2560
+ if (kind === "location")
2561
+ return "locations";
2562
+ return "props";
2563
+ }
2564
+ function assetGroupingKindIdKey(kind) {
2565
+ if (kind === "actor")
2566
+ return "actor_id";
2567
+ if (kind === "location")
2568
+ return "location_id";
2569
+ return "prop_id";
2570
+ }
2571
+ function assetGroupingKindNameKey(kind) {
2572
+ if (kind === "actor")
2573
+ return "actor_name";
2574
+ if (kind === "location")
2575
+ return "location_name";
2576
+ return "prop_name";
2577
+ }
2578
+ function assetGroupingScriptFingerprint(script, options) {
2579
+ const compact = {};
2580
+ for (const kind of ["actor", "location", "prop"]) {
2581
+ const idKey = assetGroupingKindIdKey(kind);
2582
+ const nameKey = assetGroupingKindNameKey(kind);
2583
+ compact[assetGroupingKindListKey(kind)] = asList(script[assetGroupingKindListKey(kind)])
2584
+ .filter(isDict)
2585
+ .map((asset) => ({ id: strOf(asset[idKey]), name: strOf(asset[nameKey]) }));
2586
+ }
2587
+ return sha256Text(JSON.stringify(sortDeep({
2588
+ cache_version: ASSET_GROUPING_CACHE_VERSION,
2589
+ ...assetCacheIdentity(options),
2590
+ assets: compact,
2591
+ })));
2592
+ }
2593
+ function combineAssetGroupingPlans(parts) {
2594
+ const groups = parts.flatMap((part) => asList(part["groups"]).filter(isDict));
2595
+ const duplicateMergeDecisions = parts.flatMap((part) => asList(part["duplicate_merge_decisions"]).filter(isDict));
2596
+ const normalizations = parts.flatMap((part) => asList(part["normalizations"]).filter(isDict));
2597
+ const summaries = parts.map((part) => part["summary"]).filter(isDict);
2598
+ return {
2599
+ version: 1,
2600
+ format: "asset-grouping-v1",
2601
+ groups,
2602
+ duplicate_merge_decisions: duplicateMergeDecisions,
2603
+ normalizations,
2604
+ parts: parts.map((part) => ({
2605
+ kind: part["kind"],
2606
+ expression_text: part["expression_text"],
2607
+ summary: part["summary"],
2608
+ normalizations: part["normalizations"],
2609
+ })),
2610
+ summary: {
2611
+ group_count: groups.length,
2612
+ duplicate_merge_count: duplicateMergeDecisions.length,
2613
+ normalization_count: normalizations.length,
2614
+ parts: summaries,
2615
+ },
2616
+ };
2617
+ }
2618
+ async function loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint) {
2619
+ const artifactName = `asset_grouping_parts/${kind}.json`;
2620
+ const cached = ctx.ws.readJsonSafe(artifactName);
2621
+ const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
2622
+ if (cachedPlan && isList(cachedPlan["groups"]))
2623
+ return cachedPlan;
2624
+ const startedAtMs = Date.now();
2625
+ const idKey = assetGroupingKindIdKey(kind);
2626
+ const assetCount = asList(script[assetGroupingKindListKey(kind)])
2627
+ .filter(isDict)
2628
+ .map((asset) => strOf(asset[idKey]).trim())
2629
+ .filter((id) => id).length;
2630
+ if (assetCount === 0) {
2631
+ const plan = {
2632
+ version: 1,
2633
+ format: "asset-grouping-v1",
2634
+ kind,
2635
+ expression_text: "",
2636
+ expressions: [],
2637
+ groups: [],
2638
+ duplicate_merge_decisions: [],
2639
+ summary: { input_count: 0, group_count: 0, mentioned_count: 0, omitted_duplicate_count: 0 },
2640
+ };
2641
+ ctx.ws.writeJson(artifactName, {
2642
+ version: 1,
2643
+ ...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
2644
+ script_fingerprint: fingerprint,
2645
+ parse_status: "ok",
2646
+ kind,
2647
+ context_chars: 0,
2648
+ expression_text: "",
2649
+ duration_ms: durationMs(startedAtMs, Date.now()),
2650
+ plan,
2651
+ updated_at: checkpointTimestamp(),
2652
+ });
2653
+ return plan;
2654
+ }
2655
+ const contextText = buildAssetGroupingContextText(script, kind);
2656
+ let rawText = "";
2657
+ try {
2658
+ rawText = await ctx.timing.span({ name: `asset_grouping.${kind}`, category: "chunk", parent: "asset_curation.grouping", metadata: { artifact: artifactName } }, () => groupAssets(script, contextText));
2659
+ const parsed = parseAssetGroupingExpressions(rawText);
2660
+ const plan = validateAssetGroupingPlan(script, kind, parsed);
2661
+ ctx.ws.writeJson(artifactName, {
2662
+ version: 1,
2663
+ ...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
2664
+ script_fingerprint: fingerprint,
2665
+ parse_status: "ok",
2666
+ kind,
2667
+ context_chars: contextText.length,
2668
+ expression_text: rawText,
2669
+ duration_ms: durationMs(startedAtMs, Date.now()),
2670
+ plan,
2671
+ updated_at: checkpointTimestamp(),
2672
+ });
2673
+ return plan;
2674
+ }
2675
+ catch (exc) {
2676
+ const e = exc;
2677
+ ctx.ws.writeJson(artifactName, {
2678
+ version: 1,
2679
+ ...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
2680
+ script_fingerprint: fingerprint,
2681
+ parse_status: "failed",
2682
+ kind,
2683
+ context_chars: contextText.length,
2684
+ expression_text: rawText,
2685
+ duration_ms: durationMs(startedAtMs, Date.now()),
2686
+ error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
2687
+ updated_at: checkpointTimestamp(),
2688
+ });
2689
+ throw exc;
2690
+ }
2691
+ }
2692
+ async function loadOrRunAssetGrouping(ctx, script, groupAssets, fingerprint) {
2693
+ ctx.ws.mkdir("asset_grouping_parts");
2694
+ const outcomes = await pMapWithConcurrency(["actor", "location", "prop"], ctx.options.concurrency, async (kind) => loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint));
2695
+ return combineAssetGroupingPlans(unwrapConcurrentOutcomes(outcomes));
2696
+ }
2697
+ function curationDecisionSourceKeys(plan) {
2698
+ return new Set(asList(plan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
2699
+ }
2700
+ function sourceKeysFromRejectedExpressions(rejected, candidateSourceKeys) {
2701
+ const candidateSet = new Set(candidateSourceKeys);
2702
+ const keys = [];
2703
+ for (const item of rejected) {
2704
+ const key = strOf(item["source_key"]);
2705
+ if (key && (candidateSet.size === 0 || candidateSet.has(key)))
2706
+ uniqueAdd(keys, key);
2707
+ }
2708
+ return keys;
2709
+ }
2710
+ function scopedCurationPlan(plan, allowedSourceKeys) {
2711
+ const allowed = new Set(allowedSourceKeys);
2712
+ const decisions = asList(plan["decisions"]).filter(isDict);
2713
+ const acceptedExpressions = asList(plan["accepted_expressions"]);
2714
+ const scopedDecisions = [];
2715
+ const scopedExpressions = [];
2716
+ const ignored = [];
2717
+ for (let index = 0; index < decisions.length; index += 1) {
2718
+ const decision = decisions[index];
2719
+ const sourceKey = curationDecisionSourceKey(decision);
2720
+ const expression = acceptedExpressions[index] ?? "";
2721
+ if (allowed.has(sourceKey)) {
2722
+ scopedDecisions.push(decision);
2723
+ if (expression)
2724
+ scopedExpressions.push(expression);
2725
+ }
2726
+ else {
2727
+ ignored.push({ source_key: sourceKey, expression, reason: "outside_repair_scope" });
2728
+ }
2729
+ }
2730
+ return {
2731
+ plan: {
2732
+ ...plan,
2733
+ expression_text: formatCurationExpressionText(scopedExpressions),
2734
+ expressions: scopedExpressions.length > 0 ? scopedExpressions : ["NOOP # no asset curation changes"],
2735
+ accepted_expressions: scopedExpressions,
2736
+ decisions: scopedDecisions,
2737
+ },
2738
+ ignored,
2739
+ };
2740
+ }
2741
+ function missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan) {
2742
+ const candidateSet = new Set(candidateSourceKeys);
2743
+ return missingAssetCurationRequiredSourceKeys(script, partialPlan).filter((key) => candidateSet.has(key));
2744
+ }
2745
+ function repairSourceKeysForPart(script, candidateSourceKeys, partialPlan, requireCoverage) {
2746
+ const candidateSet = new Set(candidateSourceKeys);
2747
+ const accepted = new Set(asList(partialPlan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
2748
+ const rejected = asList(partialPlan["rejected_expressions"]).filter(isDict);
2749
+ const repairKeys = [];
2750
+ let hasUnlocatedRejection = false;
2751
+ for (const item of rejected) {
2752
+ const key = strOf(item["source_key"]);
2753
+ if (key && (candidateSet.size === 0 || candidateSet.has(key)))
2754
+ uniqueAdd(repairKeys, key);
2755
+ else
2756
+ hasUnlocatedRejection = true;
2757
+ }
2758
+ if (hasUnlocatedRejection) {
2759
+ for (const key of candidateSourceKeys) {
2760
+ if (!accepted.has(key))
2761
+ uniqueAdd(repairKeys, key);
2762
+ }
2763
+ }
2764
+ if (requireCoverage) {
2765
+ for (const key of missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan)) {
2766
+ uniqueAdd(repairKeys, key);
2767
+ }
2768
+ }
2769
+ return repairKeys;
2770
+ }
2771
+ function mergeRepairedCurationPlan(rawText, partialPlan, repairPlan, repairAttempts) {
2772
+ const acceptedExpressions = asList(partialPlan["accepted_expressions"]);
2773
+ const repairExpressions = asList(repairPlan["accepted_expressions"]);
2774
+ const decisions = [
2775
+ ...asList(partialPlan["decisions"]).filter(isDict),
2776
+ ...asList(repairPlan["decisions"]).filter(isDict),
2777
+ ];
2778
+ const expressions = [...acceptedExpressions, ...repairExpressions].filter((expression) => !/^NOOP(?:\s|#|$)/i.test(expression.trim()));
2779
+ return {
2780
+ version: 3,
2781
+ format: "asset-curation-exp-v1",
2782
+ expression_text: formatCurationExpressionText(expressions),
2783
+ raw_expression_text: rawText,
2784
+ accepted_expressions: [...acceptedExpressions, ...repairExpressions],
2785
+ rejected_expressions: asList(partialPlan["rejected_expressions"]).filter(isDict),
2786
+ repair_attempts: repairAttempts,
2787
+ expressions: expressions.length > 0 ? expressions : ["NOOP # no asset curation changes"],
2788
+ decisions,
2789
+ partial: false,
2790
+ };
2791
+ }
2792
+ function formatCurationExpressionText(expressions) {
2793
+ if (expressions.length > 0)
2794
+ return expressions.map((expression) => `<exp>${expression}</exp>`).join("\n");
2795
+ return "<exp>NOOP # no asset curation changes</exp>";
2796
+ }
2797
+ function writeFailedCurationPartArtifact(opts) {
2798
+ const e = opts.error;
2799
+ opts.ctx.ws.writeJson(opts.artifactName, {
2800
+ version: 1,
2801
+ ...assetCacheArtifactFields(opts.ctx, ASSET_CURATION_CACHE_VERSION),
2802
+ script_fingerprint: opts.fingerprint,
2803
+ ...opts.metadata,
2804
+ parse_status: "failed",
2805
+ candidate_source_keys: opts.candidateSourceKeys,
2806
+ expression_text: opts.partPlan?.["expression_text"] ?? null,
2807
+ raw_expression_text: opts.partPlan?.["raw_expression_text"] ?? opts.expressionText,
2808
+ accepted_expressions: opts.partPlan?.["accepted_expressions"] ?? [],
2809
+ rejected_expressions: opts.partPlan?.["rejected_expressions"] ?? [],
2810
+ repair_attempts: opts.repairAttempts,
2811
+ duration_ms: durationMs(opts.startedAtMs, Date.now()),
2812
+ error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
2813
+ plan: opts.partPlan,
2814
+ updated_at: checkpointTimestamp(),
2815
+ });
2816
+ }
2817
+ async function loadOrRunCurationPart(ctx, script, artifactName, fingerprint, metadata, candidateSourceKeys, parentTiming, requireCoverage, contextText, run) {
2818
+ const cached = ctx.ws.readJsonSafe(artifactName);
2819
+ const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
2820
+ if (cachedPlan && isList(cachedPlan["decisions"]))
2821
+ return cachedPlan;
2822
+ const startedAtMs = Date.now();
2823
+ const baseContextText = contextText(candidateSourceKeys);
2824
+ const expressionText = await ctx.timing.span({ name: strOf(metadata["part_id"]) || artifactName, category: "chunk", parent: parentTiming, metadata: { artifact: artifactName } }, () => run(baseContextText));
2825
+ let partPlan = parseAssetCurationExpressionsTolerant(expressionText);
2826
+ if (!isDict(partPlan) || !isList(partPlan["decisions"])) {
2827
+ throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
2828
+ exitCode: EXIT_RUNTIME,
2829
+ required: ["object with decisions[]"],
2830
+ received: [typeof partPlan],
2831
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
2832
+ });
2833
+ }
2834
+ const repairAttempts = [];
2835
+ let repairSourceKeys = repairSourceKeysForPart(script, candidateSourceKeys, partPlan, requireCoverage);
2836
+ let repairRejectedContext = asList(partPlan["rejected_expressions"]).filter(isDict);
2837
+ try {
2838
+ for (let attempt = 1; repairSourceKeys.length > 0 && attempt <= MAX_CURATION_PART_REPAIR_ATTEMPTS; attempt += 1) {
2839
+ const repairContext = buildAssetCurationRepairContextText({
2840
+ baseContextText: contextText(repairSourceKeys),
2841
+ repairSourceKeys,
2842
+ rejectedExpressions: repairRejectedContext,
2843
+ acceptedDecisions: asList(partPlan["decisions"]).filter(isDict),
2844
+ });
2845
+ const repairText = await ctx.timing.span({ name: `${strOf(metadata["part_id"]) || artifactName}.repair.${pad3(attempt)}`, category: "repair", parent: parentTiming, metadata: { artifact: artifactName, repair_source_keys: repairSourceKeys } }, () => run(repairContext));
2846
+ const repairPlan = parseAssetCurationExpressionsTolerant(repairText);
2847
+ const repairRejected = asList(repairPlan["rejected_expressions"]).filter(isDict);
2848
+ const scopedRepair = scopedCurationPlan(repairPlan, repairSourceKeys);
2849
+ repairAttempts.push({
2850
+ index: attempt,
2851
+ repair_source_keys: repairSourceKeys,
2852
+ expression_text: repairText,
2853
+ accepted_expressions: scopedRepair.plan["accepted_expressions"],
2854
+ ignored_expressions: scopedRepair.ignored,
2855
+ rejected_expressions: repairRejected,
2856
+ decisions: scopedRepair.plan["decisions"],
2857
+ });
2858
+ if (repairRejected.length > 0) {
2859
+ if (asList(scopedRepair.plan["decisions"]).filter(isDict).length > 0) {
2860
+ partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
2861
+ }
2862
+ const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
2863
+ const rejectedKeys = sourceKeysFromRejectedExpressions(repairRejected, repairSourceKeys);
2864
+ const nextRepairSourceKeys = [...(rejectedKeys.length > 0 ? rejectedKeys : []), ...missingKeys]
2865
+ .filter((key, index, all) => all.indexOf(key) === index);
2866
+ if (nextRepairSourceKeys.length === 0) {
2867
+ repairSourceKeys = [];
2868
+ repairRejectedContext = [];
2869
+ continue;
2870
+ }
2871
+ if (attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
2872
+ throw invalidCurationPart("INIT FAILED: Asset curation repair returned invalid expressions", repairRejected.map((item) => `${strOf(item["source_key"]) || "-"}: ${strOf(item["error"])}`).slice(0, 40));
2873
+ }
2874
+ repairSourceKeys = nextRepairSourceKeys;
2875
+ repairRejectedContext = repairRejected;
2876
+ continue;
2877
+ }
2878
+ const finalKeys = new Set([...curationDecisionSourceKeys(partPlan), ...curationDecisionSourceKeys(scopedRepair.plan)]);
2879
+ const unrepaired = repairSourceKeys.filter((key) => !finalKeys.has(key));
2880
+ if (unrepaired.length > 0 && attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
2881
+ throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", unrepaired.slice(0, 40));
2882
+ }
2883
+ partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
2884
+ repairRejectedContext = [];
2885
+ const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
2886
+ repairSourceKeys = [...unrepaired, ...missingKeys]
2887
+ .filter((key, index, all) => all.indexOf(key) === index);
2888
+ }
2889
+ if (repairSourceKeys.length > 0) {
2890
+ throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", repairSourceKeys.slice(0, 40));
2891
+ }
2892
+ }
2893
+ catch (exc) {
2894
+ writeFailedCurationPartArtifact({
2895
+ ctx,
2896
+ artifactName,
2897
+ fingerprint,
2898
+ metadata,
2899
+ candidateSourceKeys,
2900
+ expressionText,
2901
+ partPlan,
2902
+ repairAttempts,
2903
+ startedAtMs,
2904
+ error: exc,
2905
+ });
2906
+ throw exc;
2907
+ }
2908
+ for (const [key, value] of Object.entries(metadata))
2909
+ partPlan[key] = value;
2910
+ ctx.ws.writeJson(artifactName, {
2911
+ version: 1,
2912
+ ...assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION),
2913
+ script_fingerprint: fingerprint,
2914
+ ...metadata,
2915
+ parse_status: "ok",
2916
+ candidate_source_keys: candidateSourceKeys,
2917
+ expression_text: partPlan["expression_text"],
2918
+ raw_expression_text: partPlan["raw_expression_text"] ?? expressionText,
2919
+ accepted_expressions: partPlan["accepted_expressions"],
2920
+ rejected_expressions: partPlan["rejected_expressions"],
2921
+ repair_attempts: repairAttempts,
2922
+ duration_ms: durationMs(startedAtMs, Date.now()),
2923
+ plan: partPlan,
2924
+ updated_at: checkpointTimestamp(),
2925
+ });
2926
+ return partPlan;
2927
+ }
2928
+ async function loadOrRunPrimaryCurationPart(ctx, script, curateAssets, kind, partId, fingerprint, ids) {
2929
+ const selection = curationPartSelection(kind, ids);
2930
+ return loadOrRunCurationPart(ctx, script, `asset_curation_parts/primary.${partId}.json`, fingerprint, { asset_kind: kind, part_id: partId, selection }, curationSourceKeysForIds(kind, ids), "asset_curation.primary", true, (selectedSourceKeys = curationSourceKeysForIds(kind, ids)) => {
2931
+ const selectedIds = curationIdsFromSourceKeys(kind, ids, selectedSourceKeys);
2932
+ return buildAssetCurationContextText(script, [kind], curationPartSelection(kind, selectedIds.length > 0 ? selectedIds : ids));
2933
+ }, (contextText) => curateAssets(script, contextText));
2934
+ }
2935
+ function primaryCurationChunks(script, groupingPlan) {
2936
+ const chunks = [];
2937
+ for (const kind of ["actor", "location", "prop"]) {
2938
+ const groupingGroups = groupingPlan
2939
+ ? asList(groupingPlan["groups"]).filter((group) => isDict(group) && group["kind"] === kind)
2940
+ : [];
2941
+ const kindChunks = groupingGroups.length > 0
2942
+ ? groupedAssetCurationChunks(script, kind, groupingGroups)
2943
+ : buildPrimaryAssetCurationChunks(script, kind);
2944
+ for (let index = 0; index < kindChunks.length; index += 1) {
2945
+ chunks.push({
2946
+ kind,
2947
+ partId: `${kind}.${String(index + 1).padStart(3, "0")}`,
2948
+ ids: kindChunks[index],
2949
+ });
2950
+ }
2951
+ }
2952
+ return chunks;
2953
+ }
2954
+ async function loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, fingerprint) {
2955
+ const sourceChunks = chunkStrings(assetCurationAuditSourceKeys(script, primaryPlan), 24);
2956
+ if (sourceChunks.length === 0) {
2957
+ return parseAssetCurationExpressions("<exp>NOOP # no asset curation changes</exp>");
2958
+ }
2959
+ const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
2960
+ const partId = `audit.${String(index + 1).padStart(3, "0")}`;
2961
+ return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.audit", false, (repairSourceKeys = selectedSourceKeys) => buildAssetCurationAuditContextText(script, primaryPlan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
2962
+ });
2963
+ const parts = unwrapConcurrentOutcomes(outcomes);
2964
+ return combineAssetCurationExpressionPlans(parts);
2965
+ }
2966
+ async function loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, fingerprint) {
2967
+ const sourceChunks = chunkStrings(locationCanonicalizationCandidateSourceKeys(script, plan), 24);
2968
+ if (sourceChunks.length === 0) {
2969
+ return parseAssetCurationExpressions("<exp>NOOP # no location canonicalization changes</exp>");
2970
+ }
2971
+ const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
2972
+ const partId = `location_canonical.${String(index + 1).padStart(3, "0")}`;
2973
+ return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.location_canonical", false, (repairSourceKeys = selectedSourceKeys) => buildLocationCanonicalizationAuditContextText(script, plan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
2974
+ });
2975
+ const parts = unwrapConcurrentOutcomes(outcomes);
2976
+ return combineAssetCurationExpressionPlans(parts);
2977
+ }
2134
2978
  export async function stageAssetCuration(ctx) {
2135
2979
  const { ws, state, options } = ctx;
2136
2980
  const script = ensureScript(ctx);
2981
+ let groupingPlan = null;
2982
+ let groupingApplySummary = null;
2983
+ let plan = null;
2984
+ let primaryPlan = null;
2985
+ let auditPlan = null;
2986
+ let locationCanonicalPlan = null;
2987
+ let primaryExpressionText = null;
2988
+ let primaryPartExpressionTexts = [];
2989
+ let auditExpressionText = null;
2990
+ let locationCanonicalExpressionText = null;
2137
2991
  try {
2138
2992
  ctx.updateRunState({ status: "init_running", init_stage: "asset_curation" });
2139
2993
  ctx.emitProgress("asset-curation");
2140
- const curation = curateScriptAssets(script);
2994
+ const provider = ctx.provider();
2995
+ if (!provider.curateAssets) {
2996
+ throw new CliError("INIT FAILED: Provider does not support asset curation", "Provider does not support asset curation.", {
2997
+ exitCode: EXIT_RUNTIME,
2998
+ required: ["provider.curateAssets(script, contextText)"],
2999
+ received: [`provider: ${provider.name || options.providerName}`],
3000
+ nextSteps: ["Use a direct provider that supports asset curation."],
3001
+ });
3002
+ }
3003
+ if (!provider.groupAssets) {
3004
+ throw new CliError("INIT FAILED: Provider does not support asset grouping", "Provider does not support asset grouping.", {
3005
+ exitCode: EXIT_RUNTIME,
3006
+ required: ["provider.groupAssets(script, contextText)"],
3007
+ received: [`provider: ${provider.name || options.providerName}`],
3008
+ nextSteps: ["Use a direct provider that supports asset grouping."],
3009
+ });
3010
+ }
3011
+ const groupAssets = provider.groupAssets.bind(provider);
3012
+ const curateAssets = provider.curateAssets.bind(provider);
3013
+ const groupingFingerprint = assetGroupingScriptFingerprint(script, options);
3014
+ groupingPlan = await ctx.timing.span({ name: "asset_curation.grouping", category: "substage", parent: "asset_curation" }, () => loadOrRunAssetGrouping(ctx, script, groupAssets, groupingFingerprint));
3015
+ groupingApplySummary = await ctx.timing.span({ name: "asset_curation.grouping.apply", category: "substage", parent: "asset_curation.grouping" }, () => applyAssetGroupingDuplicateMerges(script, { decisions: groupingPlan["duplicate_merge_decisions"] }));
3016
+ Object.assign(groupingPlan, assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION));
3017
+ groupingPlan["apply_summary"] = groupingApplySummary;
3018
+ groupingPlan["script_fingerprint"] = groupingFingerprint;
3019
+ ws.writeJson("asset_grouping.json", groupingPlan);
3020
+ const fingerprint = assetCurationScriptFingerprint(script, options);
3021
+ const cachedCuration = ws.readJsonSafe("asset_curation.json");
3022
+ if (!state.bypassInternalCaches && isDict(cachedCuration) && cachedCuration["script_fingerprint"] === fingerprint && isList(cachedCuration["decisions"])) {
3023
+ const curation = curateScriptAssets(script, { decisions: cachedCuration["decisions"] });
3024
+ Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
3025
+ curation["script_fingerprint"] = fingerprint;
3026
+ curation["grouping"] = groupingPlan;
3027
+ curation["grouping_apply_summary"] = groupingApplySummary;
3028
+ if (typeof cachedCuration["expression_text"] === "string")
3029
+ curation["expression_text"] = cachedCuration["expression_text"];
3030
+ if (typeof cachedCuration["primary_expression_text"] === "string")
3031
+ curation["primary_expression_text"] = cachedCuration["primary_expression_text"];
3032
+ if (typeof cachedCuration["primary_part_expression_text"] === "string")
3033
+ curation["primary_part_expression_text"] = cachedCuration["primary_part_expression_text"];
3034
+ if (typeof cachedCuration["audit_expression_text"] === "string")
3035
+ curation["audit_expression_text"] = cachedCuration["audit_expression_text"];
3036
+ if (typeof cachedCuration["location_canonical_expression_text"] === "string")
3037
+ curation["location_canonical_expression_text"] = cachedCuration["location_canonical_expression_text"];
3038
+ if (isList(cachedCuration["expressions"]))
3039
+ curation["expressions"] = cachedCuration["expressions"];
3040
+ if (isList(cachedCuration["primary_expressions"]))
3041
+ curation["primary_expressions"] = cachedCuration["primary_expressions"];
3042
+ if (isList(cachedCuration["primary_parts"]))
3043
+ curation["primary_parts"] = cachedCuration["primary_parts"];
3044
+ if (isList(cachedCuration["audit_expressions"]))
3045
+ curation["audit_expressions"] = cachedCuration["audit_expressions"];
3046
+ if (isList(cachedCuration["location_canonical_expressions"]))
3047
+ curation["location_canonical_expressions"] = cachedCuration["location_canonical_expressions"];
3048
+ if (typeof cachedCuration["format"] === "string")
3049
+ curation["format"] = cachedCuration["format"];
3050
+ if (isDict(cachedCuration["audit_summary"]))
3051
+ curation["audit_summary"] = cachedCuration["audit_summary"];
3052
+ ws.writeJson("asset_curation.json", curation);
3053
+ ws.delete("asset_curation.failed.json");
3054
+ ctx.emitProgress("asset-curation", "success");
3055
+ return;
3056
+ }
3057
+ const primaryChunkSpecs = primaryCurationChunks(script, groupingPlan);
3058
+ const primaryOutcomes = await pMapWithConcurrency(primaryChunkSpecs, options.concurrency, async (chunk) => loadOrRunPrimaryCurationPart(ctx, script, curateAssets, chunk.kind, chunk.partId, fingerprint, chunk.ids));
3059
+ const primaryParts = unwrapConcurrentOutcomes(primaryOutcomes);
3060
+ primaryPartExpressionTexts = primaryParts.map((part) => ({
3061
+ asset_kind: part["asset_kind"],
3062
+ part_id: part["part_id"],
3063
+ expression_text: part["expression_text"],
3064
+ }));
3065
+ primaryPlan = await ctx.timing.span({ name: "asset_curation.primary.combine", category: "substage", parent: "asset_curation.primary", metadata: { chunks: primaryParts.length } }, () => combinePrimaryAssetCurationPlans(primaryParts));
3066
+ primaryExpressionText = typeof primaryPlan["expression_text"] === "string" ? primaryPlan["expression_text"] : "";
3067
+ if (!isDict(primaryPlan) || !isList(primaryPlan["decisions"])) {
3068
+ throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
3069
+ exitCode: EXIT_RUNTIME,
3070
+ required: ["object with decisions[]"],
3071
+ received: [typeof primaryPlan],
3072
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
3073
+ });
3074
+ }
3075
+ const allowedNewAuditSources = [
3076
+ ...missingAssetCurationRequiredSourceKeys(script, primaryPlan),
3077
+ ...assetCurationAuditCandidateSourceKeys(script, primaryPlan),
3078
+ ];
3079
+ auditPlan = await ctx.timing.span({ name: "asset_curation.audit", category: "substage", parent: "asset_curation" }, () => loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, assetCurationPlanFingerprint(`${fingerprint}:audit-chunk-v1`, primaryPlan)));
3080
+ auditExpressionText = typeof auditPlan["expression_text"] === "string" ? auditPlan["expression_text"] : "";
3081
+ if (!isDict(auditPlan) || !isList(auditPlan["decisions"])) {
3082
+ throw new CliError("INIT FAILED: Provider returned invalid asset curation audit", "Provider returned invalid asset curation audit.", {
3083
+ exitCode: EXIT_RUNTIME,
3084
+ required: ["object with decisions[]"],
3085
+ received: [typeof auditPlan],
3086
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation audit will retry."],
3087
+ });
3088
+ }
3089
+ plan = combineAssetCurationPlans(primaryPlan, auditPlan, allowedNewAuditSources);
3090
+ const allowedNewLocationCanonicalSources = locationCanonicalizationCandidateSourceKeys(script, plan);
3091
+ locationCanonicalPlan = await ctx.timing.span({ name: "asset_curation.location_canonical", category: "substage", parent: "asset_curation" }, () => loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, assetCurationPlanFingerprint(`${fingerprint}:location-canonical-chunk-v2`, plan)));
3092
+ locationCanonicalExpressionText = typeof locationCanonicalPlan["expression_text"] === "string" ? locationCanonicalPlan["expression_text"] : "";
3093
+ if (!isDict(locationCanonicalPlan) || !isList(locationCanonicalPlan["decisions"])) {
3094
+ throw new CliError("INIT FAILED: Provider returned invalid location canonicalization audit", "Provider returned invalid location canonicalization audit.", {
3095
+ exitCode: EXIT_RUNTIME,
3096
+ required: ["object with decisions[]"],
3097
+ received: [typeof locationCanonicalPlan],
3098
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and location canonicalization will retry."],
3099
+ });
3100
+ }
3101
+ assertLocationCanonicalizationPlan(locationCanonicalPlan);
3102
+ plan = combineLocationCanonicalizationPlan(script, plan, locationCanonicalPlan, allowedNewLocationCanonicalSources);
3103
+ const curation = await ctx.timing.span({ name: "asset_curation.apply", category: "substage", parent: "asset_curation" }, () => curateScriptAssets(script, plan));
3104
+ Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
3105
+ curation["script_fingerprint"] = fingerprint;
3106
+ curation["format"] = plan["format"];
3107
+ curation["expression_text"] = plan["expression_text"];
3108
+ curation["primary_expression_text"] = plan["primary_expression_text"];
3109
+ curation["audit_expression_text"] = plan["audit_expression_text"];
3110
+ curation["location_canonical_expression_text"] = plan["location_canonical_expression_text"];
3111
+ curation["primary_part_expression_text"] = plan["primary_part_expression_text"];
3112
+ curation["primary_parts"] = plan["primary_parts"];
3113
+ curation["expressions"] = plan["expressions"];
3114
+ curation["primary_expressions"] = plan["primary_expressions"];
3115
+ curation["audit_expressions"] = plan["audit_expressions"];
3116
+ curation["location_canonical_expressions"] = plan["location_canonical_expressions"];
3117
+ curation["audit_summary"] = plan["audit_summary"];
3118
+ curation["grouping"] = groupingPlan;
3119
+ curation["grouping_apply_summary"] = groupingApplySummary;
2141
3120
  ws.writeJson("asset_curation.json", curation);
3121
+ ws.delete("asset_curation.failed.json");
2142
3122
  }
2143
3123
  catch (exc) {
3124
+ if (plan || primaryPlan || auditPlan || locationCanonicalPlan || primaryExpressionText !== null || auditExpressionText !== null || locationCanonicalExpressionText !== null) {
3125
+ const e = exc;
3126
+ ws.writeJson("asset_curation.failed.json", {
3127
+ version: 3,
3128
+ failed_at: checkpointTimestamp(),
3129
+ error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
3130
+ expression_text: plan?.["expression_text"] ?? null,
3131
+ primary_expression_text: primaryExpressionText,
3132
+ primary_part_expression_texts: primaryPartExpressionTexts,
3133
+ audit_expression_text: auditExpressionText,
3134
+ location_canonical_expression_text: locationCanonicalExpressionText,
3135
+ primary_plan: primaryPlan,
3136
+ audit_plan: auditPlan,
3137
+ location_canonical_plan: locationCanonicalPlan,
3138
+ grouping_plan: groupingPlan,
3139
+ grouping_apply_summary: groupingApplySummary,
3140
+ plan,
3141
+ });
3142
+ }
2144
3143
  stageFailure(options.workspace, exc, {
2145
3144
  title: "INIT FAILED: Asset curation failed",
2146
3145
  stage: "asset_curation",
@@ -2151,28 +3150,198 @@ export async function stageAssetCuration(ctx) {
2151
3150
  }
2152
3151
  ctx.emitProgress("asset-curation", "success");
2153
3152
  }
2154
- // --- metadata -------------------------------------------------------------
2155
- export async function stageMetadata(ctx) {
2156
- const { ws, state, options } = ctx;
3153
+ const STATE_BINDING_CHUNK_CACHE_VERSION = 1;
3154
+ function stateBindingChunkFingerprint(options, context) {
3155
+ return sha256Text(JSON.stringify(sortDeep({
3156
+ cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
3157
+ provider: options.providerName,
3158
+ model: options.model,
3159
+ context,
3160
+ })));
3161
+ }
3162
+ async function runStateBindingForScript(ctx, script, progress) {
3163
+ const { ws, options } = ctx;
3164
+ const provider = ctx.provider();
3165
+ if (!provider.bindStates) {
3166
+ throw new CliError("INIT FAILED: Provider does not support state binding", "Provider does not support state binding.", {
3167
+ exitCode: EXIT_RUNTIME,
3168
+ required: ["provider.bindStates(script, context)"],
3169
+ received: [`provider: ${provider.name || options.providerName}`],
3170
+ nextSteps: ["Use a direct provider that supports state binding."],
3171
+ });
3172
+ }
3173
+ const defaultSummary = await ctx.timing.span({ name: "state_binding.default_states", category: "substage", parent: "state_binding" }, () => ensureDefaultStates(script));
3174
+ const defaultBindingSummary = await ctx.timing.span({ name: "state_binding.default_refs", category: "substage", parent: "state_binding" }, () => bindDefaultStateRefsInScript(script));
3175
+ const chunks = buildStateBindingContexts(script, 8);
3176
+ ws.mkdir("state_binding_chunks");
3177
+ if (progress) {
3178
+ ctx.updateRunState({ state_binding_completed_chunks: 0, state_binding_total_chunks: chunks.length });
3179
+ }
3180
+ let chunkPlans = [];
3181
+ const bypassChunkCache = ctx.state.bypassInternalCaches;
3182
+ try {
3183
+ const outcomes = await pMapWithConcurrency(chunks, options.concurrency, async (context, index) => {
3184
+ const chunkArtifact = `state_binding_chunks/chunk_${pad3(index + 1)}.json`;
3185
+ const contextFingerprint = stateBindingChunkFingerprint(options, context);
3186
+ const cachedChunk = ws.readJsonSafe(chunkArtifact);
3187
+ const cachedPlanRaw = isDict(cachedChunk) ? cachedChunk["plan"] : null;
3188
+ const cachedPlan = !bypassChunkCache
3189
+ && isDict(cachedChunk)
3190
+ && cachedChunk["context_fingerprint"] === contextFingerprint
3191
+ && isDict(cachedPlanRaw)
3192
+ ? cachedPlanRaw
3193
+ : null;
3194
+ const startedAtMs = Date.now();
3195
+ const rawPlan = normalizeStateBindingPlanShape(cachedPlan ?? await ctx.timing.span({ name: `state_binding.chunk.${pad3(index + 1)}`, category: "chunk", parent: "state_binding.chunks", metadata: { index } }, () => provider.bindStates(script, context)));
3196
+ if (!isDict(rawPlan) || !isList(rawPlan["scenes"])) {
3197
+ throw new CliError("INIT FAILED: Provider returned invalid state binding", "Provider returned invalid state binding.", {
3198
+ exitCode: EXIT_RUNTIME,
3199
+ required: ["object with scenes[]"],
3200
+ received: [typeof rawPlan],
3201
+ nextSteps: ["Rerun init; completed extraction checkpoints will be reused and state binding will retry."],
3202
+ });
3203
+ }
3204
+ if (!cachedPlan) {
3205
+ ws.writeJson(chunkArtifact, {
3206
+ version: 1,
3207
+ cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
3208
+ index,
3209
+ provider: options.providerName,
3210
+ model: options.model,
3211
+ context_fingerprint: contextFingerprint,
3212
+ scene_ids: asList(context["scenes"]).map((scene) => strOf(scene["scene_id"])),
3213
+ duration_ms: durationMs(startedAtMs, Date.now()),
3214
+ plan: rawPlan,
3215
+ updated_at: checkpointTimestamp(),
3216
+ });
3217
+ }
3218
+ if (progress) {
3219
+ ctx.updateRunState({ state_binding_completed_chunks: index + 1, state_binding_total_chunks: chunks.length });
3220
+ }
3221
+ return { index, context, plan: rawPlan, contextFingerprint, reused: Boolean(cachedPlan) };
3222
+ });
3223
+ chunkPlans = unwrapConcurrentOutcomes(outcomes).sort((left, right) => left.index - right.index);
3224
+ }
3225
+ catch (exc) {
3226
+ const e = exc;
3227
+ ws.writeJson("state_binding.failed.json", {
3228
+ version: 1,
3229
+ failed_at: checkpointTimestamp(),
3230
+ chunks_total: chunks.length,
3231
+ error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
3232
+ completed_chunks: [],
3233
+ });
3234
+ throw exc;
3235
+ }
3236
+ const chunkReports = [];
3237
+ let sceneRefsBound = 0;
3238
+ let stateChangesApplied = 0;
3239
+ let invalidSceneRefsSkipped = 0;
3240
+ let invalidStateChangesSkipped = 0;
3241
+ try {
3242
+ await ctx.timing.span({ name: "state_binding.apply", category: "substage", parent: "state_binding" }, () => {
3243
+ for (const item of chunkPlans) {
3244
+ const applied = applyStateBindingPlan(script, item.plan, { invalidStateMode: "skip" });
3245
+ sceneRefsBound += Number(applied["scene_refs_bound"] ?? 0);
3246
+ stateChangesApplied += Number(applied["state_changes_applied"] ?? 0);
3247
+ invalidSceneRefsSkipped += Number(applied["invalid_scene_refs_skipped"] ?? 0);
3248
+ invalidStateChangesSkipped += Number(applied["invalid_state_changes_skipped"] ?? 0);
3249
+ chunkReports.push({
3250
+ index: item.index,
3251
+ reused: item.reused,
3252
+ context_fingerprint: item.contextFingerprint,
3253
+ scene_count: asList(item.context["scenes"]).length,
3254
+ scene_refs_bound: applied["scene_refs_bound"],
3255
+ state_changes_applied: applied["state_changes_applied"],
3256
+ invalid_scene_refs_skipped: applied["invalid_scene_refs_skipped"],
3257
+ invalid_state_changes_skipped: applied["invalid_state_changes_skipped"],
3258
+ });
3259
+ }
3260
+ });
3261
+ }
3262
+ catch (exc) {
3263
+ const e = exc;
3264
+ ws.writeJson("state_binding.failed.json", {
3265
+ version: 1,
3266
+ failed_at: checkpointTimestamp(),
3267
+ chunks_total: chunks.length,
3268
+ error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
3269
+ completed_chunks: chunkReports,
3270
+ });
3271
+ throw exc;
3272
+ }
3273
+ const summary = {
3274
+ ...defaultSummary,
3275
+ ...defaultBindingSummary,
3276
+ llm_scene_refs_bound: sceneRefsBound,
3277
+ llm_state_changes_applied: stateChangesApplied,
3278
+ invalid_scene_refs_skipped: invalidSceneRefsSkipped,
3279
+ invalid_state_changes_skipped: invalidStateChangesSkipped,
3280
+ chunks: chunks.length,
3281
+ cached_chunks: chunkPlans.filter((chunk) => chunk.reused).length,
3282
+ llm_scene_count: chunks.reduce((n, chunk) => n + asList(chunk["scenes"]).length, 0),
3283
+ };
3284
+ ws.writeJson("state_binding.json", {
3285
+ version: 1,
3286
+ summary,
3287
+ chunks: chunkReports,
3288
+ });
3289
+ ws.delete("state_binding.failed.json");
3290
+ return { summary, chunks: chunkReports };
3291
+ }
3292
+ export async function stageStateBinding(ctx) {
3293
+ const { state, options } = ctx;
2157
3294
  const script = ensureScript(ctx);
3295
+ try {
3296
+ ctx.updateRunState({ status: "init_running", init_stage: "state_binding" });
3297
+ ctx.emitProgress("state-binding");
3298
+ await runStateBindingForScript(ctx, script, true);
3299
+ }
3300
+ catch (exc) {
3301
+ stageFailure(options.workspace, exc, {
3302
+ title: "INIT FAILED: State binding failed",
3303
+ stage: "state_binding",
3304
+ required: ["provider state binding decisions for a well-formed assembled script"],
3305
+ nextSteps: ["Rerun init; completed extraction and curation checkpoints will be reused."],
3306
+ updates: { episode_completed: state.mergedResults.length },
3307
+ });
3308
+ }
3309
+ ctx.emitProgress("state-binding", "success");
3310
+ }
3311
+ async function runMetadataForScript(ctx, script, applyToScript, progress) {
3312
+ const { ws, options } = ctx;
2158
3313
  const sourceText = ensureSourceText(ctx);
2159
3314
  const assetMetadataPath = ws.pathOf("asset_metadata.json");
2160
3315
  if (options.skipMetadata) {
2161
- ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: true });
3316
+ if (progress)
3317
+ ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: true });
2162
3318
  if (exists(assetMetadataPath))
2163
3319
  deletePath(assetMetadataPath);
2164
- ctx.emitProgress("metadata", "skipped");
2165
- return;
3320
+ return { metadata: {}, skipped: true };
2166
3321
  }
2167
3322
  const provider = ctx.provider();
3323
+ let metadata = provider.extractMetadata
3324
+ ? await ctx.timing.span({ name: "metadata.extract", category: "substage", parent: "metadata_extract" }, () => provider.extractMetadata(sourceText, script))
3325
+ : {};
3326
+ if (!isDict(metadata))
3327
+ metadata = {};
3328
+ ws.writeJson("asset_metadata.json", metadata);
3329
+ if (applyToScript) {
3330
+ await ctx.timing.span({ name: "metadata.apply", category: "substage", parent: "metadata_extract" }, () => applyMetadataToScript(script, metadata));
3331
+ }
3332
+ return { metadata, skipped: false };
3333
+ }
3334
+ export async function stageMetadata(ctx) {
3335
+ const { state, options } = ctx;
3336
+ const script = ensureScript(ctx);
2168
3337
  try {
2169
3338
  ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: false });
2170
3339
  ctx.emitProgress("metadata");
2171
- let metadata = provider.extractMetadata ? await provider.extractMetadata(sourceText, script) : {};
2172
- if (!isDict(metadata))
2173
- metadata = {};
2174
- ws.writeJson("asset_metadata.json", metadata);
2175
- applyMetadataToScript(script, metadata);
3340
+ const result = await runMetadataForScript(ctx, script, true, true);
3341
+ if (result.skipped) {
3342
+ ctx.emitProgress("metadata", "skipped");
3343
+ return;
3344
+ }
2176
3345
  }
2177
3346
  catch (exc) {
2178
3347
  if (exc instanceof CliError) {
@@ -2197,16 +3366,21 @@ export async function stageMetadata(ctx) {
2197
3366
  }
2198
3367
  ctx.emitProgress("metadata", "success");
2199
3368
  }
2200
- // --- script-synopsis ------------------------------------------------------
2201
- export async function stageScriptSynopsis(ctx) {
2202
- const { ws, state, options } = ctx;
2203
- const script = ensureScript(ctx);
3369
+ function overviewFieldsFromScript(script) {
3370
+ return {
3371
+ synopsis: script["synopsis"],
3372
+ theme: script["theme"],
3373
+ logline: script["logline"],
3374
+ style: script["style"],
3375
+ main_characters: script["main_characters"],
3376
+ };
3377
+ }
3378
+ async function runScriptSynopsisForScript(ctx, script, persistScript) {
3379
+ const { ws, options } = ctx;
2204
3380
  const provider = ctx.provider();
2205
- let summaryFullDegraded = false;
2206
- let summaryDegradeReason = "";
3381
+ let degraded = false;
3382
+ let reason = "";
2207
3383
  try {
2208
- ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
2209
- ctx.emitProgress("script-synopsis");
2210
3384
  const summary = await buildScriptSynopsis(provider, script, {
2211
3385
  dd: ws.dir,
2212
3386
  thresholdChars: options.summaryThresholdChars,
@@ -2215,25 +3389,32 @@ export async function stageScriptSynopsis(ctx) {
2215
3389
  skipSummary: options.skipSummary,
2216
3390
  concurrency: options.concurrency,
2217
3391
  });
2218
- summaryFullDegraded = summary.degraded;
2219
- summaryDegradeReason = summary.reason ?? "";
3392
+ degraded = summary.degraded;
3393
+ reason = summary.reason ?? "";
2220
3394
  }
2221
3395
  catch (exc) {
2222
- summaryFullDegraded = true;
2223
- summaryDegradeReason = providerErrorSummary(exc);
3396
+ degraded = true;
3397
+ reason = providerErrorSummary(exc);
2224
3398
  }
2225
- state.summaryFullDegraded = summaryFullDegraded;
2226
- state.summaryDegradeReason = summaryDegradeReason;
3399
+ if (persistScript)
3400
+ ws.writeJson("script.initial.json", script);
3401
+ return { degraded, reason, fields: overviewFieldsFromScript(script) };
3402
+ }
3403
+ export async function stageScriptSynopsis(ctx) {
3404
+ const { ws, state, options } = ctx;
3405
+ const script = ensureScript(ctx);
3406
+ ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
3407
+ ctx.emitProgress("script-synopsis");
3408
+ const summary = await runScriptSynopsisForScript(ctx, script, true);
3409
+ state.summaryFullDegraded = summary.degraded;
3410
+ state.summaryDegradeReason = summary.reason;
2227
3411
  state.summaryDegradedEpisodes.sort((a, b) => a - b);
2228
- const summaryDegraded = summaryFullDegraded || state.summaryDegradedEpisodes.length > 0;
2229
- // Persist script.initial.json now that the overview fields are assigned — the
2230
- // same write point as the god-command (right after the synopsis stage).
2231
- ws.writeJson("script.initial.json", script);
3412
+ const summaryDegraded = summary.degraded || state.summaryDegradedEpisodes.length > 0;
2232
3413
  ctx.updateRunState({
2233
3414
  summary_skipped: options.skipSummary,
2234
3415
  summary_degraded: summaryDegraded,
2235
3416
  summary_degraded_episodes: state.summaryDegradedEpisodes,
2236
- summary_degraded_reason: summaryDegradeReason || null,
3417
+ summary_degraded_reason: summary.reason || null,
2237
3418
  });
2238
3419
  ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
2239
3420
  }
@@ -2502,6 +3683,40 @@ function renderAssetCurationSummary(curation) {
2502
3683
  `locations ${summary["locations_before"] ?? 0}->${summary["locations_after"] ?? 0} ` +
2503
3684
  `(merged ${summary["locations_merged"] ?? 0})`,
2504
3685
  ];
3686
+ const coverage = isDict(summary["coverage"]) ? summary["coverage"] : {};
3687
+ const required = isDict(coverage["required"]) ? coverage["required"] : {};
3688
+ const auditSummary = isDict(curation["audit_summary"]) ? curation["audit_summary"] : {};
3689
+ if (summary["explicit_keep_count"] !== undefined || auditSummary["audit_decisions"] !== undefined || required["prop"] !== undefined) {
3690
+ lines.push(`curation coverage: ` +
3691
+ `keep ${summary["explicit_keep_count"] ?? 0}, ` +
3692
+ `implicit ${summary["implicit_keep_count"] ?? 0}, ` +
3693
+ `audit ${auditSummary["audit_decisions"] ?? 0}, ` +
3694
+ `required actor/location/prop ${required["actor"] ?? 0}/${required["location"] ?? 0}/${required["prop"] ?? 0}`);
3695
+ }
3696
+ for (const decision of asList(curation["decisions"])) {
3697
+ if (!isDict(decision))
3698
+ continue;
3699
+ const action = strOf(decision["decision"]);
3700
+ if (!action || action === "keep")
3701
+ continue;
3702
+ const kind = strOf(decision["kind"]);
3703
+ const sourceId = strOf(decision["source_id"]);
3704
+ const targetKind = strOf(decision["target_kind"]);
3705
+ const targetId = strOf(decision["target_id"]);
3706
+ const newName = strOf(decision["new_name"]);
3707
+ const stateName = strOf(decision["state_name"]);
3708
+ const speakerKind = strOf(decision["speaker_kind"]);
3709
+ const target = targetId
3710
+ ? ` -> ${targetKind || kind}:${targetId}`
3711
+ : newName
3712
+ ? ` -> ${newName}`
3713
+ : stateName
3714
+ ? ` -> state:${stateName}`
3715
+ : speakerKind
3716
+ ? ` -> speaker:${speakerKind}`
3717
+ : "";
3718
+ lines.push(`curation ${kind}:${sourceId} ${action}${target} — ${decision["reason"] || "-"}`);
3719
+ }
2505
3720
  for (const actor of asList(curation["actors"])) {
2506
3721
  if (!isDict(actor) || actor["decision"] !== "remove")
2507
3722
  continue;
@@ -2519,6 +3734,18 @@ function renderAssetCurationSummary(curation) {
2519
3734
  }
2520
3735
  return lines;
2521
3736
  }
3737
+ function renderStateBindingSummary(binding) {
3738
+ const summary = isDict(binding["summary"]) ? binding["summary"] : {};
3739
+ return [
3740
+ `state-binding: ` +
3741
+ `default-states +${summary["default_states_added"] ?? 0}, ` +
3742
+ `llm refs ${summary["llm_scene_refs_bound"] ?? 0}, ` +
3743
+ `default refs ${summary["default_or_unique_refs_bound"] ?? 0}, ` +
3744
+ `unresolved ${summary["unresolved_stateful_refs"] ?? 0}, ` +
3745
+ `state_changes ${summary["llm_state_changes_applied"] ?? 0}, ` +
3746
+ `chunks ${summary["chunks"] ?? 0}`,
3747
+ ];
3748
+ }
2522
3749
  function renderAssetNormalizationSummary(normalization) {
2523
3750
  const summary = isDict(normalization["summary"]) ? normalization["summary"] : {};
2524
3751
  const lines = [
@@ -2631,6 +3858,7 @@ export function commandReview(opts) {
2631
3858
  // (validation issues recomputed live, plus batch extraction failures).
2632
3859
  const lines = [];
2633
3860
  const curationPath = path.join(dd, "asset_curation.json");
3861
+ const stateBindingPath = path.join(dd, "state_binding.json");
2634
3862
  const normalizationPath = path.join(dd, "asset_normalization.json");
2635
3863
  if (exists(normalizationPath)) {
2636
3864
  const normalization = readJson(normalizationPath);
@@ -2642,6 +3870,11 @@ export function commandReview(opts) {
2642
3870
  if (isDict(curation))
2643
3871
  lines.push(...renderAssetCurationSummary(curation));
2644
3872
  }
3873
+ if (exists(stateBindingPath)) {
3874
+ const binding = readJson(stateBindingPath);
3875
+ if (isDict(binding))
3876
+ lines.push(...renderStateBindingSummary(binding));
3877
+ }
2645
3878
  const batchFailures = collectBatchFailures(batchResultsDir);
2646
3879
  for (const errors of batchFailures.values()) {
2647
3880
  for (const error of errors) {
@@ -2665,7 +3898,7 @@ export function commandReview(opts) {
2665
3898
  const report = {
2666
3899
  title: "DIRECT REVIEW",
2667
3900
  result: lines.length > 0 ? lines : ["No issues, curation decisions, or batch failures to review."],
2668
- artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_curation.json"), batchResultsDir],
3901
+ artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_grouping.json"), path.join(dd, "asset_curation.json"), stateBindingPath, batchResultsDir],
2669
3902
  next: [
2670
3903
  "Run `direct review --episode <n>` for source↔extract alignment.",
2671
3904
  "Edit with `--draft` (e.g. `scriptctl rename … --draft`), then validate/export.",