@lingjingai/scriptctl 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changes/0.32.0.md +30 -0
- package/changes/0.33.0.md +17 -0
- package/dist/common.d.ts +1 -0
- package/dist/common.js +70 -0
- package/dist/common.js.map +1 -1
- package/dist/domain/direct/runner.d.ts +3 -1
- package/dist/domain/direct/runner.js +20 -4
- package/dist/domain/direct/runner.js.map +1 -1
- package/dist/domain/direct/stage.d.ts +23 -1
- package/dist/domain/direct/stage.js +3 -0
- package/dist/domain/direct/stage.js.map +1 -1
- package/dist/domain/direct/stages/asset-curation.js +3 -2
- package/dist/domain/direct/stages/asset-curation.js.map +1 -1
- package/dist/domain/direct/stages/index.d.ts +3 -1
- package/dist/domain/direct/stages/index.js +5 -1
- package/dist/domain/direct/stages/index.js.map +1 -1
- package/dist/domain/direct/stages/state-binding.d.ts +2 -0
- package/dist/domain/direct/stages/state-binding.js +9 -0
- package/dist/domain/direct/stages/state-binding.js.map +1 -0
- package/dist/domain/direct/stages/state-curation.d.ts +2 -0
- package/dist/domain/direct/stages/state-curation.js +10 -0
- package/dist/domain/direct/stages/state-curation.js.map +1 -0
- package/dist/domain/direct-core.d.ts +78 -0
- package/dist/domain/direct-core.js +4252 -239
- package/dist/domain/direct-core.js.map +1 -1
- package/dist/help-text.js +60 -1
- package/dist/help-text.js.map +1 -1
- package/dist/infra/providers.d.ts +12 -0
- package/dist/infra/providers.js +590 -3
- package/dist/infra/providers.js.map +1 -1
- package/dist/usecases/direct.d.ts +2 -0
- package/dist/usecases/direct.js +1533 -53
- package/dist/usecases/direct.js.map +1 -1
- package/package.json +1 -1
package/dist/usecases/direct.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_DIRECT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
|
|
4
|
-
import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
4
|
+
import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, applyAssetGroupingDuplicateMerges, applyStateCurationPlan, applyStateBindingPlan, bindDefaultStateRefsInScript, buildAssetCurationAuditContextText, buildAssetCurationContextText, buildAssetCurationRepairContextText, buildAssetGroupingContextText, buildLocationCanonicalizationAuditContextText, buildPrimaryAssetCurationChunks, buildStateCurationChunks, buildStateCurationContextText, groupedAssetCurationChunks, combineLocationCanonicalizationPlan, buildStateBindingContexts, assetCurationAuditSourceKeys, combineAssetCurationExpressionPlans, combinePrimaryAssetCurationPlans, combineAssetCurationPlans, curateScriptAssets, ensureDefaultStates, assetCurationAuditCandidateSourceKeys, locationCanonicalizationCandidateSourceKeys, missingAssetCurationRequiredSourceKeys, normalizeStateBindingPlanShape, parseStateCurationExpressions, parseAssetCurationExpressions, parseAssetCurationExpressionsTolerant, parseAssetGroupingExpressions, assertLocationCanonicalizationPlan, applyMetadataToScript, validateAssetGroupingPlan, stateCurationSourceKeysForRefs, assertStateCurationPlan, } from "../domain/direct-core.js";
|
|
5
5
|
import { validateScript } from "../domain/script/validate.js";
|
|
6
6
|
import { makeProvider, providerCapabilities } from "../infra/providers.js";
|
|
7
7
|
import { remoteScriptOutputStoreFromEnv } from "../infra/script-output-api.js";
|
|
@@ -118,6 +118,152 @@ export function readRunState(workspace) {
|
|
|
118
118
|
return {};
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
+
function isoFromMs(ms) {
|
|
122
|
+
return new Date(ms).toISOString();
|
|
123
|
+
}
|
|
124
|
+
function durationMs(startedAtMs, finishedAtMs) {
|
|
125
|
+
return Math.max(0, Math.round(finishedAtMs - startedAtMs));
|
|
126
|
+
}
|
|
127
|
+
function timingErrorSummary(error) {
|
|
128
|
+
const e = error;
|
|
129
|
+
return `${e?.name ?? "Error"}: ${e?.message ?? ""}`.slice(0, 240);
|
|
130
|
+
}
|
|
131
|
+
function timingSummary(events) {
|
|
132
|
+
const stages = {};
|
|
133
|
+
const branches = {};
|
|
134
|
+
const slowChunks = events
|
|
135
|
+
.filter((event) => event.category === "chunk")
|
|
136
|
+
.sort((left, right) => right.duration_ms - left.duration_ms)
|
|
137
|
+
.slice(0, 20)
|
|
138
|
+
.map((event) => ({
|
|
139
|
+
name: event.name,
|
|
140
|
+
parent: event.parent ?? "",
|
|
141
|
+
duration_ms: event.duration_ms,
|
|
142
|
+
status: event.status,
|
|
143
|
+
}));
|
|
144
|
+
for (const event of events) {
|
|
145
|
+
if (event.category === "stage")
|
|
146
|
+
stages[event.name] = event.duration_ms;
|
|
147
|
+
if (event.category === "branch")
|
|
148
|
+
branches[event.name] = event.duration_ms;
|
|
149
|
+
}
|
|
150
|
+
return { stages, branches, slow_chunks: slowChunks };
|
|
151
|
+
}
|
|
152
|
+
class JsonTimingRecorder {
|
|
153
|
+
ws;
|
|
154
|
+
workspace;
|
|
155
|
+
command = "";
|
|
156
|
+
events = [];
|
|
157
|
+
activeSpans = [];
|
|
158
|
+
counter = 0;
|
|
159
|
+
constructor(ws, workspace) {
|
|
160
|
+
this.ws = ws;
|
|
161
|
+
this.workspace = workspace;
|
|
162
|
+
}
|
|
163
|
+
reset(command) {
|
|
164
|
+
this.command = command;
|
|
165
|
+
this.events = [];
|
|
166
|
+
this.activeSpans = [];
|
|
167
|
+
this.counter = 0;
|
|
168
|
+
this.flush();
|
|
169
|
+
updateRunState(this.workspace, {
|
|
170
|
+
timing_current: null,
|
|
171
|
+
timing_current_stage: null,
|
|
172
|
+
timing_current_branch: null,
|
|
173
|
+
timing_summary: { stages: {}, branches: {}, slow_chunks: [] },
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
async span(options, run) {
|
|
177
|
+
const startedAtMs = Date.now();
|
|
178
|
+
const active = {
|
|
179
|
+
name: options.name,
|
|
180
|
+
category: options.category,
|
|
181
|
+
...(options.parent ? { parent: options.parent } : {}),
|
|
182
|
+
started_at: isoFromMs(startedAtMs),
|
|
183
|
+
};
|
|
184
|
+
this.activeSpans.push(active);
|
|
185
|
+
this.updateTimingCurrent();
|
|
186
|
+
try {
|
|
187
|
+
const value = await run();
|
|
188
|
+
this.removeActiveSpan(active);
|
|
189
|
+
this.record({
|
|
190
|
+
...options,
|
|
191
|
+
status: "success",
|
|
192
|
+
startedAtMs,
|
|
193
|
+
finishedAtMs: Date.now(),
|
|
194
|
+
});
|
|
195
|
+
return value;
|
|
196
|
+
}
|
|
197
|
+
catch (error) {
|
|
198
|
+
this.removeActiveSpan(active);
|
|
199
|
+
this.record({
|
|
200
|
+
...options,
|
|
201
|
+
status: "error",
|
|
202
|
+
startedAtMs,
|
|
203
|
+
finishedAtMs: Date.now(),
|
|
204
|
+
error: timingErrorSummary(error),
|
|
205
|
+
});
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
removeActiveSpan(active) {
|
|
210
|
+
const index = this.activeSpans.lastIndexOf(active);
|
|
211
|
+
if (index >= 0)
|
|
212
|
+
this.activeSpans.splice(index, 1);
|
|
213
|
+
}
|
|
214
|
+
activeSpanByCategory(category) {
|
|
215
|
+
for (let index = this.activeSpans.length - 1; index >= 0; index -= 1) {
|
|
216
|
+
const span = this.activeSpans[index];
|
|
217
|
+
if (span.category === category)
|
|
218
|
+
return span;
|
|
219
|
+
}
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
currentSpanPayload(span) {
|
|
223
|
+
if (!span)
|
|
224
|
+
return null;
|
|
225
|
+
return {
|
|
226
|
+
name: span.name,
|
|
227
|
+
category: span.category,
|
|
228
|
+
parent: span.parent ?? null,
|
|
229
|
+
started_at: span.started_at,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
updateTimingCurrent() {
|
|
233
|
+
updateRunState(this.workspace, {
|
|
234
|
+
timing_current: this.currentSpanPayload(this.activeSpans[this.activeSpans.length - 1] ?? null),
|
|
235
|
+
timing_current_stage: this.currentSpanPayload(this.activeSpanByCategory("stage")),
|
|
236
|
+
timing_current_branch: this.currentSpanPayload(this.activeSpanByCategory("branch")),
|
|
237
|
+
timing_summary: timingSummary(this.events),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
record(options) {
|
|
241
|
+
const event = {
|
|
242
|
+
id: `tim_${String(++this.counter).padStart(5, "0")}`,
|
|
243
|
+
name: options.name,
|
|
244
|
+
category: options.category,
|
|
245
|
+
...(options.parent ? { parent: options.parent } : {}),
|
|
246
|
+
status: options.status,
|
|
247
|
+
started_at: isoFromMs(options.startedAtMs),
|
|
248
|
+
finished_at: isoFromMs(options.finishedAtMs),
|
|
249
|
+
duration_ms: durationMs(options.startedAtMs, options.finishedAtMs),
|
|
250
|
+
...(options.metadata ? { metadata: options.metadata } : {}),
|
|
251
|
+
...(options.error ? { error: options.error } : {}),
|
|
252
|
+
};
|
|
253
|
+
this.events.push(event);
|
|
254
|
+
this.flush();
|
|
255
|
+
this.updateTimingCurrent();
|
|
256
|
+
}
|
|
257
|
+
flush() {
|
|
258
|
+
this.ws.writeJson("timings.json", {
|
|
259
|
+
version: 1,
|
|
260
|
+
command: this.command,
|
|
261
|
+
updated_at: checkpointTimestamp(),
|
|
262
|
+
events: this.events,
|
|
263
|
+
summary: timingSummary(this.events),
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
121
267
|
// ---------------------------------------------------------------------------
|
|
122
268
|
// Paths for episode/batch results
|
|
123
269
|
// ---------------------------------------------------------------------------
|
|
@@ -214,6 +360,8 @@ function isDirectStageName(value) {
|
|
|
214
360
|
case "episode_synopsis":
|
|
215
361
|
case "script_merge":
|
|
216
362
|
case "asset_curation":
|
|
363
|
+
case "state_curation":
|
|
364
|
+
case "state_binding":
|
|
217
365
|
case "metadata_extract":
|
|
218
366
|
case "script_synopsis":
|
|
219
367
|
case "validate":
|
|
@@ -254,6 +402,56 @@ function stageSnapshot(name, status, completed, total, message) {
|
|
|
254
402
|
message: message ?? "",
|
|
255
403
|
};
|
|
256
404
|
}
|
|
405
|
+
function formatDuration(ms) {
|
|
406
|
+
const value = Math.max(0, Math.round(ms));
|
|
407
|
+
if (value < 1000)
|
|
408
|
+
return `${value}ms`;
|
|
409
|
+
const seconds = value / 1000;
|
|
410
|
+
if (seconds < 60)
|
|
411
|
+
return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;
|
|
412
|
+
const minutes = Math.floor(seconds / 60);
|
|
413
|
+
const rest = Math.round(seconds % 60);
|
|
414
|
+
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
|
415
|
+
}
|
|
416
|
+
function stageTimingDurations(state) {
|
|
417
|
+
const timingSummaryRaw = state["timing_summary"];
|
|
418
|
+
const timingSummary = isDict(timingSummaryRaw) ? timingSummaryRaw : {};
|
|
419
|
+
const stagesRaw = timingSummary["stages"];
|
|
420
|
+
const stages = isDict(stagesRaw) ? stagesRaw : {};
|
|
421
|
+
const out = {};
|
|
422
|
+
for (const [key, value] of Object.entries(stages)) {
|
|
423
|
+
const n = Number(value);
|
|
424
|
+
if (!Number.isNaN(n))
|
|
425
|
+
out[key] = n;
|
|
426
|
+
}
|
|
427
|
+
return out;
|
|
428
|
+
}
|
|
429
|
+
function runningTimingElapsed(state, stageName) {
|
|
430
|
+
const stageRaw = state["timing_current_stage"];
|
|
431
|
+
const currentRaw = isDict(stageRaw) ? stageRaw : state["timing_current"];
|
|
432
|
+
const current = isDict(currentRaw) ? currentRaw : null;
|
|
433
|
+
if (!current || strOf(current["category"]) !== "stage" || strOf(current["name"]) !== stageName)
|
|
434
|
+
return null;
|
|
435
|
+
const startedAt = Date.parse(strOf(current["started_at"]));
|
|
436
|
+
if (Number.isNaN(startedAt))
|
|
437
|
+
return null;
|
|
438
|
+
return Date.now() - startedAt;
|
|
439
|
+
}
|
|
440
|
+
function applyStageTiming(stage, state, durations) {
|
|
441
|
+
const duration = durations[stage.name];
|
|
442
|
+
const elapsed = stage.status === "running" ? runningTimingElapsed(state, stage.name) : null;
|
|
443
|
+
const messageParts = [stage.message].filter((item) => item);
|
|
444
|
+
if (typeof duration === "number")
|
|
445
|
+
messageParts.push(`耗时 ${formatDuration(duration)}`);
|
|
446
|
+
else if (elapsed !== null)
|
|
447
|
+
messageParts.push(`已运行 ${formatDuration(elapsed)}`);
|
|
448
|
+
return {
|
|
449
|
+
...stage,
|
|
450
|
+
...(typeof duration === "number" ? { duration_ms: duration } : {}),
|
|
451
|
+
...(elapsed !== null ? { elapsed_ms: elapsed } : {}),
|
|
452
|
+
message: messageParts.join("; "),
|
|
453
|
+
};
|
|
454
|
+
}
|
|
257
455
|
function fileStage(name, filePath, runningStage) {
|
|
258
456
|
if (exists(filePath))
|
|
259
457
|
return stageSnapshot(name, "success", 1, 1);
|
|
@@ -304,6 +502,8 @@ function stageLabel(name) {
|
|
|
304
502
|
case "episode_synopsis": return "补充信息";
|
|
305
503
|
case "script_merge": return "生成剧本";
|
|
306
504
|
case "asset_curation": return "补充信息";
|
|
505
|
+
case "state_curation": return "补充信息";
|
|
506
|
+
case "state_binding": return "补充信息";
|
|
307
507
|
case "metadata_extract": return "补充信息";
|
|
308
508
|
case "script_synopsis": return "补充信息";
|
|
309
509
|
case "validate": return "检查结果";
|
|
@@ -364,9 +564,11 @@ function stageOrder(name) {
|
|
|
364
564
|
case "episode_synopsis": return 6;
|
|
365
565
|
case "script_merge": return 7;
|
|
366
566
|
case "asset_curation": return 8;
|
|
367
|
-
case "
|
|
368
|
-
case "
|
|
369
|
-
case "
|
|
567
|
+
case "state_curation": return 9;
|
|
568
|
+
case "state_binding": return 10;
|
|
569
|
+
case "metadata_extract": return 11;
|
|
570
|
+
case "script_synopsis": return 12;
|
|
571
|
+
case "validate": return 13;
|
|
370
572
|
}
|
|
371
573
|
}
|
|
372
574
|
function stageRatio(stage) {
|
|
@@ -602,6 +804,8 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
602
804
|
const episodeResultsDir = path.join(dd, "episode_results");
|
|
603
805
|
const trustEpisodeMergeArtifacts = canTrustStageArtifacts("episode_merge", runningStage, runStatus);
|
|
604
806
|
const trustEpisodeSynopsisArtifacts = canTrustStageArtifacts("episode_synopsis", runningStage, runStatus);
|
|
807
|
+
const trustStateCurationArtifacts = canTrustStageArtifacts("state_curation", runningStage, runStatus);
|
|
808
|
+
const trustStateBindingArtifacts = canTrustStageArtifacts("state_binding", runningStage, runStatus);
|
|
605
809
|
const trustMetadataArtifacts = canTrustStageArtifacts("metadata_extract", runningStage, runStatus);
|
|
606
810
|
const trustScriptSynopsisArtifacts = canTrustStageArtifacts("script_synopsis", runningStage, runStatus);
|
|
607
811
|
const batchesByEpisode = new Map();
|
|
@@ -741,6 +945,12 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
741
945
|
};
|
|
742
946
|
const scriptStage = orderedFileStage("script_merge", path.join(dd, "script.initial.json"), runningStage, runStatus);
|
|
743
947
|
const curationStage = orderedFileStage("asset_curation", path.join(dd, "asset_curation.json"), runningStage, runStatus);
|
|
948
|
+
const stateCurationStage = trustStateCurationArtifacts
|
|
949
|
+
? orderedFileStage("state_curation", path.join(dd, "state_curation.json"), runningStage, runStatus)
|
|
950
|
+
: stageSnapshot("state_curation", runningStage === "state_curation" ? "running" : "pending", 0, 1);
|
|
951
|
+
const stateBindingStage = trustStateBindingArtifacts
|
|
952
|
+
? orderedFileStage("state_binding", path.join(dd, "state_binding.json"), runningStage, runStatus)
|
|
953
|
+
: stageSnapshot("state_binding", runningStage === "state_binding" ? "running" : "pending", 0, 1);
|
|
744
954
|
const metadataStage = Boolean(state["metadata_skipped"]) && trustMetadataArtifacts
|
|
745
955
|
? stageSnapshot("metadata_extract", "skipped", 0, 1, "metadata: skipped (--skip-metadata)")
|
|
746
956
|
: orderedFileStage("metadata_extract", path.join(dd, "asset_metadata.json"), runningStage, runStatus);
|
|
@@ -761,6 +971,7 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
761
971
|
const validation = readJson(validationPath);
|
|
762
972
|
validationStage = stageSnapshot("validate", Boolean(validation["passed"]) ? "success" : "warning", 1, 1);
|
|
763
973
|
}
|
|
974
|
+
const timingDurations = stageTimingDurations(state);
|
|
764
975
|
const stages = [
|
|
765
976
|
sourceStage,
|
|
766
977
|
episodePlanStage,
|
|
@@ -771,10 +982,14 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
771
982
|
episodeSynopsisStage,
|
|
772
983
|
scriptStage,
|
|
773
984
|
curationStage,
|
|
985
|
+
stateCurationStage,
|
|
986
|
+
stateBindingStage,
|
|
774
987
|
metadataStage,
|
|
775
988
|
scriptSynopsisStage,
|
|
776
989
|
validationStage,
|
|
777
|
-
]
|
|
990
|
+
]
|
|
991
|
+
.map((stage) => failedStage === stage.name ? { ...stage, status: "error", error: Math.max(1, stage.error) } : stage)
|
|
992
|
+
.map((stage) => applyStageTiming(stage, state, timingDurations));
|
|
778
993
|
let overallStatus = "pending";
|
|
779
994
|
const maxStage = stages.reduce((max, stage) => statusPriority(stage.status) > statusPriority(max) ? stage.status : max, "success");
|
|
780
995
|
if (runStatus === "init_failed" || runStatus === "init_incomplete" || maxStage === "error")
|
|
@@ -1010,11 +1225,16 @@ function initFailedReport(workspace, opts) {
|
|
|
1010
1225
|
// into every stage's catch block.
|
|
1011
1226
|
function stageFailure(workspace, exc, opts) {
|
|
1012
1227
|
const e = exc;
|
|
1228
|
+
const received = exc instanceof CliError && exc.received.length > 0
|
|
1229
|
+
? exc.received
|
|
1230
|
+
: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`];
|
|
1013
1231
|
throw initFailedReport(workspace, {
|
|
1014
1232
|
title: opts.title,
|
|
1015
1233
|
stage: opts.stage,
|
|
1234
|
+
exitCode: exc instanceof CliError ? exc.exitCode : undefined,
|
|
1235
|
+
errorCode: exc instanceof CliError ? exc.errorCode : undefined,
|
|
1016
1236
|
required: opts.required,
|
|
1017
|
-
received
|
|
1237
|
+
received,
|
|
1018
1238
|
nextSteps: opts.nextSteps,
|
|
1019
1239
|
...(opts.updates ? { updates: opts.updates } : {}),
|
|
1020
1240
|
});
|
|
@@ -1042,6 +1262,12 @@ async function pMapWithConcurrency(items, concurrency, worker) {
|
|
|
1042
1262
|
}));
|
|
1043
1263
|
return out;
|
|
1044
1264
|
}
|
|
1265
|
+
function unwrapConcurrentOutcomes(outcomes) {
|
|
1266
|
+
const failed = outcomes.find((outcome) => !outcome.ok);
|
|
1267
|
+
if (failed)
|
|
1268
|
+
throw failed.error;
|
|
1269
|
+
return outcomes.map((outcome) => outcome.value);
|
|
1270
|
+
}
|
|
1045
1271
|
function providerErrorSummary(error) {
|
|
1046
1272
|
if (error instanceof CliError && error.errorCode)
|
|
1047
1273
|
return error.errorCode;
|
|
@@ -1051,6 +1277,9 @@ function providerErrorSummary(error) {
|
|
|
1051
1277
|
return "PROVIDER_CONTENT_FILTERED";
|
|
1052
1278
|
return `${e?.name ?? "Error"}: ${msg}`.slice(0, 160);
|
|
1053
1279
|
}
|
|
1280
|
+
function cloneDict(value) {
|
|
1281
|
+
return structuredClone(value);
|
|
1282
|
+
}
|
|
1054
1283
|
// Generate the plan's missing episode titles in bounded, concurrent chunks.
|
|
1055
1284
|
// Title generation is non-essential (every episode has a deterministic
|
|
1056
1285
|
// fallback), so a chunk that the provider filters or fails does NOT abort
|
|
@@ -1188,6 +1417,56 @@ function assignOverviewFields(script, o) {
|
|
|
1188
1417
|
function applyScriptSynopsisFields(script, fields) {
|
|
1189
1418
|
assignOverviewFields(script, normalizeOverviewFields(fields));
|
|
1190
1419
|
}
|
|
1420
|
+
function synopsisGroupSignature(opts) {
|
|
1421
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
1422
|
+
title: strOf(opts.title),
|
|
1423
|
+
thresholdChars: opts.thresholdChars,
|
|
1424
|
+
groupSize: opts.groupSize,
|
|
1425
|
+
level: opts.level,
|
|
1426
|
+
groupIndex: opts.groupIndex,
|
|
1427
|
+
synopses: opts.synopses,
|
|
1428
|
+
})));
|
|
1429
|
+
}
|
|
1430
|
+
function synopsisGroupPartPath(dd, signature) {
|
|
1431
|
+
return path.join(dd, "synopsis_parts", `${signature}.json`);
|
|
1432
|
+
}
|
|
1433
|
+
async function loadOrRunSynopsisGroup(provider, groupSynopses, opts) {
|
|
1434
|
+
if (groupSynopses.length <= 1)
|
|
1435
|
+
return { value: groupSynopses[0] ?? "", reused: false };
|
|
1436
|
+
const signature = synopsisGroupSignature({
|
|
1437
|
+
title: opts.title,
|
|
1438
|
+
thresholdChars: opts.thresholdChars,
|
|
1439
|
+
groupSize: opts.groupSize,
|
|
1440
|
+
level: opts.level,
|
|
1441
|
+
groupIndex: opts.groupIndex,
|
|
1442
|
+
synopses: groupSynopses,
|
|
1443
|
+
});
|
|
1444
|
+
const partPath = synopsisGroupPartPath(opts.dd, signature);
|
|
1445
|
+
if (!opts.resummarize && exists(partPath)) {
|
|
1446
|
+
const prior = readJson(partPath);
|
|
1447
|
+
if (isDict(prior) && strOf(prior["source_group_signature"]) === signature) {
|
|
1448
|
+
const synopsis = strOf(prior["synopsis"]).trim();
|
|
1449
|
+
if (synopsis)
|
|
1450
|
+
return { value: synopsis, reused: true };
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
try {
|
|
1454
|
+
const synopsis = await reduceSynopses(provider, [...groupSynopses], { title: opts.title });
|
|
1455
|
+
writeJson(partPath, {
|
|
1456
|
+
version: 1,
|
|
1457
|
+
source_group_signature: signature,
|
|
1458
|
+
level: opts.level,
|
|
1459
|
+
group_index: opts.groupIndex,
|
|
1460
|
+
source_count: groupSynopses.length,
|
|
1461
|
+
synopsis,
|
|
1462
|
+
updated_at: checkpointTimestamp(),
|
|
1463
|
+
});
|
|
1464
|
+
return { value: synopsis, reused: false };
|
|
1465
|
+
}
|
|
1466
|
+
catch (error) {
|
|
1467
|
+
return { value: groupSynopses.join(" "), error, reused: false };
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1191
1470
|
// Whole-script reduce-2: collapse the episode synopses down to a set that fits
|
|
1192
1471
|
// one final reduce, then produce the top-level overview. Reuses a prior
|
|
1193
1472
|
// synopsis.json when the episode synopses are unchanged (signature match). Soft
|
|
@@ -1244,6 +1523,7 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1244
1523
|
// whenever it can't form fewer groups than items), so `current` strictly
|
|
1245
1524
|
// shrinks each non-terminal pass and this terminates without a counter.
|
|
1246
1525
|
let current = episodeSynopses;
|
|
1526
|
+
let level = 1;
|
|
1247
1527
|
while (true) {
|
|
1248
1528
|
const plan = planSynopsisReduction(current, opts.thresholdChars, opts.groupSize);
|
|
1249
1529
|
if (plan.fitsInOnePass)
|
|
@@ -1252,16 +1532,10 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1252
1532
|
// singleton group passes through verbatim (no LLM rewrite), mirroring the
|
|
1253
1533
|
// reduce-1 passthrough. A group whose stitch FAILS degrades to its joined
|
|
1254
1534
|
// beats (soft, per-group) — one bad group never wipes the whole overview.
|
|
1255
|
-
const
|
|
1535
|
+
const currentLevel = level;
|
|
1536
|
+
const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group, groupIndex) => {
|
|
1256
1537
|
const groupSynopses = group.map((i) => current[i]).filter((s) => s.trim());
|
|
1257
|
-
|
|
1258
|
-
return { value: groupSynopses[0] ?? "" };
|
|
1259
|
-
try {
|
|
1260
|
-
return { value: await reduceSynopses(provider, groupSynopses, { title: script["title"] }) };
|
|
1261
|
-
}
|
|
1262
|
-
catch (err) {
|
|
1263
|
-
return { value: groupSynopses.join(" "), error: err };
|
|
1264
|
-
}
|
|
1538
|
+
return loadOrRunSynopsisGroup(provider, groupSynopses, { ...opts, title: script["title"], level: currentLevel, groupIndex });
|
|
1265
1539
|
});
|
|
1266
1540
|
const next = [];
|
|
1267
1541
|
for (const o of outcomes) {
|
|
@@ -1275,6 +1549,7 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1275
1549
|
next.push(o.value.value);
|
|
1276
1550
|
}
|
|
1277
1551
|
current = next;
|
|
1552
|
+
level += 1;
|
|
1278
1553
|
if (current.length === 0)
|
|
1279
1554
|
break;
|
|
1280
1555
|
}
|
|
@@ -1416,10 +1691,12 @@ function parseDirectRunOptions(opts) {
|
|
|
1416
1691
|
function buildStageContext(options) {
|
|
1417
1692
|
const ws = new Workspace(options.workspace);
|
|
1418
1693
|
let provider = null;
|
|
1694
|
+
const timing = new JsonTimingRecorder(ws, options.workspace);
|
|
1419
1695
|
const ctx = {
|
|
1420
1696
|
ws,
|
|
1421
1697
|
options,
|
|
1422
1698
|
state: emptyRunState(),
|
|
1699
|
+
timing,
|
|
1423
1700
|
provider() {
|
|
1424
1701
|
if (provider)
|
|
1425
1702
|
return provider;
|
|
@@ -1545,11 +1822,98 @@ function buildIncompleteReport(ctx) {
|
|
|
1545
1822
|
};
|
|
1546
1823
|
return [report, EXIT_RUNTIME];
|
|
1547
1824
|
}
|
|
1825
|
+
function initPreFanoutStages() {
|
|
1826
|
+
const end = STAGES.findIndex((stage) => stage.name === "state-curation");
|
|
1827
|
+
return STAGES.slice(0, end + 1);
|
|
1828
|
+
}
|
|
1829
|
+
function validateOnlyStages() {
|
|
1830
|
+
return STAGES.filter((stage) => stage.name === "validate");
|
|
1831
|
+
}
|
|
1832
|
+
function updatePostCurationBranch(ctx, name, status) {
|
|
1833
|
+
const current = readRunState(ctx.options.workspace);
|
|
1834
|
+
const rawBranches = current["post_curation_branches"];
|
|
1835
|
+
const branches = isDict(rawBranches) ? { ...rawBranches } : {};
|
|
1836
|
+
branches[name] = { status, updated_at: checkpointTimestamp() };
|
|
1837
|
+
ctx.updateRunState({ post_curation_branches: branches });
|
|
1838
|
+
}
|
|
1839
|
+
async function runPostCurationBranch(ctx, name, run) {
|
|
1840
|
+
updatePostCurationBranch(ctx, name, "running");
|
|
1841
|
+
try {
|
|
1842
|
+
const value = await ctx.timing.span({ name, category: "branch", parent: "post_curation" }, run);
|
|
1843
|
+
updatePostCurationBranch(ctx, name, "success");
|
|
1844
|
+
return value;
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
updatePostCurationBranch(ctx, name, "error");
|
|
1848
|
+
throw error;
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
async function runPostCurationFanout(ctx) {
|
|
1852
|
+
const { ws, state, options } = ctx;
|
|
1853
|
+
const baseScript = ensureScript(ctx);
|
|
1854
|
+
ctx.updateRunState({
|
|
1855
|
+
status: "init_running",
|
|
1856
|
+
init_stage: "state_binding",
|
|
1857
|
+
post_curation_parallel: true,
|
|
1858
|
+
post_curation_branches: {
|
|
1859
|
+
state_binding: { status: "running", updated_at: checkpointTimestamp() },
|
|
1860
|
+
metadata_extract: { status: options.skipMetadata ? "skipped" : "running", updated_at: checkpointTimestamp() },
|
|
1861
|
+
script_synopsis: { status: options.skipSummary ? "skipped" : "running", updated_at: checkpointTimestamp() },
|
|
1862
|
+
},
|
|
1863
|
+
});
|
|
1864
|
+
ctx.emitProgress("state-binding");
|
|
1865
|
+
ctx.emitProgress("metadata", options.skipMetadata ? "skipped" : "running");
|
|
1866
|
+
ctx.emitProgress("script-synopsis", options.skipSummary ? "skipped" : "running");
|
|
1867
|
+
const stateBindingPromise = runPostCurationBranch(ctx, "state_binding", async () => {
|
|
1868
|
+
const script = cloneDict(baseScript);
|
|
1869
|
+
const result = await runStateBindingForScript(ctx, script, false);
|
|
1870
|
+
return { ...result, script };
|
|
1871
|
+
});
|
|
1872
|
+
const metadataPromise = runPostCurationBranch(ctx, "metadata_extract", async () => {
|
|
1873
|
+
const script = cloneDict(baseScript);
|
|
1874
|
+
return runMetadataForScript(ctx, script, false, false);
|
|
1875
|
+
});
|
|
1876
|
+
const synopsisPromise = runPostCurationBranch(ctx, "script_synopsis", async () => {
|
|
1877
|
+
const script = cloneDict(baseScript);
|
|
1878
|
+
return runScriptSynopsisForScript(ctx, script, false);
|
|
1879
|
+
});
|
|
1880
|
+
const settled = await Promise.allSettled([stateBindingPromise, metadataPromise, synopsisPromise]);
|
|
1881
|
+
const failed = settled.find((item) => item.status === "rejected");
|
|
1882
|
+
if (failed && failed.status === "rejected")
|
|
1883
|
+
throw failed.reason;
|
|
1884
|
+
const stateBinding = await stateBindingPromise;
|
|
1885
|
+
const metadata = await metadataPromise;
|
|
1886
|
+
const synopsis = await synopsisPromise;
|
|
1887
|
+
await ctx.timing.span({ name: "post_curation.merge", category: "substage", parent: "post_curation" }, () => {
|
|
1888
|
+
const finalScript = stateBinding.script;
|
|
1889
|
+
if (!metadata.skipped)
|
|
1890
|
+
applyMetadataToScript(finalScript, metadata.metadata);
|
|
1891
|
+
applyScriptSynopsisFields(finalScript, synopsis.fields);
|
|
1892
|
+
ws.writeJson("script.initial.json", finalScript);
|
|
1893
|
+
state.script = finalScript;
|
|
1894
|
+
state.summaryFullDegraded = synopsis.degraded;
|
|
1895
|
+
state.summaryDegradeReason = synopsis.reason;
|
|
1896
|
+
});
|
|
1897
|
+
state.summaryDegradedEpisodes.sort((a, b) => a - b);
|
|
1898
|
+
const summaryDegraded = state.summaryFullDegraded || state.summaryDegradedEpisodes.length > 0;
|
|
1899
|
+
ctx.updateRunState({
|
|
1900
|
+
post_curation_parallel: false,
|
|
1901
|
+
summary_skipped: options.skipSummary,
|
|
1902
|
+
summary_degraded: summaryDegraded,
|
|
1903
|
+
summary_degraded_episodes: state.summaryDegradedEpisodes,
|
|
1904
|
+
summary_degraded_reason: state.summaryDegradeReason || null,
|
|
1905
|
+
metadata_skipped: options.skipMetadata,
|
|
1906
|
+
});
|
|
1907
|
+
ctx.emitProgress("state-binding", "success");
|
|
1908
|
+
ctx.emitProgress("metadata", metadata.skipped ? "skipped" : "success");
|
|
1909
|
+
ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
|
|
1910
|
+
}
|
|
1548
1911
|
async function commandInitImpl(opts) {
|
|
1549
1912
|
const options = parseDirectRunOptions(opts);
|
|
1550
1913
|
assertInitSource(options.source);
|
|
1551
1914
|
const ctx = buildStageContext(options);
|
|
1552
1915
|
ctx.ws.ensure();
|
|
1916
|
+
ctx.timing.reset("direct init");
|
|
1553
1917
|
// Seed run_state with the init header before the first stage runs (provider /
|
|
1554
1918
|
// model / source_path / metadata flag) — the god-command set these up front.
|
|
1555
1919
|
updateRunState(options.workspace, {
|
|
@@ -1563,13 +1927,9 @@ async function commandInitImpl(opts) {
|
|
|
1563
1927
|
metadata_skipped: options.skipMetadata,
|
|
1564
1928
|
});
|
|
1565
1929
|
try {
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
// in the synopsis stages), so a deterministic stage must still re-derive its
|
|
1570
|
-
// output from the (possibly reused/edited) upstream artifacts. Whole-stage
|
|
1571
|
-
// skip-on-present is a `direct run <stage>` convenience only.
|
|
1572
|
-
await runStages(ctx, STAGES, { force: true });
|
|
1930
|
+
await runStages(ctx, initPreFanoutStages(), { force: true, persistScriptAfterStages: false });
|
|
1931
|
+
await ctx.timing.span({ name: "post_curation", category: "stage" }, () => runPostCurationFanout(ctx));
|
|
1932
|
+
await runStages(ctx, validateOnlyStages(), { force: true, persistScriptAfterStages: false });
|
|
1573
1933
|
}
|
|
1574
1934
|
catch (exc) {
|
|
1575
1935
|
// The resumable "incomplete" stop is not a failure — surface the INCOMPLETE
|
|
@@ -1645,6 +2005,8 @@ export async function commandRun(opts) {
|
|
|
1645
2005
|
}
|
|
1646
2006
|
const options = parseDirectRunOptions(opts);
|
|
1647
2007
|
const ctx = buildStageContext(options);
|
|
2008
|
+
ctx.ws.ensure();
|
|
2009
|
+
ctx.timing.reset(`direct run ${stageName}`);
|
|
1648
2010
|
const force = Boolean(opts["force"]);
|
|
1649
2011
|
try {
|
|
1650
2012
|
if (force) {
|
|
@@ -2131,16 +2493,661 @@ export async function stageScriptMerge(ctx) {
|
|
|
2131
2493
|
}
|
|
2132
2494
|
}
|
|
2133
2495
|
// --- asset-curation -------------------------------------------------------
|
|
2496
|
+
const ASSET_GROUPING_CACHE_VERSION = 3;
|
|
2497
|
+
const ASSET_CURATION_CACHE_VERSION = 4;
|
|
2498
|
+
function assetCacheIdentity(options) {
|
|
2499
|
+
return {
|
|
2500
|
+
provider: options.providerName,
|
|
2501
|
+
model: options.model,
|
|
2502
|
+
};
|
|
2503
|
+
}
|
|
2504
|
+
function assetCacheArtifactFields(ctx, cacheVersion) {
|
|
2505
|
+
return {
|
|
2506
|
+
cache_version: cacheVersion,
|
|
2507
|
+
...assetCacheIdentity(ctx.options),
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
function assetCurationScriptFingerprint(script, options) {
|
|
2511
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2512
|
+
cache_version: ASSET_CURATION_CACHE_VERSION,
|
|
2513
|
+
...assetCacheIdentity(options),
|
|
2514
|
+
script: {
|
|
2515
|
+
actors: script["actors"],
|
|
2516
|
+
locations: script["locations"],
|
|
2517
|
+
props: script["props"],
|
|
2518
|
+
speakers: script["speakers"],
|
|
2519
|
+
episodes: script["episodes"],
|
|
2520
|
+
},
|
|
2521
|
+
})));
|
|
2522
|
+
}
|
|
2523
|
+
function assetCurationPlanFingerprint(scriptFingerprint, plan) {
|
|
2524
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2525
|
+
script_fingerprint: scriptFingerprint,
|
|
2526
|
+
decisions: plan["decisions"],
|
|
2527
|
+
})));
|
|
2528
|
+
}
|
|
2529
|
+
function chunkStrings(values, chunkSize) {
|
|
2530
|
+
const chunks = [];
|
|
2531
|
+
for (let index = 0; index < values.length; index += chunkSize) {
|
|
2532
|
+
chunks.push(values.slice(index, index + chunkSize));
|
|
2533
|
+
}
|
|
2534
|
+
return chunks;
|
|
2535
|
+
}
|
|
2536
|
+
function curationSourceKey(kind, id) {
|
|
2537
|
+
return `${kind}:${id}`;
|
|
2538
|
+
}
|
|
2539
|
+
function curationDecisionSourceKey(decision) {
|
|
2540
|
+
return curationSourceKey(strOf(decision["kind"]), strOf(decision["source_id"]));
|
|
2541
|
+
}
|
|
2542
|
+
function curationSourceKeysForIds(kind, ids) {
|
|
2543
|
+
return ids.map((id) => curationSourceKey(kind, id));
|
|
2544
|
+
}
|
|
2545
|
+
function curationPartSelection(kind, ids) {
|
|
2546
|
+
if (kind === "actor")
|
|
2547
|
+
return { actor: ids };
|
|
2548
|
+
if (kind === "location")
|
|
2549
|
+
return { location: ids };
|
|
2550
|
+
return { prop: ids };
|
|
2551
|
+
}
|
|
2552
|
+
function curationIdsFromSourceKeys(kind, ids, sourceKeys) {
|
|
2553
|
+
const allowed = new Set(sourceKeys);
|
|
2554
|
+
return ids.filter((id) => allowed.has(curationSourceKey(kind, id)));
|
|
2555
|
+
}
|
|
2556
|
+
function invalidCurationPart(title, received) {
|
|
2557
|
+
return new CliError(title, title, {
|
|
2558
|
+
exitCode: EXIT_RUNTIME,
|
|
2559
|
+
required: ["repairable asset curation expressions"],
|
|
2560
|
+
received,
|
|
2561
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and only invalid curation parts will retry."],
|
|
2562
|
+
});
|
|
2563
|
+
}
|
|
2564
|
+
const MAX_CURATION_PART_REPAIR_ATTEMPTS = 2;
|
|
2565
|
+
function assetGroupingKindListKey(kind) {
|
|
2566
|
+
if (kind === "actor")
|
|
2567
|
+
return "actors";
|
|
2568
|
+
if (kind === "location")
|
|
2569
|
+
return "locations";
|
|
2570
|
+
return "props";
|
|
2571
|
+
}
|
|
2572
|
+
function assetGroupingKindIdKey(kind) {
|
|
2573
|
+
if (kind === "actor")
|
|
2574
|
+
return "actor_id";
|
|
2575
|
+
if (kind === "location")
|
|
2576
|
+
return "location_id";
|
|
2577
|
+
return "prop_id";
|
|
2578
|
+
}
|
|
2579
|
+
function assetGroupingKindNameKey(kind) {
|
|
2580
|
+
if (kind === "actor")
|
|
2581
|
+
return "actor_name";
|
|
2582
|
+
if (kind === "location")
|
|
2583
|
+
return "location_name";
|
|
2584
|
+
return "prop_name";
|
|
2585
|
+
}
|
|
2586
|
+
function assetGroupingScriptFingerprint(script, options) {
|
|
2587
|
+
const compact = {};
|
|
2588
|
+
for (const kind of ["actor", "location", "prop"]) {
|
|
2589
|
+
const idKey = assetGroupingKindIdKey(kind);
|
|
2590
|
+
const nameKey = assetGroupingKindNameKey(kind);
|
|
2591
|
+
compact[assetGroupingKindListKey(kind)] = asList(script[assetGroupingKindListKey(kind)])
|
|
2592
|
+
.filter(isDict)
|
|
2593
|
+
.map((asset) => ({ id: strOf(asset[idKey]), name: strOf(asset[nameKey]) }));
|
|
2594
|
+
}
|
|
2595
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2596
|
+
cache_version: ASSET_GROUPING_CACHE_VERSION,
|
|
2597
|
+
...assetCacheIdentity(options),
|
|
2598
|
+
assets: compact,
|
|
2599
|
+
})));
|
|
2600
|
+
}
|
|
2601
|
+
function combineAssetGroupingPlans(parts) {
|
|
2602
|
+
const groups = parts.flatMap((part) => asList(part["groups"]).filter(isDict));
|
|
2603
|
+
const duplicateMergeDecisions = parts.flatMap((part) => asList(part["duplicate_merge_decisions"]).filter(isDict));
|
|
2604
|
+
const normalizations = parts.flatMap((part) => asList(part["normalizations"]).filter(isDict));
|
|
2605
|
+
const summaries = parts.map((part) => part["summary"]).filter(isDict);
|
|
2606
|
+
return {
|
|
2607
|
+
version: 1,
|
|
2608
|
+
format: "asset-grouping-v1",
|
|
2609
|
+
groups,
|
|
2610
|
+
duplicate_merge_decisions: duplicateMergeDecisions,
|
|
2611
|
+
normalizations,
|
|
2612
|
+
parts: parts.map((part) => ({
|
|
2613
|
+
kind: part["kind"],
|
|
2614
|
+
expression_text: part["expression_text"],
|
|
2615
|
+
summary: part["summary"],
|
|
2616
|
+
normalizations: part["normalizations"],
|
|
2617
|
+
})),
|
|
2618
|
+
summary: {
|
|
2619
|
+
group_count: groups.length,
|
|
2620
|
+
duplicate_merge_count: duplicateMergeDecisions.length,
|
|
2621
|
+
normalization_count: normalizations.length,
|
|
2622
|
+
parts: summaries,
|
|
2623
|
+
},
|
|
2624
|
+
};
|
|
2625
|
+
}
|
|
2626
|
+
async function loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint) {
|
|
2627
|
+
const artifactName = `asset_grouping_parts/${kind}.json`;
|
|
2628
|
+
const cached = ctx.ws.readJsonSafe(artifactName);
|
|
2629
|
+
const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
|
|
2630
|
+
if (cachedPlan && isList(cachedPlan["groups"]))
|
|
2631
|
+
return cachedPlan;
|
|
2632
|
+
const startedAtMs = Date.now();
|
|
2633
|
+
const idKey = assetGroupingKindIdKey(kind);
|
|
2634
|
+
const assetCount = asList(script[assetGroupingKindListKey(kind)])
|
|
2635
|
+
.filter(isDict)
|
|
2636
|
+
.map((asset) => strOf(asset[idKey]).trim())
|
|
2637
|
+
.filter((id) => id).length;
|
|
2638
|
+
if (assetCount === 0) {
|
|
2639
|
+
const plan = {
|
|
2640
|
+
version: 1,
|
|
2641
|
+
format: "asset-grouping-v1",
|
|
2642
|
+
kind,
|
|
2643
|
+
expression_text: "",
|
|
2644
|
+
expressions: [],
|
|
2645
|
+
groups: [],
|
|
2646
|
+
duplicate_merge_decisions: [],
|
|
2647
|
+
summary: { input_count: 0, group_count: 0, mentioned_count: 0, omitted_duplicate_count: 0 },
|
|
2648
|
+
};
|
|
2649
|
+
ctx.ws.writeJson(artifactName, {
|
|
2650
|
+
version: 1,
|
|
2651
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2652
|
+
script_fingerprint: fingerprint,
|
|
2653
|
+
parse_status: "ok",
|
|
2654
|
+
kind,
|
|
2655
|
+
context_chars: 0,
|
|
2656
|
+
expression_text: "",
|
|
2657
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2658
|
+
plan,
|
|
2659
|
+
updated_at: checkpointTimestamp(),
|
|
2660
|
+
});
|
|
2661
|
+
return plan;
|
|
2662
|
+
}
|
|
2663
|
+
const contextText = buildAssetGroupingContextText(script, kind);
|
|
2664
|
+
let rawText = "";
|
|
2665
|
+
try {
|
|
2666
|
+
rawText = await ctx.timing.span({ name: `asset_grouping.${kind}`, category: "chunk", parent: "asset_curation.grouping", metadata: { artifact: artifactName } }, () => groupAssets(script, contextText));
|
|
2667
|
+
const parsed = parseAssetGroupingExpressions(rawText);
|
|
2668
|
+
const plan = validateAssetGroupingPlan(script, kind, parsed);
|
|
2669
|
+
ctx.ws.writeJson(artifactName, {
|
|
2670
|
+
version: 1,
|
|
2671
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2672
|
+
script_fingerprint: fingerprint,
|
|
2673
|
+
parse_status: "ok",
|
|
2674
|
+
kind,
|
|
2675
|
+
context_chars: contextText.length,
|
|
2676
|
+
expression_text: rawText,
|
|
2677
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2678
|
+
plan,
|
|
2679
|
+
updated_at: checkpointTimestamp(),
|
|
2680
|
+
});
|
|
2681
|
+
return plan;
|
|
2682
|
+
}
|
|
2683
|
+
catch (exc) {
|
|
2684
|
+
const e = exc;
|
|
2685
|
+
ctx.ws.writeJson(artifactName, {
|
|
2686
|
+
version: 1,
|
|
2687
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2688
|
+
script_fingerprint: fingerprint,
|
|
2689
|
+
parse_status: "failed",
|
|
2690
|
+
kind,
|
|
2691
|
+
context_chars: contextText.length,
|
|
2692
|
+
expression_text: rawText,
|
|
2693
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2694
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
2695
|
+
updated_at: checkpointTimestamp(),
|
|
2696
|
+
});
|
|
2697
|
+
throw exc;
|
|
2698
|
+
}
|
|
2699
|
+
}
|
|
2700
|
+
async function loadOrRunAssetGrouping(ctx, script, groupAssets, fingerprint) {
|
|
2701
|
+
ctx.ws.mkdir("asset_grouping_parts");
|
|
2702
|
+
const outcomes = await pMapWithConcurrency(["actor", "location", "prop"], ctx.options.concurrency, async (kind) => loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint));
|
|
2703
|
+
return combineAssetGroupingPlans(unwrapConcurrentOutcomes(outcomes));
|
|
2704
|
+
}
|
|
2705
|
+
function curationDecisionSourceKeys(plan) {
|
|
2706
|
+
return new Set(asList(plan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
|
|
2707
|
+
}
|
|
2708
|
+
function sourceKeysFromRejectedExpressions(rejected, candidateSourceKeys) {
|
|
2709
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2710
|
+
const keys = [];
|
|
2711
|
+
for (const item of rejected) {
|
|
2712
|
+
const key = strOf(item["source_key"]);
|
|
2713
|
+
if (key && (candidateSet.size === 0 || candidateSet.has(key)))
|
|
2714
|
+
uniqueAdd(keys, key);
|
|
2715
|
+
}
|
|
2716
|
+
return keys;
|
|
2717
|
+
}
|
|
2718
|
+
function scopedCurationPlan(plan, allowedSourceKeys) {
|
|
2719
|
+
const allowed = new Set(allowedSourceKeys);
|
|
2720
|
+
const decisions = asList(plan["decisions"]).filter(isDict);
|
|
2721
|
+
const acceptedExpressions = asList(plan["accepted_expressions"]);
|
|
2722
|
+
const scopedDecisions = [];
|
|
2723
|
+
const scopedExpressions = [];
|
|
2724
|
+
const ignored = [];
|
|
2725
|
+
for (let index = 0; index < decisions.length; index += 1) {
|
|
2726
|
+
const decision = decisions[index];
|
|
2727
|
+
const sourceKey = curationDecisionSourceKey(decision);
|
|
2728
|
+
const expression = acceptedExpressions[index] ?? "";
|
|
2729
|
+
if (allowed.has(sourceKey)) {
|
|
2730
|
+
scopedDecisions.push(decision);
|
|
2731
|
+
if (expression)
|
|
2732
|
+
scopedExpressions.push(expression);
|
|
2733
|
+
}
|
|
2734
|
+
else {
|
|
2735
|
+
ignored.push({ source_key: sourceKey, expression, reason: "outside_repair_scope" });
|
|
2736
|
+
}
|
|
2737
|
+
}
|
|
2738
|
+
return {
|
|
2739
|
+
plan: {
|
|
2740
|
+
...plan,
|
|
2741
|
+
expression_text: formatCurationExpressionText(scopedExpressions),
|
|
2742
|
+
expressions: scopedExpressions.length > 0 ? scopedExpressions : ["NOOP # no asset curation changes"],
|
|
2743
|
+
accepted_expressions: scopedExpressions,
|
|
2744
|
+
decisions: scopedDecisions,
|
|
2745
|
+
},
|
|
2746
|
+
ignored,
|
|
2747
|
+
};
|
|
2748
|
+
}
|
|
2749
|
+
function missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan) {
|
|
2750
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2751
|
+
return missingAssetCurationRequiredSourceKeys(script, partialPlan).filter((key) => candidateSet.has(key));
|
|
2752
|
+
}
|
|
2753
|
+
function repairSourceKeysForPart(script, candidateSourceKeys, partialPlan, requireCoverage) {
|
|
2754
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2755
|
+
const accepted = new Set(asList(partialPlan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
|
|
2756
|
+
const rejected = asList(partialPlan["rejected_expressions"]).filter(isDict);
|
|
2757
|
+
const repairKeys = [];
|
|
2758
|
+
let hasUnlocatedRejection = false;
|
|
2759
|
+
for (const item of rejected) {
|
|
2760
|
+
const key = strOf(item["source_key"]);
|
|
2761
|
+
if (key && (candidateSet.size === 0 || candidateSet.has(key)))
|
|
2762
|
+
uniqueAdd(repairKeys, key);
|
|
2763
|
+
else
|
|
2764
|
+
hasUnlocatedRejection = true;
|
|
2765
|
+
}
|
|
2766
|
+
if (hasUnlocatedRejection) {
|
|
2767
|
+
for (const key of candidateSourceKeys) {
|
|
2768
|
+
if (!accepted.has(key))
|
|
2769
|
+
uniqueAdd(repairKeys, key);
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2772
|
+
if (requireCoverage) {
|
|
2773
|
+
for (const key of missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan)) {
|
|
2774
|
+
uniqueAdd(repairKeys, key);
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
return repairKeys;
|
|
2778
|
+
}
|
|
2779
|
+
function mergeRepairedCurationPlan(rawText, partialPlan, repairPlan, repairAttempts) {
|
|
2780
|
+
const acceptedExpressions = asList(partialPlan["accepted_expressions"]);
|
|
2781
|
+
const repairExpressions = asList(repairPlan["accepted_expressions"]);
|
|
2782
|
+
const decisions = [
|
|
2783
|
+
...asList(partialPlan["decisions"]).filter(isDict),
|
|
2784
|
+
...asList(repairPlan["decisions"]).filter(isDict),
|
|
2785
|
+
];
|
|
2786
|
+
const expressions = [...acceptedExpressions, ...repairExpressions].filter((expression) => !/^NOOP(?:\s|#|$)/i.test(expression.trim()));
|
|
2787
|
+
return {
|
|
2788
|
+
version: 3,
|
|
2789
|
+
format: "asset-curation-exp-v1",
|
|
2790
|
+
expression_text: formatCurationExpressionText(expressions),
|
|
2791
|
+
raw_expression_text: rawText,
|
|
2792
|
+
accepted_expressions: [...acceptedExpressions, ...repairExpressions],
|
|
2793
|
+
rejected_expressions: asList(partialPlan["rejected_expressions"]).filter(isDict),
|
|
2794
|
+
repair_attempts: repairAttempts,
|
|
2795
|
+
expressions: expressions.length > 0 ? expressions : ["NOOP # no asset curation changes"],
|
|
2796
|
+
decisions,
|
|
2797
|
+
partial: false,
|
|
2798
|
+
};
|
|
2799
|
+
}
|
|
2800
|
+
function formatCurationExpressionText(expressions) {
|
|
2801
|
+
if (expressions.length > 0)
|
|
2802
|
+
return expressions.map((expression) => `<exp>${expression}</exp>`).join("\n");
|
|
2803
|
+
return "<exp>NOOP # no asset curation changes</exp>";
|
|
2804
|
+
}
|
|
2805
|
+
function writeFailedCurationPartArtifact(opts) {
|
|
2806
|
+
const e = opts.error;
|
|
2807
|
+
opts.ctx.ws.writeJson(opts.artifactName, {
|
|
2808
|
+
version: 1,
|
|
2809
|
+
...assetCacheArtifactFields(opts.ctx, ASSET_CURATION_CACHE_VERSION),
|
|
2810
|
+
script_fingerprint: opts.fingerprint,
|
|
2811
|
+
...opts.metadata,
|
|
2812
|
+
parse_status: "failed",
|
|
2813
|
+
candidate_source_keys: opts.candidateSourceKeys,
|
|
2814
|
+
expression_text: opts.partPlan?.["expression_text"] ?? null,
|
|
2815
|
+
raw_expression_text: opts.partPlan?.["raw_expression_text"] ?? opts.expressionText,
|
|
2816
|
+
accepted_expressions: opts.partPlan?.["accepted_expressions"] ?? [],
|
|
2817
|
+
rejected_expressions: opts.partPlan?.["rejected_expressions"] ?? [],
|
|
2818
|
+
repair_attempts: opts.repairAttempts,
|
|
2819
|
+
duration_ms: durationMs(opts.startedAtMs, Date.now()),
|
|
2820
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
2821
|
+
plan: opts.partPlan,
|
|
2822
|
+
updated_at: checkpointTimestamp(),
|
|
2823
|
+
});
|
|
2824
|
+
}
|
|
2825
|
+
async function loadOrRunCurationPart(ctx, script, artifactName, fingerprint, metadata, candidateSourceKeys, parentTiming, requireCoverage, contextText, run) {
|
|
2826
|
+
const cached = ctx.ws.readJsonSafe(artifactName);
|
|
2827
|
+
const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
|
|
2828
|
+
if (cachedPlan && isList(cachedPlan["decisions"]))
|
|
2829
|
+
return cachedPlan;
|
|
2830
|
+
const startedAtMs = Date.now();
|
|
2831
|
+
const baseContextText = contextText(candidateSourceKeys);
|
|
2832
|
+
const expressionText = await ctx.timing.span({ name: strOf(metadata["part_id"]) || artifactName, category: "chunk", parent: parentTiming, metadata: { artifact: artifactName } }, () => run(baseContextText));
|
|
2833
|
+
let partPlan = parseAssetCurationExpressionsTolerant(expressionText);
|
|
2834
|
+
if (!isDict(partPlan) || !isList(partPlan["decisions"])) {
|
|
2835
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
|
|
2836
|
+
exitCode: EXIT_RUNTIME,
|
|
2837
|
+
required: ["object with decisions[]"],
|
|
2838
|
+
received: [typeof partPlan],
|
|
2839
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
|
|
2840
|
+
});
|
|
2841
|
+
}
|
|
2842
|
+
const repairAttempts = [];
|
|
2843
|
+
let repairSourceKeys = repairSourceKeysForPart(script, candidateSourceKeys, partPlan, requireCoverage);
|
|
2844
|
+
let repairRejectedContext = asList(partPlan["rejected_expressions"]).filter(isDict);
|
|
2845
|
+
try {
|
|
2846
|
+
for (let attempt = 1; repairSourceKeys.length > 0 && attempt <= MAX_CURATION_PART_REPAIR_ATTEMPTS; attempt += 1) {
|
|
2847
|
+
const repairContext = buildAssetCurationRepairContextText({
|
|
2848
|
+
baseContextText: contextText(repairSourceKeys),
|
|
2849
|
+
repairSourceKeys,
|
|
2850
|
+
rejectedExpressions: repairRejectedContext,
|
|
2851
|
+
acceptedDecisions: asList(partPlan["decisions"]).filter(isDict),
|
|
2852
|
+
});
|
|
2853
|
+
const repairText = await ctx.timing.span({ name: `${strOf(metadata["part_id"]) || artifactName}.repair.${pad3(attempt)}`, category: "repair", parent: parentTiming, metadata: { artifact: artifactName, repair_source_keys: repairSourceKeys } }, () => run(repairContext));
|
|
2854
|
+
const repairPlan = parseAssetCurationExpressionsTolerant(repairText);
|
|
2855
|
+
const repairRejected = asList(repairPlan["rejected_expressions"]).filter(isDict);
|
|
2856
|
+
const scopedRepair = scopedCurationPlan(repairPlan, repairSourceKeys);
|
|
2857
|
+
repairAttempts.push({
|
|
2858
|
+
index: attempt,
|
|
2859
|
+
repair_source_keys: repairSourceKeys,
|
|
2860
|
+
expression_text: repairText,
|
|
2861
|
+
accepted_expressions: scopedRepair.plan["accepted_expressions"],
|
|
2862
|
+
ignored_expressions: scopedRepair.ignored,
|
|
2863
|
+
rejected_expressions: repairRejected,
|
|
2864
|
+
decisions: scopedRepair.plan["decisions"],
|
|
2865
|
+
});
|
|
2866
|
+
if (repairRejected.length > 0) {
|
|
2867
|
+
if (asList(scopedRepair.plan["decisions"]).filter(isDict).length > 0) {
|
|
2868
|
+
partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
|
|
2869
|
+
}
|
|
2870
|
+
const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
|
|
2871
|
+
const rejectedKeys = sourceKeysFromRejectedExpressions(repairRejected, repairSourceKeys);
|
|
2872
|
+
const nextRepairSourceKeys = [...(rejectedKeys.length > 0 ? rejectedKeys : []), ...missingKeys]
|
|
2873
|
+
.filter((key, index, all) => all.indexOf(key) === index);
|
|
2874
|
+
if (nextRepairSourceKeys.length === 0) {
|
|
2875
|
+
repairSourceKeys = [];
|
|
2876
|
+
repairRejectedContext = [];
|
|
2877
|
+
continue;
|
|
2878
|
+
}
|
|
2879
|
+
if (attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
|
|
2880
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair returned invalid expressions", repairRejected.map((item) => `${strOf(item["source_key"]) || "-"}: ${strOf(item["error"])}`).slice(0, 40));
|
|
2881
|
+
}
|
|
2882
|
+
repairSourceKeys = nextRepairSourceKeys;
|
|
2883
|
+
repairRejectedContext = repairRejected;
|
|
2884
|
+
continue;
|
|
2885
|
+
}
|
|
2886
|
+
const finalKeys = new Set([...curationDecisionSourceKeys(partPlan), ...curationDecisionSourceKeys(scopedRepair.plan)]);
|
|
2887
|
+
const unrepaired = repairSourceKeys.filter((key) => !finalKeys.has(key));
|
|
2888
|
+
if (unrepaired.length > 0 && attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
|
|
2889
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", unrepaired.slice(0, 40));
|
|
2890
|
+
}
|
|
2891
|
+
partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
|
|
2892
|
+
repairRejectedContext = [];
|
|
2893
|
+
const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
|
|
2894
|
+
repairSourceKeys = [...unrepaired, ...missingKeys]
|
|
2895
|
+
.filter((key, index, all) => all.indexOf(key) === index);
|
|
2896
|
+
}
|
|
2897
|
+
if (repairSourceKeys.length > 0) {
|
|
2898
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", repairSourceKeys.slice(0, 40));
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
catch (exc) {
|
|
2902
|
+
writeFailedCurationPartArtifact({
|
|
2903
|
+
ctx,
|
|
2904
|
+
artifactName,
|
|
2905
|
+
fingerprint,
|
|
2906
|
+
metadata,
|
|
2907
|
+
candidateSourceKeys,
|
|
2908
|
+
expressionText,
|
|
2909
|
+
partPlan,
|
|
2910
|
+
repairAttempts,
|
|
2911
|
+
startedAtMs,
|
|
2912
|
+
error: exc,
|
|
2913
|
+
});
|
|
2914
|
+
throw exc;
|
|
2915
|
+
}
|
|
2916
|
+
for (const [key, value] of Object.entries(metadata))
|
|
2917
|
+
partPlan[key] = value;
|
|
2918
|
+
ctx.ws.writeJson(artifactName, {
|
|
2919
|
+
version: 1,
|
|
2920
|
+
...assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION),
|
|
2921
|
+
script_fingerprint: fingerprint,
|
|
2922
|
+
...metadata,
|
|
2923
|
+
parse_status: "ok",
|
|
2924
|
+
candidate_source_keys: candidateSourceKeys,
|
|
2925
|
+
expression_text: partPlan["expression_text"],
|
|
2926
|
+
raw_expression_text: partPlan["raw_expression_text"] ?? expressionText,
|
|
2927
|
+
accepted_expressions: partPlan["accepted_expressions"],
|
|
2928
|
+
rejected_expressions: partPlan["rejected_expressions"],
|
|
2929
|
+
repair_attempts: repairAttempts,
|
|
2930
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2931
|
+
plan: partPlan,
|
|
2932
|
+
updated_at: checkpointTimestamp(),
|
|
2933
|
+
});
|
|
2934
|
+
return partPlan;
|
|
2935
|
+
}
|
|
2936
|
+
async function loadOrRunPrimaryCurationPart(ctx, script, curateAssets, kind, partId, fingerprint, ids) {
|
|
2937
|
+
const selection = curationPartSelection(kind, ids);
|
|
2938
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/primary.${partId}.json`, fingerprint, { asset_kind: kind, part_id: partId, selection }, curationSourceKeysForIds(kind, ids), "asset_curation.primary", true, (selectedSourceKeys = curationSourceKeysForIds(kind, ids)) => {
|
|
2939
|
+
const selectedIds = curationIdsFromSourceKeys(kind, ids, selectedSourceKeys);
|
|
2940
|
+
return buildAssetCurationContextText(script, [kind], curationPartSelection(kind, selectedIds.length > 0 ? selectedIds : ids));
|
|
2941
|
+
}, (contextText) => curateAssets(script, contextText));
|
|
2942
|
+
}
|
|
2943
|
+
function primaryCurationChunks(script, groupingPlan) {
|
|
2944
|
+
const chunks = [];
|
|
2945
|
+
for (const kind of ["actor", "location", "prop"]) {
|
|
2946
|
+
const groupingGroups = groupingPlan
|
|
2947
|
+
? asList(groupingPlan["groups"]).filter((group) => isDict(group) && group["kind"] === kind)
|
|
2948
|
+
: [];
|
|
2949
|
+
const kindChunks = groupingGroups.length > 0
|
|
2950
|
+
? groupedAssetCurationChunks(script, kind, groupingGroups)
|
|
2951
|
+
: buildPrimaryAssetCurationChunks(script, kind);
|
|
2952
|
+
for (let index = 0; index < kindChunks.length; index += 1) {
|
|
2953
|
+
chunks.push({
|
|
2954
|
+
kind,
|
|
2955
|
+
partId: `${kind}.${String(index + 1).padStart(3, "0")}`,
|
|
2956
|
+
ids: kindChunks[index],
|
|
2957
|
+
});
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
return chunks;
|
|
2961
|
+
}
|
|
2962
|
+
async function loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, fingerprint) {
|
|
2963
|
+
const sourceChunks = chunkStrings(assetCurationAuditSourceKeys(script, primaryPlan), 24);
|
|
2964
|
+
if (sourceChunks.length === 0) {
|
|
2965
|
+
return parseAssetCurationExpressions("<exp>NOOP # no asset curation changes</exp>");
|
|
2966
|
+
}
|
|
2967
|
+
const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
|
|
2968
|
+
const partId = `audit.${String(index + 1).padStart(3, "0")}`;
|
|
2969
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.audit", false, (repairSourceKeys = selectedSourceKeys) => buildAssetCurationAuditContextText(script, primaryPlan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
|
|
2970
|
+
});
|
|
2971
|
+
const parts = unwrapConcurrentOutcomes(outcomes);
|
|
2972
|
+
return combineAssetCurationExpressionPlans(parts);
|
|
2973
|
+
}
|
|
2974
|
+
async function loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, fingerprint) {
|
|
2975
|
+
const sourceChunks = chunkStrings(locationCanonicalizationCandidateSourceKeys(script, plan), 24);
|
|
2976
|
+
if (sourceChunks.length === 0) {
|
|
2977
|
+
return parseAssetCurationExpressions("<exp>NOOP # no location canonicalization changes</exp>");
|
|
2978
|
+
}
|
|
2979
|
+
const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
|
|
2980
|
+
const partId = `location_canonical.${String(index + 1).padStart(3, "0")}`;
|
|
2981
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.location_canonical", false, (repairSourceKeys = selectedSourceKeys) => buildLocationCanonicalizationAuditContextText(script, plan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
|
|
2982
|
+
});
|
|
2983
|
+
const parts = unwrapConcurrentOutcomes(outcomes);
|
|
2984
|
+
return combineAssetCurationExpressionPlans(parts);
|
|
2985
|
+
}
|
|
2134
2986
|
export async function stageAssetCuration(ctx) {
|
|
2135
2987
|
const { ws, state, options } = ctx;
|
|
2136
2988
|
const script = ensureScript(ctx);
|
|
2989
|
+
let groupingPlan = null;
|
|
2990
|
+
let groupingApplySummary = null;
|
|
2991
|
+
let plan = null;
|
|
2992
|
+
let primaryPlan = null;
|
|
2993
|
+
let auditPlan = null;
|
|
2994
|
+
let locationCanonicalPlan = null;
|
|
2995
|
+
let primaryExpressionText = null;
|
|
2996
|
+
let primaryPartExpressionTexts = [];
|
|
2997
|
+
let auditExpressionText = null;
|
|
2998
|
+
let locationCanonicalExpressionText = null;
|
|
2137
2999
|
try {
|
|
2138
3000
|
ctx.updateRunState({ status: "init_running", init_stage: "asset_curation" });
|
|
2139
3001
|
ctx.emitProgress("asset-curation");
|
|
2140
|
-
const
|
|
3002
|
+
const provider = ctx.provider();
|
|
3003
|
+
if (!provider.curateAssets) {
|
|
3004
|
+
throw new CliError("INIT FAILED: Provider does not support asset curation", "Provider does not support asset curation.", {
|
|
3005
|
+
exitCode: EXIT_RUNTIME,
|
|
3006
|
+
required: ["provider.curateAssets(script, contextText)"],
|
|
3007
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3008
|
+
nextSteps: ["Use a direct provider that supports asset curation."],
|
|
3009
|
+
});
|
|
3010
|
+
}
|
|
3011
|
+
if (!provider.groupAssets) {
|
|
3012
|
+
throw new CliError("INIT FAILED: Provider does not support asset grouping", "Provider does not support asset grouping.", {
|
|
3013
|
+
exitCode: EXIT_RUNTIME,
|
|
3014
|
+
required: ["provider.groupAssets(script, contextText)"],
|
|
3015
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3016
|
+
nextSteps: ["Use a direct provider that supports asset grouping."],
|
|
3017
|
+
});
|
|
3018
|
+
}
|
|
3019
|
+
const groupAssets = provider.groupAssets.bind(provider);
|
|
3020
|
+
const curateAssets = provider.curateAssets.bind(provider);
|
|
3021
|
+
const groupingFingerprint = assetGroupingScriptFingerprint(script, options);
|
|
3022
|
+
groupingPlan = await ctx.timing.span({ name: "asset_curation.grouping", category: "substage", parent: "asset_curation" }, () => loadOrRunAssetGrouping(ctx, script, groupAssets, groupingFingerprint));
|
|
3023
|
+
groupingApplySummary = await ctx.timing.span({ name: "asset_curation.grouping.apply", category: "substage", parent: "asset_curation.grouping" }, () => applyAssetGroupingDuplicateMerges(script, { decisions: groupingPlan["duplicate_merge_decisions"] }));
|
|
3024
|
+
Object.assign(groupingPlan, assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION));
|
|
3025
|
+
groupingPlan["apply_summary"] = groupingApplySummary;
|
|
3026
|
+
groupingPlan["script_fingerprint"] = groupingFingerprint;
|
|
3027
|
+
ws.writeJson("asset_grouping.json", groupingPlan);
|
|
3028
|
+
const fingerprint = assetCurationScriptFingerprint(script, options);
|
|
3029
|
+
const cachedCuration = ws.readJsonSafe("asset_curation.json");
|
|
3030
|
+
if (!state.bypassInternalCaches && isDict(cachedCuration) && cachedCuration["script_fingerprint"] === fingerprint && isList(cachedCuration["decisions"])) {
|
|
3031
|
+
const curation = curateScriptAssets(script, { decisions: cachedCuration["decisions"] });
|
|
3032
|
+
Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
|
|
3033
|
+
curation["script_fingerprint"] = fingerprint;
|
|
3034
|
+
curation["grouping"] = groupingPlan;
|
|
3035
|
+
curation["grouping_apply_summary"] = groupingApplySummary;
|
|
3036
|
+
if (typeof cachedCuration["expression_text"] === "string")
|
|
3037
|
+
curation["expression_text"] = cachedCuration["expression_text"];
|
|
3038
|
+
if (typeof cachedCuration["primary_expression_text"] === "string")
|
|
3039
|
+
curation["primary_expression_text"] = cachedCuration["primary_expression_text"];
|
|
3040
|
+
if (typeof cachedCuration["primary_part_expression_text"] === "string")
|
|
3041
|
+
curation["primary_part_expression_text"] = cachedCuration["primary_part_expression_text"];
|
|
3042
|
+
if (typeof cachedCuration["audit_expression_text"] === "string")
|
|
3043
|
+
curation["audit_expression_text"] = cachedCuration["audit_expression_text"];
|
|
3044
|
+
if (typeof cachedCuration["location_canonical_expression_text"] === "string")
|
|
3045
|
+
curation["location_canonical_expression_text"] = cachedCuration["location_canonical_expression_text"];
|
|
3046
|
+
if (isList(cachedCuration["expressions"]))
|
|
3047
|
+
curation["expressions"] = cachedCuration["expressions"];
|
|
3048
|
+
if (isList(cachedCuration["primary_expressions"]))
|
|
3049
|
+
curation["primary_expressions"] = cachedCuration["primary_expressions"];
|
|
3050
|
+
if (isList(cachedCuration["primary_parts"]))
|
|
3051
|
+
curation["primary_parts"] = cachedCuration["primary_parts"];
|
|
3052
|
+
if (isList(cachedCuration["audit_expressions"]))
|
|
3053
|
+
curation["audit_expressions"] = cachedCuration["audit_expressions"];
|
|
3054
|
+
if (isList(cachedCuration["location_canonical_expressions"]))
|
|
3055
|
+
curation["location_canonical_expressions"] = cachedCuration["location_canonical_expressions"];
|
|
3056
|
+
if (typeof cachedCuration["format"] === "string")
|
|
3057
|
+
curation["format"] = cachedCuration["format"];
|
|
3058
|
+
if (isDict(cachedCuration["audit_summary"]))
|
|
3059
|
+
curation["audit_summary"] = cachedCuration["audit_summary"];
|
|
3060
|
+
ws.writeJson("asset_curation.json", curation);
|
|
3061
|
+
ws.delete("asset_curation.failed.json");
|
|
3062
|
+
ctx.emitProgress("asset-curation", "success");
|
|
3063
|
+
return;
|
|
3064
|
+
}
|
|
3065
|
+
const primaryChunkSpecs = primaryCurationChunks(script, groupingPlan);
|
|
3066
|
+
const primaryOutcomes = await pMapWithConcurrency(primaryChunkSpecs, options.concurrency, async (chunk) => loadOrRunPrimaryCurationPart(ctx, script, curateAssets, chunk.kind, chunk.partId, fingerprint, chunk.ids));
|
|
3067
|
+
const primaryParts = unwrapConcurrentOutcomes(primaryOutcomes);
|
|
3068
|
+
primaryPartExpressionTexts = primaryParts.map((part) => ({
|
|
3069
|
+
asset_kind: part["asset_kind"],
|
|
3070
|
+
part_id: part["part_id"],
|
|
3071
|
+
expression_text: part["expression_text"],
|
|
3072
|
+
}));
|
|
3073
|
+
primaryPlan = await ctx.timing.span({ name: "asset_curation.primary.combine", category: "substage", parent: "asset_curation.primary", metadata: { chunks: primaryParts.length } }, () => combinePrimaryAssetCurationPlans(primaryParts));
|
|
3074
|
+
primaryExpressionText = typeof primaryPlan["expression_text"] === "string" ? primaryPlan["expression_text"] : "";
|
|
3075
|
+
if (!isDict(primaryPlan) || !isList(primaryPlan["decisions"])) {
|
|
3076
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
|
|
3077
|
+
exitCode: EXIT_RUNTIME,
|
|
3078
|
+
required: ["object with decisions[]"],
|
|
3079
|
+
received: [typeof primaryPlan],
|
|
3080
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
|
|
3081
|
+
});
|
|
3082
|
+
}
|
|
3083
|
+
const allowedNewAuditSources = [
|
|
3084
|
+
...missingAssetCurationRequiredSourceKeys(script, primaryPlan),
|
|
3085
|
+
...assetCurationAuditCandidateSourceKeys(script, primaryPlan),
|
|
3086
|
+
];
|
|
3087
|
+
auditPlan = await ctx.timing.span({ name: "asset_curation.audit", category: "substage", parent: "asset_curation" }, () => loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, assetCurationPlanFingerprint(`${fingerprint}:audit-chunk-v1`, primaryPlan)));
|
|
3088
|
+
auditExpressionText = typeof auditPlan["expression_text"] === "string" ? auditPlan["expression_text"] : "";
|
|
3089
|
+
if (!isDict(auditPlan) || !isList(auditPlan["decisions"])) {
|
|
3090
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation audit", "Provider returned invalid asset curation audit.", {
|
|
3091
|
+
exitCode: EXIT_RUNTIME,
|
|
3092
|
+
required: ["object with decisions[]"],
|
|
3093
|
+
received: [typeof auditPlan],
|
|
3094
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation audit will retry."],
|
|
3095
|
+
});
|
|
3096
|
+
}
|
|
3097
|
+
plan = combineAssetCurationPlans(primaryPlan, auditPlan, allowedNewAuditSources);
|
|
3098
|
+
const allowedNewLocationCanonicalSources = locationCanonicalizationCandidateSourceKeys(script, plan);
|
|
3099
|
+
locationCanonicalPlan = await ctx.timing.span({ name: "asset_curation.location_canonical", category: "substage", parent: "asset_curation" }, () => loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, assetCurationPlanFingerprint(`${fingerprint}:location-canonical-chunk-v2`, plan)));
|
|
3100
|
+
locationCanonicalExpressionText = typeof locationCanonicalPlan["expression_text"] === "string" ? locationCanonicalPlan["expression_text"] : "";
|
|
3101
|
+
if (!isDict(locationCanonicalPlan) || !isList(locationCanonicalPlan["decisions"])) {
|
|
3102
|
+
throw new CliError("INIT FAILED: Provider returned invalid location canonicalization audit", "Provider returned invalid location canonicalization audit.", {
|
|
3103
|
+
exitCode: EXIT_RUNTIME,
|
|
3104
|
+
required: ["object with decisions[]"],
|
|
3105
|
+
received: [typeof locationCanonicalPlan],
|
|
3106
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and location canonicalization will retry."],
|
|
3107
|
+
});
|
|
3108
|
+
}
|
|
3109
|
+
assertLocationCanonicalizationPlan(locationCanonicalPlan);
|
|
3110
|
+
plan = combineLocationCanonicalizationPlan(script, plan, locationCanonicalPlan, allowedNewLocationCanonicalSources);
|
|
3111
|
+
const curation = await ctx.timing.span({ name: "asset_curation.apply", category: "substage", parent: "asset_curation" }, () => curateScriptAssets(script, plan));
|
|
3112
|
+
Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
|
|
3113
|
+
curation["script_fingerprint"] = fingerprint;
|
|
3114
|
+
curation["format"] = plan["format"];
|
|
3115
|
+
curation["expression_text"] = plan["expression_text"];
|
|
3116
|
+
curation["primary_expression_text"] = plan["primary_expression_text"];
|
|
3117
|
+
curation["audit_expression_text"] = plan["audit_expression_text"];
|
|
3118
|
+
curation["location_canonical_expression_text"] = plan["location_canonical_expression_text"];
|
|
3119
|
+
curation["primary_part_expression_text"] = plan["primary_part_expression_text"];
|
|
3120
|
+
curation["primary_parts"] = plan["primary_parts"];
|
|
3121
|
+
curation["expressions"] = plan["expressions"];
|
|
3122
|
+
curation["primary_expressions"] = plan["primary_expressions"];
|
|
3123
|
+
curation["audit_expressions"] = plan["audit_expressions"];
|
|
3124
|
+
curation["location_canonical_expressions"] = plan["location_canonical_expressions"];
|
|
3125
|
+
curation["audit_summary"] = plan["audit_summary"];
|
|
3126
|
+
curation["grouping"] = groupingPlan;
|
|
3127
|
+
curation["grouping_apply_summary"] = groupingApplySummary;
|
|
2141
3128
|
ws.writeJson("asset_curation.json", curation);
|
|
3129
|
+
ws.delete("asset_curation.failed.json");
|
|
2142
3130
|
}
|
|
2143
3131
|
catch (exc) {
|
|
3132
|
+
if (plan || primaryPlan || auditPlan || locationCanonicalPlan || primaryExpressionText !== null || auditExpressionText !== null || locationCanonicalExpressionText !== null) {
|
|
3133
|
+
const e = exc;
|
|
3134
|
+
ws.writeJson("asset_curation.failed.json", {
|
|
3135
|
+
version: 3,
|
|
3136
|
+
failed_at: checkpointTimestamp(),
|
|
3137
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3138
|
+
expression_text: plan?.["expression_text"] ?? null,
|
|
3139
|
+
primary_expression_text: primaryExpressionText,
|
|
3140
|
+
primary_part_expression_texts: primaryPartExpressionTexts,
|
|
3141
|
+
audit_expression_text: auditExpressionText,
|
|
3142
|
+
location_canonical_expression_text: locationCanonicalExpressionText,
|
|
3143
|
+
primary_plan: primaryPlan,
|
|
3144
|
+
audit_plan: auditPlan,
|
|
3145
|
+
location_canonical_plan: locationCanonicalPlan,
|
|
3146
|
+
grouping_plan: groupingPlan,
|
|
3147
|
+
grouping_apply_summary: groupingApplySummary,
|
|
3148
|
+
plan,
|
|
3149
|
+
});
|
|
3150
|
+
}
|
|
2144
3151
|
stageFailure(options.workspace, exc, {
|
|
2145
3152
|
title: "INIT FAILED: Asset curation failed",
|
|
2146
3153
|
stage: "asset_curation",
|
|
@@ -2151,28 +3158,403 @@ export async function stageAssetCuration(ctx) {
|
|
|
2151
3158
|
}
|
|
2152
3159
|
ctx.emitProgress("asset-curation", "success");
|
|
2153
3160
|
}
|
|
2154
|
-
// ---
|
|
2155
|
-
|
|
3161
|
+
// --- state-curation -------------------------------------------------------
|
|
3162
|
+
const STATE_CURATION_CACHE_VERSION = 1;
|
|
3163
|
+
function stateCurationScriptFingerprint(script, options) {
|
|
3164
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
3165
|
+
cache_version: STATE_CURATION_CACHE_VERSION,
|
|
3166
|
+
...assetCacheIdentity(options),
|
|
3167
|
+
script: {
|
|
3168
|
+
actors: script["actors"],
|
|
3169
|
+
locations: script["locations"],
|
|
3170
|
+
props: script["props"],
|
|
3171
|
+
episodes: script["episodes"],
|
|
3172
|
+
},
|
|
3173
|
+
})));
|
|
3174
|
+
}
|
|
3175
|
+
function stateCurationContextFingerprint(scriptFingerprint, partId, refs) {
|
|
3176
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
3177
|
+
script_fingerprint: scriptFingerprint,
|
|
3178
|
+
part_id: partId,
|
|
3179
|
+
refs,
|
|
3180
|
+
})));
|
|
3181
|
+
}
|
|
3182
|
+
function combineStateCurationPlans(parts) {
|
|
3183
|
+
const decisions = parts.flatMap((part) => asList(part.plan["decisions"]).filter(isDict));
|
|
3184
|
+
const expressions = parts.flatMap((part) => asList(part.plan["expressions"]));
|
|
3185
|
+
return {
|
|
3186
|
+
version: 1,
|
|
3187
|
+
format: "state-curation-exp-v1",
|
|
3188
|
+
expression_text: expressions.length > 0 ? expressions.map((expression) => `<exp>${expression}</exp>`).join("\n") : "<exp>NOOP # no state curation changes</exp>",
|
|
3189
|
+
expressions,
|
|
3190
|
+
decisions,
|
|
3191
|
+
parts: parts.map((part) => ({
|
|
3192
|
+
part_id: part.partId,
|
|
3193
|
+
refs: part.refs,
|
|
3194
|
+
context_fingerprint: part.contextFingerprint,
|
|
3195
|
+
context_chars: part.contextChars,
|
|
3196
|
+
reused: part.reused,
|
|
3197
|
+
decisions: asList(part.plan["decisions"]).length,
|
|
3198
|
+
})),
|
|
3199
|
+
};
|
|
3200
|
+
}
|
|
3201
|
+
async function loadOrRunStateCurationPart(ctx, script, curateStates, scriptFingerprint, partId, refs) {
|
|
3202
|
+
const artifactName = `state_curation_parts/${partId}.json`;
|
|
3203
|
+
const contextFingerprint = stateCurationContextFingerprint(scriptFingerprint, partId, refs);
|
|
3204
|
+
const cached = ctx.ws.readJsonSafe(artifactName);
|
|
3205
|
+
const cachedPlan = !ctx.state.bypassInternalCaches
|
|
3206
|
+
&& isDict(cached)
|
|
3207
|
+
&& cached["context_fingerprint"] === contextFingerprint
|
|
3208
|
+
&& cached["parse_status"] === "ok"
|
|
3209
|
+
&& isDict(cached["plan"])
|
|
3210
|
+
? cached["plan"]
|
|
3211
|
+
: null;
|
|
3212
|
+
if (cachedPlan && isList(cachedPlan["decisions"])) {
|
|
3213
|
+
const cachedContextChars = isDict(cached) ? Number(cached["context_chars"] ?? 0) : 0;
|
|
3214
|
+
return {
|
|
3215
|
+
partId,
|
|
3216
|
+
refs: [...refs],
|
|
3217
|
+
contextFingerprint,
|
|
3218
|
+
contextChars: cachedContextChars,
|
|
3219
|
+
plan: cachedPlan,
|
|
3220
|
+
reused: true,
|
|
3221
|
+
};
|
|
3222
|
+
}
|
|
3223
|
+
const startedAtMs = Date.now();
|
|
3224
|
+
const contextText = buildStateCurationContextText(script, refs);
|
|
3225
|
+
const requiredKeys = stateCurationSourceKeysForRefs(script, refs);
|
|
3226
|
+
let expressionText = "";
|
|
3227
|
+
try {
|
|
3228
|
+
expressionText = await ctx.timing.span({ name: `state_curation.${partId}`, category: "chunk", parent: "state_curation.chunks", metadata: { artifact: artifactName } }, () => curateStates(script, contextText));
|
|
3229
|
+
const parsed = parseStateCurationExpressions(expressionText);
|
|
3230
|
+
const plan = assertStateCurationPlan(script, parsed, requiredKeys);
|
|
3231
|
+
ctx.ws.writeJson(artifactName, {
|
|
3232
|
+
version: 1,
|
|
3233
|
+
...assetCacheArtifactFields(ctx, STATE_CURATION_CACHE_VERSION),
|
|
3234
|
+
context_fingerprint: contextFingerprint,
|
|
3235
|
+
script_fingerprint: scriptFingerprint,
|
|
3236
|
+
parse_status: "ok",
|
|
3237
|
+
part_id: partId,
|
|
3238
|
+
refs,
|
|
3239
|
+
required_state_ids: requiredKeys,
|
|
3240
|
+
context_chars: contextText.length,
|
|
3241
|
+
expression_text: expressionText,
|
|
3242
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
3243
|
+
plan,
|
|
3244
|
+
updated_at: checkpointTimestamp(),
|
|
3245
|
+
});
|
|
3246
|
+
return {
|
|
3247
|
+
partId,
|
|
3248
|
+
refs: [...refs],
|
|
3249
|
+
contextFingerprint,
|
|
3250
|
+
contextChars: contextText.length,
|
|
3251
|
+
plan,
|
|
3252
|
+
reused: false,
|
|
3253
|
+
};
|
|
3254
|
+
}
|
|
3255
|
+
catch (exc) {
|
|
3256
|
+
const e = exc;
|
|
3257
|
+
ctx.ws.writeJson(artifactName, {
|
|
3258
|
+
version: 1,
|
|
3259
|
+
...assetCacheArtifactFields(ctx, STATE_CURATION_CACHE_VERSION),
|
|
3260
|
+
context_fingerprint: contextFingerprint,
|
|
3261
|
+
script_fingerprint: scriptFingerprint,
|
|
3262
|
+
parse_status: "failed",
|
|
3263
|
+
part_id: partId,
|
|
3264
|
+
refs,
|
|
3265
|
+
required_state_ids: requiredKeys,
|
|
3266
|
+
context_chars: contextText.length,
|
|
3267
|
+
expression_text: expressionText,
|
|
3268
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
3269
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3270
|
+
updated_at: checkpointTimestamp(),
|
|
3271
|
+
});
|
|
3272
|
+
throw exc;
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
export async function stageStateCuration(ctx) {
|
|
2156
3276
|
const { ws, state, options } = ctx;
|
|
2157
3277
|
const script = ensureScript(ctx);
|
|
3278
|
+
let plan = null;
|
|
3279
|
+
try {
|
|
3280
|
+
ctx.updateRunState({ status: "init_running", init_stage: "state_curation" });
|
|
3281
|
+
ctx.emitProgress("state-curation");
|
|
3282
|
+
const scriptFingerprint = stateCurationScriptFingerprint(script, options);
|
|
3283
|
+
const cached = ws.readJsonSafe("state_curation.json");
|
|
3284
|
+
if (!state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === scriptFingerprint && isList(cached["decisions"])) {
|
|
3285
|
+
const curation = await ctx.timing.span({ name: "state_curation.apply", category: "substage", parent: "state_curation" }, () => applyStateCurationPlan(script, { decisions: cached["decisions"] }));
|
|
3286
|
+
curation["script_fingerprint"] = scriptFingerprint;
|
|
3287
|
+
curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
|
|
3288
|
+
curation["provider"] = options.providerName;
|
|
3289
|
+
curation["model"] = options.model;
|
|
3290
|
+
if (isList(cached["parts"]))
|
|
3291
|
+
curation["parts"] = cached["parts"];
|
|
3292
|
+
if (typeof cached["expression_text"] === "string")
|
|
3293
|
+
curation["expression_text"] = cached["expression_text"];
|
|
3294
|
+
if (isList(cached["expressions"]))
|
|
3295
|
+
curation["expressions"] = cached["expressions"];
|
|
3296
|
+
ws.writeJson("state_curation.json", curation);
|
|
3297
|
+
ws.delete("state_curation.failed.json");
|
|
3298
|
+
ctx.emitProgress("state-curation", "success");
|
|
3299
|
+
return;
|
|
3300
|
+
}
|
|
3301
|
+
const chunks = buildStateCurationChunks(script);
|
|
3302
|
+
if (chunks.length === 0) {
|
|
3303
|
+
const curation = applyStateCurationPlan(script, { decisions: [] });
|
|
3304
|
+
curation["script_fingerprint"] = scriptFingerprint;
|
|
3305
|
+
curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
|
|
3306
|
+
curation["provider"] = options.providerName;
|
|
3307
|
+
curation["model"] = options.model;
|
|
3308
|
+
curation["parts"] = [];
|
|
3309
|
+
curation["expression_text"] = "<exp>NOOP # no state curation changes</exp>";
|
|
3310
|
+
curation["expressions"] = ["NOOP # no state curation changes"];
|
|
3311
|
+
ws.writeJson("state_curation.json", curation);
|
|
3312
|
+
ws.delete("state_curation.failed.json");
|
|
3313
|
+
ctx.emitProgress("state-curation", "success");
|
|
3314
|
+
return;
|
|
3315
|
+
}
|
|
3316
|
+
const provider = ctx.provider();
|
|
3317
|
+
if (!provider.curateStates) {
|
|
3318
|
+
throw new CliError("INIT FAILED: Provider does not support state curation", "Provider does not support state curation.", {
|
|
3319
|
+
exitCode: EXIT_RUNTIME,
|
|
3320
|
+
required: ["provider.curateStates(script, contextText)"],
|
|
3321
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3322
|
+
nextSteps: ["Use a direct provider that supports state curation."],
|
|
3323
|
+
});
|
|
3324
|
+
}
|
|
3325
|
+
ws.mkdir("state_curation_parts");
|
|
3326
|
+
ctx.updateRunState({ state_curation_completed_chunks: 0, state_curation_total_chunks: chunks.length });
|
|
3327
|
+
const curateStates = provider.curateStates.bind(provider);
|
|
3328
|
+
let completedChunks = 0;
|
|
3329
|
+
const outcomes = await pMapWithConcurrency(chunks, options.concurrency, async (chunk) => {
|
|
3330
|
+
const part = await loadOrRunStateCurationPart(ctx, script, curateStates, scriptFingerprint, chunk.part_id, chunk.refs);
|
|
3331
|
+
completedChunks += 1;
|
|
3332
|
+
ctx.updateRunState({ state_curation_completed_chunks: completedChunks, state_curation_total_chunks: chunks.length });
|
|
3333
|
+
return part;
|
|
3334
|
+
});
|
|
3335
|
+
const parts = unwrapConcurrentOutcomes(outcomes).sort((left, right) => left.partId.localeCompare(right.partId));
|
|
3336
|
+
plan = combineStateCurationPlans(parts);
|
|
3337
|
+
const curation = await ctx.timing.span({ name: "state_curation.apply", category: "substage", parent: "state_curation" }, () => applyStateCurationPlan(script, plan));
|
|
3338
|
+
curation["script_fingerprint"] = scriptFingerprint;
|
|
3339
|
+
curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
|
|
3340
|
+
curation["provider"] = options.providerName;
|
|
3341
|
+
curation["model"] = options.model;
|
|
3342
|
+
curation["expression_text"] = plan["expression_text"];
|
|
3343
|
+
curation["expressions"] = plan["expressions"];
|
|
3344
|
+
curation["parts"] = plan["parts"];
|
|
3345
|
+
ws.writeJson("state_curation.json", curation);
|
|
3346
|
+
ws.delete("state_curation.failed.json");
|
|
3347
|
+
}
|
|
3348
|
+
catch (exc) {
|
|
3349
|
+
const e = exc;
|
|
3350
|
+
ws.writeJson("state_curation.failed.json", {
|
|
3351
|
+
version: 1,
|
|
3352
|
+
failed_at: checkpointTimestamp(),
|
|
3353
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3354
|
+
plan,
|
|
3355
|
+
});
|
|
3356
|
+
stageFailure(options.workspace, exc, {
|
|
3357
|
+
title: "INIT FAILED: State curation failed",
|
|
3358
|
+
stage: "state_curation",
|
|
3359
|
+
required: ["provider state curation decisions for the assembled script state catalog"],
|
|
3360
|
+
nextSteps: ["Rerun init; completed extraction and asset curation checkpoints will be reused."],
|
|
3361
|
+
updates: { episode_completed: state.mergedResults.length },
|
|
3362
|
+
});
|
|
3363
|
+
}
|
|
3364
|
+
ctx.emitProgress("state-curation", "success");
|
|
3365
|
+
}
|
|
3366
|
+
const STATE_BINDING_CHUNK_CACHE_VERSION = 1;
|
|
3367
|
+
function stateBindingChunkFingerprint(options, context) {
|
|
3368
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
3369
|
+
cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
|
|
3370
|
+
provider: options.providerName,
|
|
3371
|
+
model: options.model,
|
|
3372
|
+
context,
|
|
3373
|
+
})));
|
|
3374
|
+
}
|
|
3375
|
+
async function runStateBindingForScript(ctx, script, progress) {
|
|
3376
|
+
const { ws, options } = ctx;
|
|
3377
|
+
const provider = ctx.provider();
|
|
3378
|
+
if (!provider.bindStates) {
|
|
3379
|
+
throw new CliError("INIT FAILED: Provider does not support state binding", "Provider does not support state binding.", {
|
|
3380
|
+
exitCode: EXIT_RUNTIME,
|
|
3381
|
+
required: ["provider.bindStates(script, context)"],
|
|
3382
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3383
|
+
nextSteps: ["Use a direct provider that supports state binding."],
|
|
3384
|
+
});
|
|
3385
|
+
}
|
|
3386
|
+
const defaultSummary = await ctx.timing.span({ name: "state_binding.default_states", category: "substage", parent: "state_binding" }, () => ensureDefaultStates(script));
|
|
3387
|
+
const defaultBindingSummary = await ctx.timing.span({ name: "state_binding.default_refs", category: "substage", parent: "state_binding" }, () => bindDefaultStateRefsInScript(script));
|
|
3388
|
+
const chunks = buildStateBindingContexts(script, 8);
|
|
3389
|
+
ws.mkdir("state_binding_chunks");
|
|
3390
|
+
if (progress) {
|
|
3391
|
+
ctx.updateRunState({ state_binding_completed_chunks: 0, state_binding_total_chunks: chunks.length });
|
|
3392
|
+
}
|
|
3393
|
+
let chunkPlans = [];
|
|
3394
|
+
const bypassChunkCache = ctx.state.bypassInternalCaches;
|
|
3395
|
+
try {
|
|
3396
|
+
const outcomes = await pMapWithConcurrency(chunks, options.concurrency, async (context, index) => {
|
|
3397
|
+
const chunkArtifact = `state_binding_chunks/chunk_${pad3(index + 1)}.json`;
|
|
3398
|
+
const contextFingerprint = stateBindingChunkFingerprint(options, context);
|
|
3399
|
+
const cachedChunk = ws.readJsonSafe(chunkArtifact);
|
|
3400
|
+
const cachedPlanRaw = isDict(cachedChunk) ? cachedChunk["plan"] : null;
|
|
3401
|
+
const cachedPlan = !bypassChunkCache
|
|
3402
|
+
&& isDict(cachedChunk)
|
|
3403
|
+
&& cachedChunk["context_fingerprint"] === contextFingerprint
|
|
3404
|
+
&& isDict(cachedPlanRaw)
|
|
3405
|
+
? cachedPlanRaw
|
|
3406
|
+
: null;
|
|
3407
|
+
const startedAtMs = Date.now();
|
|
3408
|
+
const rawPlan = normalizeStateBindingPlanShape(cachedPlan ?? await ctx.timing.span({ name: `state_binding.chunk.${pad3(index + 1)}`, category: "chunk", parent: "state_binding.chunks", metadata: { index } }, () => provider.bindStates(script, context)));
|
|
3409
|
+
if (!isDict(rawPlan) || !isList(rawPlan["scenes"])) {
|
|
3410
|
+
throw new CliError("INIT FAILED: Provider returned invalid state binding", "Provider returned invalid state binding.", {
|
|
3411
|
+
exitCode: EXIT_RUNTIME,
|
|
3412
|
+
required: ["object with scenes[]"],
|
|
3413
|
+
received: [typeof rawPlan],
|
|
3414
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and state binding will retry."],
|
|
3415
|
+
});
|
|
3416
|
+
}
|
|
3417
|
+
if (!cachedPlan) {
|
|
3418
|
+
ws.writeJson(chunkArtifact, {
|
|
3419
|
+
version: 1,
|
|
3420
|
+
cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
|
|
3421
|
+
index,
|
|
3422
|
+
provider: options.providerName,
|
|
3423
|
+
model: options.model,
|
|
3424
|
+
context_fingerprint: contextFingerprint,
|
|
3425
|
+
scene_ids: asList(context["scenes"]).map((scene) => strOf(scene["scene_id"])),
|
|
3426
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
3427
|
+
plan: rawPlan,
|
|
3428
|
+
updated_at: checkpointTimestamp(),
|
|
3429
|
+
});
|
|
3430
|
+
}
|
|
3431
|
+
if (progress) {
|
|
3432
|
+
ctx.updateRunState({ state_binding_completed_chunks: index + 1, state_binding_total_chunks: chunks.length });
|
|
3433
|
+
}
|
|
3434
|
+
return { index, context, plan: rawPlan, contextFingerprint, reused: Boolean(cachedPlan) };
|
|
3435
|
+
});
|
|
3436
|
+
chunkPlans = unwrapConcurrentOutcomes(outcomes).sort((left, right) => left.index - right.index);
|
|
3437
|
+
}
|
|
3438
|
+
catch (exc) {
|
|
3439
|
+
const e = exc;
|
|
3440
|
+
ws.writeJson("state_binding.failed.json", {
|
|
3441
|
+
version: 1,
|
|
3442
|
+
failed_at: checkpointTimestamp(),
|
|
3443
|
+
chunks_total: chunks.length,
|
|
3444
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3445
|
+
completed_chunks: [],
|
|
3446
|
+
});
|
|
3447
|
+
throw exc;
|
|
3448
|
+
}
|
|
3449
|
+
const chunkReports = [];
|
|
3450
|
+
let sceneRefsBound = 0;
|
|
3451
|
+
let stateChangesApplied = 0;
|
|
3452
|
+
let invalidSceneRefsSkipped = 0;
|
|
3453
|
+
let invalidStateChangesSkipped = 0;
|
|
3454
|
+
try {
|
|
3455
|
+
await ctx.timing.span({ name: "state_binding.apply", category: "substage", parent: "state_binding" }, () => {
|
|
3456
|
+
for (const item of chunkPlans) {
|
|
3457
|
+
const applied = applyStateBindingPlan(script, item.plan, { invalidStateMode: "skip" });
|
|
3458
|
+
sceneRefsBound += Number(applied["scene_refs_bound"] ?? 0);
|
|
3459
|
+
stateChangesApplied += Number(applied["state_changes_applied"] ?? 0);
|
|
3460
|
+
invalidSceneRefsSkipped += Number(applied["invalid_scene_refs_skipped"] ?? 0);
|
|
3461
|
+
invalidStateChangesSkipped += Number(applied["invalid_state_changes_skipped"] ?? 0);
|
|
3462
|
+
chunkReports.push({
|
|
3463
|
+
index: item.index,
|
|
3464
|
+
reused: item.reused,
|
|
3465
|
+
context_fingerprint: item.contextFingerprint,
|
|
3466
|
+
scene_count: asList(item.context["scenes"]).length,
|
|
3467
|
+
scene_refs_bound: applied["scene_refs_bound"],
|
|
3468
|
+
state_changes_applied: applied["state_changes_applied"],
|
|
3469
|
+
invalid_scene_refs_skipped: applied["invalid_scene_refs_skipped"],
|
|
3470
|
+
invalid_state_changes_skipped: applied["invalid_state_changes_skipped"],
|
|
3471
|
+
});
|
|
3472
|
+
}
|
|
3473
|
+
});
|
|
3474
|
+
}
|
|
3475
|
+
catch (exc) {
|
|
3476
|
+
const e = exc;
|
|
3477
|
+
ws.writeJson("state_binding.failed.json", {
|
|
3478
|
+
version: 1,
|
|
3479
|
+
failed_at: checkpointTimestamp(),
|
|
3480
|
+
chunks_total: chunks.length,
|
|
3481
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3482
|
+
completed_chunks: chunkReports,
|
|
3483
|
+
});
|
|
3484
|
+
throw exc;
|
|
3485
|
+
}
|
|
3486
|
+
const summary = {
|
|
3487
|
+
...defaultSummary,
|
|
3488
|
+
...defaultBindingSummary,
|
|
3489
|
+
llm_scene_refs_bound: sceneRefsBound,
|
|
3490
|
+
llm_state_changes_applied: stateChangesApplied,
|
|
3491
|
+
invalid_scene_refs_skipped: invalidSceneRefsSkipped,
|
|
3492
|
+
invalid_state_changes_skipped: invalidStateChangesSkipped,
|
|
3493
|
+
chunks: chunks.length,
|
|
3494
|
+
cached_chunks: chunkPlans.filter((chunk) => chunk.reused).length,
|
|
3495
|
+
llm_scene_count: chunks.reduce((n, chunk) => n + asList(chunk["scenes"]).length, 0),
|
|
3496
|
+
};
|
|
3497
|
+
ws.writeJson("state_binding.json", {
|
|
3498
|
+
version: 1,
|
|
3499
|
+
summary,
|
|
3500
|
+
chunks: chunkReports,
|
|
3501
|
+
});
|
|
3502
|
+
ws.delete("state_binding.failed.json");
|
|
3503
|
+
return { summary, chunks: chunkReports };
|
|
3504
|
+
}
|
|
3505
|
+
export async function stageStateBinding(ctx) {
|
|
3506
|
+
const { state, options } = ctx;
|
|
3507
|
+
const script = ensureScript(ctx);
|
|
3508
|
+
try {
|
|
3509
|
+
ctx.updateRunState({ status: "init_running", init_stage: "state_binding" });
|
|
3510
|
+
ctx.emitProgress("state-binding");
|
|
3511
|
+
await runStateBindingForScript(ctx, script, true);
|
|
3512
|
+
}
|
|
3513
|
+
catch (exc) {
|
|
3514
|
+
stageFailure(options.workspace, exc, {
|
|
3515
|
+
title: "INIT FAILED: State binding failed",
|
|
3516
|
+
stage: "state_binding",
|
|
3517
|
+
required: ["provider state binding decisions for a well-formed assembled script"],
|
|
3518
|
+
nextSteps: ["Rerun init; completed extraction and curation checkpoints will be reused."],
|
|
3519
|
+
updates: { episode_completed: state.mergedResults.length },
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
3522
|
+
ctx.emitProgress("state-binding", "success");
|
|
3523
|
+
}
|
|
3524
|
+
async function runMetadataForScript(ctx, script, applyToScript, progress) {
|
|
3525
|
+
const { ws, options } = ctx;
|
|
2158
3526
|
const sourceText = ensureSourceText(ctx);
|
|
2159
3527
|
const assetMetadataPath = ws.pathOf("asset_metadata.json");
|
|
2160
3528
|
if (options.skipMetadata) {
|
|
2161
|
-
|
|
3529
|
+
if (progress)
|
|
3530
|
+
ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: true });
|
|
2162
3531
|
if (exists(assetMetadataPath))
|
|
2163
3532
|
deletePath(assetMetadataPath);
|
|
2164
|
-
|
|
2165
|
-
return;
|
|
3533
|
+
return { metadata: {}, skipped: true };
|
|
2166
3534
|
}
|
|
2167
3535
|
const provider = ctx.provider();
|
|
3536
|
+
let metadata = provider.extractMetadata
|
|
3537
|
+
? await ctx.timing.span({ name: "metadata.extract", category: "substage", parent: "metadata_extract" }, () => provider.extractMetadata(sourceText, script))
|
|
3538
|
+
: {};
|
|
3539
|
+
if (!isDict(metadata))
|
|
3540
|
+
metadata = {};
|
|
3541
|
+
ws.writeJson("asset_metadata.json", metadata);
|
|
3542
|
+
if (applyToScript) {
|
|
3543
|
+
await ctx.timing.span({ name: "metadata.apply", category: "substage", parent: "metadata_extract" }, () => applyMetadataToScript(script, metadata));
|
|
3544
|
+
}
|
|
3545
|
+
return { metadata, skipped: false };
|
|
3546
|
+
}
|
|
3547
|
+
export async function stageMetadata(ctx) {
|
|
3548
|
+
const { state, options } = ctx;
|
|
3549
|
+
const script = ensureScript(ctx);
|
|
2168
3550
|
try {
|
|
2169
3551
|
ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: false });
|
|
2170
3552
|
ctx.emitProgress("metadata");
|
|
2171
|
-
|
|
2172
|
-
if (
|
|
2173
|
-
metadata
|
|
2174
|
-
|
|
2175
|
-
|
|
3553
|
+
const result = await runMetadataForScript(ctx, script, true, true);
|
|
3554
|
+
if (result.skipped) {
|
|
3555
|
+
ctx.emitProgress("metadata", "skipped");
|
|
3556
|
+
return;
|
|
3557
|
+
}
|
|
2176
3558
|
}
|
|
2177
3559
|
catch (exc) {
|
|
2178
3560
|
if (exc instanceof CliError) {
|
|
@@ -2197,16 +3579,21 @@ export async function stageMetadata(ctx) {
|
|
|
2197
3579
|
}
|
|
2198
3580
|
ctx.emitProgress("metadata", "success");
|
|
2199
3581
|
}
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
3582
|
+
function overviewFieldsFromScript(script) {
|
|
3583
|
+
return {
|
|
3584
|
+
synopsis: script["synopsis"],
|
|
3585
|
+
theme: script["theme"],
|
|
3586
|
+
logline: script["logline"],
|
|
3587
|
+
style: script["style"],
|
|
3588
|
+
main_characters: script["main_characters"],
|
|
3589
|
+
};
|
|
3590
|
+
}
|
|
3591
|
+
async function runScriptSynopsisForScript(ctx, script, persistScript) {
|
|
3592
|
+
const { ws, options } = ctx;
|
|
2204
3593
|
const provider = ctx.provider();
|
|
2205
|
-
let
|
|
2206
|
-
let
|
|
3594
|
+
let degraded = false;
|
|
3595
|
+
let reason = "";
|
|
2207
3596
|
try {
|
|
2208
|
-
ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
|
|
2209
|
-
ctx.emitProgress("script-synopsis");
|
|
2210
3597
|
const summary = await buildScriptSynopsis(provider, script, {
|
|
2211
3598
|
dd: ws.dir,
|
|
2212
3599
|
thresholdChars: options.summaryThresholdChars,
|
|
@@ -2215,25 +3602,32 @@ export async function stageScriptSynopsis(ctx) {
|
|
|
2215
3602
|
skipSummary: options.skipSummary,
|
|
2216
3603
|
concurrency: options.concurrency,
|
|
2217
3604
|
});
|
|
2218
|
-
|
|
2219
|
-
|
|
3605
|
+
degraded = summary.degraded;
|
|
3606
|
+
reason = summary.reason ?? "";
|
|
2220
3607
|
}
|
|
2221
3608
|
catch (exc) {
|
|
2222
|
-
|
|
2223
|
-
|
|
3609
|
+
degraded = true;
|
|
3610
|
+
reason = providerErrorSummary(exc);
|
|
2224
3611
|
}
|
|
2225
|
-
|
|
2226
|
-
|
|
3612
|
+
if (persistScript)
|
|
3613
|
+
ws.writeJson("script.initial.json", script);
|
|
3614
|
+
return { degraded, reason, fields: overviewFieldsFromScript(script) };
|
|
3615
|
+
}
|
|
3616
|
+
export async function stageScriptSynopsis(ctx) {
|
|
3617
|
+
const { ws, state, options } = ctx;
|
|
3618
|
+
const script = ensureScript(ctx);
|
|
3619
|
+
ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
|
|
3620
|
+
ctx.emitProgress("script-synopsis");
|
|
3621
|
+
const summary = await runScriptSynopsisForScript(ctx, script, true);
|
|
3622
|
+
state.summaryFullDegraded = summary.degraded;
|
|
3623
|
+
state.summaryDegradeReason = summary.reason;
|
|
2227
3624
|
state.summaryDegradedEpisodes.sort((a, b) => a - b);
|
|
2228
|
-
const summaryDegraded =
|
|
2229
|
-
// Persist script.initial.json now that the overview fields are assigned — the
|
|
2230
|
-
// same write point as the god-command (right after the synopsis stage).
|
|
2231
|
-
ws.writeJson("script.initial.json", script);
|
|
3625
|
+
const summaryDegraded = summary.degraded || state.summaryDegradedEpisodes.length > 0;
|
|
2232
3626
|
ctx.updateRunState({
|
|
2233
3627
|
summary_skipped: options.skipSummary,
|
|
2234
3628
|
summary_degraded: summaryDegraded,
|
|
2235
3629
|
summary_degraded_episodes: state.summaryDegradedEpisodes,
|
|
2236
|
-
summary_degraded_reason:
|
|
3630
|
+
summary_degraded_reason: summary.reason || null,
|
|
2237
3631
|
});
|
|
2238
3632
|
ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
|
|
2239
3633
|
}
|
|
@@ -2502,6 +3896,40 @@ function renderAssetCurationSummary(curation) {
|
|
|
2502
3896
|
`locations ${summary["locations_before"] ?? 0}->${summary["locations_after"] ?? 0} ` +
|
|
2503
3897
|
`(merged ${summary["locations_merged"] ?? 0})`,
|
|
2504
3898
|
];
|
|
3899
|
+
const coverage = isDict(summary["coverage"]) ? summary["coverage"] : {};
|
|
3900
|
+
const required = isDict(coverage["required"]) ? coverage["required"] : {};
|
|
3901
|
+
const auditSummary = isDict(curation["audit_summary"]) ? curation["audit_summary"] : {};
|
|
3902
|
+
if (summary["explicit_keep_count"] !== undefined || auditSummary["audit_decisions"] !== undefined || required["prop"] !== undefined) {
|
|
3903
|
+
lines.push(`curation coverage: ` +
|
|
3904
|
+
`keep ${summary["explicit_keep_count"] ?? 0}, ` +
|
|
3905
|
+
`implicit ${summary["implicit_keep_count"] ?? 0}, ` +
|
|
3906
|
+
`audit ${auditSummary["audit_decisions"] ?? 0}, ` +
|
|
3907
|
+
`required actor/location/prop ${required["actor"] ?? 0}/${required["location"] ?? 0}/${required["prop"] ?? 0}`);
|
|
3908
|
+
}
|
|
3909
|
+
for (const decision of asList(curation["decisions"])) {
|
|
3910
|
+
if (!isDict(decision))
|
|
3911
|
+
continue;
|
|
3912
|
+
const action = strOf(decision["decision"]);
|
|
3913
|
+
if (!action || action === "keep")
|
|
3914
|
+
continue;
|
|
3915
|
+
const kind = strOf(decision["kind"]);
|
|
3916
|
+
const sourceId = strOf(decision["source_id"]);
|
|
3917
|
+
const targetKind = strOf(decision["target_kind"]);
|
|
3918
|
+
const targetId = strOf(decision["target_id"]);
|
|
3919
|
+
const newName = strOf(decision["new_name"]);
|
|
3920
|
+
const stateName = strOf(decision["state_name"]);
|
|
3921
|
+
const speakerKind = strOf(decision["speaker_kind"]);
|
|
3922
|
+
const target = targetId
|
|
3923
|
+
? ` -> ${targetKind || kind}:${targetId}`
|
|
3924
|
+
: newName
|
|
3925
|
+
? ` -> ${newName}`
|
|
3926
|
+
: stateName
|
|
3927
|
+
? ` -> state:${stateName}`
|
|
3928
|
+
: speakerKind
|
|
3929
|
+
? ` -> speaker:${speakerKind}`
|
|
3930
|
+
: "";
|
|
3931
|
+
lines.push(`curation ${kind}:${sourceId} ${action}${target} — ${decision["reason"] || "-"}`);
|
|
3932
|
+
}
|
|
2505
3933
|
for (const actor of asList(curation["actors"])) {
|
|
2506
3934
|
if (!isDict(actor) || actor["decision"] !== "remove")
|
|
2507
3935
|
continue;
|
|
@@ -2519,6 +3947,46 @@ function renderAssetCurationSummary(curation) {
|
|
|
2519
3947
|
}
|
|
2520
3948
|
return lines;
|
|
2521
3949
|
}
|
|
3950
|
+
function renderStateBindingSummary(binding) {
|
|
3951
|
+
const summary = isDict(binding["summary"]) ? binding["summary"] : {};
|
|
3952
|
+
return [
|
|
3953
|
+
`state-binding: ` +
|
|
3954
|
+
`default-states +${summary["default_states_added"] ?? 0}, ` +
|
|
3955
|
+
`llm refs ${summary["llm_scene_refs_bound"] ?? 0}, ` +
|
|
3956
|
+
`default refs ${summary["default_or_unique_refs_bound"] ?? 0}, ` +
|
|
3957
|
+
`unresolved ${summary["unresolved_stateful_refs"] ?? 0}, ` +
|
|
3958
|
+
`state_changes ${summary["llm_state_changes_applied"] ?? 0}, ` +
|
|
3959
|
+
`chunks ${summary["chunks"] ?? 0}`,
|
|
3960
|
+
];
|
|
3961
|
+
}
|
|
3962
|
+
function renderStateCurationSummary(curation) {
|
|
3963
|
+
const summary = isDict(curation["summary"]) ? curation["summary"] : {};
|
|
3964
|
+
const decisionSummary = isDict(summary["decisions"]) ? summary["decisions"] : {};
|
|
3965
|
+
const lines = [
|
|
3966
|
+
`state-curation: ` +
|
|
3967
|
+
`states ${summary["states_before"] ?? 0}->${summary["states_after"] ?? 0} ` +
|
|
3968
|
+
`(removed ${summary["states_removed"] ?? 0}), ` +
|
|
3969
|
+
`actor/location/prop ${summary["actor_states_after"] ?? 0}/${summary["location_states_after"] ?? 0}/${summary["prop_states_after"] ?? 0}, ` +
|
|
3970
|
+
`merge ${decisionSummary["merge"] ?? 0}, drop ${decisionSummary["drop"] ?? 0}, action ${decisionSummary["action_desc"] ?? 0}, rename ${decisionSummary["rename"] ?? 0}`,
|
|
3971
|
+
];
|
|
3972
|
+
for (const decision of asList(curation["decisions"])) {
|
|
3973
|
+
if (!isDict(decision))
|
|
3974
|
+
continue;
|
|
3975
|
+
const action = strOf(decision["decision"]);
|
|
3976
|
+
if (!action || action === "keep")
|
|
3977
|
+
continue;
|
|
3978
|
+
const kind = strOf(decision["kind"]);
|
|
3979
|
+
const assetId = strOf(decision["asset_id"]);
|
|
3980
|
+
const stateId = strOf(decision["state_id"]);
|
|
3981
|
+
const target = action === "merge"
|
|
3982
|
+
? ` -> ${decision["target_state_id"] || "-"}`
|
|
3983
|
+
: action === "rename"
|
|
3984
|
+
? ` -> ${decision["new_name"] || "-"}`
|
|
3985
|
+
: "";
|
|
3986
|
+
lines.push(`state-curation ${kind}:${assetId}/${stateId} ${action}${target} — ${decision["reason"] || "-"}`);
|
|
3987
|
+
}
|
|
3988
|
+
return lines;
|
|
3989
|
+
}
|
|
2522
3990
|
function renderAssetNormalizationSummary(normalization) {
|
|
2523
3991
|
const summary = isDict(normalization["summary"]) ? normalization["summary"] : {};
|
|
2524
3992
|
const lines = [
|
|
@@ -2631,6 +4099,8 @@ export function commandReview(opts) {
|
|
|
2631
4099
|
// (validation issues recomputed live, plus batch extraction failures).
|
|
2632
4100
|
const lines = [];
|
|
2633
4101
|
const curationPath = path.join(dd, "asset_curation.json");
|
|
4102
|
+
const stateCurationPath = path.join(dd, "state_curation.json");
|
|
4103
|
+
const stateBindingPath = path.join(dd, "state_binding.json");
|
|
2634
4104
|
const normalizationPath = path.join(dd, "asset_normalization.json");
|
|
2635
4105
|
if (exists(normalizationPath)) {
|
|
2636
4106
|
const normalization = readJson(normalizationPath);
|
|
@@ -2642,6 +4112,16 @@ export function commandReview(opts) {
|
|
|
2642
4112
|
if (isDict(curation))
|
|
2643
4113
|
lines.push(...renderAssetCurationSummary(curation));
|
|
2644
4114
|
}
|
|
4115
|
+
if (exists(stateCurationPath)) {
|
|
4116
|
+
const curation = readJson(stateCurationPath);
|
|
4117
|
+
if (isDict(curation))
|
|
4118
|
+
lines.push(...renderStateCurationSummary(curation));
|
|
4119
|
+
}
|
|
4120
|
+
if (exists(stateBindingPath)) {
|
|
4121
|
+
const binding = readJson(stateBindingPath);
|
|
4122
|
+
if (isDict(binding))
|
|
4123
|
+
lines.push(...renderStateBindingSummary(binding));
|
|
4124
|
+
}
|
|
2645
4125
|
const batchFailures = collectBatchFailures(batchResultsDir);
|
|
2646
4126
|
for (const errors of batchFailures.values()) {
|
|
2647
4127
|
for (const error of errors) {
|
|
@@ -2665,7 +4145,7 @@ export function commandReview(opts) {
|
|
|
2665
4145
|
const report = {
|
|
2666
4146
|
title: "DIRECT REVIEW",
|
|
2667
4147
|
result: lines.length > 0 ? lines : ["No issues, curation decisions, or batch failures to review."],
|
|
2668
|
-
artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_curation.json"), batchResultsDir],
|
|
4148
|
+
artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_grouping.json"), path.join(dd, "asset_curation.json"), stateCurationPath, stateBindingPath, batchResultsDir],
|
|
2669
4149
|
next: [
|
|
2670
4150
|
"Run `direct review --episode <n>` for source↔extract alignment.",
|
|
2671
4151
|
"Edit with `--draft` (e.g. `scriptctl rename … --draft`), then validate/export.",
|