@lingjingai/scriptctl 0.30.2 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changes/0.31.0.md +13 -0
- package/changes/0.32.0.md +30 -0
- package/dist/cli.js +1 -4
- package/dist/cli.js.map +1 -1
- package/dist/common.d.ts +2 -3
- package/dist/common.js +70 -0
- package/dist/common.js.map +1 -1
- package/dist/domain/direct/runner.d.ts +3 -1
- package/dist/domain/direct/runner.js +20 -4
- package/dist/domain/direct/runner.js.map +1 -1
- package/dist/domain/direct/stage.d.ts +23 -1
- package/dist/domain/direct/stage.js +2 -0
- package/dist/domain/direct/stage.js.map +1 -1
- package/dist/domain/direct/stages/asset-curation.js +3 -2
- package/dist/domain/direct/stages/asset-curation.js.map +1 -1
- package/dist/domain/direct/stages/index.d.ts +2 -1
- package/dist/domain/direct/stages/index.js +3 -1
- package/dist/domain/direct/stages/index.js.map +1 -1
- package/dist/domain/direct/stages/state-binding.d.ts +2 -0
- package/dist/domain/direct/stages/state-binding.js +9 -0
- package/dist/domain/direct/stages/state-binding.js.map +1 -0
- package/dist/domain/direct-core.d.ts +57 -0
- package/dist/domain/direct-core.js +3469 -112
- package/dist/domain/direct-core.js.map +1 -1
- package/dist/help-text.js +55 -1
- package/dist/help-text.js.map +1 -1
- package/dist/infra/providers.d.ts +9 -0
- package/dist/infra/providers.js +442 -3
- package/dist/infra/providers.js.map +1 -1
- package/dist/output.d.ts +1 -1
- package/dist/output.js +11 -10
- package/dist/output.js.map +1 -1
- package/dist/usecases/direct.d.ts +1 -0
- package/dist/usecases/direct.js +1304 -88
- package/dist/usecases/direct.js.map +1 -1
- package/dist/usecases/parse.js +8 -24
- package/dist/usecases/parse.js.map +1 -1
- package/dist/usecases/script/actions.js +1 -1
- package/dist/usecases/script/actions.js.map +1 -1
- package/dist/usecases/script/actors.js +1 -1
- package/dist/usecases/script/actors.js.map +1 -1
- package/dist/usecases/script/assets.js +1 -1
- package/dist/usecases/script/assets.js.map +1 -1
- package/dist/usecases/script/do.js +7 -13
- package/dist/usecases/script/do.js.map +1 -1
- package/dist/usecases/script/episodes.js +1 -1
- package/dist/usecases/script/episodes.js.map +1 -1
- package/dist/usecases/script/export.js +2 -8
- package/dist/usecases/script/export.js.map +1 -1
- package/dist/usecases/script/extend.js +1 -4
- package/dist/usecases/script/extend.js.map +1 -1
- package/dist/usecases/script/issues.js +1 -1
- package/dist/usecases/script/issues.js.map +1 -1
- package/dist/usecases/script/lib.d.ts +1 -1
- package/dist/usecases/script/lib.js +16 -30
- package/dist/usecases/script/lib.js.map +1 -1
- package/dist/usecases/script/locations.js +1 -1
- package/dist/usecases/script/locations.js.map +1 -1
- package/dist/usecases/script/props.js +1 -1
- package/dist/usecases/script/props.js.map +1 -1
- package/dist/usecases/script/refs.js +1 -1
- package/dist/usecases/script/refs.js.map +1 -1
- package/dist/usecases/script/scenes.js +1 -1
- package/dist/usecases/script/scenes.js.map +1 -1
- package/dist/usecases/script/script-patch.js +9 -23
- package/dist/usecases/script/script-patch.js.map +1 -1
- package/dist/usecases/script/script-validate.js +4 -11
- package/dist/usecases/script/script-validate.js.map +1 -1
- package/dist/usecases/script/speakers.js +1 -1
- package/dist/usecases/script/speakers.js.map +1 -1
- package/dist/usecases/script/states.js +1 -4
- package/dist/usecases/script/states.js.map +1 -1
- package/dist/usecases/script/summary.js +1 -1
- package/dist/usecases/script/summary.js.map +1 -1
- package/dist/usecases/script/synopsis-generate.js +5 -13
- package/dist/usecases/script/synopsis-generate.js.map +1 -1
- package/dist/usecases/update.d.ts +0 -1
- package/dist/usecases/update.js +0 -15
- package/dist/usecases/update.js.map +1 -1
- package/package.json +1 -1
package/dist/usecases/direct.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_DIRECT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
|
|
4
|
-
import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
4
|
+
import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, applyAssetGroupingDuplicateMerges, applyStateBindingPlan, bindDefaultStateRefsInScript, buildAssetCurationAuditContextText, buildAssetCurationContextText, buildAssetCurationRepairContextText, buildAssetGroupingContextText, buildLocationCanonicalizationAuditContextText, buildPrimaryAssetCurationChunks, groupedAssetCurationChunks, combineLocationCanonicalizationPlan, buildStateBindingContexts, assetCurationAuditSourceKeys, combineAssetCurationExpressionPlans, combinePrimaryAssetCurationPlans, combineAssetCurationPlans, curateScriptAssets, ensureDefaultStates, assetCurationAuditCandidateSourceKeys, locationCanonicalizationCandidateSourceKeys, missingAssetCurationRequiredSourceKeys, normalizeStateBindingPlanShape, parseAssetCurationExpressions, parseAssetCurationExpressionsTolerant, parseAssetGroupingExpressions, assertLocationCanonicalizationPlan, applyMetadataToScript, validateAssetGroupingPlan, } from "../domain/direct-core.js";
|
|
5
5
|
import { validateScript } from "../domain/script/validate.js";
|
|
6
6
|
import { makeProvider, providerCapabilities } from "../infra/providers.js";
|
|
7
7
|
import { remoteScriptOutputStoreFromEnv } from "../infra/script-output-api.js";
|
|
@@ -118,6 +118,152 @@ export function readRunState(workspace) {
|
|
|
118
118
|
return {};
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
+
function isoFromMs(ms) {
|
|
122
|
+
return new Date(ms).toISOString();
|
|
123
|
+
}
|
|
124
|
+
function durationMs(startedAtMs, finishedAtMs) {
|
|
125
|
+
return Math.max(0, Math.round(finishedAtMs - startedAtMs));
|
|
126
|
+
}
|
|
127
|
+
function timingErrorSummary(error) {
|
|
128
|
+
const e = error;
|
|
129
|
+
return `${e?.name ?? "Error"}: ${e?.message ?? ""}`.slice(0, 240);
|
|
130
|
+
}
|
|
131
|
+
function timingSummary(events) {
|
|
132
|
+
const stages = {};
|
|
133
|
+
const branches = {};
|
|
134
|
+
const slowChunks = events
|
|
135
|
+
.filter((event) => event.category === "chunk")
|
|
136
|
+
.sort((left, right) => right.duration_ms - left.duration_ms)
|
|
137
|
+
.slice(0, 20)
|
|
138
|
+
.map((event) => ({
|
|
139
|
+
name: event.name,
|
|
140
|
+
parent: event.parent ?? "",
|
|
141
|
+
duration_ms: event.duration_ms,
|
|
142
|
+
status: event.status,
|
|
143
|
+
}));
|
|
144
|
+
for (const event of events) {
|
|
145
|
+
if (event.category === "stage")
|
|
146
|
+
stages[event.name] = event.duration_ms;
|
|
147
|
+
if (event.category === "branch")
|
|
148
|
+
branches[event.name] = event.duration_ms;
|
|
149
|
+
}
|
|
150
|
+
return { stages, branches, slow_chunks: slowChunks };
|
|
151
|
+
}
|
|
152
|
+
class JsonTimingRecorder {
|
|
153
|
+
ws;
|
|
154
|
+
workspace;
|
|
155
|
+
command = "";
|
|
156
|
+
events = [];
|
|
157
|
+
activeSpans = [];
|
|
158
|
+
counter = 0;
|
|
159
|
+
constructor(ws, workspace) {
|
|
160
|
+
this.ws = ws;
|
|
161
|
+
this.workspace = workspace;
|
|
162
|
+
}
|
|
163
|
+
reset(command) {
|
|
164
|
+
this.command = command;
|
|
165
|
+
this.events = [];
|
|
166
|
+
this.activeSpans = [];
|
|
167
|
+
this.counter = 0;
|
|
168
|
+
this.flush();
|
|
169
|
+
updateRunState(this.workspace, {
|
|
170
|
+
timing_current: null,
|
|
171
|
+
timing_current_stage: null,
|
|
172
|
+
timing_current_branch: null,
|
|
173
|
+
timing_summary: { stages: {}, branches: {}, slow_chunks: [] },
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
async span(options, run) {
|
|
177
|
+
const startedAtMs = Date.now();
|
|
178
|
+
const active = {
|
|
179
|
+
name: options.name,
|
|
180
|
+
category: options.category,
|
|
181
|
+
...(options.parent ? { parent: options.parent } : {}),
|
|
182
|
+
started_at: isoFromMs(startedAtMs),
|
|
183
|
+
};
|
|
184
|
+
this.activeSpans.push(active);
|
|
185
|
+
this.updateTimingCurrent();
|
|
186
|
+
try {
|
|
187
|
+
const value = await run();
|
|
188
|
+
this.removeActiveSpan(active);
|
|
189
|
+
this.record({
|
|
190
|
+
...options,
|
|
191
|
+
status: "success",
|
|
192
|
+
startedAtMs,
|
|
193
|
+
finishedAtMs: Date.now(),
|
|
194
|
+
});
|
|
195
|
+
return value;
|
|
196
|
+
}
|
|
197
|
+
catch (error) {
|
|
198
|
+
this.removeActiveSpan(active);
|
|
199
|
+
this.record({
|
|
200
|
+
...options,
|
|
201
|
+
status: "error",
|
|
202
|
+
startedAtMs,
|
|
203
|
+
finishedAtMs: Date.now(),
|
|
204
|
+
error: timingErrorSummary(error),
|
|
205
|
+
});
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
removeActiveSpan(active) {
|
|
210
|
+
const index = this.activeSpans.lastIndexOf(active);
|
|
211
|
+
if (index >= 0)
|
|
212
|
+
this.activeSpans.splice(index, 1);
|
|
213
|
+
}
|
|
214
|
+
activeSpanByCategory(category) {
|
|
215
|
+
for (let index = this.activeSpans.length - 1; index >= 0; index -= 1) {
|
|
216
|
+
const span = this.activeSpans[index];
|
|
217
|
+
if (span.category === category)
|
|
218
|
+
return span;
|
|
219
|
+
}
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
currentSpanPayload(span) {
|
|
223
|
+
if (!span)
|
|
224
|
+
return null;
|
|
225
|
+
return {
|
|
226
|
+
name: span.name,
|
|
227
|
+
category: span.category,
|
|
228
|
+
parent: span.parent ?? null,
|
|
229
|
+
started_at: span.started_at,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
updateTimingCurrent() {
|
|
233
|
+
updateRunState(this.workspace, {
|
|
234
|
+
timing_current: this.currentSpanPayload(this.activeSpans[this.activeSpans.length - 1] ?? null),
|
|
235
|
+
timing_current_stage: this.currentSpanPayload(this.activeSpanByCategory("stage")),
|
|
236
|
+
timing_current_branch: this.currentSpanPayload(this.activeSpanByCategory("branch")),
|
|
237
|
+
timing_summary: timingSummary(this.events),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
record(options) {
|
|
241
|
+
const event = {
|
|
242
|
+
id: `tim_${String(++this.counter).padStart(5, "0")}`,
|
|
243
|
+
name: options.name,
|
|
244
|
+
category: options.category,
|
|
245
|
+
...(options.parent ? { parent: options.parent } : {}),
|
|
246
|
+
status: options.status,
|
|
247
|
+
started_at: isoFromMs(options.startedAtMs),
|
|
248
|
+
finished_at: isoFromMs(options.finishedAtMs),
|
|
249
|
+
duration_ms: durationMs(options.startedAtMs, options.finishedAtMs),
|
|
250
|
+
...(options.metadata ? { metadata: options.metadata } : {}),
|
|
251
|
+
...(options.error ? { error: options.error } : {}),
|
|
252
|
+
};
|
|
253
|
+
this.events.push(event);
|
|
254
|
+
this.flush();
|
|
255
|
+
this.updateTimingCurrent();
|
|
256
|
+
}
|
|
257
|
+
flush() {
|
|
258
|
+
this.ws.writeJson("timings.json", {
|
|
259
|
+
version: 1,
|
|
260
|
+
command: this.command,
|
|
261
|
+
updated_at: checkpointTimestamp(),
|
|
262
|
+
events: this.events,
|
|
263
|
+
summary: timingSummary(this.events),
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
121
267
|
// ---------------------------------------------------------------------------
|
|
122
268
|
// Paths for episode/batch results
|
|
123
269
|
// ---------------------------------------------------------------------------
|
|
@@ -214,6 +360,7 @@ function isDirectStageName(value) {
|
|
|
214
360
|
case "episode_synopsis":
|
|
215
361
|
case "script_merge":
|
|
216
362
|
case "asset_curation":
|
|
363
|
+
case "state_binding":
|
|
217
364
|
case "metadata_extract":
|
|
218
365
|
case "script_synopsis":
|
|
219
366
|
case "validate":
|
|
@@ -254,6 +401,56 @@ function stageSnapshot(name, status, completed, total, message) {
|
|
|
254
401
|
message: message ?? "",
|
|
255
402
|
};
|
|
256
403
|
}
|
|
404
|
+
function formatDuration(ms) {
|
|
405
|
+
const value = Math.max(0, Math.round(ms));
|
|
406
|
+
if (value < 1000)
|
|
407
|
+
return `${value}ms`;
|
|
408
|
+
const seconds = value / 1000;
|
|
409
|
+
if (seconds < 60)
|
|
410
|
+
return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;
|
|
411
|
+
const minutes = Math.floor(seconds / 60);
|
|
412
|
+
const rest = Math.round(seconds % 60);
|
|
413
|
+
return `${minutes}m${String(rest).padStart(2, "0")}s`;
|
|
414
|
+
}
|
|
415
|
+
function stageTimingDurations(state) {
|
|
416
|
+
const timingSummaryRaw = state["timing_summary"];
|
|
417
|
+
const timingSummary = isDict(timingSummaryRaw) ? timingSummaryRaw : {};
|
|
418
|
+
const stagesRaw = timingSummary["stages"];
|
|
419
|
+
const stages = isDict(stagesRaw) ? stagesRaw : {};
|
|
420
|
+
const out = {};
|
|
421
|
+
for (const [key, value] of Object.entries(stages)) {
|
|
422
|
+
const n = Number(value);
|
|
423
|
+
if (!Number.isNaN(n))
|
|
424
|
+
out[key] = n;
|
|
425
|
+
}
|
|
426
|
+
return out;
|
|
427
|
+
}
|
|
428
|
+
function runningTimingElapsed(state, stageName) {
|
|
429
|
+
const stageRaw = state["timing_current_stage"];
|
|
430
|
+
const currentRaw = isDict(stageRaw) ? stageRaw : state["timing_current"];
|
|
431
|
+
const current = isDict(currentRaw) ? currentRaw : null;
|
|
432
|
+
if (!current || strOf(current["category"]) !== "stage" || strOf(current["name"]) !== stageName)
|
|
433
|
+
return null;
|
|
434
|
+
const startedAt = Date.parse(strOf(current["started_at"]));
|
|
435
|
+
if (Number.isNaN(startedAt))
|
|
436
|
+
return null;
|
|
437
|
+
return Date.now() - startedAt;
|
|
438
|
+
}
|
|
439
|
+
function applyStageTiming(stage, state, durations) {
|
|
440
|
+
const duration = durations[stage.name];
|
|
441
|
+
const elapsed = stage.status === "running" ? runningTimingElapsed(state, stage.name) : null;
|
|
442
|
+
const messageParts = [stage.message].filter((item) => item);
|
|
443
|
+
if (typeof duration === "number")
|
|
444
|
+
messageParts.push(`耗时 ${formatDuration(duration)}`);
|
|
445
|
+
else if (elapsed !== null)
|
|
446
|
+
messageParts.push(`已运行 ${formatDuration(elapsed)}`);
|
|
447
|
+
return {
|
|
448
|
+
...stage,
|
|
449
|
+
...(typeof duration === "number" ? { duration_ms: duration } : {}),
|
|
450
|
+
...(elapsed !== null ? { elapsed_ms: elapsed } : {}),
|
|
451
|
+
message: messageParts.join("; "),
|
|
452
|
+
};
|
|
453
|
+
}
|
|
257
454
|
function fileStage(name, filePath, runningStage) {
|
|
258
455
|
if (exists(filePath))
|
|
259
456
|
return stageSnapshot(name, "success", 1, 1);
|
|
@@ -304,6 +501,7 @@ function stageLabel(name) {
|
|
|
304
501
|
case "episode_synopsis": return "补充信息";
|
|
305
502
|
case "script_merge": return "生成剧本";
|
|
306
503
|
case "asset_curation": return "补充信息";
|
|
504
|
+
case "state_binding": return "补充信息";
|
|
307
505
|
case "metadata_extract": return "补充信息";
|
|
308
506
|
case "script_synopsis": return "补充信息";
|
|
309
507
|
case "validate": return "检查结果";
|
|
@@ -364,9 +562,10 @@ function stageOrder(name) {
|
|
|
364
562
|
case "episode_synopsis": return 6;
|
|
365
563
|
case "script_merge": return 7;
|
|
366
564
|
case "asset_curation": return 8;
|
|
367
|
-
case "
|
|
368
|
-
case "
|
|
369
|
-
case "
|
|
565
|
+
case "state_binding": return 9;
|
|
566
|
+
case "metadata_extract": return 10;
|
|
567
|
+
case "script_synopsis": return 11;
|
|
568
|
+
case "validate": return 12;
|
|
370
569
|
}
|
|
371
570
|
}
|
|
372
571
|
function stageRatio(stage) {
|
|
@@ -602,6 +801,7 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
602
801
|
const episodeResultsDir = path.join(dd, "episode_results");
|
|
603
802
|
const trustEpisodeMergeArtifacts = canTrustStageArtifacts("episode_merge", runningStage, runStatus);
|
|
604
803
|
const trustEpisodeSynopsisArtifacts = canTrustStageArtifacts("episode_synopsis", runningStage, runStatus);
|
|
804
|
+
const trustStateBindingArtifacts = canTrustStageArtifacts("state_binding", runningStage, runStatus);
|
|
605
805
|
const trustMetadataArtifacts = canTrustStageArtifacts("metadata_extract", runningStage, runStatus);
|
|
606
806
|
const trustScriptSynopsisArtifacts = canTrustStageArtifacts("script_synopsis", runningStage, runStatus);
|
|
607
807
|
const batchesByEpisode = new Map();
|
|
@@ -741,6 +941,9 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
741
941
|
};
|
|
742
942
|
const scriptStage = orderedFileStage("script_merge", path.join(dd, "script.initial.json"), runningStage, runStatus);
|
|
743
943
|
const curationStage = orderedFileStage("asset_curation", path.join(dd, "asset_curation.json"), runningStage, runStatus);
|
|
944
|
+
const stateBindingStage = trustStateBindingArtifacts
|
|
945
|
+
? orderedFileStage("state_binding", path.join(dd, "state_binding.json"), runningStage, runStatus)
|
|
946
|
+
: stageSnapshot("state_binding", runningStage === "state_binding" ? "running" : "pending", 0, 1);
|
|
744
947
|
const metadataStage = Boolean(state["metadata_skipped"]) && trustMetadataArtifacts
|
|
745
948
|
? stageSnapshot("metadata_extract", "skipped", 0, 1, "metadata: skipped (--skip-metadata)")
|
|
746
949
|
: orderedFileStage("metadata_extract", path.join(dd, "asset_metadata.json"), runningStage, runStatus);
|
|
@@ -761,6 +964,7 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
761
964
|
const validation = readJson(validationPath);
|
|
762
965
|
validationStage = stageSnapshot("validate", Boolean(validation["passed"]) ? "success" : "warning", 1, 1);
|
|
763
966
|
}
|
|
967
|
+
const timingDurations = stageTimingDurations(state);
|
|
764
968
|
const stages = [
|
|
765
969
|
sourceStage,
|
|
766
970
|
episodePlanStage,
|
|
@@ -771,10 +975,13 @@ function buildDirectStatusSnapshot(workspace) {
|
|
|
771
975
|
episodeSynopsisStage,
|
|
772
976
|
scriptStage,
|
|
773
977
|
curationStage,
|
|
978
|
+
stateBindingStage,
|
|
774
979
|
metadataStage,
|
|
775
980
|
scriptSynopsisStage,
|
|
776
981
|
validationStage,
|
|
777
|
-
]
|
|
982
|
+
]
|
|
983
|
+
.map((stage) => failedStage === stage.name ? { ...stage, status: "error", error: Math.max(1, stage.error) } : stage)
|
|
984
|
+
.map((stage) => applyStageTiming(stage, state, timingDurations));
|
|
778
985
|
let overallStatus = "pending";
|
|
779
986
|
const maxStage = stages.reduce((max, stage) => statusPriority(stage.status) > statusPriority(max) ? stage.status : max, "success");
|
|
780
987
|
if (runStatus === "init_failed" || runStatus === "init_incomplete" || maxStage === "error")
|
|
@@ -1010,11 +1217,16 @@ function initFailedReport(workspace, opts) {
|
|
|
1010
1217
|
// into every stage's catch block.
|
|
1011
1218
|
function stageFailure(workspace, exc, opts) {
|
|
1012
1219
|
const e = exc;
|
|
1220
|
+
const received = exc instanceof CliError && exc.received.length > 0
|
|
1221
|
+
? exc.received
|
|
1222
|
+
: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`];
|
|
1013
1223
|
throw initFailedReport(workspace, {
|
|
1014
1224
|
title: opts.title,
|
|
1015
1225
|
stage: opts.stage,
|
|
1226
|
+
exitCode: exc instanceof CliError ? exc.exitCode : undefined,
|
|
1227
|
+
errorCode: exc instanceof CliError ? exc.errorCode : undefined,
|
|
1016
1228
|
required: opts.required,
|
|
1017
|
-
received
|
|
1229
|
+
received,
|
|
1018
1230
|
nextSteps: opts.nextSteps,
|
|
1019
1231
|
...(opts.updates ? { updates: opts.updates } : {}),
|
|
1020
1232
|
});
|
|
@@ -1042,6 +1254,12 @@ async function pMapWithConcurrency(items, concurrency, worker) {
|
|
|
1042
1254
|
}));
|
|
1043
1255
|
return out;
|
|
1044
1256
|
}
|
|
1257
|
+
function unwrapConcurrentOutcomes(outcomes) {
|
|
1258
|
+
const failed = outcomes.find((outcome) => !outcome.ok);
|
|
1259
|
+
if (failed)
|
|
1260
|
+
throw failed.error;
|
|
1261
|
+
return outcomes.map((outcome) => outcome.value);
|
|
1262
|
+
}
|
|
1045
1263
|
function providerErrorSummary(error) {
|
|
1046
1264
|
if (error instanceof CliError && error.errorCode)
|
|
1047
1265
|
return error.errorCode;
|
|
@@ -1051,6 +1269,9 @@ function providerErrorSummary(error) {
|
|
|
1051
1269
|
return "PROVIDER_CONTENT_FILTERED";
|
|
1052
1270
|
return `${e?.name ?? "Error"}: ${msg}`.slice(0, 160);
|
|
1053
1271
|
}
|
|
1272
|
+
function cloneDict(value) {
|
|
1273
|
+
return structuredClone(value);
|
|
1274
|
+
}
|
|
1054
1275
|
// Generate the plan's missing episode titles in bounded, concurrent chunks.
|
|
1055
1276
|
// Title generation is non-essential (every episode has a deterministic
|
|
1056
1277
|
// fallback), so a chunk that the provider filters or fails does NOT abort
|
|
@@ -1188,6 +1409,56 @@ function assignOverviewFields(script, o) {
|
|
|
1188
1409
|
function applyScriptSynopsisFields(script, fields) {
|
|
1189
1410
|
assignOverviewFields(script, normalizeOverviewFields(fields));
|
|
1190
1411
|
}
|
|
1412
|
+
function synopsisGroupSignature(opts) {
|
|
1413
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
1414
|
+
title: strOf(opts.title),
|
|
1415
|
+
thresholdChars: opts.thresholdChars,
|
|
1416
|
+
groupSize: opts.groupSize,
|
|
1417
|
+
level: opts.level,
|
|
1418
|
+
groupIndex: opts.groupIndex,
|
|
1419
|
+
synopses: opts.synopses,
|
|
1420
|
+
})));
|
|
1421
|
+
}
|
|
1422
|
+
function synopsisGroupPartPath(dd, signature) {
|
|
1423
|
+
return path.join(dd, "synopsis_parts", `${signature}.json`);
|
|
1424
|
+
}
|
|
1425
|
+
async function loadOrRunSynopsisGroup(provider, groupSynopses, opts) {
|
|
1426
|
+
if (groupSynopses.length <= 1)
|
|
1427
|
+
return { value: groupSynopses[0] ?? "", reused: false };
|
|
1428
|
+
const signature = synopsisGroupSignature({
|
|
1429
|
+
title: opts.title,
|
|
1430
|
+
thresholdChars: opts.thresholdChars,
|
|
1431
|
+
groupSize: opts.groupSize,
|
|
1432
|
+
level: opts.level,
|
|
1433
|
+
groupIndex: opts.groupIndex,
|
|
1434
|
+
synopses: groupSynopses,
|
|
1435
|
+
});
|
|
1436
|
+
const partPath = synopsisGroupPartPath(opts.dd, signature);
|
|
1437
|
+
if (!opts.resummarize && exists(partPath)) {
|
|
1438
|
+
const prior = readJson(partPath);
|
|
1439
|
+
if (isDict(prior) && strOf(prior["source_group_signature"]) === signature) {
|
|
1440
|
+
const synopsis = strOf(prior["synopsis"]).trim();
|
|
1441
|
+
if (synopsis)
|
|
1442
|
+
return { value: synopsis, reused: true };
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
try {
|
|
1446
|
+
const synopsis = await reduceSynopses(provider, [...groupSynopses], { title: opts.title });
|
|
1447
|
+
writeJson(partPath, {
|
|
1448
|
+
version: 1,
|
|
1449
|
+
source_group_signature: signature,
|
|
1450
|
+
level: opts.level,
|
|
1451
|
+
group_index: opts.groupIndex,
|
|
1452
|
+
source_count: groupSynopses.length,
|
|
1453
|
+
synopsis,
|
|
1454
|
+
updated_at: checkpointTimestamp(),
|
|
1455
|
+
});
|
|
1456
|
+
return { value: synopsis, reused: false };
|
|
1457
|
+
}
|
|
1458
|
+
catch (error) {
|
|
1459
|
+
return { value: groupSynopses.join(" "), error, reused: false };
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1191
1462
|
// Whole-script reduce-2: collapse the episode synopses down to a set that fits
|
|
1192
1463
|
// one final reduce, then produce the top-level overview. Reuses a prior
|
|
1193
1464
|
// synopsis.json when the episode synopses are unchanged (signature match). Soft
|
|
@@ -1244,6 +1515,7 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1244
1515
|
// whenever it can't form fewer groups than items), so `current` strictly
|
|
1245
1516
|
// shrinks each non-terminal pass and this terminates without a counter.
|
|
1246
1517
|
let current = episodeSynopses;
|
|
1518
|
+
let level = 1;
|
|
1247
1519
|
while (true) {
|
|
1248
1520
|
const plan = planSynopsisReduction(current, opts.thresholdChars, opts.groupSize);
|
|
1249
1521
|
if (plan.fitsInOnePass)
|
|
@@ -1252,16 +1524,10 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1252
1524
|
// singleton group passes through verbatim (no LLM rewrite), mirroring the
|
|
1253
1525
|
// reduce-1 passthrough. A group whose stitch FAILS degrades to its joined
|
|
1254
1526
|
// beats (soft, per-group) — one bad group never wipes the whole overview.
|
|
1255
|
-
const
|
|
1527
|
+
const currentLevel = level;
|
|
1528
|
+
const outcomes = await pMapWithConcurrency(plan.groups, opts.concurrency, async (group, groupIndex) => {
|
|
1256
1529
|
const groupSynopses = group.map((i) => current[i]).filter((s) => s.trim());
|
|
1257
|
-
|
|
1258
|
-
return { value: groupSynopses[0] ?? "" };
|
|
1259
|
-
try {
|
|
1260
|
-
return { value: await reduceSynopses(provider, groupSynopses, { title: script["title"] }) };
|
|
1261
|
-
}
|
|
1262
|
-
catch (err) {
|
|
1263
|
-
return { value: groupSynopses.join(" "), error: err };
|
|
1264
|
-
}
|
|
1530
|
+
return loadOrRunSynopsisGroup(provider, groupSynopses, { ...opts, title: script["title"], level: currentLevel, groupIndex });
|
|
1265
1531
|
});
|
|
1266
1532
|
const next = [];
|
|
1267
1533
|
for (const o of outcomes) {
|
|
@@ -1275,6 +1541,7 @@ async function buildScriptSynopsis(provider, script, opts) {
|
|
|
1275
1541
|
next.push(o.value.value);
|
|
1276
1542
|
}
|
|
1277
1543
|
current = next;
|
|
1544
|
+
level += 1;
|
|
1278
1545
|
if (current.length === 0)
|
|
1279
1546
|
break;
|
|
1280
1547
|
}
|
|
@@ -1416,10 +1683,12 @@ function parseDirectRunOptions(opts) {
|
|
|
1416
1683
|
function buildStageContext(options) {
|
|
1417
1684
|
const ws = new Workspace(options.workspace);
|
|
1418
1685
|
let provider = null;
|
|
1686
|
+
const timing = new JsonTimingRecorder(ws, options.workspace);
|
|
1419
1687
|
const ctx = {
|
|
1420
1688
|
ws,
|
|
1421
1689
|
options,
|
|
1422
1690
|
state: emptyRunState(),
|
|
1691
|
+
timing,
|
|
1423
1692
|
provider() {
|
|
1424
1693
|
if (provider)
|
|
1425
1694
|
return provider;
|
|
@@ -1489,43 +1758,29 @@ function buildInitReport(ctx) {
|
|
|
1489
1758
|
const statesFromNames = Number(normSummary["location_states_from_names"] ?? 0);
|
|
1490
1759
|
const statesFromTime = Number(normSummary["location_states_from_time"] ?? 0);
|
|
1491
1760
|
const reviewItems = Number(normSummary["review_items"] ?? 0);
|
|
1761
|
+
const norm = merges || statesFromNames || statesFromTime || reviewItems;
|
|
1492
1762
|
const report = {
|
|
1493
1763
|
title,
|
|
1494
1764
|
result: [
|
|
1495
|
-
|
|
1496
|
-
`
|
|
1497
|
-
|
|
1498
|
-
`
|
|
1499
|
-
`provider: ${options.providerName}`,
|
|
1500
|
-
`episodes: ${state.mergedResults.length}/${state.totalEpisodes} complete`,
|
|
1501
|
-
`content: ${state.doneBatches}/${state.totalBatches} done`,
|
|
1502
|
-
`normalization: merges=${merges}, states=${statesFromNames + statesFromTime}, review=${reviewItems}`,
|
|
1765
|
+
`${stats["episodes"] ?? 0} episodes / ${stats["scenes"] ?? 0} scenes / ${stats["actions"] ?? 0} actions; validation ${passed ? "passed" : "needs repair"}`,
|
|
1766
|
+
`provider ${options.providerName}; ${state.mergedResults.length}/${state.totalEpisodes} episodes, ${state.doneBatches}/${state.totalBatches} batches`,
|
|
1767
|
+
// Deviation-only lines: omit when at default/clean state.
|
|
1768
|
+
...(norm ? [`normalization: merges=${merges}, states=${statesFromNames + statesFromTime}, review=${reviewItems}`] : []),
|
|
1503
1769
|
...(options.skipMetadata ? ["metadata: skipped"] : []),
|
|
1504
1770
|
...(state.titleDegraded.length > 0 ? [`titles degraded (deterministic): ${state.titleDegraded.join(", ")}`] : []),
|
|
1505
1771
|
...(summaryLine ? [summaryLine] : []),
|
|
1506
1772
|
],
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
path.join(dd, "
|
|
1511
|
-
path.join(dd, "
|
|
1512
|
-
path.join(dd, "batch_results"),
|
|
1513
|
-
path.join(dd, "episode_results"),
|
|
1514
|
-
path.join(dd, "asset_normalization.json"),
|
|
1515
|
-
path.join(dd, "asset_curation.json"),
|
|
1516
|
-
...(options.skipMetadata ? [] : [path.join(dd, "asset_metadata.json")]),
|
|
1517
|
-
path.join(dd, "synopsis.json"),
|
|
1518
|
-
path.join(dd, "script.initial.json"),
|
|
1519
|
-
path.join(dd, "validation.json"),
|
|
1520
|
-
path.join(dd, "run_state.json"),
|
|
1521
|
-
],
|
|
1773
|
+
// The produced script is the handle for `direct review` / validate / export;
|
|
1774
|
+
// validation.json only matters when there are issues to inspect.
|
|
1775
|
+
artifacts: passed
|
|
1776
|
+
? [path.join(dd, "script.initial.json")]
|
|
1777
|
+
: [path.join(dd, "script.initial.json"), path.join(dd, "validation.json")],
|
|
1522
1778
|
issues: summarizeIssues(asList(validation["issues"])),
|
|
1779
|
+
// init is the entrypoint of a multi-command pipeline, so the next handoff is
|
|
1780
|
+
// genuinely non-obvious — keep it concise.
|
|
1523
1781
|
next: options.providerName === "mock"
|
|
1524
|
-
? [
|
|
1525
|
-
|
|
1526
|
-
"Do not export mock-provider results for delivery.",
|
|
1527
|
-
]
|
|
1528
|
-
: ["Run `direct review` (add --episode <n> for source↔extract); edit with --draft if needed; then validate/export."],
|
|
1782
|
+
? ["Run `direct review`, then validate/export.", "Do not export mock-provider results for delivery."]
|
|
1783
|
+
: ["Run `direct review`, then validate/export."],
|
|
1529
1784
|
};
|
|
1530
1785
|
return [report, passed ? EXIT_OK : EXIT_NEEDS_AGENT];
|
|
1531
1786
|
}
|
|
@@ -1559,11 +1814,98 @@ function buildIncompleteReport(ctx) {
|
|
|
1559
1814
|
};
|
|
1560
1815
|
return [report, EXIT_RUNTIME];
|
|
1561
1816
|
}
|
|
1817
|
+
function initPreFanoutStages() {
|
|
1818
|
+
const end = STAGES.findIndex((stage) => stage.name === "asset-curation");
|
|
1819
|
+
return STAGES.slice(0, end + 1);
|
|
1820
|
+
}
|
|
1821
|
+
function validateOnlyStages() {
|
|
1822
|
+
return STAGES.filter((stage) => stage.name === "validate");
|
|
1823
|
+
}
|
|
1824
|
+
function updatePostCurationBranch(ctx, name, status) {
|
|
1825
|
+
const current = readRunState(ctx.options.workspace);
|
|
1826
|
+
const rawBranches = current["post_curation_branches"];
|
|
1827
|
+
const branches = isDict(rawBranches) ? { ...rawBranches } : {};
|
|
1828
|
+
branches[name] = { status, updated_at: checkpointTimestamp() };
|
|
1829
|
+
ctx.updateRunState({ post_curation_branches: branches });
|
|
1830
|
+
}
|
|
1831
|
+
async function runPostCurationBranch(ctx, name, run) {
|
|
1832
|
+
updatePostCurationBranch(ctx, name, "running");
|
|
1833
|
+
try {
|
|
1834
|
+
const value = await ctx.timing.span({ name, category: "branch", parent: "post_curation" }, run);
|
|
1835
|
+
updatePostCurationBranch(ctx, name, "success");
|
|
1836
|
+
return value;
|
|
1837
|
+
}
|
|
1838
|
+
catch (error) {
|
|
1839
|
+
updatePostCurationBranch(ctx, name, "error");
|
|
1840
|
+
throw error;
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
async function runPostCurationFanout(ctx) {
|
|
1844
|
+
const { ws, state, options } = ctx;
|
|
1845
|
+
const baseScript = ensureScript(ctx);
|
|
1846
|
+
ctx.updateRunState({
|
|
1847
|
+
status: "init_running",
|
|
1848
|
+
init_stage: "state_binding",
|
|
1849
|
+
post_curation_parallel: true,
|
|
1850
|
+
post_curation_branches: {
|
|
1851
|
+
state_binding: { status: "running", updated_at: checkpointTimestamp() },
|
|
1852
|
+
metadata_extract: { status: options.skipMetadata ? "skipped" : "running", updated_at: checkpointTimestamp() },
|
|
1853
|
+
script_synopsis: { status: options.skipSummary ? "skipped" : "running", updated_at: checkpointTimestamp() },
|
|
1854
|
+
},
|
|
1855
|
+
});
|
|
1856
|
+
ctx.emitProgress("state-binding");
|
|
1857
|
+
ctx.emitProgress("metadata", options.skipMetadata ? "skipped" : "running");
|
|
1858
|
+
ctx.emitProgress("script-synopsis", options.skipSummary ? "skipped" : "running");
|
|
1859
|
+
const stateBindingPromise = runPostCurationBranch(ctx, "state_binding", async () => {
|
|
1860
|
+
const script = cloneDict(baseScript);
|
|
1861
|
+
const result = await runStateBindingForScript(ctx, script, false);
|
|
1862
|
+
return { ...result, script };
|
|
1863
|
+
});
|
|
1864
|
+
const metadataPromise = runPostCurationBranch(ctx, "metadata_extract", async () => {
|
|
1865
|
+
const script = cloneDict(baseScript);
|
|
1866
|
+
return runMetadataForScript(ctx, script, false, false);
|
|
1867
|
+
});
|
|
1868
|
+
const synopsisPromise = runPostCurationBranch(ctx, "script_synopsis", async () => {
|
|
1869
|
+
const script = cloneDict(baseScript);
|
|
1870
|
+
return runScriptSynopsisForScript(ctx, script, false);
|
|
1871
|
+
});
|
|
1872
|
+
const settled = await Promise.allSettled([stateBindingPromise, metadataPromise, synopsisPromise]);
|
|
1873
|
+
const failed = settled.find((item) => item.status === "rejected");
|
|
1874
|
+
if (failed && failed.status === "rejected")
|
|
1875
|
+
throw failed.reason;
|
|
1876
|
+
const stateBinding = await stateBindingPromise;
|
|
1877
|
+
const metadata = await metadataPromise;
|
|
1878
|
+
const synopsis = await synopsisPromise;
|
|
1879
|
+
await ctx.timing.span({ name: "post_curation.merge", category: "substage", parent: "post_curation" }, () => {
|
|
1880
|
+
const finalScript = stateBinding.script;
|
|
1881
|
+
if (!metadata.skipped)
|
|
1882
|
+
applyMetadataToScript(finalScript, metadata.metadata);
|
|
1883
|
+
applyScriptSynopsisFields(finalScript, synopsis.fields);
|
|
1884
|
+
ws.writeJson("script.initial.json", finalScript);
|
|
1885
|
+
state.script = finalScript;
|
|
1886
|
+
state.summaryFullDegraded = synopsis.degraded;
|
|
1887
|
+
state.summaryDegradeReason = synopsis.reason;
|
|
1888
|
+
});
|
|
1889
|
+
state.summaryDegradedEpisodes.sort((a, b) => a - b);
|
|
1890
|
+
const summaryDegraded = state.summaryFullDegraded || state.summaryDegradedEpisodes.length > 0;
|
|
1891
|
+
ctx.updateRunState({
|
|
1892
|
+
post_curation_parallel: false,
|
|
1893
|
+
summary_skipped: options.skipSummary,
|
|
1894
|
+
summary_degraded: summaryDegraded,
|
|
1895
|
+
summary_degraded_episodes: state.summaryDegradedEpisodes,
|
|
1896
|
+
summary_degraded_reason: state.summaryDegradeReason || null,
|
|
1897
|
+
metadata_skipped: options.skipMetadata,
|
|
1898
|
+
});
|
|
1899
|
+
ctx.emitProgress("state-binding", "success");
|
|
1900
|
+
ctx.emitProgress("metadata", metadata.skipped ? "skipped" : "success");
|
|
1901
|
+
ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
|
|
1902
|
+
}
|
|
1562
1903
|
async function commandInitImpl(opts) {
|
|
1563
1904
|
const options = parseDirectRunOptions(opts);
|
|
1564
1905
|
assertInitSource(options.source);
|
|
1565
1906
|
const ctx = buildStageContext(options);
|
|
1566
1907
|
ctx.ws.ensure();
|
|
1908
|
+
ctx.timing.reset("direct init");
|
|
1567
1909
|
// Seed run_state with the init header before the first stage runs (provider /
|
|
1568
1910
|
// model / source_path / metadata flag) — the god-command set these up front.
|
|
1569
1911
|
updateRunState(options.workspace, {
|
|
@@ -1577,13 +1919,9 @@ async function commandInitImpl(opts) {
|
|
|
1577
1919
|
metadata_skipped: options.skipMetadata,
|
|
1578
1920
|
});
|
|
1579
1921
|
try {
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
// in the synopsis stages), so a deterministic stage must still re-derive its
|
|
1584
|
-
// output from the (possibly reused/edited) upstream artifacts. Whole-stage
|
|
1585
|
-
// skip-on-present is a `direct run <stage>` convenience only.
|
|
1586
|
-
await runStages(ctx, STAGES, { force: true });
|
|
1922
|
+
await runStages(ctx, initPreFanoutStages(), { force: true, persistScriptAfterStages: false });
|
|
1923
|
+
await ctx.timing.span({ name: "post_curation", category: "stage" }, () => runPostCurationFanout(ctx));
|
|
1924
|
+
await runStages(ctx, validateOnlyStages(), { force: true, persistScriptAfterStages: false });
|
|
1587
1925
|
}
|
|
1588
1926
|
catch (exc) {
|
|
1589
1927
|
// The resumable "incomplete" stop is not a failure — surface the INCOMPLETE
|
|
@@ -1659,6 +1997,8 @@ export async function commandRun(opts) {
|
|
|
1659
1997
|
}
|
|
1660
1998
|
const options = parseDirectRunOptions(opts);
|
|
1661
1999
|
const ctx = buildStageContext(options);
|
|
2000
|
+
ctx.ws.ensure();
|
|
2001
|
+
ctx.timing.reset(`direct run ${stageName}`);
|
|
1662
2002
|
const force = Boolean(opts["force"]);
|
|
1663
2003
|
try {
|
|
1664
2004
|
if (force) {
|
|
@@ -2145,16 +2485,661 @@ export async function stageScriptMerge(ctx) {
|
|
|
2145
2485
|
}
|
|
2146
2486
|
}
|
|
2147
2487
|
// --- asset-curation -------------------------------------------------------
|
|
2488
|
+
const ASSET_GROUPING_CACHE_VERSION = 2;
|
|
2489
|
+
const ASSET_CURATION_CACHE_VERSION = 2;
|
|
2490
|
+
function assetCacheIdentity(options) {
|
|
2491
|
+
return {
|
|
2492
|
+
provider: options.providerName,
|
|
2493
|
+
model: options.model,
|
|
2494
|
+
};
|
|
2495
|
+
}
|
|
2496
|
+
function assetCacheArtifactFields(ctx, cacheVersion) {
|
|
2497
|
+
return {
|
|
2498
|
+
cache_version: cacheVersion,
|
|
2499
|
+
...assetCacheIdentity(ctx.options),
|
|
2500
|
+
};
|
|
2501
|
+
}
|
|
2502
|
+
function assetCurationScriptFingerprint(script, options) {
|
|
2503
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2504
|
+
cache_version: ASSET_CURATION_CACHE_VERSION,
|
|
2505
|
+
...assetCacheIdentity(options),
|
|
2506
|
+
script: {
|
|
2507
|
+
actors: script["actors"],
|
|
2508
|
+
locations: script["locations"],
|
|
2509
|
+
props: script["props"],
|
|
2510
|
+
speakers: script["speakers"],
|
|
2511
|
+
episodes: script["episodes"],
|
|
2512
|
+
},
|
|
2513
|
+
})));
|
|
2514
|
+
}
|
|
2515
|
+
function assetCurationPlanFingerprint(scriptFingerprint, plan) {
|
|
2516
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2517
|
+
script_fingerprint: scriptFingerprint,
|
|
2518
|
+
decisions: plan["decisions"],
|
|
2519
|
+
})));
|
|
2520
|
+
}
|
|
2521
|
+
function chunkStrings(values, chunkSize) {
|
|
2522
|
+
const chunks = [];
|
|
2523
|
+
for (let index = 0; index < values.length; index += chunkSize) {
|
|
2524
|
+
chunks.push(values.slice(index, index + chunkSize));
|
|
2525
|
+
}
|
|
2526
|
+
return chunks;
|
|
2527
|
+
}
|
|
2528
|
+
function curationSourceKey(kind, id) {
|
|
2529
|
+
return `${kind}:${id}`;
|
|
2530
|
+
}
|
|
2531
|
+
function curationDecisionSourceKey(decision) {
|
|
2532
|
+
return curationSourceKey(strOf(decision["kind"]), strOf(decision["source_id"]));
|
|
2533
|
+
}
|
|
2534
|
+
function curationSourceKeysForIds(kind, ids) {
|
|
2535
|
+
return ids.map((id) => curationSourceKey(kind, id));
|
|
2536
|
+
}
|
|
2537
|
+
function curationPartSelection(kind, ids) {
|
|
2538
|
+
if (kind === "actor")
|
|
2539
|
+
return { actor: ids };
|
|
2540
|
+
if (kind === "location")
|
|
2541
|
+
return { location: ids };
|
|
2542
|
+
return { prop: ids };
|
|
2543
|
+
}
|
|
2544
|
+
function curationIdsFromSourceKeys(kind, ids, sourceKeys) {
|
|
2545
|
+
const allowed = new Set(sourceKeys);
|
|
2546
|
+
return ids.filter((id) => allowed.has(curationSourceKey(kind, id)));
|
|
2547
|
+
}
|
|
2548
|
+
function invalidCurationPart(title, received) {
|
|
2549
|
+
return new CliError(title, title, {
|
|
2550
|
+
exitCode: EXIT_RUNTIME,
|
|
2551
|
+
required: ["repairable asset curation expressions"],
|
|
2552
|
+
received,
|
|
2553
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and only invalid curation parts will retry."],
|
|
2554
|
+
});
|
|
2555
|
+
}
|
|
2556
|
+
const MAX_CURATION_PART_REPAIR_ATTEMPTS = 2;
|
|
2557
|
+
function assetGroupingKindListKey(kind) {
|
|
2558
|
+
if (kind === "actor")
|
|
2559
|
+
return "actors";
|
|
2560
|
+
if (kind === "location")
|
|
2561
|
+
return "locations";
|
|
2562
|
+
return "props";
|
|
2563
|
+
}
|
|
2564
|
+
function assetGroupingKindIdKey(kind) {
|
|
2565
|
+
if (kind === "actor")
|
|
2566
|
+
return "actor_id";
|
|
2567
|
+
if (kind === "location")
|
|
2568
|
+
return "location_id";
|
|
2569
|
+
return "prop_id";
|
|
2570
|
+
}
|
|
2571
|
+
function assetGroupingKindNameKey(kind) {
|
|
2572
|
+
if (kind === "actor")
|
|
2573
|
+
return "actor_name";
|
|
2574
|
+
if (kind === "location")
|
|
2575
|
+
return "location_name";
|
|
2576
|
+
return "prop_name";
|
|
2577
|
+
}
|
|
2578
|
+
function assetGroupingScriptFingerprint(script, options) {
|
|
2579
|
+
const compact = {};
|
|
2580
|
+
for (const kind of ["actor", "location", "prop"]) {
|
|
2581
|
+
const idKey = assetGroupingKindIdKey(kind);
|
|
2582
|
+
const nameKey = assetGroupingKindNameKey(kind);
|
|
2583
|
+
compact[assetGroupingKindListKey(kind)] = asList(script[assetGroupingKindListKey(kind)])
|
|
2584
|
+
.filter(isDict)
|
|
2585
|
+
.map((asset) => ({ id: strOf(asset[idKey]), name: strOf(asset[nameKey]) }));
|
|
2586
|
+
}
|
|
2587
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
2588
|
+
cache_version: ASSET_GROUPING_CACHE_VERSION,
|
|
2589
|
+
...assetCacheIdentity(options),
|
|
2590
|
+
assets: compact,
|
|
2591
|
+
})));
|
|
2592
|
+
}
|
|
2593
|
+
function combineAssetGroupingPlans(parts) {
|
|
2594
|
+
const groups = parts.flatMap((part) => asList(part["groups"]).filter(isDict));
|
|
2595
|
+
const duplicateMergeDecisions = parts.flatMap((part) => asList(part["duplicate_merge_decisions"]).filter(isDict));
|
|
2596
|
+
const normalizations = parts.flatMap((part) => asList(part["normalizations"]).filter(isDict));
|
|
2597
|
+
const summaries = parts.map((part) => part["summary"]).filter(isDict);
|
|
2598
|
+
return {
|
|
2599
|
+
version: 1,
|
|
2600
|
+
format: "asset-grouping-v1",
|
|
2601
|
+
groups,
|
|
2602
|
+
duplicate_merge_decisions: duplicateMergeDecisions,
|
|
2603
|
+
normalizations,
|
|
2604
|
+
parts: parts.map((part) => ({
|
|
2605
|
+
kind: part["kind"],
|
|
2606
|
+
expression_text: part["expression_text"],
|
|
2607
|
+
summary: part["summary"],
|
|
2608
|
+
normalizations: part["normalizations"],
|
|
2609
|
+
})),
|
|
2610
|
+
summary: {
|
|
2611
|
+
group_count: groups.length,
|
|
2612
|
+
duplicate_merge_count: duplicateMergeDecisions.length,
|
|
2613
|
+
normalization_count: normalizations.length,
|
|
2614
|
+
parts: summaries,
|
|
2615
|
+
},
|
|
2616
|
+
};
|
|
2617
|
+
}
|
|
2618
|
+
async function loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint) {
|
|
2619
|
+
const artifactName = `asset_grouping_parts/${kind}.json`;
|
|
2620
|
+
const cached = ctx.ws.readJsonSafe(artifactName);
|
|
2621
|
+
const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
|
|
2622
|
+
if (cachedPlan && isList(cachedPlan["groups"]))
|
|
2623
|
+
return cachedPlan;
|
|
2624
|
+
const startedAtMs = Date.now();
|
|
2625
|
+
const idKey = assetGroupingKindIdKey(kind);
|
|
2626
|
+
const assetCount = asList(script[assetGroupingKindListKey(kind)])
|
|
2627
|
+
.filter(isDict)
|
|
2628
|
+
.map((asset) => strOf(asset[idKey]).trim())
|
|
2629
|
+
.filter((id) => id).length;
|
|
2630
|
+
if (assetCount === 0) {
|
|
2631
|
+
const plan = {
|
|
2632
|
+
version: 1,
|
|
2633
|
+
format: "asset-grouping-v1",
|
|
2634
|
+
kind,
|
|
2635
|
+
expression_text: "",
|
|
2636
|
+
expressions: [],
|
|
2637
|
+
groups: [],
|
|
2638
|
+
duplicate_merge_decisions: [],
|
|
2639
|
+
summary: { input_count: 0, group_count: 0, mentioned_count: 0, omitted_duplicate_count: 0 },
|
|
2640
|
+
};
|
|
2641
|
+
ctx.ws.writeJson(artifactName, {
|
|
2642
|
+
version: 1,
|
|
2643
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2644
|
+
script_fingerprint: fingerprint,
|
|
2645
|
+
parse_status: "ok",
|
|
2646
|
+
kind,
|
|
2647
|
+
context_chars: 0,
|
|
2648
|
+
expression_text: "",
|
|
2649
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2650
|
+
plan,
|
|
2651
|
+
updated_at: checkpointTimestamp(),
|
|
2652
|
+
});
|
|
2653
|
+
return plan;
|
|
2654
|
+
}
|
|
2655
|
+
const contextText = buildAssetGroupingContextText(script, kind);
|
|
2656
|
+
let rawText = "";
|
|
2657
|
+
try {
|
|
2658
|
+
rawText = await ctx.timing.span({ name: `asset_grouping.${kind}`, category: "chunk", parent: "asset_curation.grouping", metadata: { artifact: artifactName } }, () => groupAssets(script, contextText));
|
|
2659
|
+
const parsed = parseAssetGroupingExpressions(rawText);
|
|
2660
|
+
const plan = validateAssetGroupingPlan(script, kind, parsed);
|
|
2661
|
+
ctx.ws.writeJson(artifactName, {
|
|
2662
|
+
version: 1,
|
|
2663
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2664
|
+
script_fingerprint: fingerprint,
|
|
2665
|
+
parse_status: "ok",
|
|
2666
|
+
kind,
|
|
2667
|
+
context_chars: contextText.length,
|
|
2668
|
+
expression_text: rawText,
|
|
2669
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2670
|
+
plan,
|
|
2671
|
+
updated_at: checkpointTimestamp(),
|
|
2672
|
+
});
|
|
2673
|
+
return plan;
|
|
2674
|
+
}
|
|
2675
|
+
catch (exc) {
|
|
2676
|
+
const e = exc;
|
|
2677
|
+
ctx.ws.writeJson(artifactName, {
|
|
2678
|
+
version: 1,
|
|
2679
|
+
...assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION),
|
|
2680
|
+
script_fingerprint: fingerprint,
|
|
2681
|
+
parse_status: "failed",
|
|
2682
|
+
kind,
|
|
2683
|
+
context_chars: contextText.length,
|
|
2684
|
+
expression_text: rawText,
|
|
2685
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2686
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
2687
|
+
updated_at: checkpointTimestamp(),
|
|
2688
|
+
});
|
|
2689
|
+
throw exc;
|
|
2690
|
+
}
|
|
2691
|
+
}
|
|
2692
|
+
async function loadOrRunAssetGrouping(ctx, script, groupAssets, fingerprint) {
|
|
2693
|
+
ctx.ws.mkdir("asset_grouping_parts");
|
|
2694
|
+
const outcomes = await pMapWithConcurrency(["actor", "location", "prop"], ctx.options.concurrency, async (kind) => loadOrRunAssetGroupingPart(ctx, script, groupAssets, kind, fingerprint));
|
|
2695
|
+
return combineAssetGroupingPlans(unwrapConcurrentOutcomes(outcomes));
|
|
2696
|
+
}
|
|
2697
|
+
function curationDecisionSourceKeys(plan) {
|
|
2698
|
+
return new Set(asList(plan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
|
|
2699
|
+
}
|
|
2700
|
+
function sourceKeysFromRejectedExpressions(rejected, candidateSourceKeys) {
|
|
2701
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2702
|
+
const keys = [];
|
|
2703
|
+
for (const item of rejected) {
|
|
2704
|
+
const key = strOf(item["source_key"]);
|
|
2705
|
+
if (key && (candidateSet.size === 0 || candidateSet.has(key)))
|
|
2706
|
+
uniqueAdd(keys, key);
|
|
2707
|
+
}
|
|
2708
|
+
return keys;
|
|
2709
|
+
}
|
|
2710
|
+
function scopedCurationPlan(plan, allowedSourceKeys) {
|
|
2711
|
+
const allowed = new Set(allowedSourceKeys);
|
|
2712
|
+
const decisions = asList(plan["decisions"]).filter(isDict);
|
|
2713
|
+
const acceptedExpressions = asList(plan["accepted_expressions"]);
|
|
2714
|
+
const scopedDecisions = [];
|
|
2715
|
+
const scopedExpressions = [];
|
|
2716
|
+
const ignored = [];
|
|
2717
|
+
for (let index = 0; index < decisions.length; index += 1) {
|
|
2718
|
+
const decision = decisions[index];
|
|
2719
|
+
const sourceKey = curationDecisionSourceKey(decision);
|
|
2720
|
+
const expression = acceptedExpressions[index] ?? "";
|
|
2721
|
+
if (allowed.has(sourceKey)) {
|
|
2722
|
+
scopedDecisions.push(decision);
|
|
2723
|
+
if (expression)
|
|
2724
|
+
scopedExpressions.push(expression);
|
|
2725
|
+
}
|
|
2726
|
+
else {
|
|
2727
|
+
ignored.push({ source_key: sourceKey, expression, reason: "outside_repair_scope" });
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2730
|
+
return {
|
|
2731
|
+
plan: {
|
|
2732
|
+
...plan,
|
|
2733
|
+
expression_text: formatCurationExpressionText(scopedExpressions),
|
|
2734
|
+
expressions: scopedExpressions.length > 0 ? scopedExpressions : ["NOOP # no asset curation changes"],
|
|
2735
|
+
accepted_expressions: scopedExpressions,
|
|
2736
|
+
decisions: scopedDecisions,
|
|
2737
|
+
},
|
|
2738
|
+
ignored,
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
function missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan) {
|
|
2742
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2743
|
+
return missingAssetCurationRequiredSourceKeys(script, partialPlan).filter((key) => candidateSet.has(key));
|
|
2744
|
+
}
|
|
2745
|
+
function repairSourceKeysForPart(script, candidateSourceKeys, partialPlan, requireCoverage) {
|
|
2746
|
+
const candidateSet = new Set(candidateSourceKeys);
|
|
2747
|
+
const accepted = new Set(asList(partialPlan["decisions"]).filter(isDict).map((decision) => curationDecisionSourceKey(decision)));
|
|
2748
|
+
const rejected = asList(partialPlan["rejected_expressions"]).filter(isDict);
|
|
2749
|
+
const repairKeys = [];
|
|
2750
|
+
let hasUnlocatedRejection = false;
|
|
2751
|
+
for (const item of rejected) {
|
|
2752
|
+
const key = strOf(item["source_key"]);
|
|
2753
|
+
if (key && (candidateSet.size === 0 || candidateSet.has(key)))
|
|
2754
|
+
uniqueAdd(repairKeys, key);
|
|
2755
|
+
else
|
|
2756
|
+
hasUnlocatedRejection = true;
|
|
2757
|
+
}
|
|
2758
|
+
if (hasUnlocatedRejection) {
|
|
2759
|
+
for (const key of candidateSourceKeys) {
|
|
2760
|
+
if (!accepted.has(key))
|
|
2761
|
+
uniqueAdd(repairKeys, key);
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
if (requireCoverage) {
|
|
2765
|
+
for (const key of missingRequiredSourceKeysForPart(script, candidateSourceKeys, partialPlan)) {
|
|
2766
|
+
uniqueAdd(repairKeys, key);
|
|
2767
|
+
}
|
|
2768
|
+
}
|
|
2769
|
+
return repairKeys;
|
|
2770
|
+
}
|
|
2771
|
+
function mergeRepairedCurationPlan(rawText, partialPlan, repairPlan, repairAttempts) {
|
|
2772
|
+
const acceptedExpressions = asList(partialPlan["accepted_expressions"]);
|
|
2773
|
+
const repairExpressions = asList(repairPlan["accepted_expressions"]);
|
|
2774
|
+
const decisions = [
|
|
2775
|
+
...asList(partialPlan["decisions"]).filter(isDict),
|
|
2776
|
+
...asList(repairPlan["decisions"]).filter(isDict),
|
|
2777
|
+
];
|
|
2778
|
+
const expressions = [...acceptedExpressions, ...repairExpressions].filter((expression) => !/^NOOP(?:\s|#|$)/i.test(expression.trim()));
|
|
2779
|
+
return {
|
|
2780
|
+
version: 3,
|
|
2781
|
+
format: "asset-curation-exp-v1",
|
|
2782
|
+
expression_text: formatCurationExpressionText(expressions),
|
|
2783
|
+
raw_expression_text: rawText,
|
|
2784
|
+
accepted_expressions: [...acceptedExpressions, ...repairExpressions],
|
|
2785
|
+
rejected_expressions: asList(partialPlan["rejected_expressions"]).filter(isDict),
|
|
2786
|
+
repair_attempts: repairAttempts,
|
|
2787
|
+
expressions: expressions.length > 0 ? expressions : ["NOOP # no asset curation changes"],
|
|
2788
|
+
decisions,
|
|
2789
|
+
partial: false,
|
|
2790
|
+
};
|
|
2791
|
+
}
|
|
2792
|
+
function formatCurationExpressionText(expressions) {
|
|
2793
|
+
if (expressions.length > 0)
|
|
2794
|
+
return expressions.map((expression) => `<exp>${expression}</exp>`).join("\n");
|
|
2795
|
+
return "<exp>NOOP # no asset curation changes</exp>";
|
|
2796
|
+
}
|
|
2797
|
+
function writeFailedCurationPartArtifact(opts) {
|
|
2798
|
+
const e = opts.error;
|
|
2799
|
+
opts.ctx.ws.writeJson(opts.artifactName, {
|
|
2800
|
+
version: 1,
|
|
2801
|
+
...assetCacheArtifactFields(opts.ctx, ASSET_CURATION_CACHE_VERSION),
|
|
2802
|
+
script_fingerprint: opts.fingerprint,
|
|
2803
|
+
...opts.metadata,
|
|
2804
|
+
parse_status: "failed",
|
|
2805
|
+
candidate_source_keys: opts.candidateSourceKeys,
|
|
2806
|
+
expression_text: opts.partPlan?.["expression_text"] ?? null,
|
|
2807
|
+
raw_expression_text: opts.partPlan?.["raw_expression_text"] ?? opts.expressionText,
|
|
2808
|
+
accepted_expressions: opts.partPlan?.["accepted_expressions"] ?? [],
|
|
2809
|
+
rejected_expressions: opts.partPlan?.["rejected_expressions"] ?? [],
|
|
2810
|
+
repair_attempts: opts.repairAttempts,
|
|
2811
|
+
duration_ms: durationMs(opts.startedAtMs, Date.now()),
|
|
2812
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
2813
|
+
plan: opts.partPlan,
|
|
2814
|
+
updated_at: checkpointTimestamp(),
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2817
|
+
async function loadOrRunCurationPart(ctx, script, artifactName, fingerprint, metadata, candidateSourceKeys, parentTiming, requireCoverage, contextText, run) {
|
|
2818
|
+
const cached = ctx.ws.readJsonSafe(artifactName);
|
|
2819
|
+
const cachedPlan = !ctx.state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === fingerprint && cached["parse_status"] === "ok" && isDict(cached["plan"]) ? cached["plan"] : null;
|
|
2820
|
+
if (cachedPlan && isList(cachedPlan["decisions"]))
|
|
2821
|
+
return cachedPlan;
|
|
2822
|
+
const startedAtMs = Date.now();
|
|
2823
|
+
const baseContextText = contextText(candidateSourceKeys);
|
|
2824
|
+
const expressionText = await ctx.timing.span({ name: strOf(metadata["part_id"]) || artifactName, category: "chunk", parent: parentTiming, metadata: { artifact: artifactName } }, () => run(baseContextText));
|
|
2825
|
+
let partPlan = parseAssetCurationExpressionsTolerant(expressionText);
|
|
2826
|
+
if (!isDict(partPlan) || !isList(partPlan["decisions"])) {
|
|
2827
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
|
|
2828
|
+
exitCode: EXIT_RUNTIME,
|
|
2829
|
+
required: ["object with decisions[]"],
|
|
2830
|
+
received: [typeof partPlan],
|
|
2831
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
const repairAttempts = [];
|
|
2835
|
+
let repairSourceKeys = repairSourceKeysForPart(script, candidateSourceKeys, partPlan, requireCoverage);
|
|
2836
|
+
let repairRejectedContext = asList(partPlan["rejected_expressions"]).filter(isDict);
|
|
2837
|
+
try {
|
|
2838
|
+
for (let attempt = 1; repairSourceKeys.length > 0 && attempt <= MAX_CURATION_PART_REPAIR_ATTEMPTS; attempt += 1) {
|
|
2839
|
+
const repairContext = buildAssetCurationRepairContextText({
|
|
2840
|
+
baseContextText: contextText(repairSourceKeys),
|
|
2841
|
+
repairSourceKeys,
|
|
2842
|
+
rejectedExpressions: repairRejectedContext,
|
|
2843
|
+
acceptedDecisions: asList(partPlan["decisions"]).filter(isDict),
|
|
2844
|
+
});
|
|
2845
|
+
const repairText = await ctx.timing.span({ name: `${strOf(metadata["part_id"]) || artifactName}.repair.${pad3(attempt)}`, category: "repair", parent: parentTiming, metadata: { artifact: artifactName, repair_source_keys: repairSourceKeys } }, () => run(repairContext));
|
|
2846
|
+
const repairPlan = parseAssetCurationExpressionsTolerant(repairText);
|
|
2847
|
+
const repairRejected = asList(repairPlan["rejected_expressions"]).filter(isDict);
|
|
2848
|
+
const scopedRepair = scopedCurationPlan(repairPlan, repairSourceKeys);
|
|
2849
|
+
repairAttempts.push({
|
|
2850
|
+
index: attempt,
|
|
2851
|
+
repair_source_keys: repairSourceKeys,
|
|
2852
|
+
expression_text: repairText,
|
|
2853
|
+
accepted_expressions: scopedRepair.plan["accepted_expressions"],
|
|
2854
|
+
ignored_expressions: scopedRepair.ignored,
|
|
2855
|
+
rejected_expressions: repairRejected,
|
|
2856
|
+
decisions: scopedRepair.plan["decisions"],
|
|
2857
|
+
});
|
|
2858
|
+
if (repairRejected.length > 0) {
|
|
2859
|
+
if (asList(scopedRepair.plan["decisions"]).filter(isDict).length > 0) {
|
|
2860
|
+
partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
|
|
2861
|
+
}
|
|
2862
|
+
const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
|
|
2863
|
+
const rejectedKeys = sourceKeysFromRejectedExpressions(repairRejected, repairSourceKeys);
|
|
2864
|
+
const nextRepairSourceKeys = [...(rejectedKeys.length > 0 ? rejectedKeys : []), ...missingKeys]
|
|
2865
|
+
.filter((key, index, all) => all.indexOf(key) === index);
|
|
2866
|
+
if (nextRepairSourceKeys.length === 0) {
|
|
2867
|
+
repairSourceKeys = [];
|
|
2868
|
+
repairRejectedContext = [];
|
|
2869
|
+
continue;
|
|
2870
|
+
}
|
|
2871
|
+
if (attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
|
|
2872
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair returned invalid expressions", repairRejected.map((item) => `${strOf(item["source_key"]) || "-"}: ${strOf(item["error"])}`).slice(0, 40));
|
|
2873
|
+
}
|
|
2874
|
+
repairSourceKeys = nextRepairSourceKeys;
|
|
2875
|
+
repairRejectedContext = repairRejected;
|
|
2876
|
+
continue;
|
|
2877
|
+
}
|
|
2878
|
+
const finalKeys = new Set([...curationDecisionSourceKeys(partPlan), ...curationDecisionSourceKeys(scopedRepair.plan)]);
|
|
2879
|
+
const unrepaired = repairSourceKeys.filter((key) => !finalKeys.has(key));
|
|
2880
|
+
if (unrepaired.length > 0 && attempt >= MAX_CURATION_PART_REPAIR_ATTEMPTS) {
|
|
2881
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", unrepaired.slice(0, 40));
|
|
2882
|
+
}
|
|
2883
|
+
partPlan = mergeRepairedCurationPlan(expressionText, partPlan, scopedRepair.plan, repairAttempts);
|
|
2884
|
+
repairRejectedContext = [];
|
|
2885
|
+
const missingKeys = requireCoverage ? missingRequiredSourceKeysForPart(script, candidateSourceKeys, partPlan) : [];
|
|
2886
|
+
repairSourceKeys = [...unrepaired, ...missingKeys]
|
|
2887
|
+
.filter((key, index, all) => all.indexOf(key) === index);
|
|
2888
|
+
}
|
|
2889
|
+
if (repairSourceKeys.length > 0) {
|
|
2890
|
+
throw invalidCurationPart("INIT FAILED: Asset curation repair omitted required source decisions", repairSourceKeys.slice(0, 40));
|
|
2891
|
+
}
|
|
2892
|
+
}
|
|
2893
|
+
catch (exc) {
|
|
2894
|
+
writeFailedCurationPartArtifact({
|
|
2895
|
+
ctx,
|
|
2896
|
+
artifactName,
|
|
2897
|
+
fingerprint,
|
|
2898
|
+
metadata,
|
|
2899
|
+
candidateSourceKeys,
|
|
2900
|
+
expressionText,
|
|
2901
|
+
partPlan,
|
|
2902
|
+
repairAttempts,
|
|
2903
|
+
startedAtMs,
|
|
2904
|
+
error: exc,
|
|
2905
|
+
});
|
|
2906
|
+
throw exc;
|
|
2907
|
+
}
|
|
2908
|
+
for (const [key, value] of Object.entries(metadata))
|
|
2909
|
+
partPlan[key] = value;
|
|
2910
|
+
ctx.ws.writeJson(artifactName, {
|
|
2911
|
+
version: 1,
|
|
2912
|
+
...assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION),
|
|
2913
|
+
script_fingerprint: fingerprint,
|
|
2914
|
+
...metadata,
|
|
2915
|
+
parse_status: "ok",
|
|
2916
|
+
candidate_source_keys: candidateSourceKeys,
|
|
2917
|
+
expression_text: partPlan["expression_text"],
|
|
2918
|
+
raw_expression_text: partPlan["raw_expression_text"] ?? expressionText,
|
|
2919
|
+
accepted_expressions: partPlan["accepted_expressions"],
|
|
2920
|
+
rejected_expressions: partPlan["rejected_expressions"],
|
|
2921
|
+
repair_attempts: repairAttempts,
|
|
2922
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
2923
|
+
plan: partPlan,
|
|
2924
|
+
updated_at: checkpointTimestamp(),
|
|
2925
|
+
});
|
|
2926
|
+
return partPlan;
|
|
2927
|
+
}
|
|
2928
|
+
async function loadOrRunPrimaryCurationPart(ctx, script, curateAssets, kind, partId, fingerprint, ids) {
|
|
2929
|
+
const selection = curationPartSelection(kind, ids);
|
|
2930
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/primary.${partId}.json`, fingerprint, { asset_kind: kind, part_id: partId, selection }, curationSourceKeysForIds(kind, ids), "asset_curation.primary", true, (selectedSourceKeys = curationSourceKeysForIds(kind, ids)) => {
|
|
2931
|
+
const selectedIds = curationIdsFromSourceKeys(kind, ids, selectedSourceKeys);
|
|
2932
|
+
return buildAssetCurationContextText(script, [kind], curationPartSelection(kind, selectedIds.length > 0 ? selectedIds : ids));
|
|
2933
|
+
}, (contextText) => curateAssets(script, contextText));
|
|
2934
|
+
}
|
|
2935
|
+
function primaryCurationChunks(script, groupingPlan) {
|
|
2936
|
+
const chunks = [];
|
|
2937
|
+
for (const kind of ["actor", "location", "prop"]) {
|
|
2938
|
+
const groupingGroups = groupingPlan
|
|
2939
|
+
? asList(groupingPlan["groups"]).filter((group) => isDict(group) && group["kind"] === kind)
|
|
2940
|
+
: [];
|
|
2941
|
+
const kindChunks = groupingGroups.length > 0
|
|
2942
|
+
? groupedAssetCurationChunks(script, kind, groupingGroups)
|
|
2943
|
+
: buildPrimaryAssetCurationChunks(script, kind);
|
|
2944
|
+
for (let index = 0; index < kindChunks.length; index += 1) {
|
|
2945
|
+
chunks.push({
|
|
2946
|
+
kind,
|
|
2947
|
+
partId: `${kind}.${String(index + 1).padStart(3, "0")}`,
|
|
2948
|
+
ids: kindChunks[index],
|
|
2949
|
+
});
|
|
2950
|
+
}
|
|
2951
|
+
}
|
|
2952
|
+
return chunks;
|
|
2953
|
+
}
|
|
2954
|
+
async function loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, fingerprint) {
|
|
2955
|
+
const sourceChunks = chunkStrings(assetCurationAuditSourceKeys(script, primaryPlan), 24);
|
|
2956
|
+
if (sourceChunks.length === 0) {
|
|
2957
|
+
return parseAssetCurationExpressions("<exp>NOOP # no asset curation changes</exp>");
|
|
2958
|
+
}
|
|
2959
|
+
const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
|
|
2960
|
+
const partId = `audit.${String(index + 1).padStart(3, "0")}`;
|
|
2961
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.audit", false, (repairSourceKeys = selectedSourceKeys) => buildAssetCurationAuditContextText(script, primaryPlan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
|
|
2962
|
+
});
|
|
2963
|
+
const parts = unwrapConcurrentOutcomes(outcomes);
|
|
2964
|
+
return combineAssetCurationExpressionPlans(parts);
|
|
2965
|
+
}
|
|
2966
|
+
async function loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, fingerprint) {
|
|
2967
|
+
const sourceChunks = chunkStrings(locationCanonicalizationCandidateSourceKeys(script, plan), 24);
|
|
2968
|
+
if (sourceChunks.length === 0) {
|
|
2969
|
+
return parseAssetCurationExpressions("<exp>NOOP # no location canonicalization changes</exp>");
|
|
2970
|
+
}
|
|
2971
|
+
const outcomes = await pMapWithConcurrency(sourceChunks, ctx.options.concurrency, async (selectedSourceKeys, index) => {
|
|
2972
|
+
const partId = `location_canonical.${String(index + 1).padStart(3, "0")}`;
|
|
2973
|
+
return loadOrRunCurationPart(ctx, script, `asset_curation_parts/${partId}.json`, fingerprint, { part_id: partId, selected_source_keys: selectedSourceKeys }, selectedSourceKeys, "asset_curation.location_canonical", false, (repairSourceKeys = selectedSourceKeys) => buildLocationCanonicalizationAuditContextText(script, plan, repairSourceKeys), (contextText) => curateAssets(script, contextText));
|
|
2974
|
+
});
|
|
2975
|
+
const parts = unwrapConcurrentOutcomes(outcomes);
|
|
2976
|
+
return combineAssetCurationExpressionPlans(parts);
|
|
2977
|
+
}
|
|
2148
2978
|
export async function stageAssetCuration(ctx) {
|
|
2149
2979
|
const { ws, state, options } = ctx;
|
|
2150
2980
|
const script = ensureScript(ctx);
|
|
2981
|
+
let groupingPlan = null;
|
|
2982
|
+
let groupingApplySummary = null;
|
|
2983
|
+
let plan = null;
|
|
2984
|
+
let primaryPlan = null;
|
|
2985
|
+
let auditPlan = null;
|
|
2986
|
+
let locationCanonicalPlan = null;
|
|
2987
|
+
let primaryExpressionText = null;
|
|
2988
|
+
let primaryPartExpressionTexts = [];
|
|
2989
|
+
let auditExpressionText = null;
|
|
2990
|
+
let locationCanonicalExpressionText = null;
|
|
2151
2991
|
try {
|
|
2152
2992
|
ctx.updateRunState({ status: "init_running", init_stage: "asset_curation" });
|
|
2153
2993
|
ctx.emitProgress("asset-curation");
|
|
2154
|
-
const
|
|
2994
|
+
const provider = ctx.provider();
|
|
2995
|
+
if (!provider.curateAssets) {
|
|
2996
|
+
throw new CliError("INIT FAILED: Provider does not support asset curation", "Provider does not support asset curation.", {
|
|
2997
|
+
exitCode: EXIT_RUNTIME,
|
|
2998
|
+
required: ["provider.curateAssets(script, contextText)"],
|
|
2999
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3000
|
+
nextSteps: ["Use a direct provider that supports asset curation."],
|
|
3001
|
+
});
|
|
3002
|
+
}
|
|
3003
|
+
if (!provider.groupAssets) {
|
|
3004
|
+
throw new CliError("INIT FAILED: Provider does not support asset grouping", "Provider does not support asset grouping.", {
|
|
3005
|
+
exitCode: EXIT_RUNTIME,
|
|
3006
|
+
required: ["provider.groupAssets(script, contextText)"],
|
|
3007
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3008
|
+
nextSteps: ["Use a direct provider that supports asset grouping."],
|
|
3009
|
+
});
|
|
3010
|
+
}
|
|
3011
|
+
const groupAssets = provider.groupAssets.bind(provider);
|
|
3012
|
+
const curateAssets = provider.curateAssets.bind(provider);
|
|
3013
|
+
const groupingFingerprint = assetGroupingScriptFingerprint(script, options);
|
|
3014
|
+
groupingPlan = await ctx.timing.span({ name: "asset_curation.grouping", category: "substage", parent: "asset_curation" }, () => loadOrRunAssetGrouping(ctx, script, groupAssets, groupingFingerprint));
|
|
3015
|
+
groupingApplySummary = await ctx.timing.span({ name: "asset_curation.grouping.apply", category: "substage", parent: "asset_curation.grouping" }, () => applyAssetGroupingDuplicateMerges(script, { decisions: groupingPlan["duplicate_merge_decisions"] }));
|
|
3016
|
+
Object.assign(groupingPlan, assetCacheArtifactFields(ctx, ASSET_GROUPING_CACHE_VERSION));
|
|
3017
|
+
groupingPlan["apply_summary"] = groupingApplySummary;
|
|
3018
|
+
groupingPlan["script_fingerprint"] = groupingFingerprint;
|
|
3019
|
+
ws.writeJson("asset_grouping.json", groupingPlan);
|
|
3020
|
+
const fingerprint = assetCurationScriptFingerprint(script, options);
|
|
3021
|
+
const cachedCuration = ws.readJsonSafe("asset_curation.json");
|
|
3022
|
+
if (!state.bypassInternalCaches && isDict(cachedCuration) && cachedCuration["script_fingerprint"] === fingerprint && isList(cachedCuration["decisions"])) {
|
|
3023
|
+
const curation = curateScriptAssets(script, { decisions: cachedCuration["decisions"] });
|
|
3024
|
+
Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
|
|
3025
|
+
curation["script_fingerprint"] = fingerprint;
|
|
3026
|
+
curation["grouping"] = groupingPlan;
|
|
3027
|
+
curation["grouping_apply_summary"] = groupingApplySummary;
|
|
3028
|
+
if (typeof cachedCuration["expression_text"] === "string")
|
|
3029
|
+
curation["expression_text"] = cachedCuration["expression_text"];
|
|
3030
|
+
if (typeof cachedCuration["primary_expression_text"] === "string")
|
|
3031
|
+
curation["primary_expression_text"] = cachedCuration["primary_expression_text"];
|
|
3032
|
+
if (typeof cachedCuration["primary_part_expression_text"] === "string")
|
|
3033
|
+
curation["primary_part_expression_text"] = cachedCuration["primary_part_expression_text"];
|
|
3034
|
+
if (typeof cachedCuration["audit_expression_text"] === "string")
|
|
3035
|
+
curation["audit_expression_text"] = cachedCuration["audit_expression_text"];
|
|
3036
|
+
if (typeof cachedCuration["location_canonical_expression_text"] === "string")
|
|
3037
|
+
curation["location_canonical_expression_text"] = cachedCuration["location_canonical_expression_text"];
|
|
3038
|
+
if (isList(cachedCuration["expressions"]))
|
|
3039
|
+
curation["expressions"] = cachedCuration["expressions"];
|
|
3040
|
+
if (isList(cachedCuration["primary_expressions"]))
|
|
3041
|
+
curation["primary_expressions"] = cachedCuration["primary_expressions"];
|
|
3042
|
+
if (isList(cachedCuration["primary_parts"]))
|
|
3043
|
+
curation["primary_parts"] = cachedCuration["primary_parts"];
|
|
3044
|
+
if (isList(cachedCuration["audit_expressions"]))
|
|
3045
|
+
curation["audit_expressions"] = cachedCuration["audit_expressions"];
|
|
3046
|
+
if (isList(cachedCuration["location_canonical_expressions"]))
|
|
3047
|
+
curation["location_canonical_expressions"] = cachedCuration["location_canonical_expressions"];
|
|
3048
|
+
if (typeof cachedCuration["format"] === "string")
|
|
3049
|
+
curation["format"] = cachedCuration["format"];
|
|
3050
|
+
if (isDict(cachedCuration["audit_summary"]))
|
|
3051
|
+
curation["audit_summary"] = cachedCuration["audit_summary"];
|
|
3052
|
+
ws.writeJson("asset_curation.json", curation);
|
|
3053
|
+
ws.delete("asset_curation.failed.json");
|
|
3054
|
+
ctx.emitProgress("asset-curation", "success");
|
|
3055
|
+
return;
|
|
3056
|
+
}
|
|
3057
|
+
const primaryChunkSpecs = primaryCurationChunks(script, groupingPlan);
|
|
3058
|
+
const primaryOutcomes = await pMapWithConcurrency(primaryChunkSpecs, options.concurrency, async (chunk) => loadOrRunPrimaryCurationPart(ctx, script, curateAssets, chunk.kind, chunk.partId, fingerprint, chunk.ids));
|
|
3059
|
+
const primaryParts = unwrapConcurrentOutcomes(primaryOutcomes);
|
|
3060
|
+
primaryPartExpressionTexts = primaryParts.map((part) => ({
|
|
3061
|
+
asset_kind: part["asset_kind"],
|
|
3062
|
+
part_id: part["part_id"],
|
|
3063
|
+
expression_text: part["expression_text"],
|
|
3064
|
+
}));
|
|
3065
|
+
primaryPlan = await ctx.timing.span({ name: "asset_curation.primary.combine", category: "substage", parent: "asset_curation.primary", metadata: { chunks: primaryParts.length } }, () => combinePrimaryAssetCurationPlans(primaryParts));
|
|
3066
|
+
primaryExpressionText = typeof primaryPlan["expression_text"] === "string" ? primaryPlan["expression_text"] : "";
|
|
3067
|
+
if (!isDict(primaryPlan) || !isList(primaryPlan["decisions"])) {
|
|
3068
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation", "Provider returned invalid asset curation.", {
|
|
3069
|
+
exitCode: EXIT_RUNTIME,
|
|
3070
|
+
required: ["object with decisions[]"],
|
|
3071
|
+
received: [typeof primaryPlan],
|
|
3072
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation will retry."],
|
|
3073
|
+
});
|
|
3074
|
+
}
|
|
3075
|
+
const allowedNewAuditSources = [
|
|
3076
|
+
...missingAssetCurationRequiredSourceKeys(script, primaryPlan),
|
|
3077
|
+
...assetCurationAuditCandidateSourceKeys(script, primaryPlan),
|
|
3078
|
+
];
|
|
3079
|
+
auditPlan = await ctx.timing.span({ name: "asset_curation.audit", category: "substage", parent: "asset_curation" }, () => loadOrRunAuditCurationParts(ctx, script, curateAssets, primaryPlan, assetCurationPlanFingerprint(`${fingerprint}:audit-chunk-v1`, primaryPlan)));
|
|
3080
|
+
auditExpressionText = typeof auditPlan["expression_text"] === "string" ? auditPlan["expression_text"] : "";
|
|
3081
|
+
if (!isDict(auditPlan) || !isList(auditPlan["decisions"])) {
|
|
3082
|
+
throw new CliError("INIT FAILED: Provider returned invalid asset curation audit", "Provider returned invalid asset curation audit.", {
|
|
3083
|
+
exitCode: EXIT_RUNTIME,
|
|
3084
|
+
required: ["object with decisions[]"],
|
|
3085
|
+
received: [typeof auditPlan],
|
|
3086
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and curation audit will retry."],
|
|
3087
|
+
});
|
|
3088
|
+
}
|
|
3089
|
+
plan = combineAssetCurationPlans(primaryPlan, auditPlan, allowedNewAuditSources);
|
|
3090
|
+
const allowedNewLocationCanonicalSources = locationCanonicalizationCandidateSourceKeys(script, plan);
|
|
3091
|
+
locationCanonicalPlan = await ctx.timing.span({ name: "asset_curation.location_canonical", category: "substage", parent: "asset_curation" }, () => loadOrRunLocationCanonicalizationParts(ctx, script, curateAssets, plan, assetCurationPlanFingerprint(`${fingerprint}:location-canonical-chunk-v2`, plan)));
|
|
3092
|
+
locationCanonicalExpressionText = typeof locationCanonicalPlan["expression_text"] === "string" ? locationCanonicalPlan["expression_text"] : "";
|
|
3093
|
+
if (!isDict(locationCanonicalPlan) || !isList(locationCanonicalPlan["decisions"])) {
|
|
3094
|
+
throw new CliError("INIT FAILED: Provider returned invalid location canonicalization audit", "Provider returned invalid location canonicalization audit.", {
|
|
3095
|
+
exitCode: EXIT_RUNTIME,
|
|
3096
|
+
required: ["object with decisions[]"],
|
|
3097
|
+
received: [typeof locationCanonicalPlan],
|
|
3098
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and location canonicalization will retry."],
|
|
3099
|
+
});
|
|
3100
|
+
}
|
|
3101
|
+
assertLocationCanonicalizationPlan(locationCanonicalPlan);
|
|
3102
|
+
plan = combineLocationCanonicalizationPlan(script, plan, locationCanonicalPlan, allowedNewLocationCanonicalSources);
|
|
3103
|
+
const curation = await ctx.timing.span({ name: "asset_curation.apply", category: "substage", parent: "asset_curation" }, () => curateScriptAssets(script, plan));
|
|
3104
|
+
Object.assign(curation, assetCacheArtifactFields(ctx, ASSET_CURATION_CACHE_VERSION));
|
|
3105
|
+
curation["script_fingerprint"] = fingerprint;
|
|
3106
|
+
curation["format"] = plan["format"];
|
|
3107
|
+
curation["expression_text"] = plan["expression_text"];
|
|
3108
|
+
curation["primary_expression_text"] = plan["primary_expression_text"];
|
|
3109
|
+
curation["audit_expression_text"] = plan["audit_expression_text"];
|
|
3110
|
+
curation["location_canonical_expression_text"] = plan["location_canonical_expression_text"];
|
|
3111
|
+
curation["primary_part_expression_text"] = plan["primary_part_expression_text"];
|
|
3112
|
+
curation["primary_parts"] = plan["primary_parts"];
|
|
3113
|
+
curation["expressions"] = plan["expressions"];
|
|
3114
|
+
curation["primary_expressions"] = plan["primary_expressions"];
|
|
3115
|
+
curation["audit_expressions"] = plan["audit_expressions"];
|
|
3116
|
+
curation["location_canonical_expressions"] = plan["location_canonical_expressions"];
|
|
3117
|
+
curation["audit_summary"] = plan["audit_summary"];
|
|
3118
|
+
curation["grouping"] = groupingPlan;
|
|
3119
|
+
curation["grouping_apply_summary"] = groupingApplySummary;
|
|
2155
3120
|
ws.writeJson("asset_curation.json", curation);
|
|
3121
|
+
ws.delete("asset_curation.failed.json");
|
|
2156
3122
|
}
|
|
2157
3123
|
catch (exc) {
|
|
3124
|
+
if (plan || primaryPlan || auditPlan || locationCanonicalPlan || primaryExpressionText !== null || auditExpressionText !== null || locationCanonicalExpressionText !== null) {
|
|
3125
|
+
const e = exc;
|
|
3126
|
+
ws.writeJson("asset_curation.failed.json", {
|
|
3127
|
+
version: 3,
|
|
3128
|
+
failed_at: checkpointTimestamp(),
|
|
3129
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3130
|
+
expression_text: plan?.["expression_text"] ?? null,
|
|
3131
|
+
primary_expression_text: primaryExpressionText,
|
|
3132
|
+
primary_part_expression_texts: primaryPartExpressionTexts,
|
|
3133
|
+
audit_expression_text: auditExpressionText,
|
|
3134
|
+
location_canonical_expression_text: locationCanonicalExpressionText,
|
|
3135
|
+
primary_plan: primaryPlan,
|
|
3136
|
+
audit_plan: auditPlan,
|
|
3137
|
+
location_canonical_plan: locationCanonicalPlan,
|
|
3138
|
+
grouping_plan: groupingPlan,
|
|
3139
|
+
grouping_apply_summary: groupingApplySummary,
|
|
3140
|
+
plan,
|
|
3141
|
+
});
|
|
3142
|
+
}
|
|
2158
3143
|
stageFailure(options.workspace, exc, {
|
|
2159
3144
|
title: "INIT FAILED: Asset curation failed",
|
|
2160
3145
|
stage: "asset_curation",
|
|
@@ -2165,28 +3150,198 @@ export async function stageAssetCuration(ctx) {
|
|
|
2165
3150
|
}
|
|
2166
3151
|
ctx.emitProgress("asset-curation", "success");
|
|
2167
3152
|
}
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
3153
|
+
const STATE_BINDING_CHUNK_CACHE_VERSION = 1;
|
|
3154
|
+
function stateBindingChunkFingerprint(options, context) {
|
|
3155
|
+
return sha256Text(JSON.stringify(sortDeep({
|
|
3156
|
+
cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
|
|
3157
|
+
provider: options.providerName,
|
|
3158
|
+
model: options.model,
|
|
3159
|
+
context,
|
|
3160
|
+
})));
|
|
3161
|
+
}
|
|
3162
|
+
async function runStateBindingForScript(ctx, script, progress) {
|
|
3163
|
+
const { ws, options } = ctx;
|
|
3164
|
+
const provider = ctx.provider();
|
|
3165
|
+
if (!provider.bindStates) {
|
|
3166
|
+
throw new CliError("INIT FAILED: Provider does not support state binding", "Provider does not support state binding.", {
|
|
3167
|
+
exitCode: EXIT_RUNTIME,
|
|
3168
|
+
required: ["provider.bindStates(script, context)"],
|
|
3169
|
+
received: [`provider: ${provider.name || options.providerName}`],
|
|
3170
|
+
nextSteps: ["Use a direct provider that supports state binding."],
|
|
3171
|
+
});
|
|
3172
|
+
}
|
|
3173
|
+
const defaultSummary = await ctx.timing.span({ name: "state_binding.default_states", category: "substage", parent: "state_binding" }, () => ensureDefaultStates(script));
|
|
3174
|
+
const defaultBindingSummary = await ctx.timing.span({ name: "state_binding.default_refs", category: "substage", parent: "state_binding" }, () => bindDefaultStateRefsInScript(script));
|
|
3175
|
+
const chunks = buildStateBindingContexts(script, 8);
|
|
3176
|
+
ws.mkdir("state_binding_chunks");
|
|
3177
|
+
if (progress) {
|
|
3178
|
+
ctx.updateRunState({ state_binding_completed_chunks: 0, state_binding_total_chunks: chunks.length });
|
|
3179
|
+
}
|
|
3180
|
+
let chunkPlans = [];
|
|
3181
|
+
const bypassChunkCache = ctx.state.bypassInternalCaches;
|
|
3182
|
+
try {
|
|
3183
|
+
const outcomes = await pMapWithConcurrency(chunks, options.concurrency, async (context, index) => {
|
|
3184
|
+
const chunkArtifact = `state_binding_chunks/chunk_${pad3(index + 1)}.json`;
|
|
3185
|
+
const contextFingerprint = stateBindingChunkFingerprint(options, context);
|
|
3186
|
+
const cachedChunk = ws.readJsonSafe(chunkArtifact);
|
|
3187
|
+
const cachedPlanRaw = isDict(cachedChunk) ? cachedChunk["plan"] : null;
|
|
3188
|
+
const cachedPlan = !bypassChunkCache
|
|
3189
|
+
&& isDict(cachedChunk)
|
|
3190
|
+
&& cachedChunk["context_fingerprint"] === contextFingerprint
|
|
3191
|
+
&& isDict(cachedPlanRaw)
|
|
3192
|
+
? cachedPlanRaw
|
|
3193
|
+
: null;
|
|
3194
|
+
const startedAtMs = Date.now();
|
|
3195
|
+
const rawPlan = normalizeStateBindingPlanShape(cachedPlan ?? await ctx.timing.span({ name: `state_binding.chunk.${pad3(index + 1)}`, category: "chunk", parent: "state_binding.chunks", metadata: { index } }, () => provider.bindStates(script, context)));
|
|
3196
|
+
if (!isDict(rawPlan) || !isList(rawPlan["scenes"])) {
|
|
3197
|
+
throw new CliError("INIT FAILED: Provider returned invalid state binding", "Provider returned invalid state binding.", {
|
|
3198
|
+
exitCode: EXIT_RUNTIME,
|
|
3199
|
+
required: ["object with scenes[]"],
|
|
3200
|
+
received: [typeof rawPlan],
|
|
3201
|
+
nextSteps: ["Rerun init; completed extraction checkpoints will be reused and state binding will retry."],
|
|
3202
|
+
});
|
|
3203
|
+
}
|
|
3204
|
+
if (!cachedPlan) {
|
|
3205
|
+
ws.writeJson(chunkArtifact, {
|
|
3206
|
+
version: 1,
|
|
3207
|
+
cache_version: STATE_BINDING_CHUNK_CACHE_VERSION,
|
|
3208
|
+
index,
|
|
3209
|
+
provider: options.providerName,
|
|
3210
|
+
model: options.model,
|
|
3211
|
+
context_fingerprint: contextFingerprint,
|
|
3212
|
+
scene_ids: asList(context["scenes"]).map((scene) => strOf(scene["scene_id"])),
|
|
3213
|
+
duration_ms: durationMs(startedAtMs, Date.now()),
|
|
3214
|
+
plan: rawPlan,
|
|
3215
|
+
updated_at: checkpointTimestamp(),
|
|
3216
|
+
});
|
|
3217
|
+
}
|
|
3218
|
+
if (progress) {
|
|
3219
|
+
ctx.updateRunState({ state_binding_completed_chunks: index + 1, state_binding_total_chunks: chunks.length });
|
|
3220
|
+
}
|
|
3221
|
+
return { index, context, plan: rawPlan, contextFingerprint, reused: Boolean(cachedPlan) };
|
|
3222
|
+
});
|
|
3223
|
+
chunkPlans = unwrapConcurrentOutcomes(outcomes).sort((left, right) => left.index - right.index);
|
|
3224
|
+
}
|
|
3225
|
+
catch (exc) {
|
|
3226
|
+
const e = exc;
|
|
3227
|
+
ws.writeJson("state_binding.failed.json", {
|
|
3228
|
+
version: 1,
|
|
3229
|
+
failed_at: checkpointTimestamp(),
|
|
3230
|
+
chunks_total: chunks.length,
|
|
3231
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3232
|
+
completed_chunks: [],
|
|
3233
|
+
});
|
|
3234
|
+
throw exc;
|
|
3235
|
+
}
|
|
3236
|
+
const chunkReports = [];
|
|
3237
|
+
let sceneRefsBound = 0;
|
|
3238
|
+
let stateChangesApplied = 0;
|
|
3239
|
+
let invalidSceneRefsSkipped = 0;
|
|
3240
|
+
let invalidStateChangesSkipped = 0;
|
|
3241
|
+
try {
|
|
3242
|
+
await ctx.timing.span({ name: "state_binding.apply", category: "substage", parent: "state_binding" }, () => {
|
|
3243
|
+
for (const item of chunkPlans) {
|
|
3244
|
+
const applied = applyStateBindingPlan(script, item.plan, { invalidStateMode: "skip" });
|
|
3245
|
+
sceneRefsBound += Number(applied["scene_refs_bound"] ?? 0);
|
|
3246
|
+
stateChangesApplied += Number(applied["state_changes_applied"] ?? 0);
|
|
3247
|
+
invalidSceneRefsSkipped += Number(applied["invalid_scene_refs_skipped"] ?? 0);
|
|
3248
|
+
invalidStateChangesSkipped += Number(applied["invalid_state_changes_skipped"] ?? 0);
|
|
3249
|
+
chunkReports.push({
|
|
3250
|
+
index: item.index,
|
|
3251
|
+
reused: item.reused,
|
|
3252
|
+
context_fingerprint: item.contextFingerprint,
|
|
3253
|
+
scene_count: asList(item.context["scenes"]).length,
|
|
3254
|
+
scene_refs_bound: applied["scene_refs_bound"],
|
|
3255
|
+
state_changes_applied: applied["state_changes_applied"],
|
|
3256
|
+
invalid_scene_refs_skipped: applied["invalid_scene_refs_skipped"],
|
|
3257
|
+
invalid_state_changes_skipped: applied["invalid_state_changes_skipped"],
|
|
3258
|
+
});
|
|
3259
|
+
}
|
|
3260
|
+
});
|
|
3261
|
+
}
|
|
3262
|
+
catch (exc) {
|
|
3263
|
+
const e = exc;
|
|
3264
|
+
ws.writeJson("state_binding.failed.json", {
|
|
3265
|
+
version: 1,
|
|
3266
|
+
failed_at: checkpointTimestamp(),
|
|
3267
|
+
chunks_total: chunks.length,
|
|
3268
|
+
error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
|
|
3269
|
+
completed_chunks: chunkReports,
|
|
3270
|
+
});
|
|
3271
|
+
throw exc;
|
|
3272
|
+
}
|
|
3273
|
+
const summary = {
|
|
3274
|
+
...defaultSummary,
|
|
3275
|
+
...defaultBindingSummary,
|
|
3276
|
+
llm_scene_refs_bound: sceneRefsBound,
|
|
3277
|
+
llm_state_changes_applied: stateChangesApplied,
|
|
3278
|
+
invalid_scene_refs_skipped: invalidSceneRefsSkipped,
|
|
3279
|
+
invalid_state_changes_skipped: invalidStateChangesSkipped,
|
|
3280
|
+
chunks: chunks.length,
|
|
3281
|
+
cached_chunks: chunkPlans.filter((chunk) => chunk.reused).length,
|
|
3282
|
+
llm_scene_count: chunks.reduce((n, chunk) => n + asList(chunk["scenes"]).length, 0),
|
|
3283
|
+
};
|
|
3284
|
+
ws.writeJson("state_binding.json", {
|
|
3285
|
+
version: 1,
|
|
3286
|
+
summary,
|
|
3287
|
+
chunks: chunkReports,
|
|
3288
|
+
});
|
|
3289
|
+
ws.delete("state_binding.failed.json");
|
|
3290
|
+
return { summary, chunks: chunkReports };
|
|
3291
|
+
}
|
|
3292
|
+
export async function stageStateBinding(ctx) {
|
|
3293
|
+
const { state, options } = ctx;
|
|
2171
3294
|
const script = ensureScript(ctx);
|
|
3295
|
+
try {
|
|
3296
|
+
ctx.updateRunState({ status: "init_running", init_stage: "state_binding" });
|
|
3297
|
+
ctx.emitProgress("state-binding");
|
|
3298
|
+
await runStateBindingForScript(ctx, script, true);
|
|
3299
|
+
}
|
|
3300
|
+
catch (exc) {
|
|
3301
|
+
stageFailure(options.workspace, exc, {
|
|
3302
|
+
title: "INIT FAILED: State binding failed",
|
|
3303
|
+
stage: "state_binding",
|
|
3304
|
+
required: ["provider state binding decisions for a well-formed assembled script"],
|
|
3305
|
+
nextSteps: ["Rerun init; completed extraction and curation checkpoints will be reused."],
|
|
3306
|
+
updates: { episode_completed: state.mergedResults.length },
|
|
3307
|
+
});
|
|
3308
|
+
}
|
|
3309
|
+
ctx.emitProgress("state-binding", "success");
|
|
3310
|
+
}
|
|
3311
|
+
async function runMetadataForScript(ctx, script, applyToScript, progress) {
|
|
3312
|
+
const { ws, options } = ctx;
|
|
2172
3313
|
const sourceText = ensureSourceText(ctx);
|
|
2173
3314
|
const assetMetadataPath = ws.pathOf("asset_metadata.json");
|
|
2174
3315
|
if (options.skipMetadata) {
|
|
2175
|
-
|
|
3316
|
+
if (progress)
|
|
3317
|
+
ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: true });
|
|
2176
3318
|
if (exists(assetMetadataPath))
|
|
2177
3319
|
deletePath(assetMetadataPath);
|
|
2178
|
-
|
|
2179
|
-
return;
|
|
3320
|
+
return { metadata: {}, skipped: true };
|
|
2180
3321
|
}
|
|
2181
3322
|
const provider = ctx.provider();
|
|
3323
|
+
let metadata = provider.extractMetadata
|
|
3324
|
+
? await ctx.timing.span({ name: "metadata.extract", category: "substage", parent: "metadata_extract" }, () => provider.extractMetadata(sourceText, script))
|
|
3325
|
+
: {};
|
|
3326
|
+
if (!isDict(metadata))
|
|
3327
|
+
metadata = {};
|
|
3328
|
+
ws.writeJson("asset_metadata.json", metadata);
|
|
3329
|
+
if (applyToScript) {
|
|
3330
|
+
await ctx.timing.span({ name: "metadata.apply", category: "substage", parent: "metadata_extract" }, () => applyMetadataToScript(script, metadata));
|
|
3331
|
+
}
|
|
3332
|
+
return { metadata, skipped: false };
|
|
3333
|
+
}
|
|
3334
|
+
export async function stageMetadata(ctx) {
|
|
3335
|
+
const { state, options } = ctx;
|
|
3336
|
+
const script = ensureScript(ctx);
|
|
2182
3337
|
try {
|
|
2183
3338
|
ctx.updateRunState({ status: "init_running", init_stage: "metadata_extract", metadata_skipped: false });
|
|
2184
3339
|
ctx.emitProgress("metadata");
|
|
2185
|
-
|
|
2186
|
-
if (
|
|
2187
|
-
metadata
|
|
2188
|
-
|
|
2189
|
-
|
|
3340
|
+
const result = await runMetadataForScript(ctx, script, true, true);
|
|
3341
|
+
if (result.skipped) {
|
|
3342
|
+
ctx.emitProgress("metadata", "skipped");
|
|
3343
|
+
return;
|
|
3344
|
+
}
|
|
2190
3345
|
}
|
|
2191
3346
|
catch (exc) {
|
|
2192
3347
|
if (exc instanceof CliError) {
|
|
@@ -2211,16 +3366,21 @@ export async function stageMetadata(ctx) {
|
|
|
2211
3366
|
}
|
|
2212
3367
|
ctx.emitProgress("metadata", "success");
|
|
2213
3368
|
}
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
3369
|
+
function overviewFieldsFromScript(script) {
|
|
3370
|
+
return {
|
|
3371
|
+
synopsis: script["synopsis"],
|
|
3372
|
+
theme: script["theme"],
|
|
3373
|
+
logline: script["logline"],
|
|
3374
|
+
style: script["style"],
|
|
3375
|
+
main_characters: script["main_characters"],
|
|
3376
|
+
};
|
|
3377
|
+
}
|
|
3378
|
+
async function runScriptSynopsisForScript(ctx, script, persistScript) {
|
|
3379
|
+
const { ws, options } = ctx;
|
|
2218
3380
|
const provider = ctx.provider();
|
|
2219
|
-
let
|
|
2220
|
-
let
|
|
3381
|
+
let degraded = false;
|
|
3382
|
+
let reason = "";
|
|
2221
3383
|
try {
|
|
2222
|
-
ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
|
|
2223
|
-
ctx.emitProgress("script-synopsis");
|
|
2224
3384
|
const summary = await buildScriptSynopsis(provider, script, {
|
|
2225
3385
|
dd: ws.dir,
|
|
2226
3386
|
thresholdChars: options.summaryThresholdChars,
|
|
@@ -2229,25 +3389,32 @@ export async function stageScriptSynopsis(ctx) {
|
|
|
2229
3389
|
skipSummary: options.skipSummary,
|
|
2230
3390
|
concurrency: options.concurrency,
|
|
2231
3391
|
});
|
|
2232
|
-
|
|
2233
|
-
|
|
3392
|
+
degraded = summary.degraded;
|
|
3393
|
+
reason = summary.reason ?? "";
|
|
2234
3394
|
}
|
|
2235
3395
|
catch (exc) {
|
|
2236
|
-
|
|
2237
|
-
|
|
3396
|
+
degraded = true;
|
|
3397
|
+
reason = providerErrorSummary(exc);
|
|
2238
3398
|
}
|
|
2239
|
-
|
|
2240
|
-
|
|
3399
|
+
if (persistScript)
|
|
3400
|
+
ws.writeJson("script.initial.json", script);
|
|
3401
|
+
return { degraded, reason, fields: overviewFieldsFromScript(script) };
|
|
3402
|
+
}
|
|
3403
|
+
export async function stageScriptSynopsis(ctx) {
|
|
3404
|
+
const { ws, state, options } = ctx;
|
|
3405
|
+
const script = ensureScript(ctx);
|
|
3406
|
+
ctx.updateRunState({ status: "init_running", init_stage: "script_synopsis" });
|
|
3407
|
+
ctx.emitProgress("script-synopsis");
|
|
3408
|
+
const summary = await runScriptSynopsisForScript(ctx, script, true);
|
|
3409
|
+
state.summaryFullDegraded = summary.degraded;
|
|
3410
|
+
state.summaryDegradeReason = summary.reason;
|
|
2241
3411
|
state.summaryDegradedEpisodes.sort((a, b) => a - b);
|
|
2242
|
-
const summaryDegraded =
|
|
2243
|
-
// Persist script.initial.json now that the overview fields are assigned — the
|
|
2244
|
-
// same write point as the god-command (right after the synopsis stage).
|
|
2245
|
-
ws.writeJson("script.initial.json", script);
|
|
3412
|
+
const summaryDegraded = summary.degraded || state.summaryDegradedEpisodes.length > 0;
|
|
2246
3413
|
ctx.updateRunState({
|
|
2247
3414
|
summary_skipped: options.skipSummary,
|
|
2248
3415
|
summary_degraded: summaryDegraded,
|
|
2249
3416
|
summary_degraded_episodes: state.summaryDegradedEpisodes,
|
|
2250
|
-
summary_degraded_reason:
|
|
3417
|
+
summary_degraded_reason: summary.reason || null,
|
|
2251
3418
|
});
|
|
2252
3419
|
ctx.emitProgress("script-synopsis", summaryDegraded ? "warning" : options.skipSummary ? "skipped" : "success");
|
|
2253
3420
|
}
|
|
@@ -2342,14 +3509,11 @@ export function commandValidate(opts) {
|
|
|
2342
3509
|
const passed = Boolean(validation["passed"]);
|
|
2343
3510
|
const report = {
|
|
2344
3511
|
title: passed ? "VALIDATE PASSED: Script is ready" : "VALIDATE NEEDS AGENT: Repair issues found",
|
|
2345
|
-
result: [
|
|
2346
|
-
`episodes: ${stats["episodes"] ?? 0}`,
|
|
2347
|
-
`scenes: ${stats["scenes"] ?? 0}`,
|
|
2348
|
-
`actions: ${stats["actions"] ?? 0}`,
|
|
2349
|
-
],
|
|
3512
|
+
result: [`${stats["episodes"] ?? 0} episodes / ${stats["scenes"] ?? 0} scenes / ${stats["actions"] ?? 0} actions`],
|
|
2350
3513
|
artifacts: [path.join(directDir(workspace), "validation.json")],
|
|
2351
3514
|
issues: summarizeIssues(asList(validation["issues"])),
|
|
2352
|
-
|
|
3515
|
+
// Success needs no guidance; only repair gets a next step.
|
|
3516
|
+
next: passed ? undefined : ["Inspect issues and apply structured patches."],
|
|
2353
3517
|
};
|
|
2354
3518
|
return [report, passed ? EXIT_OK : EXIT_NEEDS_AGENT];
|
|
2355
3519
|
}
|
|
@@ -2519,6 +3683,40 @@ function renderAssetCurationSummary(curation) {
|
|
|
2519
3683
|
`locations ${summary["locations_before"] ?? 0}->${summary["locations_after"] ?? 0} ` +
|
|
2520
3684
|
`(merged ${summary["locations_merged"] ?? 0})`,
|
|
2521
3685
|
];
|
|
3686
|
+
const coverage = isDict(summary["coverage"]) ? summary["coverage"] : {};
|
|
3687
|
+
const required = isDict(coverage["required"]) ? coverage["required"] : {};
|
|
3688
|
+
const auditSummary = isDict(curation["audit_summary"]) ? curation["audit_summary"] : {};
|
|
3689
|
+
if (summary["explicit_keep_count"] !== undefined || auditSummary["audit_decisions"] !== undefined || required["prop"] !== undefined) {
|
|
3690
|
+
lines.push(`curation coverage: ` +
|
|
3691
|
+
`keep ${summary["explicit_keep_count"] ?? 0}, ` +
|
|
3692
|
+
`implicit ${summary["implicit_keep_count"] ?? 0}, ` +
|
|
3693
|
+
`audit ${auditSummary["audit_decisions"] ?? 0}, ` +
|
|
3694
|
+
`required actor/location/prop ${required["actor"] ?? 0}/${required["location"] ?? 0}/${required["prop"] ?? 0}`);
|
|
3695
|
+
}
|
|
3696
|
+
for (const decision of asList(curation["decisions"])) {
|
|
3697
|
+
if (!isDict(decision))
|
|
3698
|
+
continue;
|
|
3699
|
+
const action = strOf(decision["decision"]);
|
|
3700
|
+
if (!action || action === "keep")
|
|
3701
|
+
continue;
|
|
3702
|
+
const kind = strOf(decision["kind"]);
|
|
3703
|
+
const sourceId = strOf(decision["source_id"]);
|
|
3704
|
+
const targetKind = strOf(decision["target_kind"]);
|
|
3705
|
+
const targetId = strOf(decision["target_id"]);
|
|
3706
|
+
const newName = strOf(decision["new_name"]);
|
|
3707
|
+
const stateName = strOf(decision["state_name"]);
|
|
3708
|
+
const speakerKind = strOf(decision["speaker_kind"]);
|
|
3709
|
+
const target = targetId
|
|
3710
|
+
? ` -> ${targetKind || kind}:${targetId}`
|
|
3711
|
+
: newName
|
|
3712
|
+
? ` -> ${newName}`
|
|
3713
|
+
: stateName
|
|
3714
|
+
? ` -> state:${stateName}`
|
|
3715
|
+
: speakerKind
|
|
3716
|
+
? ` -> speaker:${speakerKind}`
|
|
3717
|
+
: "";
|
|
3718
|
+
lines.push(`curation ${kind}:${sourceId} ${action}${target} — ${decision["reason"] || "-"}`);
|
|
3719
|
+
}
|
|
2522
3720
|
for (const actor of asList(curation["actors"])) {
|
|
2523
3721
|
if (!isDict(actor) || actor["decision"] !== "remove")
|
|
2524
3722
|
continue;
|
|
@@ -2536,6 +3734,18 @@ function renderAssetCurationSummary(curation) {
|
|
|
2536
3734
|
}
|
|
2537
3735
|
return lines;
|
|
2538
3736
|
}
|
|
3737
|
+
function renderStateBindingSummary(binding) {
|
|
3738
|
+
const summary = isDict(binding["summary"]) ? binding["summary"] : {};
|
|
3739
|
+
return [
|
|
3740
|
+
`state-binding: ` +
|
|
3741
|
+
`default-states +${summary["default_states_added"] ?? 0}, ` +
|
|
3742
|
+
`llm refs ${summary["llm_scene_refs_bound"] ?? 0}, ` +
|
|
3743
|
+
`default refs ${summary["default_or_unique_refs_bound"] ?? 0}, ` +
|
|
3744
|
+
`unresolved ${summary["unresolved_stateful_refs"] ?? 0}, ` +
|
|
3745
|
+
`state_changes ${summary["llm_state_changes_applied"] ?? 0}, ` +
|
|
3746
|
+
`chunks ${summary["chunks"] ?? 0}`,
|
|
3747
|
+
];
|
|
3748
|
+
}
|
|
2539
3749
|
function renderAssetNormalizationSummary(normalization) {
|
|
2540
3750
|
const summary = isDict(normalization["summary"]) ? normalization["summary"] : {};
|
|
2541
3751
|
const lines = [
|
|
@@ -2648,6 +3858,7 @@ export function commandReview(opts) {
|
|
|
2648
3858
|
// (validation issues recomputed live, plus batch extraction failures).
|
|
2649
3859
|
const lines = [];
|
|
2650
3860
|
const curationPath = path.join(dd, "asset_curation.json");
|
|
3861
|
+
const stateBindingPath = path.join(dd, "state_binding.json");
|
|
2651
3862
|
const normalizationPath = path.join(dd, "asset_normalization.json");
|
|
2652
3863
|
if (exists(normalizationPath)) {
|
|
2653
3864
|
const normalization = readJson(normalizationPath);
|
|
@@ -2659,6 +3870,11 @@ export function commandReview(opts) {
|
|
|
2659
3870
|
if (isDict(curation))
|
|
2660
3871
|
lines.push(...renderAssetCurationSummary(curation));
|
|
2661
3872
|
}
|
|
3873
|
+
if (exists(stateBindingPath)) {
|
|
3874
|
+
const binding = readJson(stateBindingPath);
|
|
3875
|
+
if (isDict(binding))
|
|
3876
|
+
lines.push(...renderStateBindingSummary(binding));
|
|
3877
|
+
}
|
|
2662
3878
|
const batchFailures = collectBatchFailures(batchResultsDir);
|
|
2663
3879
|
for (const errors of batchFailures.values()) {
|
|
2664
3880
|
for (const error of errors) {
|
|
@@ -2682,7 +3898,7 @@ export function commandReview(opts) {
|
|
|
2682
3898
|
const report = {
|
|
2683
3899
|
title: "DIRECT REVIEW",
|
|
2684
3900
|
result: lines.length > 0 ? lines : ["No issues, curation decisions, or batch failures to review."],
|
|
2685
|
-
artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_curation.json"), batchResultsDir],
|
|
3901
|
+
artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_grouping.json"), path.join(dd, "asset_curation.json"), stateBindingPath, batchResultsDir],
|
|
2686
3902
|
next: [
|
|
2687
3903
|
"Run `direct review --episode <n>` for source↔extract alignment.",
|
|
2688
3904
|
"Edit with `--draft` (e.g. `scriptctl rename … --draft`), then validate/export.",
|