@lingjingai/scriptctl 0.32.0 → 0.33.0

package/dist/usecases/direct.js

@@ -1,7 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_DIRECT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_SUMMARY_GROUP_SIZE, DEFAULT_SUMMARY_REDUCE_THRESHOLD_CHARS, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, sha256Text, writeJson, } from "../common.js";
-import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, applyAssetGroupingDuplicateMerges, applyStateBindingPlan, bindDefaultStateRefsInScript, buildAssetCurationAuditContextText, buildAssetCurationContextText, buildAssetCurationRepairContextText, buildAssetGroupingContextText, buildLocationCanonicalizationAuditContextText, buildPrimaryAssetCurationChunks, groupedAssetCurationChunks, combineLocationCanonicalizationPlan, buildStateBindingContexts, assetCurationAuditSourceKeys, combineAssetCurationExpressionPlans, combinePrimaryAssetCurationPlans, combineAssetCurationPlans, curateScriptAssets, ensureDefaultStates, assetCurationAuditCandidateSourceKeys, locationCanonicalizationCandidateSourceKeys, missingAssetCurationRequiredSourceKeys, normalizeStateBindingPlanShape, parseAssetCurationExpressions, parseAssetCurationExpressionsTolerant, parseAssetGroupingExpressions, assertLocationCanonicalizationPlan, applyMetadataToScript, validateAssetGroupingPlan, } from "../domain/direct-core.js";
+import { compactBatchResult, compactEpisodeResult, expandCompactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, cleanSynopsisList, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResultsWithReport, normalizeEpisodeResult, planSynopsisReduction, readCarriedSynopsis, sortDeep, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, applyAssetGroupingDuplicateMerges, applyStateCurationPlan, applyStateBindingPlan, bindDefaultStateRefsInScript, buildAssetCurationAuditContextText, buildAssetCurationContextText, buildAssetCurationRepairContextText, buildAssetGroupingContextText, buildLocationCanonicalizationAuditContextText, buildPrimaryAssetCurationChunks, buildStateCurationChunks, buildStateCurationContextText, groupedAssetCurationChunks, combineLocationCanonicalizationPlan, buildStateBindingContexts, assetCurationAuditSourceKeys, combineAssetCurationExpressionPlans, combinePrimaryAssetCurationPlans, combineAssetCurationPlans, curateScriptAssets, ensureDefaultStates, assetCurationAuditCandidateSourceKeys, locationCanonicalizationCandidateSourceKeys, missingAssetCurationRequiredSourceKeys, normalizeStateBindingPlanShape, parseStateCurationExpressions, parseAssetCurationExpressions, parseAssetCurationExpressionsTolerant, parseAssetGroupingExpressions, assertLocationCanonicalizationPlan, applyMetadataToScript, validateAssetGroupingPlan, stateCurationSourceKeysForRefs, assertStateCurationPlan, } from "../domain/direct-core.js";
 import { validateScript } from "../domain/script/validate.js";
 import { makeProvider, providerCapabilities } from "../infra/providers.js";
 import { remoteScriptOutputStoreFromEnv } from "../infra/script-output-api.js";
@@ -360,6 +360,7 @@ function isDirectStageName(value) {
         case "episode_synopsis":
         case "script_merge":
         case "asset_curation":
+        case "state_curation":
         case "state_binding":
         case "metadata_extract":
         case "script_synopsis":
@@ -501,6 +502,7 @@ function stageLabel(name) {
         case "episode_synopsis": return "补充信息";
         case "script_merge": return "生成剧本";
         case "asset_curation": return "补充信息";
+        case "state_curation": return "补充信息";
         case "state_binding": return "补充信息";
         case "metadata_extract": return "补充信息";
         case "script_synopsis": return "补充信息";
@@ -562,10 +564,11 @@ function stageOrder(name) {
         case "episode_synopsis": return 6;
         case "script_merge": return 7;
         case "asset_curation": return 8;
-        case "state_binding": return 9;
-        case "metadata_extract": return 10;
-        case "script_synopsis": return 11;
-        case "validate": return 12;
+        case "state_curation": return 9;
+        case "state_binding": return 10;
+        case "metadata_extract": return 11;
+        case "script_synopsis": return 12;
+        case "validate": return 13;
     }
 }
 function stageRatio(stage) {
@@ -801,6 +804,7 @@ function buildDirectStatusSnapshot(workspace) {
     const episodeResultsDir = path.join(dd, "episode_results");
     const trustEpisodeMergeArtifacts = canTrustStageArtifacts("episode_merge", runningStage, runStatus);
     const trustEpisodeSynopsisArtifacts = canTrustStageArtifacts("episode_synopsis", runningStage, runStatus);
+    const trustStateCurationArtifacts = canTrustStageArtifacts("state_curation", runningStage, runStatus);
     const trustStateBindingArtifacts = canTrustStageArtifacts("state_binding", runningStage, runStatus);
     const trustMetadataArtifacts = canTrustStageArtifacts("metadata_extract", runningStage, runStatus);
     const trustScriptSynopsisArtifacts = canTrustStageArtifacts("script_synopsis", runningStage, runStatus);
@@ -941,6 +945,9 @@ function buildDirectStatusSnapshot(workspace) {
     };
     const scriptStage = orderedFileStage("script_merge", path.join(dd, "script.initial.json"), runningStage, runStatus);
     const curationStage = orderedFileStage("asset_curation", path.join(dd, "asset_curation.json"), runningStage, runStatus);
+    const stateCurationStage = trustStateCurationArtifacts
+        ? orderedFileStage("state_curation", path.join(dd, "state_curation.json"), runningStage, runStatus)
+        : stageSnapshot("state_curation", runningStage === "state_curation" ? "running" : "pending", 0, 1);
     const stateBindingStage = trustStateBindingArtifacts
         ? orderedFileStage("state_binding", path.join(dd, "state_binding.json"), runningStage, runStatus)
         : stageSnapshot("state_binding", runningStage === "state_binding" ? "running" : "pending", 0, 1);
@@ -975,6 +982,7 @@ function buildDirectStatusSnapshot(workspace) {
         episodeSynopsisStage,
         scriptStage,
         curationStage,
+        stateCurationStage,
         stateBindingStage,
         metadataStage,
         scriptSynopsisStage,
@@ -1815,7 +1823,7 @@ function buildIncompleteReport(ctx) {
     return [report, EXIT_RUNTIME];
 }
 function initPreFanoutStages() {
-    const end = STAGES.findIndex((stage) => stage.name === "asset-curation");
+    const end = STAGES.findIndex((stage) => stage.name === "state-curation");
     return STAGES.slice(0, end + 1);
 }
 function validateOnlyStages() {
@@ -2485,8 +2493,8 @@ export async function stageScriptMerge(ctx) {
     }
 }
 // --- asset-curation -------------------------------------------------------
-const ASSET_GROUPING_CACHE_VERSION = 2;
-const ASSET_CURATION_CACHE_VERSION = 2;
+const ASSET_GROUPING_CACHE_VERSION = 3;
+const ASSET_CURATION_CACHE_VERSION = 4;
 function assetCacheIdentity(options) {
     return {
         provider: options.providerName,
@@ -3150,6 +3158,211 @@ export async function stageAssetCuration(ctx) {
     }
     ctx.emitProgress("asset-curation", "success");
 }
+// --- state-curation -------------------------------------------------------
+const STATE_CURATION_CACHE_VERSION = 1;
+function stateCurationScriptFingerprint(script, options) {
+    return sha256Text(JSON.stringify(sortDeep({
+        cache_version: STATE_CURATION_CACHE_VERSION,
+        ...assetCacheIdentity(options),
+        script: {
+            actors: script["actors"],
+            locations: script["locations"],
+            props: script["props"],
+            episodes: script["episodes"],
+        },
+    })));
+}
+function stateCurationContextFingerprint(scriptFingerprint, partId, refs) {
+    return sha256Text(JSON.stringify(sortDeep({
+        script_fingerprint: scriptFingerprint,
+        part_id: partId,
+        refs,
+    })));
+}
+function combineStateCurationPlans(parts) {
+    const decisions = parts.flatMap((part) => asList(part.plan["decisions"]).filter(isDict));
+    const expressions = parts.flatMap((part) => asList(part.plan["expressions"]));
+    return {
+        version: 1,
+        format: "state-curation-exp-v1",
+        expression_text: expressions.length > 0 ? expressions.map((expression) => `<exp>${expression}</exp>`).join("\n") : "<exp>NOOP # no state curation changes</exp>",
+        expressions,
+        decisions,
+        parts: parts.map((part) => ({
+            part_id: part.partId,
+            refs: part.refs,
+            context_fingerprint: part.contextFingerprint,
+            context_chars: part.contextChars,
+            reused: part.reused,
+            decisions: asList(part.plan["decisions"]).length,
+        })),
+    };
+}
+async function loadOrRunStateCurationPart(ctx, script, curateStates, scriptFingerprint, partId, refs) {
+    const artifactName = `state_curation_parts/${partId}.json`;
+    const contextFingerprint = stateCurationContextFingerprint(scriptFingerprint, partId, refs);
+    const cached = ctx.ws.readJsonSafe(artifactName);
+    const cachedPlan = !ctx.state.bypassInternalCaches
+        && isDict(cached)
+        && cached["context_fingerprint"] === contextFingerprint
+        && cached["parse_status"] === "ok"
+        && isDict(cached["plan"])
+        ? cached["plan"]
+        : null;
+    if (cachedPlan && isList(cachedPlan["decisions"])) {
+        const cachedContextChars = isDict(cached) ? Number(cached["context_chars"] ?? 0) : 0;
+        return {
+            partId,
+            refs: [...refs],
+            contextFingerprint,
+            contextChars: cachedContextChars,
+            plan: cachedPlan,
+            reused: true,
+        };
+    }
+    const startedAtMs = Date.now();
+    const contextText = buildStateCurationContextText(script, refs);
+    const requiredKeys = stateCurationSourceKeysForRefs(script, refs);
+    let expressionText = "";
+    try {
+        expressionText = await ctx.timing.span({ name: `state_curation.${partId}`, category: "chunk", parent: "state_curation.chunks", metadata: { artifact: artifactName } }, () => curateStates(script, contextText));
+        const parsed = parseStateCurationExpressions(expressionText);
+        const plan = assertStateCurationPlan(script, parsed, requiredKeys);
+        ctx.ws.writeJson(artifactName, {
+            version: 1,
+            ...assetCacheArtifactFields(ctx, STATE_CURATION_CACHE_VERSION),
+            context_fingerprint: contextFingerprint,
+            script_fingerprint: scriptFingerprint,
+            parse_status: "ok",
+            part_id: partId,
+            refs,
+            required_state_ids: requiredKeys,
+            context_chars: contextText.length,
+            expression_text: expressionText,
+            duration_ms: durationMs(startedAtMs, Date.now()),
+            plan,
+            updated_at: checkpointTimestamp(),
+        });
+        return {
+            partId,
+            refs: [...refs],
+            contextFingerprint,
+            contextChars: contextText.length,
+            plan,
+            reused: false,
+        };
+    }
+    catch (exc) {
+        const e = exc;
+        ctx.ws.writeJson(artifactName, {
+            version: 1,
+            ...assetCacheArtifactFields(ctx, STATE_CURATION_CACHE_VERSION),
+            context_fingerprint: contextFingerprint,
+            script_fingerprint: scriptFingerprint,
+            parse_status: "failed",
+            part_id: partId,
+            refs,
+            required_state_ids: requiredKeys,
+            context_chars: contextText.length,
+            expression_text: expressionText,
+            duration_ms: durationMs(startedAtMs, Date.now()),
+            error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
+            updated_at: checkpointTimestamp(),
+        });
+        throw exc;
+    }
+}
+export async function stageStateCuration(ctx) {
+    const { ws, state, options } = ctx;
+    const script = ensureScript(ctx);
+    let plan = null;
+    try {
+        ctx.updateRunState({ status: "init_running", init_stage: "state_curation" });
+        ctx.emitProgress("state-curation");
+        const scriptFingerprint = stateCurationScriptFingerprint(script, options);
+        const cached = ws.readJsonSafe("state_curation.json");
+        if (!state.bypassInternalCaches && isDict(cached) && cached["script_fingerprint"] === scriptFingerprint && isList(cached["decisions"])) {
+            const curation = await ctx.timing.span({ name: "state_curation.apply", category: "substage", parent: "state_curation" }, () => applyStateCurationPlan(script, { decisions: cached["decisions"] }));
+            curation["script_fingerprint"] = scriptFingerprint;
+            curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
+            curation["provider"] = options.providerName;
+            curation["model"] = options.model;
+            if (isList(cached["parts"]))
+                curation["parts"] = cached["parts"];
+            if (typeof cached["expression_text"] === "string")
+                curation["expression_text"] = cached["expression_text"];
+            if (isList(cached["expressions"]))
+                curation["expressions"] = cached["expressions"];
+            ws.writeJson("state_curation.json", curation);
+            ws.delete("state_curation.failed.json");
+            ctx.emitProgress("state-curation", "success");
+            return;
+        }
+        const chunks = buildStateCurationChunks(script);
+        if (chunks.length === 0) {
+            const curation = applyStateCurationPlan(script, { decisions: [] });
+            curation["script_fingerprint"] = scriptFingerprint;
+            curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
+            curation["provider"] = options.providerName;
+            curation["model"] = options.model;
+            curation["parts"] = [];
+            curation["expression_text"] = "<exp>NOOP # no state curation changes</exp>";
+            curation["expressions"] = ["NOOP # no state curation changes"];
+            ws.writeJson("state_curation.json", curation);
+            ws.delete("state_curation.failed.json");
+            ctx.emitProgress("state-curation", "success");
+            return;
+        }
+        const provider = ctx.provider();
+        if (!provider.curateStates) {
+            throw new CliError("INIT FAILED: Provider does not support state curation", "Provider does not support state curation.", {
+                exitCode: EXIT_RUNTIME,
+                required: ["provider.curateStates(script, contextText)"],
+                received: [`provider: ${provider.name || options.providerName}`],
+                nextSteps: ["Use a direct provider that supports state curation."],
+            });
+        }
+        ws.mkdir("state_curation_parts");
+        ctx.updateRunState({ state_curation_completed_chunks: 0, state_curation_total_chunks: chunks.length });
+        const curateStates = provider.curateStates.bind(provider);
+        let completedChunks = 0;
+        const outcomes = await pMapWithConcurrency(chunks, options.concurrency, async (chunk) => {
+            const part = await loadOrRunStateCurationPart(ctx, script, curateStates, scriptFingerprint, chunk.part_id, chunk.refs);
+            completedChunks += 1;
+            ctx.updateRunState({ state_curation_completed_chunks: completedChunks, state_curation_total_chunks: chunks.length });
+            return part;
+        });
+        const parts = unwrapConcurrentOutcomes(outcomes).sort((left, right) => left.partId.localeCompare(right.partId));
+        plan = combineStateCurationPlans(parts);
+        const curation = await ctx.timing.span({ name: "state_curation.apply", category: "substage", parent: "state_curation" }, () => applyStateCurationPlan(script, plan));
+        curation["script_fingerprint"] = scriptFingerprint;
+        curation["cache_version"] = STATE_CURATION_CACHE_VERSION;
+        curation["provider"] = options.providerName;
+        curation["model"] = options.model;
+        curation["expression_text"] = plan["expression_text"];
+        curation["expressions"] = plan["expressions"];
+        curation["parts"] = plan["parts"];
+        ws.writeJson("state_curation.json", curation);
+        ws.delete("state_curation.failed.json");
+    }
+    catch (exc) {
+        const e = exc;
+        ws.writeJson("state_curation.failed.json", {
+            version: 1,
+            failed_at: checkpointTimestamp(),
+            error: `${e?.name ?? "Error"}: ${e?.message ?? ""}`,
+            plan,
+        });
+        stageFailure(options.workspace, exc, {
+            title: "INIT FAILED: State curation failed",
+            stage: "state_curation",
+            required: ["provider state curation decisions for the assembled script state catalog"],
+            nextSteps: ["Rerun init; completed extraction and asset curation checkpoints will be reused."],
+            updates: { episode_completed: state.mergedResults.length },
+        });
+    }
+    ctx.emitProgress("state-curation", "success");
+}
 const STATE_BINDING_CHUNK_CACHE_VERSION = 1;
 function stateBindingChunkFingerprint(options, context) {
     return sha256Text(JSON.stringify(sortDeep({
@@ -3746,6 +3959,34 @@ function renderStateBindingSummary(binding) {
             `chunks ${summary["chunks"] ?? 0}`,
     ];
 }
+function renderStateCurationSummary(curation) {
+    const summary = isDict(curation["summary"]) ? curation["summary"] : {};
+    const decisionSummary = isDict(summary["decisions"]) ? summary["decisions"] : {};
+    const lines = [
+        `state-curation: ` +
+            `states ${summary["states_before"] ?? 0}->${summary["states_after"] ?? 0} ` +
+            `(removed ${summary["states_removed"] ?? 0}), ` +
+            `actor/location/prop ${summary["actor_states_after"] ?? 0}/${summary["location_states_after"] ?? 0}/${summary["prop_states_after"] ?? 0}, ` +
+            `merge ${decisionSummary["merge"] ?? 0}, drop ${decisionSummary["drop"] ?? 0}, action ${decisionSummary["action_desc"] ?? 0}, rename ${decisionSummary["rename"] ?? 0}`,
+    ];
+    for (const decision of asList(curation["decisions"])) {
+        if (!isDict(decision))
+            continue;
+        const action = strOf(decision["decision"]);
+        if (!action || action === "keep")
+            continue;
+        const kind = strOf(decision["kind"]);
+        const assetId = strOf(decision["asset_id"]);
+        const stateId = strOf(decision["state_id"]);
+        const target = action === "merge"
+            ? ` -> ${decision["target_state_id"] || "-"}`
+            : action === "rename"
+                ? ` -> ${decision["new_name"] || "-"}`
+                : "";
+        lines.push(`state-curation ${kind}:${assetId}/${stateId} ${action}${target} — ${decision["reason"] || "-"}`);
+    }
+    return lines;
+}
 function renderAssetNormalizationSummary(normalization) {
     const summary = isDict(normalization["summary"]) ? normalization["summary"] : {};
     const lines = [
@@ -3858,6 +4099,7 @@ export function commandReview(opts) {
     // (validation issues recomputed live, plus batch extraction failures).
     const lines = [];
     const curationPath = path.join(dd, "asset_curation.json");
+    const stateCurationPath = path.join(dd, "state_curation.json");
     const stateBindingPath = path.join(dd, "state_binding.json");
     const normalizationPath = path.join(dd, "asset_normalization.json");
     if (exists(normalizationPath)) {
@@ -3870,6 +4112,11 @@ export function commandReview(opts) {
         if (isDict(curation))
             lines.push(...renderAssetCurationSummary(curation));
     }
+    if (exists(stateCurationPath)) {
+        const curation = readJson(stateCurationPath);
+        if (isDict(curation))
+            lines.push(...renderStateCurationSummary(curation));
+    }
     if (exists(stateBindingPath)) {
         const binding = readJson(stateBindingPath);
         if (isDict(binding))
@@ -3898,7 +4145,7 @@ export function commandReview(opts) {
     const report = {
         title: "DIRECT REVIEW",
         result: lines.length > 0 ? lines : ["No issues, curation decisions, or batch failures to review."],
-        artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_grouping.json"), path.join(dd, "asset_curation.json"), stateBindingPath, batchResultsDir],
+        artifacts: [scriptPath, normalizationPath, path.join(dd, "asset_grouping.json"), path.join(dd, "asset_curation.json"), stateCurationPath, stateBindingPath, batchResultsDir],
         next: [
             "Run `direct review --episode <n>` for source↔extract alignment.",
             "Edit with `--draft` (e.g. `scriptctl rename … --draft`), then validate/export.",