@lingjingai/scriptctl 0.11.5 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +6 -10
- package/dist/cli.js.map +1 -1
- package/dist/common.d.ts +1 -1
- package/dist/common.js +1 -4
- package/dist/common.js.map +1 -1
- package/dist/domain/direct-core.d.ts +3 -3
- package/dist/domain/direct-core.js +32 -45
- package/dist/domain/direct-core.js.map +1 -1
- package/dist/help-text.js +35 -4
- package/dist/help-text.js.map +1 -1
- package/dist/infra/providers.js +2 -2
- package/dist/infra/providers.js.map +1 -1
- package/dist/usecases/direct.d.ts +0 -4
- package/dist/usecases/direct.js +290 -515
- package/dist/usecases/direct.js.map +1 -1
- package/dist/usecases/script.js +3 -25
- package/dist/usecases/script.js.map +1 -1
- package/package.json +1 -1
package/dist/usecases/direct.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER,
|
|
4
|
-
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan,
|
|
3
|
+
import { CliError, DEFAULT_BATCH_MAX_CHARS, DEFAULT_BATCH_MIN_LINES, DEFAULT_BATCH_MODE, DEFAULT_BATCH_TARGET_LINES, DEFAULT_CONCURRENCY, DEFAULT_MODEL, DEFAULT_PROVIDER, EXIT_INPUT, EXIT_NEEDS_AGENT, EXIT_OK, EXIT_RUNTIME, EXIT_USAGE, REVIEW_TARGETS, SUPPORTED_EXTS, deletePath, directDir, exists, fmtId, readJson, readText, writeJson, } from "../common.js";
|
|
4
|
+
import { compactBatchResult, compactEpisodeResult, buildBatchPlan, buildEpisodePlan, DEFAULT_TITLE_CHUNK_SIZE, chunkEpisodesNeedingTitles, applyEpisodeTitlesToPlan, episodeTitleResponseMap, extractBatchWithRecovery, mergeEpisodeResults, normalizeEpisodeResult, normalizeInt, uniqueAdd, validateEpisodeExtractionQuality, _md_push_asset, curateScriptAssets, applyMetadataToScript, } from "../domain/direct-core.js";
|
|
5
5
|
import { validateScript } from "../domain/script-core.js";
|
|
6
6
|
import { makeProvider } from "../infra/providers.js";
|
|
7
7
|
import { makeSourceManifest, prepareSource, } from "../infra/converters.js";
|
|
@@ -23,6 +23,12 @@ function checkpointTimestamp() {
|
|
|
23
23
|
const now = new Date();
|
|
24
24
|
return now.toISOString().replace(/\.\d+Z$/, "Z");
|
|
25
25
|
}
|
|
26
|
+
function parseIntList(v) {
|
|
27
|
+
return strOf(v).split(",").map((s) => parseInt(s.trim(), 10)).filter((n) => !Number.isNaN(n));
|
|
28
|
+
}
|
|
29
|
+
function parseStrList(v) {
|
|
30
|
+
return strOf(v).split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
31
|
+
}
|
|
26
32
|
// ---------------------------------------------------------------------------
|
|
27
33
|
// run_state.json
|
|
28
34
|
// ---------------------------------------------------------------------------
|
|
@@ -163,53 +169,42 @@ function persistBatchResult(dir, batch, result) {
|
|
|
163
169
|
deletePath(mdPath);
|
|
164
170
|
}
|
|
165
171
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
// Per-batch result files ARE the state. A batch with `<key>.json` is done; with
|
|
174
|
+
// `<key>.error.json` it failed; with neither it is pending. Reuse = file
|
|
175
|
+
// presence — the agent drives what to (re)run via `direct status` + --episodes/
|
|
176
|
+
// --batches selection. No content hashes, no global checkpoint, no destructive
|
|
177
|
+
// reset: re-running a unit only ever touches that unit's own files, so the other
|
|
178
|
+
// episodes can never be lost.
|
|
179
|
+
// ---------------------------------------------------------------------------
|
|
180
|
+
// Read a persisted batch result (reused or freshly written) for assembly.
|
|
181
|
+
// Returns null if absent or unreadable, in which case the batch counts as
|
|
182
|
+
// not-done and will be (re)run.
|
|
183
|
+
function readBatchResult(dir, batch) {
|
|
184
|
+
const p = batchResultPath(dir, batch);
|
|
185
|
+
if (!exists(p))
|
|
174
186
|
return null;
|
|
175
187
|
try {
|
|
176
|
-
const
|
|
177
|
-
|
|
188
|
+
const result = normalizeEpisodeResult(readJson(p), batch);
|
|
189
|
+
result["_batch_id"] = batchResultKey(batch);
|
|
190
|
+
result["_batch_part"] = Number(batch["part"]);
|
|
191
|
+
result["_starts_inside_scene"] = Boolean(batch["starts_inside_scene"]);
|
|
192
|
+
return result;
|
|
178
193
|
}
|
|
179
194
|
catch {
|
|
180
195
|
return null;
|
|
181
196
|
}
|
|
182
197
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
function
|
|
188
|
-
if (
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
schema: 1, key: episodeResultKey(ep), episode: Number(ep["episode"]),
|
|
194
|
-
input_hash: inputHash, provenance, status: "ok", provider: providerName, model, extracted_at: checkpointTimestamp(),
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
function stampBatchMeta(dir, batch, inputHash, provenance, providerName, model) {
|
|
198
|
-
writeUnitMeta(batchMetaPath(dir, batch), {
|
|
199
|
-
schema: 1, key: batchResultKey(batch), episode: Number(batch["episode"]), part: Number(batch["part"]),
|
|
200
|
-
input_hash: inputHash, provenance, status: "ok", provider: providerName, model, extracted_at: checkpointTimestamp(),
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
// Content-address a single episode/batch plan unit: the contract version, the
|
|
204
|
-
// exact source span text, the title-stable plan item, and provider/model. Any
|
|
205
|
-
// change to what would alter extraction rotates the hash for THAT unit only.
|
|
206
|
-
export function computeUnitHash(sourceText, unit, providerName, model) {
|
|
207
|
-
const span = isDict(unit["source_span"]) ? unit["source_span"] : {};
|
|
208
|
-
const start = Number(span["start"] ?? 0);
|
|
209
|
-
const end = Number(span["end"] ?? 0);
|
|
210
|
-
const spanText = sourceText.slice(start, end);
|
|
211
|
-
const planText = JSON.stringify(unit, checkpointReplacer());
|
|
212
|
-
return sha256Text([String(DIRECT_CONTRACT_VERSION), spanText, planText, providerName ?? "", model ?? ""].join("\u0000"));
|
|
198
|
+
// Single source of truth for a batch's on-disk state, shared by init (what to
|
|
199
|
+
// (re)run) and status (what to report) so the two can never disagree. "done"
|
|
200
|
+
// requires a result that actually PARSES — a present-but-corrupt result is
|
|
201
|
+
// pending, so it self-heals on the next run instead of being silently trusted.
|
|
202
|
+
function classifyBatch(dir, batch) {
|
|
203
|
+
if (readBatchResult(dir, batch) !== null)
|
|
204
|
+
return "done";
|
|
205
|
+
if (exists(batchErrorPath(dir, batch)))
|
|
206
|
+
return "error";
|
|
207
|
+
return "pending";
|
|
213
208
|
}
|
|
214
209
|
// Delete result/meta/error/markdown files whose unit key is no longer in the
|
|
215
210
|
// current plan (e.g. the source shed an episode). Pure function of the plan —
|
|
@@ -235,69 +230,6 @@ function gcOrphanUnits(dir, liveKeys) {
|
|
|
235
230
|
}
|
|
236
231
|
return removed;
|
|
237
232
|
}
|
|
238
|
-
// Title fields are LLM-mutated downstream by enrichEpisodePlanTitles, so they
|
|
239
|
-
// must be excluded from unit hashes — otherwise every rerun gets a fresh SHA,
|
|
240
|
-
// the cached unit never matches, and that unit re-extracts from scratch.
|
|
241
|
-
const CHECKPOINT_UNSTABLE_KEYS = new Set(["title", "generated_title", "title_status", "title_source"]);
|
|
242
|
-
function checkpointReplacer() {
|
|
243
|
-
// Python's json.dumps(sort_keys=True) sorts keys recursively. Replicate by walking and sorting.
|
|
244
|
-
return function (key, value) {
|
|
245
|
-
if (CHECKPOINT_UNSTABLE_KEYS.has(key))
|
|
246
|
-
return undefined;
|
|
247
|
-
if (isDict(value)) {
|
|
248
|
-
const sorted = {};
|
|
249
|
-
for (const k of Object.keys(value).sort()) {
|
|
250
|
-
if (CHECKPOINT_UNSTABLE_KEYS.has(k))
|
|
251
|
-
continue;
|
|
252
|
-
sorted[k] = value[k];
|
|
253
|
-
}
|
|
254
|
-
return sorted;
|
|
255
|
-
}
|
|
256
|
-
return value;
|
|
257
|
-
};
|
|
258
|
-
}
|
|
259
|
-
// Non-destructive reuse: a cached episode result is reusable iff its sidecar
|
|
260
|
-
// meta records the same input_hash we compute for the current plan unit. The
|
|
261
|
-
// hash subsumes the old provider / source_span / episode-number / contract
|
|
262
|
-
// checks — any of those changing rotates the hash. On any mismatch or read
|
|
263
|
-
// failure we return null and let the caller re-extract and overwrite; we NEVER
|
|
264
|
-
// delete the cached file pre-emptively (that was the data-loss root cause).
|
|
265
|
-
export function loadCachedEpisode(sourceText, episodeResultsDir, ep, expectedHash) {
|
|
266
|
-
const meta = readUnitMeta(episodeMetaPath(episodeResultsDir, ep));
|
|
267
|
-
if (!meta || meta["input_hash"] !== expectedHash)
|
|
268
|
-
return null;
|
|
269
|
-
if (meta["status"] === "terminal")
|
|
270
|
-
return null;
|
|
271
|
-
const p = episodeResultPath(episodeResultsDir, ep);
|
|
272
|
-
if (!exists(p))
|
|
273
|
-
return null;
|
|
274
|
-
try {
|
|
275
|
-
const result = normalizeEpisodeResult(readJson(p), ep);
|
|
276
|
-
validateEpisodeExtractionQuality(sourceText, ep, result);
|
|
277
|
-
return result;
|
|
278
|
-
}
|
|
279
|
-
catch {
|
|
280
|
-
return null;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
export function loadCachedBatch(sourceText, batchResultsDir, batch, expectedHash) {
|
|
284
|
-
const meta = readUnitMeta(batchMetaPath(batchResultsDir, batch));
|
|
285
|
-
if (!meta || meta["input_hash"] !== expectedHash)
|
|
286
|
-
return null;
|
|
287
|
-
if (meta["status"] === "terminal")
|
|
288
|
-
return null;
|
|
289
|
-
const p = batchResultPath(batchResultsDir, batch);
|
|
290
|
-
if (!exists(p))
|
|
291
|
-
return null;
|
|
292
|
-
try {
|
|
293
|
-
const result = normalizeEpisodeResult(readJson(p), batch);
|
|
294
|
-
validateBatchExtractionQuality(sourceText, batch, result);
|
|
295
|
-
return result;
|
|
296
|
-
}
|
|
297
|
-
catch {
|
|
298
|
-
return null;
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
233
|
function mergeScene(target, source) {
|
|
302
234
|
if ((target["location_name"] === "" || target["location_name"] === "未知场景" || target["location_name"] === null || target["location_name"] === undefined) &&
|
|
303
235
|
source["location_name"]) {
|
|
@@ -389,9 +321,19 @@ async function providerExtractAssetCurationLocal(provider, sourceText, script) {
|
|
|
389
321
|
}
|
|
390
322
|
return {};
|
|
391
323
|
}
|
|
392
|
-
|
|
324
|
+
// Record a batch failure: write `<key>.error.json` (which marks the batch as
|
|
325
|
+
// "tried and failed" → sticky on plain resume) and drop any stale result file.
|
|
326
|
+
// The error_code (e.g. PROVIDER_CONTENT_FILTERED) is surfaced so the agent can
|
|
327
|
+
// see WHY in `direct status` / `direct inspect` and decide whether to re-run.
|
|
328
|
+
//
|
|
329
|
+
// Deliberate content-filter policy split: batch extraction is HARD (sticky
|
|
330
|
+
// error, episode blocked until an explicit re-run) because batch content is the
|
|
331
|
+
// deliverable and cannot be fabricated; episode TITLES are SOFT (degrade to a
|
|
332
|
+
// deterministic title, recorded as title_degraded, never blocking) because a
|
|
333
|
+
// title is non-essential and always has a safe fallback. Same provider signal,
|
|
334
|
+
// two intentional policies — see enrichEpisodeTitlesChunked.
|
|
335
|
+
function writeBatchFailure(dir, batch, exc) {
|
|
393
336
|
const err = exc;
|
|
394
|
-
const terminal = classifyProviderError(exc) === "terminal";
|
|
395
337
|
const error = {
|
|
396
338
|
batch_id: batchResultKey(batch),
|
|
397
339
|
episode: Number(batch["episode"]),
|
|
@@ -400,11 +342,11 @@ function writeBatchFailure(dir, batch, exc, inputHash, providerName, model) {
|
|
|
400
342
|
line_range: batch["line_range"],
|
|
401
343
|
error_type: err?.name || "Error",
|
|
402
344
|
message: (err?.message || err?.name || "Error").slice(0, 500),
|
|
403
|
-
terminal,
|
|
404
|
-
input_hash: inputHash,
|
|
405
345
|
failed_at: checkpointTimestamp(),
|
|
406
346
|
};
|
|
407
347
|
if (exc instanceof CliError) {
|
|
348
|
+
if (exc.errorCode)
|
|
349
|
+
error["error_code"] = exc.errorCode;
|
|
408
350
|
if (exc.required.length > 0)
|
|
409
351
|
error["required"] = exc.required;
|
|
410
352
|
if (exc.received.length > 0)
|
|
@@ -415,16 +357,6 @@ function writeBatchFailure(dir, batch, exc, inputHash, providerName, model) {
|
|
|
415
357
|
const resultPath = batchResultPath(dir, batch);
|
|
416
358
|
if (exists(resultPath))
|
|
417
359
|
deletePath(resultPath);
|
|
418
|
-
if (terminal) {
|
|
419
|
-
writeUnitMeta(batchMetaPath(dir, batch), {
|
|
420
|
-
schema: 1, key: batchResultKey(batch), episode: Number(batch["episode"]), part: Number(batch["part"]),
|
|
421
|
-
input_hash: inputHash, provenance: "extracted", status: "terminal",
|
|
422
|
-
provider: providerName, model, extracted_at: checkpointTimestamp(),
|
|
423
|
-
});
|
|
424
|
-
}
|
|
425
|
-
else {
|
|
426
|
-
removeUnitMeta(batchMetaPath(dir, batch));
|
|
427
|
-
}
|
|
428
360
|
writeJson(batchErrorPath(dir, batch), error);
|
|
429
361
|
return error;
|
|
430
362
|
}
|
|
@@ -468,6 +400,64 @@ async function pMapWithConcurrency(items, concurrency, worker) {
|
|
|
468
400
|
}));
|
|
469
401
|
return out;
|
|
470
402
|
}
|
|
403
|
+
function providerErrorSummary(error) {
|
|
404
|
+
if (error instanceof CliError && error.errorCode)
|
|
405
|
+
return error.errorCode;
|
|
406
|
+
const e = error;
|
|
407
|
+
const msg = strOf(e?.message).replace(/\s+/g, " ").trim();
|
|
408
|
+
if (/prohibited content|content filter|refused/i.test(msg))
|
|
409
|
+
return "PROVIDER_CONTENT_FILTERED";
|
|
410
|
+
return `${e?.name ?? "Error"}: ${msg}`.slice(0, 160);
|
|
411
|
+
}
|
|
412
|
+
// Generate the plan's missing episode titles in bounded, concurrent chunks.
|
|
413
|
+
// Title generation is non-essential (every episode has a deterministic
|
|
414
|
+
// fallback), so a chunk that the provider filters or fails does NOT abort
|
|
415
|
+
// init: its episodes are retried one-by-one to isolate the poisoned excerpt,
|
|
416
|
+
// and whatever still fails degrades to a deterministic title. The plan is
|
|
417
|
+
// mutated in place and persisted by the caller; degradation is recorded to
|
|
418
|
+
// run_state so it's visible without being a hard failure.
|
|
419
|
+
async function enrichEpisodeTitlesChunked(workspace, sourceText, plan, provider, concurrency) {
|
|
420
|
+
const chunks = chunkEpisodesNeedingTitles(plan, DEFAULT_TITLE_CHUNK_SIZE);
|
|
421
|
+
if (chunks.length === 0 || !provider.extractEpisodeTitles) {
|
|
422
|
+
// Nothing to title, or a provider with no title support: deterministic for all.
|
|
423
|
+
return applyEpisodeTitlesToPlan(sourceText, plan, new Map());
|
|
424
|
+
}
|
|
425
|
+
const extract = provider.extractEpisodeTitles.bind(provider);
|
|
426
|
+
const generated = new Map();
|
|
427
|
+
const collect = (outcomes) => {
|
|
428
|
+
for (const o of outcomes) {
|
|
429
|
+
if (o.ok)
|
|
430
|
+
for (const [num, title] of o.value)
|
|
431
|
+
generated.set(num, title);
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
// Phase 1: chunked requests.
|
|
435
|
+
const chunkOutcomes = await pMapWithConcurrency(chunks, concurrency, async (chunk) => episodeTitleResponseMap(await extract(sourceText, { episodes: chunk })));
|
|
436
|
+
collect(chunkOutcomes);
|
|
437
|
+
// Phase 2: any episode still WITHOUT a title — whether its chunk threw or the
|
|
438
|
+
// chunk returned 200 but omitted it — gets one isolated single-episode request,
|
|
439
|
+
// so a single poisoned/omitted excerpt can't deny its chunk-mates a real title.
|
|
440
|
+
const missing = chunks.flat().filter((ep) => !generated.has(Number(ep["episode"] ?? 0)));
|
|
441
|
+
let reason;
|
|
442
|
+
if (missing.length > 0) {
|
|
443
|
+
const singleOutcomes = await pMapWithConcurrency(missing, concurrency, async (ep) => episodeTitleResponseMap(await extract(sourceText, { episodes: [ep] })));
|
|
444
|
+
collect(singleOutcomes);
|
|
445
|
+
const firstError = singleOutcomes.find((o) => !o.ok);
|
|
446
|
+
if (firstError)
|
|
447
|
+
reason = providerErrorSummary(firstError.error);
|
|
448
|
+
}
|
|
449
|
+
const degraded = applyEpisodeTitlesToPlan(sourceText, plan, generated);
|
|
450
|
+
if (degraded.length > 0) {
|
|
451
|
+
updateRunState(workspace, {
|
|
452
|
+
title_degraded: {
|
|
453
|
+
count: degraded.length,
|
|
454
|
+
episodes: degraded,
|
|
455
|
+
reason: reason ?? "provider returned no title for these episodes; used deterministic titles",
|
|
456
|
+
},
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
return degraded;
|
|
460
|
+
}
|
|
471
461
|
// ---------------------------------------------------------------------------
|
|
472
462
|
// command_init
|
|
473
463
|
// ---------------------------------------------------------------------------
|
|
@@ -479,9 +469,14 @@ export async function commandInit(opts) {
|
|
|
479
469
|
const workspace = strOf(opts["workspace_path"] || "workspace");
|
|
480
470
|
const providerName = strOf(opts["provider"] || DEFAULT_PROVIDER);
|
|
481
471
|
const model = strOf(opts["model"] || process.env.SCRIPTCTL_ANTHROPIC_MODEL || DEFAULT_MODEL);
|
|
482
|
-
//
|
|
483
|
-
//
|
|
484
|
-
|
|
472
|
+
// Explicit selection: the agent decides what to (re)run. Default is "resume"
|
|
473
|
+
// — only batches never attempted run. --episodes / --batches force-rerun the
|
|
474
|
+
// named units, --retry-errors force-reruns everything currently errored, --all
|
|
475
|
+
// reruns everything. Forced units have their stale result/error deleted first.
|
|
476
|
+
const selEpisodes = new Set(parseIntList(opts["episodes"]));
|
|
477
|
+
const selBatches = new Set(parseStrList(opts["batches"]));
|
|
478
|
+
const retryErrors = Boolean(opts["retry_errors"]);
|
|
479
|
+
const runAll = Boolean(opts["all"]);
|
|
485
480
|
let concurrency;
|
|
486
481
|
try {
|
|
487
482
|
concurrency = parseInt(strOf(opts["concurrency"] || DEFAULT_CONCURRENCY), 10);
|
|
@@ -636,29 +631,7 @@ export async function commandInit(opts) {
|
|
|
636
631
|
throw exc;
|
|
637
632
|
}
|
|
638
633
|
updateRunState(workspace, { status: "init_running", init_stage: "episode_titles" });
|
|
639
|
-
|
|
640
|
-
plan = await enrichEpisodePlanTitles(sourceText, plan, provider);
|
|
641
|
-
}
|
|
642
|
-
catch (exc) {
|
|
643
|
-
if (exc instanceof CliError) {
|
|
644
|
-
throw initFailedReport(workspace, {
|
|
645
|
-
title: exc.title,
|
|
646
|
-
stage: "episode_titles",
|
|
647
|
-
exitCode: exc.exitCode,
|
|
648
|
-
required: exc.required.length > 0 ? exc.required : ["episode titles generated from source text"],
|
|
649
|
-
received: exc.received.length > 0 ? exc.received : [String(exc.message).slice(0, 160)],
|
|
650
|
-
nextSteps: exc.nextSteps.length > 0 ? exc.nextSteps : ["Rerun init after checking source episode headers."],
|
|
651
|
-
});
|
|
652
|
-
}
|
|
653
|
-
const e = exc;
|
|
654
|
-
throw initFailedReport(workspace, {
|
|
655
|
-
title: "INIT FAILED: Episode title planning failed",
|
|
656
|
-
stage: "episode_titles",
|
|
657
|
-
required: ["episode titles generated from source text"],
|
|
658
|
-
received: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`],
|
|
659
|
-
nextSteps: ["Inspect workspace/source.txt and episode_plan.json, then rerun init."],
|
|
660
|
-
});
|
|
661
|
-
}
|
|
634
|
+
const titleDegraded = await enrichEpisodeTitlesChunked(workspace, sourceText, plan, provider, concurrency);
|
|
662
635
|
let batchPlan;
|
|
663
636
|
try {
|
|
664
637
|
batchPlan = buildBatchPlan(sourceText, plan, {
|
|
@@ -691,20 +664,6 @@ export async function commandInit(opts) {
|
|
|
691
664
|
// the legacy v3 index.json. There is no whole-directory reset any more.
|
|
692
665
|
gcOrphanUnits(episodeResultsDir, new Set(asList(plan["episodes"]).map((ep) => episodeResultKey(ep))));
|
|
693
666
|
gcOrphanUnits(batchResultsDir, new Set(asList(batchPlan["batches"]).map((b) => batchResultKey(b))));
|
|
694
|
-
updateRunState(workspace, {
|
|
695
|
-
status: "init_running",
|
|
696
|
-
init_stage: "batch_extract",
|
|
697
|
-
batch_mode: batchMode,
|
|
698
|
-
batch_target_lines: batchTargetLines,
|
|
699
|
-
batch_max_chars: batchMaxChars,
|
|
700
|
-
batch_min_lines: batchMinLines,
|
|
701
|
-
episode_total: asList(plan["episodes"]).length,
|
|
702
|
-
batch_total: asList(batchPlan["batches"]).length,
|
|
703
|
-
});
|
|
704
|
-
const results = [];
|
|
705
|
-
const skipped = [];
|
|
706
|
-
let skippedEpisodeBatchCount = 0;
|
|
707
|
-
const pendingBatches = [];
|
|
708
667
|
const batchesByEpisode = new Map();
|
|
709
668
|
for (const batch of asList(batchPlan["batches"])) {
|
|
710
669
|
const epNum = Number(batch["episode"]);
|
|
@@ -712,126 +671,93 @@ export async function commandInit(opts) {
|
|
|
712
671
|
batchesByEpisode.set(epNum, []);
|
|
713
672
|
batchesByEpisode.get(epNum).push(batch);
|
|
714
673
|
}
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
skipped.push(Number(episode["episode"]));
|
|
723
|
-
const cachedBatches = batchesByEpisode.get(Number(episode["episode"])) ?? [];
|
|
724
|
-
skippedEpisodeBatchCount += cachedBatches.length;
|
|
725
|
-
for (const cachedBatch of cachedBatches) {
|
|
726
|
-
if (!exists(batchResultPath(batchResultsDir, cachedBatch))) {
|
|
727
|
-
const backfilled = recoverBatchFromSource(sourceText, cachedBatch);
|
|
728
|
-
persistBatchResult(batchResultsDir, cachedBatch, backfilled);
|
|
729
|
-
stampBatchMeta(batchResultsDir, cachedBatch, computeUnitHash(sourceText, cachedBatch, providerName, model), "recovered", providerName, model);
|
|
730
|
-
}
|
|
731
|
-
const errorPath = batchErrorPath(batchResultsDir, cachedBatch);
|
|
732
|
-
if (exists(errorPath))
|
|
733
|
-
deletePath(errorPath);
|
|
734
|
-
}
|
|
735
|
-
}
|
|
736
|
-
else {
|
|
737
|
-
pendingBatches.push(...(batchesByEpisode.get(Number(episode["episode"])) ?? []));
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
const batchResults = [];
|
|
741
|
-
const skippedBatches = [];
|
|
742
|
-
const terminalSkipped = [];
|
|
674
|
+
const totalEpisodes = asList(plan["episodes"]).length;
|
|
675
|
+
const totalBatches = asList(batchPlan["batches"]).length;
|
|
676
|
+
// Decide which batches to run. State (classifyBatch) is shared with `direct
|
|
677
|
+
// status` so the two never disagree: a "done" batch is reused, an "error"
|
|
678
|
+
// batch is sticky (skipped on a plain rerun so a content filter can't loop),
|
|
679
|
+
// a "pending" (never-run or corrupt) batch runs. The agent overrides with
|
|
680
|
+
// explicit selection.
|
|
743
681
|
const pending = [];
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
const
|
|
750
|
-
|
|
751
|
-
|
|
682
|
+
let reusedBatchCount = 0;
|
|
683
|
+
let stickyErrorCount = 0;
|
|
684
|
+
for (const batch of asList(batchPlan["batches"])) {
|
|
685
|
+
const epNum = Number(batch["episode"]);
|
|
686
|
+
const key = batchResultKey(batch);
|
|
687
|
+
const errorPath = batchErrorPath(batchResultsDir, batch);
|
|
688
|
+
const forced = runAll || selEpisodes.has(epNum) || selBatches.has(key) || (retryErrors && exists(errorPath));
|
|
689
|
+
if (forced) {
|
|
690
|
+
const resultPath = batchResultPath(batchResultsDir, batch);
|
|
691
|
+
if (exists(resultPath))
|
|
692
|
+
deletePath(resultPath);
|
|
693
|
+
if (exists(errorPath))
|
|
694
|
+
deletePath(errorPath);
|
|
695
|
+
pending.push(batch);
|
|
752
696
|
continue;
|
|
753
697
|
}
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
else {
|
|
763
|
-
pending.push(batch);
|
|
698
|
+
switch (classifyBatch(batchResultsDir, batch)) {
|
|
699
|
+
case "done":
|
|
700
|
+
reusedBatchCount++;
|
|
701
|
+
break;
|
|
702
|
+
case "error":
|
|
703
|
+
stickyErrorCount++;
|
|
704
|
+
break; // tried and failed; left until explicitly re-run
|
|
705
|
+
default: pending.push(batch); // never attempted (or corrupt → self-heal)
|
|
764
706
|
}
|
|
765
707
|
}
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
708
|
+
updateRunState(workspace, {
|
|
709
|
+
status: "init_running",
|
|
710
|
+
init_stage: "batch_extract",
|
|
711
|
+
batch_mode: batchMode,
|
|
712
|
+
batch_target_lines: batchTargetLines,
|
|
713
|
+
batch_max_chars: batchMaxChars,
|
|
714
|
+
batch_min_lines: batchMinLines,
|
|
715
|
+
episode_total: totalEpisodes,
|
|
716
|
+
batch_total: totalBatches,
|
|
769
717
|
});
|
|
718
|
+
const failures = [];
|
|
719
|
+
const outcomes = await pMapWithConcurrency(pending, concurrency, async (batch) => extractBatchWithRecovery(provider, sourceText, batch));
|
|
770
720
|
for (let i = 0; i < outcomes.length; i++) {
|
|
771
721
|
const outcome = outcomes[i];
|
|
772
722
|
const batch = pending[i];
|
|
773
|
-
const errorPath = batchErrorPath(batchResultsDir, batch);
|
|
774
723
|
if (outcome.ok) {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
result["_batch_part"] = Number(batch["part"]);
|
|
778
|
-
result["_starts_inside_scene"] = Boolean(batch["starts_inside_scene"]);
|
|
779
|
-
batchResults.push(result);
|
|
780
|
-
persistBatchResult(batchResultsDir, batch, result);
|
|
781
|
-
stampBatchMeta(batchResultsDir, batch, computeUnitHash(sourceText, batch, providerName, model), "extracted", providerName, model);
|
|
724
|
+
persistBatchResult(batchResultsDir, batch, outcome.value);
|
|
725
|
+
const errorPath = batchErrorPath(batchResultsDir, batch);
|
|
782
726
|
if (exists(errorPath))
|
|
783
727
|
deletePath(errorPath);
|
|
784
728
|
}
|
|
785
729
|
else {
|
|
786
|
-
failures.push(writeBatchFailure(batchResultsDir, batch, outcome.error
|
|
730
|
+
failures.push(writeBatchFailure(batchResultsDir, batch, outcome.error));
|
|
787
731
|
}
|
|
788
732
|
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
});
|
|
797
|
-
failures.sort((a, b) => {
|
|
798
|
-
const ea = Number(a["episode"] ?? 0);
|
|
799
|
-
const eb = Number(b["episode"] ?? 0);
|
|
800
|
-
if (ea !== eb)
|
|
801
|
-
return ea - eb;
|
|
802
|
-
return Number(a["part"] ?? 0) - Number(b["part"] ?? 0);
|
|
803
|
-
});
|
|
804
|
-
const completedBatches = skippedEpisodeBatchCount + batchResults.length;
|
|
805
|
-
const transientFailures = failures.filter((it) => !it["terminal"]);
|
|
806
|
-
const terminalFailures = failures.filter((it) => Boolean(it["terminal"]));
|
|
807
|
-
const skippedSet = new Set(skipped);
|
|
808
|
-
// Merge every fully-completed, non-cached episode into an episode_results
|
|
809
|
-
// checkpoint. Episodes still missing a batch (a failure this run, or a batch
|
|
810
|
-
// a prior run marked terminal and we skipped) are left unmerged so a rerun or
|
|
811
|
-
// an override can complete them.
|
|
812
|
-
const batchResultsByEpisode = new Map();
|
|
813
|
-
for (const result of batchResults) {
|
|
814
|
-
const ep = Number(result["episode"] ?? 0);
|
|
815
|
-
if (!batchResultsByEpisode.has(ep))
|
|
816
|
-
batchResultsByEpisode.set(ep, []);
|
|
817
|
-
batchResultsByEpisode.get(ep).push(result);
|
|
818
|
-
}
|
|
733
|
+
const doneBatches = reusedBatchCount + outcomes.filter((o) => o.ok).length;
|
|
734
|
+
const erroredBatches = stickyErrorCount + failures.length;
|
|
735
|
+
// Assemble every episode whose batches are all present on disk (reused +
|
|
736
|
+
// freshly extracted, read uniformly from disk). Incomplete episodes are
|
|
737
|
+
// reported; assembling them is left to an explicit re-run.
|
|
738
|
+
const results = [];
|
|
739
|
+
const incompleteEpisodes = [];
|
|
819
740
|
try {
|
|
820
741
|
for (const episode of asList(plan["episodes"])) {
|
|
821
742
|
const episodeNum = Number(episode["episode"]);
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
743
|
+
const epBatches = batchesByEpisode.get(episodeNum) ?? [];
|
|
744
|
+
const batchResults = epBatches.map((b) => readBatchResult(batchResultsDir, b)).filter((r) => r !== null);
|
|
745
|
+
if (epBatches.length === 0 || batchResults.length !== epBatches.length) {
|
|
746
|
+
incompleteEpisodes.push(episodeNum);
|
|
747
|
+
// Drop any stale episode_results/ep_NNN.json from a prior complete run so
|
|
748
|
+
// the derived artifact never claims an episode is complete when it isn't.
|
|
749
|
+
const stale = episodeResultPath(episodeResultsDir, episode);
|
|
750
|
+
if (exists(stale))
|
|
751
|
+
deletePath(stale);
|
|
826
752
|
continue;
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
const
|
|
833
|
-
if (exists(
|
|
834
|
-
deletePath(
|
|
753
|
+
}
|
|
754
|
+
const merged = mergeBatchResultsForEpisode(episode, batchResults);
|
|
755
|
+
validateEpisodeExtractionQuality(sourceText, episode, merged);
|
|
756
|
+
results.push(merged);
|
|
757
|
+
writeJson(episodeResultPath(episodeResultsDir, episode), compactEpisodeResult(merged));
|
|
758
|
+
const errPath = episodeErrorPath(episodeResultsDir, episode);
|
|
759
|
+
if (exists(errPath))
|
|
760
|
+
deletePath(errPath);
|
|
835
761
|
}
|
|
836
762
|
}
|
|
837
763
|
catch (exc) {
|
|
@@ -839,109 +765,61 @@ export async function commandInit(opts) {
|
|
|
839
765
|
throw initFailedReport(workspace, {
|
|
840
766
|
title: "INIT FAILED: Episode merge failed",
|
|
841
767
|
stage: "episode_merge",
|
|
842
|
-
required: ["
|
|
768
|
+
required: ["batch_results/*.json that can merge into episode_results/*.json"],
|
|
843
769
|
received: [`${e?.name ?? "Error"}: ${(e?.message ?? "").slice(0, 160)}`],
|
|
844
|
-
nextSteps: ["Rerun init; completed
|
|
845
|
-
updates: { batch_completed: completedBatches },
|
|
770
|
+
nextSteps: ["Rerun init; completed batches are reused and episode merge will retry."],
|
|
846
771
|
});
|
|
847
772
|
}
|
|
848
773
|
results.sort((a, b) => Number(a["episode"] ?? 0) - Number(b["episode"] ?? 0));
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
// An episode blocked only by terminal (content-filtered) batches is held out:
|
|
852
|
-
// the rest of the script ships, and the operator overrides the blocked unit.
|
|
853
|
-
const completedEpisodeNums = new Set(results.map((r) => Number(r["episode"])));
|
|
854
|
-
const transientEpisodeSet = new Set(transientFailures.map((it) => Number(it["episode"])));
|
|
855
|
-
const incompleteEpisodes = asList(plan["episodes"]).map((ep) => Number(ep["episode"])).filter((n) => !completedEpisodeNums.has(n));
|
|
856
|
-
const transientBlocked = incompleteEpisodes.filter((n) => transientEpisodeSet.has(n)).sort((a, b) => a - b);
|
|
857
|
-
const heldOutEpisodes = incompleteEpisodes.filter((n) => !transientEpisodeSet.has(n)).sort((a, b) => a - b);
|
|
858
|
-
if (transientBlocked.length > 0) {
|
|
774
|
+
incompleteEpisodes.sort((a, b) => a - b);
|
|
775
|
+
if (incompleteEpisodes.length > 0) {
|
|
859
776
|
updateRunState(workspace, {
|
|
860
777
|
status: "init_incomplete",
|
|
861
778
|
init_stage: "batch_extract",
|
|
862
|
-
episode_total:
|
|
779
|
+
episode_total: totalEpisodes,
|
|
863
780
|
episode_completed: results.length,
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
batch_completed: completedBatches,
|
|
870
|
-
batch_reused: skippedEpisodeBatchCount + skippedBatches.length,
|
|
871
|
-
batch_failed: failures.length,
|
|
872
|
-
batch_terminal: terminalFailures.length,
|
|
873
|
-
transient_failed_batches: transientFailures.map((it) => strOf(it["batch_id"])),
|
|
874
|
-
terminal_failed_batches: terminalFailures.map((it) => strOf(it["batch_id"])),
|
|
875
|
-
last_error: { title: "INIT INCOMPLETE: Batch extraction failed", failed_at: checkpointTimestamp() },
|
|
781
|
+
incomplete_episodes: incompleteEpisodes,
|
|
782
|
+
batch_total: totalBatches,
|
|
783
|
+
batch_completed: doneBatches,
|
|
784
|
+
batch_failed: erroredBatches,
|
|
785
|
+
last_error: { title: "INIT INCOMPLETE: Some episodes have unextracted batches", failed_at: checkpointTimestamp() },
|
|
876
786
|
});
|
|
877
|
-
const issues = failures.slice(0, 5).map((it) => `${it["batch_id"]} episode ${it["episode"]} part ${it["part"]} [${it["terminal"] ? "terminal" : "transient"}]: ${it["error_type"]} - ${it["message"]}`);
|
|
878
|
-
const next = [
|
|
879
|
-
"Run direct inspect --target issue to review failed batches.",
|
|
880
|
-
"Rerun the same init to retry transient failures; completed units are reused.",
|
|
881
|
-
];
|
|
882
|
-
if (terminalFailures.length > 0) {
|
|
883
|
-
next.push("Terminal (content-filtered) batches will not clear on retry — use `direct override <unit> --from <file>` or soften the source.");
|
|
884
|
-
}
|
|
885
787
|
const report = {
|
|
886
|
-
title: "INIT INCOMPLETE:
|
|
788
|
+
title: "INIT INCOMPLETE: Some episodes have unextracted batches",
|
|
887
789
|
result: [
|
|
888
|
-
`episodes
|
|
889
|
-
`
|
|
890
|
-
`
|
|
891
|
-
`
|
|
892
|
-
`batches: ${completedBatches}/${asList(batchPlan["batches"]).length} completed, ${transientFailures.length} transient, ${terminalFailures.length} terminal`,
|
|
790
|
+
`episodes: ${results.length}/${totalEpisodes} complete`,
|
|
791
|
+
`incomplete episodes: ${incompleteEpisodes.join(", ")}`,
|
|
792
|
+
`batches: ${doneBatches}/${totalBatches} done, ${erroredBatches} error`,
|
|
793
|
+
...(titleDegraded.length > 0 ? [`titles degraded (deterministic): ${titleDegraded.join(", ")}`] : []),
|
|
893
794
|
`provider: ${providerName}`,
|
|
894
795
|
],
|
|
895
796
|
artifacts: [
|
|
896
797
|
path.join(workspace, "source.txt"),
|
|
897
|
-
path.join(dd, "source_manifest.json"),
|
|
898
798
|
path.join(dd, "episode_plan.json"),
|
|
899
799
|
path.join(dd, "batch_plan.json"),
|
|
900
800
|
batchResultsDir,
|
|
901
801
|
episodeResultsDir,
|
|
902
802
|
path.join(dd, "run_state.json"),
|
|
903
803
|
],
|
|
904
|
-
|
|
905
|
-
|
|
804
|
+
next: [
|
|
805
|
+
"Run `direct status` to see per-episode batch state.",
|
|
806
|
+
"Re-run one episode with `direct init --episodes <n>`, one batch with `--batches <key>`, or all errors with `--retry-errors`.",
|
|
807
|
+
"Errored batches are left untouched on a plain rerun; select them to retry.",
|
|
808
|
+
],
|
|
906
809
|
};
|
|
907
810
|
return [report, EXIT_RUNTIME];
|
|
908
811
|
}
|
|
909
812
|
updateRunState(workspace, {
|
|
910
813
|
status: "init_running",
|
|
911
814
|
init_stage: "episode_merge",
|
|
912
|
-
episode_total:
|
|
815
|
+
episode_total: totalEpisodes,
|
|
913
816
|
episode_completed: results.length,
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
batch_total: asList(batchPlan["batches"]).length,
|
|
919
|
-
batch_completed: completedBatches,
|
|
920
|
-
batch_reused: skippedEpisodeBatchCount + skippedBatches.length,
|
|
921
|
-
batch_failed: terminalFailures.length,
|
|
922
|
-
batch_terminal: terminalFailures.length,
|
|
817
|
+
incomplete_episodes: [],
|
|
818
|
+
batch_total: totalBatches,
|
|
819
|
+
batch_completed: doneBatches,
|
|
820
|
+
batch_failed: erroredBatches,
|
|
923
821
|
last_error: null,
|
|
924
822
|
});
|
|
925
|
-
// Drop transient/cleared error markers, but KEEP terminal ones so `direct
|
|
926
|
-
// status` and export gating can see which episodes are held out.
|
|
927
|
-
for (const dir of [batchResultsDir, episodeResultsDir]) {
|
|
928
|
-
if (!exists(dir))
|
|
929
|
-
continue;
|
|
930
|
-
for (const name of fs.readdirSync(dir)) {
|
|
931
|
-
if (!name.endsWith(".error.json"))
|
|
932
|
-
continue;
|
|
933
|
-
const errPath = path.join(dir, name);
|
|
934
|
-
try {
|
|
935
|
-
const err = readJson(errPath);
|
|
936
|
-
if (!isDict(err) || !err["terminal"])
|
|
937
|
-
deletePath(errPath);
|
|
938
|
-
}
|
|
939
|
-
catch {
|
|
940
|
-
deletePath(errPath);
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
}
|
|
944
|
-
results.sort((a, b) => Number(a["episode"] ?? 0) - Number(b["episode"] ?? 0));
|
|
945
823
|
let script;
|
|
946
824
|
try {
|
|
947
825
|
updateRunState(workspace, { status: "init_running", init_stage: "script_merge" });
|
|
@@ -1050,17 +928,13 @@ export async function commandInit(opts) {
|
|
|
1050
928
|
source_path: path.resolve(source),
|
|
1051
929
|
script_path: scriptPath,
|
|
1052
930
|
validation_path: path.join(dd, "validation.json"),
|
|
1053
|
-
episode_total:
|
|
931
|
+
episode_total: totalEpisodes,
|
|
1054
932
|
episode_completed: results.length,
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
batch_completed: completedBatches,
|
|
1061
|
-
batch_reused: skippedEpisodeBatchCount + skippedBatches.length,
|
|
1062
|
-
batch_failed: terminalFailures.length,
|
|
1063
|
-
batch_terminal: terminalFailures.length,
|
|
933
|
+
incomplete_episodes: [],
|
|
934
|
+
batch_total: totalBatches,
|
|
935
|
+
batch_completed: doneBatches,
|
|
936
|
+
batch_reused: reusedBatchCount,
|
|
937
|
+
batch_failed: erroredBatches,
|
|
1064
938
|
last_error: null,
|
|
1065
939
|
review_status: "pending",
|
|
1066
940
|
review_missing: [...REVIEW_TARGETS],
|
|
@@ -1079,9 +953,9 @@ export async function commandInit(opts) {
|
|
|
1079
953
|
`actions: ${stats["actions"] ?? 0}`,
|
|
1080
954
|
`validation: ${passed ? "passed" : "needs repair"}`,
|
|
1081
955
|
`provider: ${providerName}`,
|
|
1082
|
-
`episodes
|
|
1083
|
-
`batches: ${
|
|
1084
|
-
`
|
|
956
|
+
`episodes: ${results.length}/${totalEpisodes} complete`,
|
|
957
|
+
`batches: ${doneBatches}/${totalBatches} done (reused ${reusedBatchCount})`,
|
|
958
|
+
...(titleDegraded.length > 0 ? [`titles degraded (deterministic): ${titleDegraded.join(", ")}`] : []),
|
|
1085
959
|
"agent_review: pending",
|
|
1086
960
|
],
|
|
1087
961
|
artifacts: [
|
|
@@ -1108,138 +982,10 @@ export async function commandInit(opts) {
|
|
|
1108
982
|
return [report, passed ? EXIT_OK : EXIT_NEEDS_AGENT];
|
|
1109
983
|
}
|
|
1110
984
|
// ---------------------------------------------------------------------------
|
|
1111
|
-
//
|
|
1112
|
-
//
|
|
1113
|
-
//
|
|
1114
|
-
//
|
|
1115
|
-
// ourselves, so the operator never hand-edits source_span.
|
|
1116
|
-
// ---------------------------------------------------------------------------
|
|
1117
|
-
export function commandOverride(opts) {
|
|
1118
|
-
const workspace = strOf(opts["workspace_path"] || "workspace");
|
|
1119
|
-
const unit = strOf(asList(opts["_args"])[0]).trim();
|
|
1120
|
-
const fromPath = strOf(opts["from"]).trim();
|
|
1121
|
-
const dd = directDir(workspace);
|
|
1122
|
-
const state = readRunState(workspace);
|
|
1123
|
-
const providerName = strOf(opts["provider"] || state["provider"] || DEFAULT_PROVIDER);
|
|
1124
|
-
const model = strOf(opts["model"] || state["model"] || DEFAULT_MODEL);
|
|
1125
|
-
const isEpisode = /^ep_\d+$/.test(unit);
|
|
1126
|
-
if (!isEpisode && !/^bat_\d+$/.test(unit)) {
|
|
1127
|
-
throw new CliError("OVERRIDE BLOCKED: Invalid unit", "Invalid unit key.", {
|
|
1128
|
-
exitCode: EXIT_USAGE,
|
|
1129
|
-
required: ["<unit>: ep_NNN or bat_NNNN"],
|
|
1130
|
-
received: [`<unit>: ${unit || "<empty>"}`],
|
|
1131
|
-
nextSteps: ["Pass an episode (ep_007) or batch (bat_0012) key shown by direct status."],
|
|
1132
|
-
});
|
|
1133
|
-
}
|
|
1134
|
-
if (!fromPath || !exists(fromPath)) {
|
|
1135
|
-
throw new CliError("OVERRIDE BLOCKED: --from not found", "Override source file not found.", {
|
|
1136
|
-
exitCode: EXIT_INPUT,
|
|
1137
|
-
required: ["--from <path>: readable JSON extraction for the unit"],
|
|
1138
|
-
received: [`--from: ${fromPath || "<missing>"}`],
|
|
1139
|
-
nextSteps: ["Provide a JSON file with scenes/actions for the unit."],
|
|
1140
|
-
});
|
|
1141
|
-
}
|
|
1142
|
-
const planPath = path.join(dd, isEpisode ? "episode_plan.json" : "batch_plan.json");
|
|
1143
|
-
if (!exists(planPath)) {
|
|
1144
|
-
throw new CliError("OVERRIDE BLOCKED: Plan not found", "Plan not found.", {
|
|
1145
|
-
exitCode: EXIT_INPUT,
|
|
1146
|
-
required: [isEpisode ? "episode_plan.json" : "batch_plan.json"],
|
|
1147
|
-
received: [planPath],
|
|
1148
|
-
nextSteps: ["Run scriptctl direct init first."],
|
|
1149
|
-
});
|
|
1150
|
-
}
|
|
1151
|
-
const plan = readJson(planPath);
|
|
1152
|
-
const planUnits = asList(plan[isEpisode ? "episodes" : "batches"]);
|
|
1153
|
-
const planItem = planUnits.find((u) => (isEpisode ? episodeResultKey(u) : batchResultKey(u)) === unit) ?? null;
|
|
1154
|
-
if (!planItem) {
|
|
1155
|
-
throw new CliError("OVERRIDE BLOCKED: Unit not in current plan", "Unit not in current plan.", {
|
|
1156
|
-
exitCode: EXIT_INPUT,
|
|
1157
|
-
required: [`${unit} present in ${isEpisode ? "episode_plan.json" : "batch_plan.json"}`],
|
|
1158
|
-
received: [`${unit}: not found among ${planUnits.length} units`],
|
|
1159
|
-
nextSteps: ["Use a unit key from direct status; rerun init if the plan changed."],
|
|
1160
|
-
});
|
|
1161
|
-
}
|
|
1162
|
-
const sourceTextPath = path.join(workspace, "source.txt");
|
|
1163
|
-
if (!exists(sourceTextPath)) {
|
|
1164
|
-
throw new CliError("OVERRIDE BLOCKED: source.txt missing", "source.txt missing.", {
|
|
1165
|
-
exitCode: EXIT_INPUT,
|
|
1166
|
-
required: [sourceTextPath],
|
|
1167
|
-
received: ["<missing>"],
|
|
1168
|
-
nextSteps: ["Run scriptctl direct init first."],
|
|
1169
|
-
});
|
|
1170
|
-
}
|
|
1171
|
-
const sourceText = readText(sourceTextPath);
|
|
1172
|
-
let data;
|
|
1173
|
-
try {
|
|
1174
|
-
data = readJson(fromPath);
|
|
1175
|
-
}
|
|
1176
|
-
catch (exc) {
|
|
1177
|
-
throw new CliError("OVERRIDE BLOCKED: --from invalid JSON", "Override JSON invalid.", {
|
|
1178
|
-
exitCode: EXIT_INPUT,
|
|
1179
|
-
required: ["valid extraction JSON"],
|
|
1180
|
-
received: [`${fromPath}: ${exc.message}`],
|
|
1181
|
-
nextSteps: ["Fix the JSON and retry."],
|
|
1182
|
-
});
|
|
1183
|
-
}
|
|
1184
|
-
const result = normalizeEpisodeResult(data, planItem);
|
|
1185
|
-
if (Number(result["episode"]) !== Number(planItem["episode"])) {
|
|
1186
|
-
throw new CliError("OVERRIDE BLOCKED: Episode mismatch", "Episode mismatch.", {
|
|
1187
|
-
exitCode: EXIT_USAGE,
|
|
1188
|
-
required: [`episode ${Number(planItem["episode"])}`],
|
|
1189
|
-
received: [`episode ${Number(result["episode"])}`],
|
|
1190
|
-
nextSteps: ["Provide an extraction for the correct episode."],
|
|
1191
|
-
});
|
|
1192
|
-
}
|
|
1193
|
-
try {
|
|
1194
|
-
if (isEpisode)
|
|
1195
|
-
validateEpisodeExtractionQuality(sourceText, planItem, result);
|
|
1196
|
-
else
|
|
1197
|
-
validateBatchExtractionQuality(sourceText, planItem, result);
|
|
1198
|
-
}
|
|
1199
|
-
catch (exc) {
|
|
1200
|
-
if (exc instanceof CliError)
|
|
1201
|
-
throw exc;
|
|
1202
|
-
throw new CliError("OVERRIDE BLOCKED: Extraction invalid", "Extraction invalid.", {
|
|
1203
|
-
exitCode: EXIT_USAGE,
|
|
1204
|
-
required: ["valid action types (dialogue/inner_thought/action)"],
|
|
1205
|
-
received: [exc.message.slice(0, 160)],
|
|
1206
|
-
nextSteps: ["Fix the override extraction and retry."],
|
|
1207
|
-
});
|
|
1208
|
-
}
|
|
1209
|
-
const dir = path.join(dd, isEpisode ? "episode_results" : "batch_results");
|
|
1210
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
1211
|
-
const hash = computeUnitHash(sourceText, planItem, providerName, model);
|
|
1212
|
-
if (isEpisode) {
|
|
1213
|
-
writeJson(episodeResultPath(dir, planItem), compactEpisodeResult(result));
|
|
1214
|
-
stampEpisodeMeta(dir, planItem, hash, "override", providerName, model);
|
|
1215
|
-
const errPath = episodeErrorPath(dir, planItem);
|
|
1216
|
-
if (exists(errPath))
|
|
1217
|
-
deletePath(errPath);
|
|
1218
|
-
}
|
|
1219
|
-
else {
|
|
1220
|
-
persistBatchResult(dir, planItem, result);
|
|
1221
|
-
stampBatchMeta(dir, planItem, hash, "override", providerName, model);
|
|
1222
|
-
const errPath = batchErrorPath(dir, planItem);
|
|
1223
|
-
if (exists(errPath))
|
|
1224
|
-
deletePath(errPath);
|
|
1225
|
-
}
|
|
1226
|
-
const report = {
|
|
1227
|
-
title: "OVERRIDE COMPLETE: Unit extraction injected",
|
|
1228
|
-
result: [
|
|
1229
|
-
`unit: ${unit}`,
|
|
1230
|
-
`kind: ${isEpisode ? "episode" : "batch"}`,
|
|
1231
|
-
`provenance: override`,
|
|
1232
|
-
`provider/model: ${providerName} / ${model}`,
|
|
1233
|
-
`scenes: ${asList(result["scenes"]).length}`,
|
|
1234
|
-
],
|
|
1235
|
-
artifacts: [dir, path.join(dd, "run_state.json")],
|
|
1236
|
-
next: ["Rerun scriptctl direct init — the override is reused without re-calling the provider."],
|
|
1237
|
-
};
|
|
1238
|
-
return [report, EXIT_OK];
|
|
1239
|
-
}
|
|
1240
|
-
// ---------------------------------------------------------------------------
|
|
1241
|
-
// command_status — rebuild the progress view from on-disk meta/error sidecars.
|
|
1242
|
-
// run_state is just a cache of this; deleting it loses nothing.
|
|
985
|
+
// command_status — the agent-facing state table, derived entirely from disk: a
|
|
986
|
+
// batch with `<key>.json` is done, with `<key>.error.json` it failed, with
|
|
987
|
+
// neither it is pending. run_state is just a denormalized cache of this, so the
|
|
988
|
+
// agent can read state, then pick exactly which episodes/batches to (re)run.
|
|
1243
989
|
// ---------------------------------------------------------------------------
|
|
1244
990
|
export function commandStatus(opts) {
|
|
1245
991
|
const workspace = strOf(opts["workspace_path"] || "workspace");
|
|
@@ -1256,56 +1002,85 @@ export function commandStatus(opts) {
|
|
|
1256
1002
|
}
|
|
1257
1003
|
const episodes = asList(readJson(episodePlanPath)["episodes"]);
|
|
1258
1004
|
const batches = asList(readJson(batchPlanPath)["batches"]);
|
|
1259
|
-
const episodeResultsDir = path.join(dd, "episode_results");
|
|
1260
1005
|
const batchResultsDir = path.join(dd, "batch_results");
|
|
1261
|
-
const
|
|
1006
|
+
const byEp = new Map();
|
|
1007
|
+
let doneTotal = 0;
|
|
1008
|
+
let errorTotal = 0;
|
|
1262
1009
|
for (const batch of batches) {
|
|
1263
|
-
const
|
|
1264
|
-
if (!
|
|
1265
|
-
|
|
1010
|
+
const epNum = Number(batch["episode"]);
|
|
1011
|
+
if (!byEp.has(epNum))
|
|
1012
|
+
byEp.set(epNum, { total: 0, done: 0, error: 0, pending: 0, errorKeys: [], reasons: new Set() });
|
|
1013
|
+
const st = byEp.get(epNum);
|
|
1014
|
+
st.total++;
|
|
1015
|
+
const state = classifyBatch(batchResultsDir, batch);
|
|
1016
|
+
if (state === "done") {
|
|
1017
|
+
st.done++;
|
|
1018
|
+
doneTotal++;
|
|
1266
1019
|
continue;
|
|
1267
1020
|
}
|
|
1268
|
-
if (
|
|
1269
|
-
|
|
1270
|
-
|
|
1021
|
+
if (state === "error") {
|
|
1022
|
+
st.error++;
|
|
1023
|
+
errorTotal++;
|
|
1024
|
+
st.errorKeys.push(batchResultKey(batch));
|
|
1025
|
+
try {
|
|
1026
|
+
const err = readJson(batchErrorPath(batchResultsDir, batch));
|
|
1027
|
+
st.reasons.add(strOf(err["error_code"] || err["error_type"] || "Error"));
|
|
1028
|
+
}
|
|
1029
|
+
catch {
|
|
1030
|
+
st.reasons.add("Error");
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
else {
|
|
1034
|
+
st.pending++;
|
|
1271
1035
|
}
|
|
1272
|
-
count.ok++;
|
|
1273
|
-
if (meta["provenance"] === "override")
|
|
1274
|
-
count.override++;
|
|
1275
|
-
else if (meta["provenance"] === "recovered")
|
|
1276
|
-
count.recovered++;
|
|
1277
1036
|
}
|
|
1278
|
-
const
|
|
1037
|
+
const lines = [];
|
|
1038
|
+
let completeEpisodes = 0;
|
|
1039
|
+
const errorEpisodes = [];
|
|
1279
1040
|
for (const ep of episodes) {
|
|
1280
|
-
const
|
|
1281
|
-
|
|
1282
|
-
|
|
1041
|
+
const epNum = Number(ep["episode"]);
|
|
1042
|
+
const st = byEp.get(epNum) ?? { total: 0, done: 0, error: 0, pending: 0, errorKeys: [], reasons: new Set() };
|
|
1043
|
+
let label;
|
|
1044
|
+
if (st.total > 0 && st.done === st.total) {
|
|
1045
|
+
label = "done";
|
|
1046
|
+
completeEpisodes++;
|
|
1047
|
+
}
|
|
1048
|
+
else if (st.error > 0) {
|
|
1049
|
+
label = `ERROR (${st.done}/${st.total} done, ${st.error} err: ${[...st.reasons].join(",")}) [${st.errorKeys.join(",")}]`;
|
|
1050
|
+
errorEpisodes.push(epNum);
|
|
1051
|
+
}
|
|
1052
|
+
else {
|
|
1053
|
+
label = `pending (${st.done}/${st.total} done)`;
|
|
1054
|
+
}
|
|
1055
|
+
lines.push(`${episodeResultKey(ep)} ${label}`);
|
|
1283
1056
|
}
|
|
1284
|
-
//
|
|
1285
|
-
|
|
1286
|
-
const
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1057
|
+
// Surface degraded titles (deterministic fallback) recorded by init — they are
|
|
1058
|
+
// not a failure but the agent should know.
|
|
1059
|
+
const state = readRunState(workspace);
|
|
1060
|
+
const titleDeg = isDict(state["title_degraded"]) ? state["title_degraded"] : null;
|
|
1061
|
+
const result = [
|
|
1062
|
+
`episodes: ${completeEpisodes}/${episodes.length} done`,
|
|
1063
|
+
`batches: ${doneTotal}/${batches.length} done, ${errorTotal} error`,
|
|
1064
|
+
`error episodes: ${errorEpisodes.length === 0 ? "-" : errorEpisodes.join(", ")}`,
|
|
1065
|
+
];
|
|
1066
|
+
if (titleDeg && Number(titleDeg["count"] ?? 0) > 0) {
|
|
1067
|
+
result.push(`titles degraded (deterministic): ${asList(titleDeg["episodes"]).join(", ")}`);
|
|
1292
1068
|
}
|
|
1293
|
-
|
|
1069
|
+
result.push(...lines);
|
|
1070
|
+
const allDone = completeEpisodes === episodes.length;
|
|
1294
1071
|
const report = {
|
|
1295
1072
|
title: "DIRECT STATUS",
|
|
1296
|
-
result
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
`
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
artifacts: [batchResultsDir, episodeResultsDir, path.join(dd, "run_state.json")],
|
|
1304
|
-
next: heldOutEpisodes.length > 0
|
|
1305
|
-
? ["Override held-out episodes with direct override, or export 32/33 with direct export --allow-incomplete."]
|
|
1306
|
-
: ["All units accounted for."],
|
|
1073
|
+
result,
|
|
1074
|
+
artifacts: [batchResultsDir, path.join(dd, "episode_results"), path.join(dd, "run_state.json")],
|
|
1075
|
+
next: errorEpisodes.length > 0
|
|
1076
|
+
? [`Re-run an errored episode with: direct init --episodes ${errorEpisodes[0]} (or --batches <key>, or --retry-errors).`]
|
|
1077
|
+
: allDone
|
|
1078
|
+
? ["All episodes complete."]
|
|
1079
|
+
: ["Pending batches remain — re-run: direct init (resume) or --episodes <N>."],
|
|
1307
1080
|
};
|
|
1308
|
-
|
|
1081
|
+
// Non-zero when the workspace is not fully extracted, so automation can gate
|
|
1082
|
+
// on the exit code instead of scraping stdout.
|
|
1083
|
+
return [report, allDone ? EXIT_OK : EXIT_NEEDS_AGENT];
|
|
1309
1084
|
}
|
|
1310
1085
|
export function summarizeIssues(issues) {
|
|
1311
1086
|
if (issues.length === 0)
|