@lingjingai/scriptctl 0.49.4 → 0.49.6

package/changes/0.49.5.md

@@ -0,0 +1,13 @@
+# 0.49.5
+
+## Added
+- None.
+
+## Changed
+- `scriptctl ingest` text-normalize task max output tokens raised from 16k to 32k so longer episode bodies are less likely to truncate mid-JSON.
+
+## Fixed
+- None.
+
+## Breaking
+- None.

package/changes/0.49.6.md

@@ -0,0 +1,13 @@
+# 0.49.6
+
+## Added
+- None.
+
+## Changed
+- None.
+
+## Fixed
+- `scriptctl ingest` now bounds chunks for line-oriented text without blank separators and no longer creates an empty trailing chunk; text workspaces from earlier pipeline versions must be rebuilt with `--force`.
+
+## Breaking
+- None.

package/dist/domain/ingest/chunker.js

@@ -1,9 +1,11 @@
 // Deterministic source chunker (§pass1-prepare). Splits normalized (LF) source
-// text into contiguous, non-overlapping chunks at paragraph (blank-line)
-// boundaries under a char budget. It does NOT interpret episode/scene structure
-// — that is pass2-normalize's job. Lines carry stable 0-based indices; a chunk's
-// core range is the half-open [core_line_start, core_line_end). Cuts fall only on
-// line boundaries, so no Unicode code point is ever split.
+// text into contiguous, non-overlapping chunks under a char budget. Blank lines
+// stay with the preceding content, so paragraph boundaries are preserved when
+// they occur near a cut; line-oriented sources without blank separators fall
+// back to line boundaries. It does NOT interpret episode/scene structure — that
+// is pass2-normalize's job. Lines carry stable 0-based indices; a chunk's core
+// range is the half-open [core_line_start, core_line_end). A single overlong line
+// may exceed the budget because cuts never split a Unicode line.
 const DEFAULT_MAX_CHARS = 500;
 export function chunkSource(text, opts = {}) {
     const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
@@ -12,25 +14,34 @@ export function chunkSource(text, opts = {}) {
     let start = 0;
     let acc = 0;
     const push = (end) => {
+        const chunkText = lines.slice(start, end).join("\n");
+        if (chunkText.trim() === "")
+            return false;
         chunks.push({
             index: chunks.length,
             core_line_start: start,
             core_line_end: end,
-            text: lines.slice(start, end).join("\n"),
+            text: chunkText,
         });
         start = end;
         acc = 0;
+        return true;
     };
     for (let i = 0; i < lines.length; i++) {
-        acc += lines[i].length + 1;
-        const next = lines[i + 1];
-        // Cut once over budget, but only at a paragraph boundary (blank next line or
-        // end of source) so a paragraph is never split across chunks.
-        if (acc >= maxChars && (next === undefined || next.trim() === ""))
-            push(i + 1);
+        const line = lines[i];
+        const separatorChars = i === start ? 0 : 1;
+        const projected = acc + separatorChars + line.length;
+        // Keep blank separators with the preceding content. Before the next
+        // non-blank line crosses the budget, cut at its line boundary. If the source
+        // has no blank lines this still bounds every chunk to roughly maxChars.
+        if (line.trim() !== "" && i > start && projected > maxChars && push(i)) {
+            acc = line.length;
+        }
+        else {
+            acc = projected;
+        }
     }
-    if (start < lines.length)
-        push(lines.length);
+    push(lines.length);
     return chunks;
 }
 // ponytail: no context-only windows yet — normalize sees whole chunk text.

package/dist/domain/ingest/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/domain/ingest/chunker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,yEAAyE;AACzE,gFAAgF;AAChF,iFAAiF;AACjF,kFAAkF;AAClF,2DAA2D;AAS3D,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAA8B,EAAE;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,eAAe,EAAE,KAAK;YACtB,aAAa,EAAE,GAAG;YAClB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC;QACH,KAAK,GAAG,GAAG,CAAC;QACZ,GAAG,GAAG,CAAC,CAAC;IACV,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,GAAG,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,6EAA6E;QAC7E,8DAA8D;QAC9D,IAAI,GAAG,IAAI,QAAQ,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM;QAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,2EAA2E;AAC3E,+EAA+E"}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/domain/ingest/chunker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,6EAA6E;AAC7E,gFAAgF;AAChF,+EAA+E;AAC/E,kFAAkF;AAClF,iEAAiE;AASjE,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAA8B,EAAE;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,MAAM,IAAI,GAAG,CAAC,GAAW,EAAW,EAAE;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,eAAe,EAAE,KAAK;YACtB,aAAa,EAAE,GAAG;YAClB,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;QACH,KAAK,GAAG,GAAG,CAAC;QACZ,GAAG,GAAG,CAAC,CAAC;QACR,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,MAAM,cAAc,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,GAAG,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QAErD,oEAAoE;QACpE,6EAA6E;QAC7E,wEAAwE;QACxE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,SAAS,CAAC;QAClB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,2EAA2E;AAC3E,+EAA+E"}

package/dist/llm/registry.js

@@ -31,7 +31,7 @@ export const LLM_TASK_REGISTRY = {
     // Fires rarely (only when the synopsis exceeds budget); output is a short
     // paragraph, so a modest budget suffices (headroom for a reasoning model).
     "ingest.arc-summary": spec("ingest.arc-summary", "text", 4096),
-    "ingest.text-normalize": spec("ingest.text-normalize", "text", 16000),
+    "ingest.text-normalize": spec("ingest.text-normalize", "text", 32000),
     // Serial per-WINDOW asset/state annotation: model reads the running context + a
     // token-bounded window of 1+ episodes (~10k body tokens), returns a structured
     // Delta (SERIAL_DELTA_SCHEMA) covering all their scenes via native tool-calling —

package/dist/usecases/ingest/pipeline.js

@@ -23,7 +23,7 @@ import { mapSettled } from "./fanout.js";
 import { runSerialGovern } from "./serial-govern.js";
 import { Timings } from "./timings.js";
 import { writeUsage } from "./usage-report.js";
-const PIPELINE_VERSION = "3";
+const PIPELINE_VERSION = "4";
 const pad = (n) => String(n).padStart(3, "0");
 const serializeJsonl = (records) => records.map((r) => JSON.stringify(r)).join("\n") + "\n";
 function checkLifecycle(store, sourceSha, force) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lingjingai/scriptctl",
-  "version": "0.49.4",
+  "version": "0.49.6",
   "description": "剧本阶段统一 CLI：直转外部素材到 script.json，以及当前最终剧本的读取、校验、原子编辑与批量 patch 精修。",
   "type": "module",
   "bin": {