@lingjingai/scriptctl 0.49.5 → 0.49.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# 0.49.6
|
|
2
|
+
|
|
3
|
+
## Added
|
|
4
|
+
- None.
|
|
5
|
+
|
|
6
|
+
## Changed
|
|
7
|
+
- None.
|
|
8
|
+
|
|
9
|
+
## Fixed
|
|
10
|
+
- `scriptctl ingest` now bounds chunks for line-oriented text without blank separators and no longer creates an empty trailing chunk; text workspaces from earlier pipeline versions must be rebuilt with `--force`.
|
|
11
|
+
|
|
12
|
+
## Breaking
|
|
13
|
+
- None.
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// Deterministic source chunker (§pass1-prepare). Splits normalized (LF) source
|
|
2
|
-
// text into contiguous, non-overlapping chunks
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
2
|
+
// text into contiguous, non-overlapping chunks under a char budget. Blank lines
|
|
3
|
+
// stay with the preceding content, so paragraph boundaries are preserved when
|
|
4
|
+
// they occur near a cut; line-oriented sources without blank separators fall
|
|
5
|
+
// back to line boundaries. It does NOT interpret episode/scene structure — that
|
|
6
|
+
// is pass2-normalize's job. Lines carry stable 0-based indices; a chunk's core
|
|
7
|
+
// range is the half-open [core_line_start, core_line_end). A single overlong line
|
|
8
|
+
// may exceed the budget because cuts never split a Unicode line.
|
|
7
9
|
const DEFAULT_MAX_CHARS = 500;
|
|
8
10
|
export function chunkSource(text, opts = {}) {
|
|
9
11
|
const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
|
|
@@ -12,25 +14,34 @@ export function chunkSource(text, opts = {}) {
|
|
|
12
14
|
let start = 0;
|
|
13
15
|
let acc = 0;
|
|
14
16
|
const push = (end) => {
|
|
17
|
+
const chunkText = lines.slice(start, end).join("\n");
|
|
18
|
+
if (chunkText.trim() === "")
|
|
19
|
+
return false;
|
|
15
20
|
chunks.push({
|
|
16
21
|
index: chunks.length,
|
|
17
22
|
core_line_start: start,
|
|
18
23
|
core_line_end: end,
|
|
19
|
-
text:
|
|
24
|
+
text: chunkText,
|
|
20
25
|
});
|
|
21
26
|
start = end;
|
|
22
27
|
acc = 0;
|
|
28
|
+
return true;
|
|
23
29
|
};
|
|
24
30
|
for (let i = 0; i < lines.length; i++) {
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
//
|
|
29
|
-
|
|
30
|
-
|
|
31
|
+
const line = lines[i];
|
|
32
|
+
const separatorChars = i === start ? 0 : 1;
|
|
33
|
+
const projected = acc + separatorChars + line.length;
|
|
34
|
+
// Keep blank separators with the preceding content. Before the next
|
|
35
|
+
// non-blank line crosses the budget, cut at its line boundary. If the source
|
|
36
|
+
// has no blank lines this still bounds every chunk to roughly maxChars.
|
|
37
|
+
if (line.trim() !== "" && i > start && projected > maxChars && push(i)) {
|
|
38
|
+
acc = line.length;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
acc = projected;
|
|
42
|
+
}
|
|
31
43
|
}
|
|
32
|
-
|
|
33
|
-
push(lines.length);
|
|
44
|
+
push(lines.length);
|
|
34
45
|
return chunks;
|
|
35
46
|
}
|
|
36
47
|
// ponytail: no context-only windows yet — normalize sees whole chunk text.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/domain/ingest/chunker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/domain/ingest/chunker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,6EAA6E;AAC7E,gFAAgF;AAChF,+EAA+E;AAC/E,kFAAkF;AAClF,iEAAiE;AASjE,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAA8B,EAAE;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,MAAM,IAAI,GAAG,CAAC,GAAW,EAAW,EAAE;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,eAAe,EAAE,KAAK;YACtB,aAAa,EAAE,GAAG;YAClB,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;QACH,KAAK,GAAG,GAAG,CAAC;QACZ,GAAG,GAAG,CAAC,CAAC;QACR,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,MAAM,cAAc,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,GAAG,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QAErD,oEAAoE;QACpE,6EAA6E;QAC7E,wEAAwE;QACxE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,SAAS,CAAC;QAClB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,2EAA2E;AAC3E,+EAA+E"}
|
|
@@ -23,7 +23,7 @@ import { mapSettled } from "./fanout.js";
|
|
|
23
23
|
import { runSerialGovern } from "./serial-govern.js";
|
|
24
24
|
import { Timings } from "./timings.js";
|
|
25
25
|
import { writeUsage } from "./usage-report.js";
|
|
26
|
-
const PIPELINE_VERSION = "
|
|
26
|
+
const PIPELINE_VERSION = "4";
|
|
27
27
|
const pad = (n) => String(n).padStart(3, "0");
|
|
28
28
|
const serializeJsonl = (records) => records.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
29
29
|
function checkLifecycle(store, sourceSha, force) {
|