@dzhechkov/skills-book-digitizer 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +18 -10
- package/README.md +14 -5
- package/book-ingest/SKILL.md +2 -2
- package/book-ingest/scripts/book-ingest.mjs +259 -0
- package/book-knowledge-extract/SKILL.md +2 -0
- package/book-pack-assemble/SKILL.md +1 -1
- package/book-pack-assemble/scripts/shingling-check.mjs +271 -0
- package/book-skill-distill/SKILL.md +2 -0
- package/digitize-book/SKILL.md +10 -1
- package/package.json +6 -6
- package/sbom.json +29 -9
- package/scripts/book-ingest.mjs +2 -248
- package/scripts/shingling-check.mjs +2 -270
package/.dz-manifest.json
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
{
|
|
11
11
|
"path": "README.md",
|
|
12
|
-
"sha256": "
|
|
12
|
+
"sha256": "57ca26d13ee6314e05c18d8edfc612809e53bc4c7123bf7764a348f0ddd0a008"
|
|
13
13
|
},
|
|
14
14
|
{
|
|
15
15
|
"path": "book-brain-register/SKILL.md",
|
|
@@ -17,7 +17,11 @@
|
|
|
17
17
|
},
|
|
18
18
|
{
|
|
19
19
|
"path": "book-ingest/SKILL.md",
|
|
20
|
-
"sha256": "
|
|
20
|
+
"sha256": "9926b6c4c5d9ffbf1afa385f8c4f8e907357faccd5aef67c71ff5c0e4d495133"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"path": "book-ingest/scripts/book-ingest.mjs",
|
|
24
|
+
"sha256": "3aaa3ebe1711d03e02579040c7f692fd5443a3c94c20b9833d9cc0a8fb7fda0b"
|
|
21
25
|
},
|
|
22
26
|
{
|
|
23
27
|
"path": "book-kb-index/SKILL.md",
|
|
@@ -25,31 +29,35 @@
|
|
|
25
29
|
},
|
|
26
30
|
{
|
|
27
31
|
"path": "book-knowledge-extract/SKILL.md",
|
|
28
|
-
"sha256": "
|
|
32
|
+
"sha256": "17bc471b739d6e591df5c233bb12697ffd4fb3d919a723100c6e8290d5c1b11f"
|
|
29
33
|
},
|
|
30
34
|
{
|
|
31
35
|
"path": "book-pack-assemble/SKILL.md",
|
|
32
|
-
"sha256": "
|
|
36
|
+
"sha256": "c2d7b98a0d6893a507986c1faa766b98168a5b1df23f7a0ea54317ac5e4f0574"
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"path": "book-pack-assemble/scripts/shingling-check.mjs",
|
|
40
|
+
"sha256": "0f4ff1fbcba61593b73c4cae036c49f9ca22bd2ea561b00ccb16fd273863a915"
|
|
33
41
|
},
|
|
34
42
|
{
|
|
35
43
|
"path": "book-skill-distill/SKILL.md",
|
|
36
|
-
"sha256": "
|
|
44
|
+
"sha256": "e600ecf3130e7ff1685ceb5935890da0bdeaae933610659a886bf73f23fbb2d3"
|
|
37
45
|
},
|
|
38
46
|
{
|
|
39
47
|
"path": "digitize-book/SKILL.md",
|
|
40
|
-
"sha256": "
|
|
48
|
+
"sha256": "2b87fbe918bbfe70dae9fb0e20f6a75d4aefff964dfb0a7b2ff5f500bfd3aa48"
|
|
41
49
|
},
|
|
42
50
|
{
|
|
43
51
|
"path": "package.json",
|
|
44
|
-
"sha256": "
|
|
52
|
+
"sha256": "9d4c3033a5dfb34f99b90345826bcd9c1d3a352d97f397e61159198e86aac545"
|
|
45
53
|
},
|
|
46
54
|
{
|
|
47
55
|
"path": "scripts/book-ingest.mjs",
|
|
48
|
-
"sha256": "
|
|
56
|
+
"sha256": "779c90d88f8274e42c0b6a37c191e1cbc8bb98cb818da58e9e95877d839f927f"
|
|
49
57
|
},
|
|
50
58
|
{
|
|
51
59
|
"path": "scripts/shingling-check.mjs",
|
|
52
|
-
"sha256": "
|
|
60
|
+
"sha256": "fc87155f612fc02ac5cee17cf74041037e79b0e31836c870bc9a6047482db6e4"
|
|
53
61
|
},
|
|
54
62
|
{
|
|
55
63
|
"path": "source-brain-ingest/SKILL.md",
|
|
@@ -61,5 +69,5 @@
|
|
|
61
69
|
}
|
|
62
70
|
]
|
|
63
71
|
},
|
|
64
|
-
"signature": "
|
|
72
|
+
"signature": "d/hiyXZrbZV2SdLp0Vc+1gQIX43O8HpXNbZlFzYXeZfdTIr2hSAsC9cfaYX8itOUnrA9HnVOnh8+ZHB/hzBADA=="
|
|
65
73
|
}
|
package/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# @dzhechkov/skills-book-digitizer
|
|
2
2
|
|
|
3
|
+
Site: https://aicoding.space · Source: https://github.com/djd1m/dz-harness/tree/main/packages/skills-book-digitizer
|
|
4
|
+
|
|
3
5
|
Turn an engineering **book / textbook / monograph** into an **installable skill pack** whose skills
|
|
4
6
|
make AI coders (Claude Code, Codex, …) *work within the book's methodologies* — plus a searchable
|
|
5
7
|
Knowledge Base behind them. dz-original; design & rationale in
|
|
@@ -27,13 +29,20 @@ methods at the decisions it's about — e.g. DDIA → `ddia-replication-topology
|
|
|
27
29
|
|
|
28
30
|
## Install & use
|
|
29
31
|
|
|
32
|
+
In a new project with the DZ CLI installed:
|
|
33
|
+
|
|
30
34
|
```bash
|
|
31
|
-
dz init --target
|
|
32
|
-
# then, in your agent: «оцифруй вот эту книгу — хочу, чтобы агент проектировал по её методикам»
|
|
35
|
+
dz init --target codex --select digitize-book --no-hooks --no-integrations
|
|
33
36
|
```
|
|
34
37
|
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
Use `--target claude-code` for Claude Code. The two opt-outs keep this install focused on skills.
|
|
39
|
+
Selection installs the orchestrator, all book stages
|
|
40
|
+
and the two supporting skills from the meta pack. Open a new session in that project and ask:
|
|
41
|
+
“Use $digitize-book to ingest this PDF and show me the chapters at the first checkpoint.”
|
|
42
|
+
|
|
43
|
+
Install Poppler (`pdftotext`, `pdfinfo`, `pdfimages`) for PDF ingest. The first checkpoint writes
|
|
44
|
+
a page-anchored corpus and manifest locally. For the later KB stage, set up the AgentDB memory
|
|
45
|
+
backend with `dz setup --target codex --memory agentdb` for a Codex project.
|
|
37
46
|
|
|
38
47
|
## Usage scenarios
|
|
39
48
|
|
|
@@ -242,7 +251,7 @@ Two kinds of guarantee, and the difference matters:
|
|
|
242
251
|
auto-published; distribution is your explicit CP5 decision. `files[]` whitelist + no-`origin`
|
|
243
252
|
`sources.json`. The ingest is deterministic. The book KB is a separate namespace (book KUs never
|
|
244
253
|
pollute `dz recommend`). The **word-shingling verbatim-check is now a shipped deterministic gate**
|
|
245
|
-
(`scripts/shingling-check.mjs`) run at pack assembly: any uncited verbatim run ≥8 words fails the
|
|
254
|
+
(`book-pack-assemble/scripts/shingling-check.mjs`) run at pack assembly: any uncited verbatim run ≥8 words fails the
|
|
246
255
|
pack (cited ≤25-word quotes with a page anchor are exempt) — the paraphrase rule is mechanically
|
|
247
256
|
enforced, no longer merely agent-enforced.
|
|
248
257
|
- **Protocol (agent-enforced, fail-open — NOT mechanically proven):** KU provenance is
|
package/book-ingest/SKILL.md
CHANGED
|
@@ -30,10 +30,10 @@ pdftotext -v && pdfinfo -v # poppler-utils; on Debian/Ubuntu: apt-get install
|
|
|
30
30
|
|
|
31
31
|
## Protocol
|
|
32
32
|
|
|
33
|
-
Run the bundled deterministic script (
|
|
33
|
+
Run the bundled deterministic script (resolve `scripts/` relative to this `SKILL.md`):
|
|
34
34
|
|
|
35
35
|
```bash
|
|
36
|
-
node <
|
|
36
|
+
node <book-ingest-skill-dir>/scripts/book-ingest.mjs <book.pdf> --out <workspace> \
|
|
37
37
|
--book <slug> --isbn <isbn> [--max-chunk-tokens 70000]
|
|
38
38
|
```
|
|
39
39
|
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// book-ingest — deterministic PDF → chunked corpus + manifest (ADR-001 v2, book-knowledge-digitizer).
|
|
3
|
+
// NO LLM: poppler (pdfinfo/pdftotext/pdfimages) + heuristics only, so ingest is reproducible.
|
|
4
|
+
// node book-ingest.mjs <book.pdf> --out <dir> [--book <slug>] [--isbn <isbn>] [--max-chunk-tokens N]
|
|
5
|
+
// Emits <out>/corpus/NN-<slug>.md (with [p.N] page anchors) + <out>/manifest.json.
|
|
6
|
+
// Design: token-budget chunker (not structure-only), per-page quality matrix, figure/table
|
|
7
|
+
// inventory, front-matter listed-not-skipped, corpus_version + per-chunk source_hash/watermarks.
|
|
8
|
+
|
|
9
|
+
import { execFile } from 'node:child_process';
|
|
10
|
+
import { mkdirSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
|
|
11
|
+
import { join, basename } from 'node:path';
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { promisify } from 'node:util';
|
|
14
|
+
|
|
15
|
+
// ---- args -------------------------------------------------------------------
|
|
16
|
+
const argv = process.argv.slice(2);
|
|
17
|
+
const pdf = argv[0];
|
|
18
|
+
const opt = (name, def) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : def; };
|
|
19
|
+
if (!pdf || !existsSync(pdf)) { console.error('usage: book-ingest <book.pdf> --out <dir> [--book <slug>] [--isbn <isbn>] [--max-chunk-tokens N]'); process.exit(2); }
|
|
20
|
+
const outDir = opt('out', join(process.cwd(), 'book-corpus'));
|
|
21
|
+
const MAX_CHUNK_TOKENS = parseInt(opt('max-chunk-tokens', '70000'), 10);
|
|
22
|
+
const CHUNK_BUDGET = MAX_CHUNK_TOKENS * 0.8; // QE P3: safety margin so estimator error can't overflow the extract agent
|
|
23
|
+
const slugify = (s) => s.toLowerCase().normalize('NFKD').replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '').slice(0, 40);
|
|
24
|
+
|
|
25
|
+
// ---- poppler helpers --------------------------------------------------------
|
|
26
|
+
// `strict` rethrows on a non-zero exit (used for the PRIMARY extraction — QE P2: a poppler crash
|
|
27
|
+
// must NOT masquerade as an empty-but-successful book). Non-strict tolerates partial stdout.
|
|
28
|
+
// Codex's Linux workspace sandbox can return EPERM from spawnSync even after Poppler exits 0.
|
|
29
|
+
// Use its asynchronous equivalent, which also preserves genuine nonzero failures.
|
|
30
|
+
const execPoppler = promisify(execFile);
|
|
31
|
+
async function run(cmd, args, strict) {
|
|
32
|
+
try {
|
|
33
|
+
const { stdout } = await execPoppler(cmd, args, { encoding: 'utf-8', maxBuffer: 512 * 1024 * 1024 });
|
|
34
|
+
return stdout;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
if (strict) throw new Error(`${cmd} failed (${error.code ?? error.signal ?? 'unknown'}): ${String(error.stderr || error.message).slice(0, 200)}`);
|
|
37
|
+
return error.stdout ?? '';
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function popplerVersion() { // Included in corpus_version to identify cross-machine drift.
|
|
41
|
+
let output;
|
|
42
|
+
try { output = await execPoppler('pdftotext', ['-v'], { encoding: 'utf-8' }); }
|
|
43
|
+
catch (error) { output = error; }
|
|
44
|
+
return `${output.stdout ?? ''}\n${output.stderr ?? ''}`.match(/version\s+([\d.]+)/i)?.[1] ?? 'unknown';
|
|
45
|
+
}
|
|
46
|
+
async function pdfInfo(file) {
|
|
47
|
+
const out = await run('pdfinfo', [file]);
|
|
48
|
+
const g = (k) => { const m = new RegExp(`^${k}:\\s*(.+)$`, 'm').exec(out); return m ? m[1].trim() : ''; };
|
|
49
|
+
return { pages: parseInt(g('Pages') || '0', 10), title: g('Title'), encrypted: g('Encrypted') };
|
|
50
|
+
}
|
|
51
|
+
// Whole-book text with per-page form-feed (\f) separators → array indexed by page-1.
|
|
52
|
+
async function pagesOf(file, layout) {
|
|
53
|
+
const args = layout ? ['-layout', file, '-'] : [file, '-'];
|
|
54
|
+
const txt = await run('pdftotext', args, !layout); // primary (non-layout) run is strict
|
|
55
|
+
const parts = txt.split('\f');
|
|
56
|
+
if (parts.length && parts[parts.length - 1].trim() === '') parts.pop();
|
|
57
|
+
return parts;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ---- per-page quality matrix (cheap, deterministic) -------------------------
|
|
61
|
+
const CYR_LAT = /[\p{Script=Cyrillic}\p{Script=Latin}]/u;
|
|
62
|
+
function pageQuality(text, layoutText, layoutAvailable) {
|
|
63
|
+
const chars = text.length;
|
|
64
|
+
const words = (text.match(/[\p{L}]{2,}/gu) || []).length;
|
|
65
|
+
const letters = (text.match(/[\p{L}]/gu) || []).length;
|
|
66
|
+
const mathish = (text.match(/[=≤≥±∑∏∫√∞·×÷→∈∉⊂∪∩αβγδλμσΣΩθπ∂∇]/gu) || []).length;
|
|
67
|
+
const mathDensity = chars > 0 ? mathish / chars : 0;
|
|
68
|
+
const coherence = chars > 0 ? letters / chars : 0;
|
|
69
|
+
// two-column signal (QE P2): only when the -layout counterpart is actually available and this
|
|
70
|
+
// page has real text; require a strong divergence. This alone is advisory — a run of consecutive
|
|
71
|
+
// flagged pages (computed after) is what the summary should trust.
|
|
72
|
+
const lines = text.split('\n').length, lLines = (layoutText || '').split('\n').length;
|
|
73
|
+
const columnSignal = (layoutAvailable && chars > 200 && lines > 0) ? Math.abs(lLines - lines) / lines : 0;
|
|
74
|
+
const flags = [];
|
|
75
|
+
if (chars < 40) flags.push('empty');
|
|
76
|
+
else if (coherence < 0.55 && chars > 200) flags.push('low_coherence');
|
|
77
|
+
if (mathDensity > 0.02) flags.push('math_dense');
|
|
78
|
+
if (columnSignal > 0.6) flags.push('two_column_maybe'); // renamed: raw per-page hint, confirmed by runs
|
|
79
|
+
return { chars, words, mathDensity: +mathDensity.toFixed(4), coherence: +coherence.toFixed(3), flags };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// ---- structure detection ----------------------------------------------------
|
|
83
|
+
// Chapter opener (QE P2 — broadened beyond "Глава/Chapter + digit"): arabic OR roman OR
|
|
84
|
+
// spelled-out numeral, plus «Раздел» and «§». The capture is normalized to an index afterwards.
|
|
85
|
+
const NUM_WORD = { один: 1, первая: 1, два: 2, вторая: 2, три: 3, третья: 3, четыре: 4, четвертая: 4, пять: 5, пятая: 5, one: 1, first: 1, two: 2, second: 2, three: 3, third: 3 };
|
|
86
|
+
const CHAPTER_RE = /^\s*(?:Глава|ГЛАВА|Chapter|CHAPTER|Раздел|РАЗДЕЛ|§)\s+([IVXLC]+|\d+|[A-Za-zА-Яа-я]+)\b/;
|
|
87
|
+
const HEADING_RE = /^\s*(?:\d+\.\d+\.?|§\s*\d)\s+\S/; // sub-section like "5.3 Репликация" or "§ 5.3"
|
|
88
|
+
const romanToInt = (r) => ({ I: 1, V: 5, X: 10, L: 50, C: 100 } && r.split('').reduce((a, c, i, s) => { const v = { I: 1, V: 5, X: 10, L: 50, C: 100 }[c] || 0; const n = { I: 1, V: 5, X: 10, L: 50, C: 100 }[s[i + 1]] || 0; return a + (v < n ? -v : v); }, 0));
|
|
89
|
+
function chapterIndex(token, fallback) {
|
|
90
|
+
if (/^\d+$/.test(token)) return parseInt(token, 10);
|
|
91
|
+
if (/^[IVXLC]+$/.test(token)) return romanToInt(token) || fallback;
|
|
92
|
+
return NUM_WORD[token.toLowerCase()] ?? fallback;
|
|
93
|
+
}
|
|
94
|
+
const FIG_TAB_RE = /(?:Рис(?:\.|унок)|Таблиц[аы]|Табл\.|Figure|Table)\s*\.?\s*\d+([.\-]\d+)?/gu;
|
|
95
|
+
|
|
96
|
+
function detectChapters(pages) {
|
|
97
|
+
const chapters = [];
|
|
98
|
+
const DOT_LEADER = /\.{4,}\s*\d+\s*$/; // "…title.......... 53" — a TOC/contents line
|
|
99
|
+
pages.forEach((text, i) => {
|
|
100
|
+
const lines = text.split('\n');
|
|
101
|
+
// TOC/contents pages list many chapter refs with dot leaders — never a chapter START.
|
|
102
|
+
if (lines.filter((l) => DOT_LEADER.test(l)).length >= 3) return;
|
|
103
|
+
// A real chapter opener has "Глава N" in the first 3 non-empty lines.
|
|
104
|
+
const topNonEmpty = lines.filter((l) => l.trim()).slice(0, 3);
|
|
105
|
+
const hLine = topNonEmpty.find((l) => CHAPTER_RE.test(l));
|
|
106
|
+
if (!hLine) return;
|
|
107
|
+
const num = chapterIndex(CHAPTER_RE.exec(hLine)[1], chapters.length + 1);
|
|
108
|
+
// Descriptive name: first substantial line after the heading, not a dot-leader/bullet/number,
|
|
109
|
+
// trailing page number stripped ("… языки запросов 55" → "… языки запросов").
|
|
110
|
+
let name = '';
|
|
111
|
+
const after = lines.slice(lines.indexOf(hLine) + 1, lines.indexOf(hLine) + 8);
|
|
112
|
+
for (const l of after) {
|
|
113
|
+
if (DOT_LEADER.test(l)) continue;
|
|
114
|
+
const s = l.replace(/\s+\d+\s*$/, '').replace(/\s{2,}\d+$/, '').replace(/^[•·▪\-\s]+/, '').trim();
|
|
115
|
+
const isCaption = /^(?:Рис|Табл|Figure|Table|Листинг|Listing)\b/.test(s) || /^\d+\.\d/.test(s); // figure/subsection, not a title
|
|
116
|
+
if (s.length >= 6 && (s.match(/[\p{L}]{2,}/gu) || []).length >= 2 && !/^[•·\-\d.\s]+$/.test(s) && !CHAPTER_RE.test(s) && !isCaption) { name = s; break; }
|
|
117
|
+
}
|
|
118
|
+
// Sanity: a good title is a short-ish noun phrase. Reject long/garbled captures (extraction
|
|
119
|
+
// reading-order sometimes surfaces a body line first) → fall back to the bare "Глава N".
|
|
120
|
+
const good = name && name.length <= 70 && !/^\d/.test(name) && (name.match(/\s/g) || []).length <= 8;
|
|
121
|
+
chapters.push({ num, startPage: i + 1, title: good ? `Глава ${num}. ${name}` : `Глава ${num}` });
|
|
122
|
+
});
|
|
123
|
+
// de-dup consecutive same-number hits (running headers); keep first occurrence per number
|
|
124
|
+
const seen = new Set(); const uniq = [];
|
|
125
|
+
for (const c of chapters) { if (!seen.has(c.num)) { seen.add(c.num); uniq.push(c); } }
|
|
126
|
+
uniq.sort((a, b) => a.startPage - b.startPage);
|
|
127
|
+
for (let k = 0; k < uniq.length; k++) uniq[k].endPage = (k + 1 < uniq.length ? uniq[k + 1].startPage - 1 : pages.length);
|
|
128
|
+
return uniq;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ---- token-budget chunker within a chapter ----------------------------------
|
|
132
|
+
// Cyrillic tokenizes ~2x denser than Latin (QE P3): compute chars/token from the actual script
|
|
133
|
+
// mix so a "70k-token" chunk isn't really 100k and overflowing the extract agent's context.
|
|
134
|
+
function charsPerToken(text) {
|
|
135
|
+
const cyr = (text.match(/\p{Script=Cyrillic}/gu) || []).length;
|
|
136
|
+
const lat = (text.match(/\p{Script=Latin}/gu) || []).length;
|
|
137
|
+
const tot = cyr + lat;
|
|
138
|
+
if (tot === 0) return 3.0;
|
|
139
|
+
const cyrShare = cyr / tot; // Cyrillic ≈ 1.9 chars/token, Latin ≈ 4.0
|
|
140
|
+
return 1.9 * cyrShare + 4.0 * (1 - cyrShare);
|
|
141
|
+
}
|
|
142
|
+
let CPT = 2.5; // set per-book after extraction
|
|
143
|
+
const estTokens = (s) => Math.ceil(s.length / CPT);
|
|
144
|
+
function chunkPages(pages, startPage, endPage) {
|
|
145
|
+
const chunks = []; let cur = []; let curTokens = 0;
|
|
146
|
+
for (let p = startPage; p <= endPage; p++) {
|
|
147
|
+
const t = estTokens(pages[p - 1] || '');
|
|
148
|
+
if (t > CHUNK_BUDGET) process.stderr.write(` ⚠ page ${p} alone ≈${t} tokens > budget ${Math.round(CHUNK_BUDGET)} — extract this page carefully\n`);
|
|
149
|
+
const startsHeading = HEADING_RE.test((pages[p - 1] || '').split('\n')[0] || '');
|
|
150
|
+
if (cur.length && (curTokens + t > CHUNK_BUDGET || (startsHeading && curTokens > CHUNK_BUDGET * 0.6))) {
|
|
151
|
+
chunks.push([cur[0], cur[cur.length - 1]]); cur = []; curTokens = 0;
|
|
152
|
+
}
|
|
153
|
+
cur.push(p); curTokens += t;
|
|
154
|
+
}
|
|
155
|
+
if (cur.length) chunks.push([cur[0], cur[cur.length - 1]]);
|
|
156
|
+
return chunks;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ---- main -------------------------------------------------------------------
|
|
160
|
+
const info = await pdfInfo(pdf);
|
|
161
|
+
if (!info.pages) { console.error('pdfinfo: 0 pages (encrypted? not a PDF?)'); process.exit(1); }
|
|
162
|
+
const bookSlug = opt('book', slugify(info.title || basename(pdf, '.pdf')));
|
|
163
|
+
const isbn = opt('isbn', '');
|
|
164
|
+
const pages = await pagesOf(pdf, false);
|
|
165
|
+
const layoutPages = await pagesOf(pdf, true);
|
|
166
|
+
// QE P2: only trust the two-column signal when the -layout run aligns page-for-page.
|
|
167
|
+
const layoutAligned = layoutPages.length === pages.length;
|
|
168
|
+
CPT = charsPerToken(pages.join(''));
|
|
169
|
+
const quality = pages.map((t, i) => pageQuality(t, layoutPages[i], layoutAligned));
|
|
170
|
+
|
|
171
|
+
// QE P2: fail LOUD instead of writing a green manifest for an empty/broken extraction.
|
|
172
|
+
const emptyShare = quality.filter((q) => q.flags.includes('empty')).length / Math.max(1, pages.length);
|
|
173
|
+
if (pages.length === 0) { console.error('ERROR: 0 pages of text extracted. Encrypted or no text layer? Run OCR (pdftoppm + tesseract) and re-ingest.'); process.exit(1); }
|
|
174
|
+
if (info.encrypted && info.encrypted !== 'no' && emptyShare > 0.5) { console.error(`ERROR: PDF is encrypted (${info.encrypted}) and ${Math.round(emptyShare * 100)}% pages have no text — decrypt or OCR first.`); process.exit(1); }
|
|
175
|
+
if (emptyShare > 0.9) { console.error(`ERROR: ${Math.round(emptyShare * 100)}% of pages have no extractable text — this looks like a scanned book. OCR (pdftoppm + tesseract) and re-ingest.`); process.exit(1); }
|
|
176
|
+
|
|
177
|
+
const chapters = detectChapters(pages);
|
|
178
|
+
const structureType = chapters.length >= 3 ? 'chaptered' : (chapters.length ? 'monolithic' : 'monolithic');
|
|
179
|
+
const firstContentPage = chapters.length ? chapters[0].startPage : 1;
|
|
180
|
+
|
|
181
|
+
// front matter = pages before the first chapter (LISTED, never silently dropped)
|
|
182
|
+
const frontMatter = [];
|
|
183
|
+
for (let p = 1; p < firstContentPage; p++) {
|
|
184
|
+
const q = quality[p - 1];
|
|
185
|
+
frontMatter.push({ page: p, reason: q.chars < 40 ? 'empty-text' : 'pre-first-chapter' });
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// build chunks
|
|
189
|
+
mkdirSync(join(outDir, 'corpus'), { recursive: true });
|
|
190
|
+
const ranges = chapters.length
|
|
191
|
+
? chapters.flatMap((c) => chunkPages(pages, c.startPage, c.endPage).map((r) => ({ r, chapter: c })))
|
|
192
|
+
: chunkPages(pages, firstContentPage, pages.length).map((r) => ({ r, chapter: { num: 1, title: bookSlug, startPage: firstContentPage, endPage: pages.length } }));
|
|
193
|
+
|
|
194
|
+
const manifestChunks = [];
|
|
195
|
+
ranges.forEach(({ r, chapter }, i) => {
|
|
196
|
+
const [a, b] = r;
|
|
197
|
+
let body = '';
|
|
198
|
+
let figTab = 0;
|
|
199
|
+
for (let p = a; p <= b; p++) {
|
|
200
|
+
const pt = pages[p - 1] || '';
|
|
201
|
+
body += `\n\n[p.${p}]\n${pt}`;
|
|
202
|
+
figTab += (pt.match(FIG_TAB_RE) || []).length;
|
|
203
|
+
}
|
|
204
|
+
const seq = String(i + 1).padStart(3, '0');
|
|
205
|
+
const chunkSlug = `${seq}-${slugify(chapter.title || `chapter-${chapter.num}`)}`;
|
|
206
|
+
const file = join('corpus', `${chunkSlug}.md`);
|
|
207
|
+
const header = `# ${chapter.title || `Chapter ${chapter.num}`} [pp.${a}-${b}]\n`;
|
|
208
|
+
const content = header + body;
|
|
209
|
+
writeFileSync(join(outDir, file), content);
|
|
210
|
+
const pageFlags = {};
|
|
211
|
+
for (let p = a; p <= b; p++) if (quality[p - 1].flags.length) pageFlags[p] = quality[p - 1].flags;
|
|
212
|
+
manifestChunks.push({
|
|
213
|
+
id: chunkSlug,
|
|
214
|
+
file,
|
|
215
|
+
parent_chapter: chapter.num,
|
|
216
|
+
chapter_title: chapter.title || null,
|
|
217
|
+
pages: [a, b],
|
|
218
|
+
token_estimate: estTokens(body),
|
|
219
|
+
figure_table_mentions: figTab,
|
|
220
|
+
page_flags: pageFlags,
|
|
221
|
+
source_hash: createHash('sha256').update(content).digest('hex').slice(0, 16),
|
|
222
|
+
watermark: { extracted: false, reduced: false, indexed: false },
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
const poppler = await popplerVersion();
|
|
227
|
+
const corpusVersion = createHash('sha256')
|
|
228
|
+
.update(JSON.stringify(manifestChunks.map((c) => [c.pages, c.source_hash])) + `|maxchunk=${MAX_CHUNK_TOKENS}|poppler=${poppler}`)
|
|
229
|
+
.digest('hex').slice(0, 16);
|
|
230
|
+
|
|
231
|
+
const flagCounts = quality.flatMap((q) => q.flags).reduce((m, f) => ((m[f] = (m[f] || 0) + 1), m), {});
|
|
232
|
+
const manifest = {
|
|
233
|
+
book: { slug: bookSlug, isbn, title: info.title || basename(pdf), pages: info.pages, source_file: basename(pdf) },
|
|
234
|
+
structure_type: structureType,
|
|
235
|
+
structure_note: chapters.length ? null : 'no chapter markers detected — single monolithic chunk-set; confirm scope at CP1',
|
|
236
|
+
corpus_version: corpusVersion,
|
|
237
|
+
ingested_settings: { max_chunk_tokens: MAX_CHUNK_TOKENS, chunk_budget: Math.round(CHUNK_BUDGET), chars_per_token: +CPT.toFixed(2), poppler_version: poppler },
|
|
238
|
+
chapters: chapters.map((c) => ({ num: c.num, title: c.title, pages: [c.startPage, c.endPage] })),
|
|
239
|
+
front_matter: frontMatter, // listed for CP1 confirmation — never silently skipped
|
|
240
|
+
chunks: manifestChunks,
|
|
241
|
+
page_quality_summary: {
|
|
242
|
+
total_pages: pages.length,
|
|
243
|
+
pages_declared: info.pages, // QE P2: vs extracted, so a mismatch is visible at CP1
|
|
244
|
+
pages_extracted: pages.length,
|
|
245
|
+
layout_aligned: layoutAligned,
|
|
246
|
+
flag_counts: flagCounts, // empty/low_coherence/math_dense/two_column_maybe tallies
|
|
247
|
+
total_figure_table_mentions: manifestChunks.reduce((s, c) => s + c.figure_table_mentions, 0),
|
|
248
|
+
},
|
|
249
|
+
phase_state: { extracted: 0, reduced: false, distilled: false, packed: false },
|
|
250
|
+
};
|
|
251
|
+
writeFileSync(join(outDir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
252
|
+
|
|
253
|
+
// human summary to stdout (the skill relays this at CP1)
|
|
254
|
+
console.log(`book-ingest: "${manifest.book.title}"`);
|
|
255
|
+
console.log(` ${info.pages} pages · structure=${structureType} · ${chapters.length} chapters · ${manifestChunks.length} chunks · corpus_version=${corpusVersion}`);
|
|
256
|
+
console.log(` front matter: ${frontMatter.length} pages listed (not dropped)`);
|
|
257
|
+
console.log(` quality flags: ${Object.entries(flagCounts).map(([k, v]) => `${k}=${v}`).join(' ') || 'none'}`);
|
|
258
|
+
console.log(` figure/table mentions: ${manifest.page_quality_summary.total_figure_table_mentions}`);
|
|
259
|
+
console.log(` → ${join(outDir, 'manifest.json')}`);
|
|
@@ -79,7 +79,7 @@ Distilled skills + their KUs + routing evals from `book-skill-distill`. `dz benc
|
|
|
79
79
|
there — install a subset»). At CP4 with `target=copilot`, surface the aggregate instruction-file
|
|
80
80
|
byte count.
|
|
81
81
|
8. **Gate + smoke**: first the **shingling gate** —
|
|
82
|
-
`node <pack>/scripts/shingling-check.mjs --source <corpus> --output <pack>` (`--shingle 8`).
|
|
82
|
+
`node <book-pack-assemble-skill-dir>/scripts/shingling-check.mjs --source <corpus> --output <pack>` (`--shingle 8`).
|
|
83
83
|
ANY uncited verbatim run ≥8 words FAILS the pack (IP violation) — fix the offending KU
|
|
84
84
|
(re-paraphrase) and re-run. Cited ≤25-word quotes with a page anchor are exempt. Only once it
|
|
85
85
|
passes: `dz benchmark <pack> --all` (honest L0 grade per skill — never fabricate schemas/evals to
|