@alisio/plugin-semantic 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.es.md +168 -0
- package/README.md +160 -2
- package/THIRD_PARTY_NOTICES.md +42 -0
- package/assets/architecture.svg +1 -0
- package/assets/ask-flow.svg +1 -0
- package/cover.svg +53 -0
- package/dist/apps.d.ts +26 -0
- package/dist/apps.d.ts.map +1 -0
- package/dist/apps.js +141 -0
- package/dist/apps.js.map +1 -0
- package/dist/embedded.html +3043 -0
- package/dist/engine/download.d.ts +77 -0
- package/dist/engine/download.d.ts.map +1 -0
- package/dist/engine/download.js +213 -0
- package/dist/engine/download.js.map +1 -0
- package/dist/engine/embedder.d.ts +55 -0
- package/dist/engine/embedder.d.ts.map +1 -0
- package/dist/engine/embedder.js +182 -0
- package/dist/engine/embedder.js.map +1 -0
- package/dist/engine/infer.d.ts +22 -0
- package/dist/engine/infer.d.ts.map +1 -0
- package/dist/engine/infer.js +53 -0
- package/dist/engine/infer.js.map +1 -0
- package/dist/engine/models.d.ts +45 -0
- package/dist/engine/models.d.ts.map +1 -0
- package/dist/engine/models.js +132 -0
- package/dist/engine/models.js.map +1 -0
- package/dist/engine/runtime.d.ts +46 -0
- package/dist/engine/runtime.d.ts.map +1 -0
- package/dist/engine/runtime.js +172 -0
- package/dist/engine/runtime.js.map +1 -0
- package/dist/engine/setup.d.ts +82 -0
- package/dist/engine/setup.d.ts.map +1 -0
- package/dist/engine/setup.js +183 -0
- package/dist/engine/setup.js.map +1 -0
- package/dist/engine/store.d.ts +46 -0
- package/dist/engine/store.d.ts.map +1 -0
- package/dist/engine/store.js +288 -0
- package/dist/engine/store.js.map +1 -0
- package/dist/engine/tar.d.ts +18 -0
- package/dist/engine/tar.d.ts.map +1 -0
- package/dist/engine/tar.js +198 -0
- package/dist/engine/tar.js.map +1 -0
- package/dist/engine/worker.d.ts +2 -0
- package/dist/engine/worker.d.ts.map +1 -0
- package/dist/engine/worker.js +64 -0
- package/dist/engine/worker.js.map +1 -0
- package/dist/errors.d.ts +15 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +59 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +269 -0
- package/dist/index.js.map +1 -0
- package/dist/messages.d.ts +53 -0
- package/dist/messages.d.ts.map +1 -0
- package/dist/messages.js +118 -0
- package/dist/messages.js.map +1 -0
- package/dist/notebook/answer.d.ts +35 -0
- package/dist/notebook/answer.d.ts.map +1 -0
- package/dist/notebook/answer.js +88 -0
- package/dist/notebook/answer.js.map +1 -0
- package/dist/notebook/chunk.d.ts +23 -0
- package/dist/notebook/chunk.d.ts.map +1 -0
- package/dist/notebook/chunk.js +124 -0
- package/dist/notebook/chunk.js.map +1 -0
- package/dist/notebook/db.d.ts +107 -0
- package/dist/notebook/db.d.ts.map +1 -0
- package/dist/notebook/db.js +356 -0
- package/dist/notebook/db.js.map +1 -0
- package/dist/notebook/docx.d.ts +3 -0
- package/dist/notebook/docx.d.ts.map +1 -0
- package/dist/notebook/docx.js +16 -0
- package/dist/notebook/docx.js.map +1 -0
- package/dist/notebook/extract.d.ts +11 -0
- package/dist/notebook/extract.d.ts.map +1 -0
- package/dist/notebook/extract.js +99 -0
- package/dist/notebook/extract.js.map +1 -0
- package/dist/notebook/jobs.d.ts +31 -0
- package/dist/notebook/jobs.d.ts.map +1 -0
- package/dist/notebook/jobs.js +84 -0
- package/dist/notebook/jobs.js.map +1 -0
- package/dist/notebook/ooxml.d.ts +6 -0
- package/dist/notebook/ooxml.d.ts.map +1 -0
- package/dist/notebook/ooxml.js +131 -0
- package/dist/notebook/ooxml.js.map +1 -0
- package/dist/notebook/retrieve.d.ts +14 -0
- package/dist/notebook/retrieve.d.ts.map +1 -0
- package/dist/notebook/retrieve.js +37 -0
- package/dist/notebook/retrieve.js.map +1 -0
- package/dist/notebook/service.d.ts +92 -0
- package/dist/notebook/service.d.ts.map +1 -0
- package/dist/notebook/service.js +752 -0
- package/dist/notebook/service.js.map +1 -0
- package/dist/notebook/workspace.d.ts +17 -0
- package/dist/notebook/workspace.d.ts.map +1 -0
- package/dist/notebook/workspace.js +75 -0
- package/dist/notebook/workspace.js.map +1 -0
- package/dist/notebook/zip.d.ts +33 -0
- package/dist/notebook/zip.d.ts.map +1 -0
- package/dist/notebook/zip.js +154 -0
- package/dist/notebook/zip.js.map +1 -0
- package/dist/paths.d.ts +17 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +28 -0
- package/dist/paths.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/icon.svg +16 -0
- package/package.json +62 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/notebook/chunk.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,YAAY,OAAO,CAAC;AACjC,eAAO,MAAM,aAAa,MAAM,CAAC;AACjC,eAAO,MAAM,SAAS,OAAO,CAAC;AAC9B,+EAA+E;AAC/E,eAAO,MAAM,iBAAiB,SAAS,CAAC;AAExC,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AACD,MAAM,WAAW,KAAK;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AA6FD,sEAAsE;AACtE,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,KAAK,EAAE,CAAA;CAAE,CAmBpF"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Pure chunker (§7.4): paragraphs packed to ~1 200 characters with ~200 characters of overlap,
|
|
2
|
+
// never across a PDF page, hard cap 2 000 (long paragraphs split by sentence, then whitespace).
|
|
3
|
+
export const CHUNK_TARGET = 1200;
|
|
4
|
+
export const CHUNK_OVERLAP = 200;
|
|
5
|
+
export const CHUNK_CAP = 2000;
|
|
6
|
+
/** Separator placed between segments (PDF pages) in the stored source text. */
|
|
7
|
+
export const SEGMENT_SEPARATOR = "\n\n";
|
|
8
|
+
const isSpace = (c) => c !== undefined && /\s/.test(c);
|
|
9
|
+
/** Trims whitespace off a range. */
|
|
10
|
+
function trim(text, [start, end]) {
|
|
11
|
+
while (start < end && isSpace(text[start]))
|
|
12
|
+
start++;
|
|
13
|
+
while (end > start && isSpace(text[end - 1]))
|
|
14
|
+
end--;
|
|
15
|
+
return [start, end];
|
|
16
|
+
}
|
|
17
|
+
/** Splits `range` into pieces no longer than `limit`, preferring the given boundary pattern. */
|
|
18
|
+
function splitBy(text, [start, end], pattern) {
|
|
19
|
+
const pieces = [];
|
|
20
|
+
let at = start;
|
|
21
|
+
pattern.lastIndex = 0;
|
|
22
|
+
const slice = text.slice(start, end);
|
|
23
|
+
for (const match of slice.matchAll(pattern)) {
|
|
24
|
+
const cut = start + (match.index ?? 0) + match[0].length;
|
|
25
|
+
if (cut > at && cut < end) {
|
|
26
|
+
pieces.push([at, cut]);
|
|
27
|
+
at = cut;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
pieces.push([at, end]);
|
|
31
|
+
return pieces.map((r) => trim(text, r)).filter(([s, e]) => e > s);
|
|
32
|
+
}
|
|
33
|
+
/** Units no longer than the target: paragraphs, else lines, else sentences, else words, else cuts. */
|
|
34
|
+
function units(text, range) {
|
|
35
|
+
const out = [];
|
|
36
|
+
for (const paragraph of splitBy(text, range, /\n\s*\n/g)) {
|
|
37
|
+
if (paragraph[1] - paragraph[0] <= CHUNK_TARGET)
|
|
38
|
+
out.push(paragraph);
|
|
39
|
+
// Line-based text (spreadsheet rows, lists) keeps whole lines together.
|
|
40
|
+
else
|
|
41
|
+
out.push(...splitBy(text, paragraph, /\n/g));
|
|
42
|
+
}
|
|
43
|
+
return out.flatMap((unit) => (unit[1] - unit[0] <= CHUNK_TARGET ? [unit] : longUnit(text, unit)));
|
|
44
|
+
}
|
|
45
|
+
/** A line longer than the target: sentences, else words, else hard cuts. */
|
|
46
|
+
function longUnit(text, paragraph) {
|
|
47
|
+
const out = [];
|
|
48
|
+
for (const sentence of splitBy(text, paragraph, /[.!?;。!?]+["'”’)\]]*\s+/g)) {
|
|
49
|
+
if (sentence[1] - sentence[0] <= CHUNK_TARGET) {
|
|
50
|
+
out.push(sentence);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
let at = sentence[0];
|
|
54
|
+
while (at < sentence[1]) {
|
|
55
|
+
let cut = Math.min(sentence[1], at + CHUNK_TARGET);
|
|
56
|
+
if (cut < sentence[1]) {
|
|
57
|
+
let space = cut;
|
|
58
|
+
while (space > at + CHUNK_TARGET / 2 && !isSpace(text[space]))
|
|
59
|
+
space--;
|
|
60
|
+
if (space > at + CHUNK_TARGET / 2)
|
|
61
|
+
cut = space;
|
|
62
|
+
}
|
|
63
|
+
const piece = trim(text, [at, cut]);
|
|
64
|
+
if (piece[1] > piece[0])
|
|
65
|
+
out.push(piece);
|
|
66
|
+
at = cut;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
/** Packs units into overlapping chunks inside one segment. */
|
|
72
|
+
function pack(text, list) {
|
|
73
|
+
const chunks = [];
|
|
74
|
+
let i = 0;
|
|
75
|
+
let start = list[0]?.[0] ?? 0;
|
|
76
|
+
while (i < list.length) {
|
|
77
|
+
let end = list[i][1];
|
|
78
|
+
let j = i + 1;
|
|
79
|
+
while (j < list.length && list[j][1] - start <= CHUNK_TARGET) {
|
|
80
|
+
end = list[j][1];
|
|
81
|
+
j++;
|
|
82
|
+
}
|
|
83
|
+
chunks.push(trim(text, [start, end]));
|
|
84
|
+
if (j >= list.length)
|
|
85
|
+
break;
|
|
86
|
+
// Overlap: restart up to CHUNK_OVERLAP characters before the end, at a word boundary, but
|
|
87
|
+
// never before the start of the next unit minus the overlap, so the next chunk fits the target.
|
|
88
|
+
const next = list[j];
|
|
89
|
+
const floor = Math.max(start + 1, end - CHUNK_OVERLAP, next[1] - CHUNK_TARGET);
|
|
90
|
+
// Prefer restarting at a unit boundary (a line or paragraph) inside the overlap window.
|
|
91
|
+
const aligned = list.slice(i, j).find(([s]) => s >= floor && s < next[0])?.[0];
|
|
92
|
+
let overlap = aligned ?? floor;
|
|
93
|
+
if (aligned === undefined)
|
|
94
|
+
while (overlap < next[0] && !isSpace(text[overlap - 1]))
|
|
95
|
+
overlap++;
|
|
96
|
+
start = Math.min(overlap, next[0]);
|
|
97
|
+
i = j;
|
|
98
|
+
}
|
|
99
|
+
return chunks.filter(([s, e]) => e > s);
|
|
100
|
+
}
|
|
101
|
+
/** Joins segments with blank lines and chunks each one on its own. */
|
|
102
|
+
export function chunkSegments(segments) {
|
|
103
|
+
const chunks = [];
|
|
104
|
+
let text = "";
|
|
105
|
+
segments.forEach((segment, index) => {
|
|
106
|
+
if (index > 0)
|
|
107
|
+
text += SEGMENT_SEPARATOR;
|
|
108
|
+
const offset = text.length;
|
|
109
|
+
text += segment.text;
|
|
110
|
+
const range = trim(text, [offset, text.length]);
|
|
111
|
+
if (range[1] <= range[0])
|
|
112
|
+
return;
|
|
113
|
+
for (const [start, end] of pack(text, units(text, range)))
|
|
114
|
+
chunks.push({
|
|
115
|
+
ordinal: chunks.length,
|
|
116
|
+
text: text.slice(start, end),
|
|
117
|
+
page: segment.page,
|
|
118
|
+
charStart: start,
|
|
119
|
+
charEnd: end,
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
return { text, chunks };
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=chunk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/notebook/chunk.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,gGAAgG;AAEhG,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC;AACjC,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AACjC,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC;AAC9B,+EAA+E;AAC/E,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAiBxC,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAE3E,oCAAoC;AACpC,SAAS,IAAI,CAAC,IAAY,EAAE,CAAC,KAAK,EAAE,GAAG,CAAQ;IAC7C,OAAO,KAAK,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,KAAK,EAAE,CAAC;IACpD,OAAO,GAAG,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAAE,GAAG,EAAE,CAAC;IACpD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,gGAAgG;AAChG,SAAS,OAAO,CAAC,IAAY,EAAE,CAAC,KAAK,EAAE,GAAG,CAAQ,EAAE,OAAe;IACjE,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,EAAE,GAAG,KAAK,CAAC;IACf,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACzD,IAAI,GAAG,GAAG,EAAE,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IACvB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,sGAAsG;AACtG,SAAS,KAAK,CAAC,IAAY,EAAE,KAAY;IACvC,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;QACzD,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,YAAY;YAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrE,wEAAwE;;YACnE,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,4EAA4E;AAC5E,SAAS,QAAQ,CAAC,IAAY,EAAE,SAAgB;IAC9C,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,0BAA0B,CAAC,EAAE,CAAC;QAC5E,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QACD,IAAI,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,CAAC;YACnD,IAAI,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,GAAG,GAAG,CAAC;gBAChB,OAAO,KAAK,GAAG,EAAE,GAAG,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAAE,KAAK,EAAE,CAAC;gBACvE,IAAI,KAAK,GAAG,EAAE,GAAG,YAAY,GAAG,CAAC;oBAAE,GAAG,GAAG,KAAK,CAAC;YACjD,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YACpC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzC,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8DAA8D;AAC9D,SAAS,IAAI,CAAC,IAAY,EAAE,IAAa;IACvC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,IAAI,GAAG,GAAI,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,IAAK,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,YAAY,EAAE,CAAC;YACxE,GAAG,GAAI,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC,EAAE,CAAC;QACN,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM;QAC5B,0FAA0F;QAC1F,gGAAgG;QAChG,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAU,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC;QAC/E,wFAAwF;QACxF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,OAAO,GAAG,OAAO,IAAI,KAAK,CAAC;QAC/B,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAAE,OAAO,EAAE,CAAC;QAC9F,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC,GAAG,CAAC,CAAC;IACR,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,aAAa,CAAC,QAAmB;IAC/C,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAClC,IAAI,KAAK,GAAG,CAAC;YAAE,IAAI,IAAI,iBAAiB,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,OAAO;QACjC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,MAAM,CAAC,MAAM;gBACtB,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC5B,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,SAAS,EAAE,KAAK;gBAChB,OAAO,EAAE,GAAG;aACb,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import type { SqlDatabase } from "@alisio/sdk";
|
|
2
|
+
import type { Chunk } from "./chunk.js";
|
|
3
|
+
export interface NotebookRow {
|
|
4
|
+
id: string;
|
|
5
|
+
title: string;
|
|
6
|
+
createdAt: number;
|
|
7
|
+
updatedAt: number;
|
|
8
|
+
remoteNoticeAck: boolean;
|
|
9
|
+
/** `<provider>/<model>` chosen for this notebook, or null for Alisio's active model. */
|
|
10
|
+
model: string | null;
|
|
11
|
+
guide: Record<string, unknown> | null;
|
|
12
|
+
excluded: string[];
|
|
13
|
+
}
|
|
14
|
+
export type SourceStatus = "indexing" | "ready" | "ready_keyword" | "failed";
|
|
15
|
+
export interface SourceRow {
|
|
16
|
+
id: string;
|
|
17
|
+
notebookId: string;
|
|
18
|
+
name: string;
|
|
19
|
+
kind: string;
|
|
20
|
+
mime: string;
|
|
21
|
+
sha256: string;
|
|
22
|
+
bytes: number;
|
|
23
|
+
status: SourceStatus;
|
|
24
|
+
error: string | null;
|
|
25
|
+
chunkCount: number;
|
|
26
|
+
createdAt: number;
|
|
27
|
+
summary: Record<string, unknown> | null;
|
|
28
|
+
}
|
|
29
|
+
export interface ChunkRow {
|
|
30
|
+
id: number;
|
|
31
|
+
sourceId: string;
|
|
32
|
+
ordinal: number;
|
|
33
|
+
text: string;
|
|
34
|
+
page: number | null;
|
|
35
|
+
charStart: number;
|
|
36
|
+
charEnd: number;
|
|
37
|
+
}
|
|
38
|
+
export interface MessageRow {
|
|
39
|
+
id: number;
|
|
40
|
+
notebookId: string;
|
|
41
|
+
role: "user" | "assistant";
|
|
42
|
+
text: string;
|
|
43
|
+
meta: Record<string, unknown>;
|
|
44
|
+
createdAt: number;
|
|
45
|
+
}
|
|
46
|
+
export type JobState = "queued" | "running" | "done" | "failed" | "cancelled";
|
|
47
|
+
export interface JobRow {
|
|
48
|
+
id: string;
|
|
49
|
+
notebookId: string;
|
|
50
|
+
kind: string;
|
|
51
|
+
state: JobState;
|
|
52
|
+
progress: number;
|
|
53
|
+
error: string | null;
|
|
54
|
+
result: unknown;
|
|
55
|
+
sourceId: string | null;
|
|
56
|
+
createdAt: number;
|
|
57
|
+
updatedAt: number;
|
|
58
|
+
}
|
|
59
|
+
/** The Notebook's own SQLite store (D10): sources, chunks + FTS5, chat and jobs. */
|
|
60
|
+
export declare class NotebookDb {
|
|
61
|
+
readonly sql: SqlDatabase;
|
|
62
|
+
private readonly now;
|
|
63
|
+
constructor(sql: SqlDatabase, now?: () => number);
|
|
64
|
+
private migrate;
|
|
65
|
+
close(): void;
|
|
66
|
+
createNotebook(id: string, title: string): void;
|
|
67
|
+
notebook(id: string): NotebookRow | undefined;
|
|
68
|
+
notebooks(): Array<NotebookRow & {
|
|
69
|
+
sourceCount: number;
|
|
70
|
+
}>;
|
|
71
|
+
findNotebook(reference: string): NotebookRow | undefined;
|
|
72
|
+
renameNotebook(id: string, title: string): void;
|
|
73
|
+
touchNotebook(id: string): void;
|
|
74
|
+
setModel(id: string, model: string | null): void;
|
|
75
|
+
ackNotice(id: string): void;
|
|
76
|
+
deleteNotebook(id: string): void;
|
|
77
|
+
countNotebooks(): number;
|
|
78
|
+
setGuide(id: string, guide: Record<string, unknown> | null): void;
|
|
79
|
+
setExcluded(id: string, excluded: string[]): void;
|
|
80
|
+
setSourceSummary(id: string, summary: Record<string, unknown> | null): void;
|
|
81
|
+
insertSource(row: Omit<SourceRow, "chunkCount" | "createdAt" | "error" | "summary">): void;
|
|
82
|
+
source(id: string): SourceRow | undefined;
|
|
83
|
+
sources(notebookId: string): SourceRow[];
|
|
84
|
+
sourceBySha(notebookId: string, sha256: string): SourceRow | undefined;
|
|
85
|
+
setSourceStatus(id: string, status: SourceStatus, error?: string | null): void;
|
|
86
|
+
deleteSource(id: string): void;
|
|
87
|
+
replaceChunks(sourceId: string, chunks: Chunk[]): void;
|
|
88
|
+
clearChunks(sourceId: string): void;
|
|
89
|
+
chunks(sourceId: string): ChunkRow[];
|
|
90
|
+
chunk(sourceId: string, ordinal: number): ChunkRow | undefined;
|
|
91
|
+
notebookChunkCount(notebookId: string): number;
|
|
92
|
+
/** BM25 ranking over the notebook's chunks, as `<sourceId>:<ordinal>` keys. */
|
|
93
|
+
keywordSearch(notebookId: string, match: string, limit: number, sourceIds?: string[]): string[];
|
|
94
|
+
addMessage(notebookId: string, role: "user" | "assistant", text: string, meta?: Record<string, unknown>): number;
|
|
95
|
+
messages(notebookId: string, limit: number, before?: number): MessageRow[];
|
|
96
|
+
clearMessages(notebookId: string): void;
|
|
97
|
+
insertJob(id: string, notebookId: string, kind: string, sourceId: string | null): void;
|
|
98
|
+
updateJob(id: string, patch: {
|
|
99
|
+
state?: JobState;
|
|
100
|
+
progress?: number;
|
|
101
|
+
error?: string | null;
|
|
102
|
+
result?: unknown;
|
|
103
|
+
}): void;
|
|
104
|
+
job(id: string): JobRow | undefined;
|
|
105
|
+
activeJobs(notebookId: string): JobRow[];
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=db.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/notebook/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAU,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAyExC,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,OAAO,CAAC;IACzB,wFAAwF;IACxF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACtC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AACD,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,OAAO,GAAG,eAAe,GAAG,QAAQ,CAAC;AAC7E,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACzC;AACD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,CAAC;AAC9E,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAwDD,oFAAoF;AACpF,qBAAa,UAAU;IACrB,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAEnC,YAAY,GAAG,EAAE,WAAW,EAAE,GAAG,GAAE,MAAM,MAAiB,EAezD;IAED,OAAO,CAAC,OAAO;IASf,KAAK,SAEJ;IAGD,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAKvC;IACD,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAG5C;IACD,SAAS,IAAI,KAAK,CAAC,WAAW,GAAG;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAOxD;IACD,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CASvD;IACD,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAIvC;IACD,aAAa,CAAC,EAAE,EAAE,MAAM,QAEvB;IACD,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,QAExC;IACD,SAAS,CAAC,EAAE,EAAE,MAAM,QAEnB;IACD,cAAc,CAAC,EAAE,EAAE,MAAM,QAYxB;IACD,cAAc,IAAI,MAAM,CAEvB;IAGD,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,QAIzD;IACD,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAIzC;IACD,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,QAInE;IACD,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,GAAG,WAAW,GAAG,OAAO,GAAG,SAAS,CAAC,QAgBlF;IACD,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAGxC;IACD,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,CAKvC;IACD,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAKrE;IACD,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,GAAE,MAAM,GAAG,IAAW,QAI5E;IACD,YAAY,CAAC,EAAE,EAAE,MAAM,QAKtB;IAGD,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAY9C;IACD,WAAW,CAAC,QAAQ,EAAE,MAAM,QAE3B;IACD,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE,CAKnC;IACD,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CAK7D;IACD,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAQ7C;IACD,+EAA+E;IAC/E,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAkB9F;IAGD,UAAU,CACR,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,GAAG,WAAW,EAC1B,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,UAQnC;IACD,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAczE;IACD,aAAa,CAAC,UAAU,EAAE,MAAM,QAE/B;IAGD,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,QAO9E;IACD,SAAS,CACP,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QAAE,KAAK,CAAC,EAAE,QAAQ,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,QAgBxF;IACD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGlC;IACD,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAOvC;CACF"}
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/** Numbered migrations (§7.1); `PRAGMA user_version` records the last one applied. */
|
|
2
|
+
const MIGRATIONS = [
|
|
3
|
+
`CREATE TABLE notebooks (
|
|
4
|
+
id TEXT PRIMARY KEY,
|
|
5
|
+
title TEXT NOT NULL,
|
|
6
|
+
created_at INTEGER NOT NULL,
|
|
7
|
+
updated_at INTEGER NOT NULL,
|
|
8
|
+
remote_notice_ack INTEGER NOT NULL DEFAULT 0
|
|
9
|
+
);
|
|
10
|
+
CREATE TABLE sources (
|
|
11
|
+
id TEXT PRIMARY KEY,
|
|
12
|
+
notebook_id TEXT NOT NULL,
|
|
13
|
+
name TEXT NOT NULL,
|
|
14
|
+
kind TEXT NOT NULL,
|
|
15
|
+
mime TEXT NOT NULL,
|
|
16
|
+
sha256 TEXT NOT NULL,
|
|
17
|
+
bytes INTEGER NOT NULL,
|
|
18
|
+
status TEXT NOT NULL,
|
|
19
|
+
error TEXT,
|
|
20
|
+
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
21
|
+
created_at INTEGER NOT NULL
|
|
22
|
+
);
|
|
23
|
+
CREATE INDEX sources_notebook ON sources(notebook_id);
|
|
24
|
+
CREATE TABLE chunks (
|
|
25
|
+
id INTEGER PRIMARY KEY,
|
|
26
|
+
source_id TEXT NOT NULL,
|
|
27
|
+
ordinal INTEGER NOT NULL,
|
|
28
|
+
text TEXT NOT NULL,
|
|
29
|
+
page INTEGER,
|
|
30
|
+
char_start INTEGER NOT NULL,
|
|
31
|
+
char_end INTEGER NOT NULL,
|
|
32
|
+
UNIQUE (source_id, ordinal)
|
|
33
|
+
);
|
|
34
|
+
CREATE VIRTUAL TABLE chunks_fts USING fts5(
|
|
35
|
+
text, content='chunks', content_rowid='id', tokenize='unicode61 remove_diacritics 2'
|
|
36
|
+
);
|
|
37
|
+
CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
|
|
38
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
|
|
39
|
+
END;
|
|
40
|
+
CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
|
|
41
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
42
|
+
END;
|
|
43
|
+
CREATE TABLE messages (
|
|
44
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
45
|
+
notebook_id TEXT NOT NULL,
|
|
46
|
+
role TEXT NOT NULL,
|
|
47
|
+
text TEXT NOT NULL,
|
|
48
|
+
citations_json TEXT,
|
|
49
|
+
created_at INTEGER NOT NULL
|
|
50
|
+
);
|
|
51
|
+
CREATE INDEX messages_notebook ON messages(notebook_id, id);
|
|
52
|
+
CREATE TABLE jobs (
|
|
53
|
+
id TEXT PRIMARY KEY,
|
|
54
|
+
notebook_id TEXT NOT NULL,
|
|
55
|
+
kind TEXT NOT NULL,
|
|
56
|
+
state TEXT NOT NULL,
|
|
57
|
+
progress REAL NOT NULL DEFAULT 0,
|
|
58
|
+
error TEXT,
|
|
59
|
+
result_json TEXT,
|
|
60
|
+
source_id TEXT,
|
|
61
|
+
created_at INTEGER NOT NULL,
|
|
62
|
+
updated_at INTEGER NOT NULL
|
|
63
|
+
);`,
|
|
64
|
+
// 2 (0.1.1): per-notebook model choice; null means Alisio's active model.
|
|
65
|
+
"ALTER TABLE notebooks ADD COLUMN model TEXT;",
|
|
66
|
+
// 3 (0.1.1, O15): the generated guide, the excluded sources and per-source summaries persist.
|
|
67
|
+
`ALTER TABLE notebooks ADD COLUMN guide_json TEXT;
|
|
68
|
+
ALTER TABLE notebooks ADD COLUMN excluded_json TEXT;
|
|
69
|
+
ALTER TABLE sources ADD COLUMN summary_json TEXT;`,
|
|
70
|
+
];
|
|
71
|
+
const notebook = (r) => ({
|
|
72
|
+
id: r.id,
|
|
73
|
+
title: r.title,
|
|
74
|
+
createdAt: Number(r.created_at),
|
|
75
|
+
updatedAt: Number(r.updated_at),
|
|
76
|
+
remoteNoticeAck: Number(r.remote_notice_ack) === 1,
|
|
77
|
+
model: typeof r.model === "string" && r.model ? r.model : null,
|
|
78
|
+
guide: parse(r.guide_json) ?? null,
|
|
79
|
+
excluded: Array.isArray(parse(r.excluded_json)) ? parse(r.excluded_json) : [],
|
|
80
|
+
});
|
|
81
|
+
const source = (r) => ({
|
|
82
|
+
id: r.id,
|
|
83
|
+
notebookId: r.notebook_id,
|
|
84
|
+
name: r.name,
|
|
85
|
+
kind: r.kind,
|
|
86
|
+
mime: r.mime,
|
|
87
|
+
sha256: r.sha256,
|
|
88
|
+
bytes: Number(r.bytes),
|
|
89
|
+
status: r.status,
|
|
90
|
+
error: r.error ?? null,
|
|
91
|
+
chunkCount: Number(r.chunk_count),
|
|
92
|
+
createdAt: Number(r.created_at),
|
|
93
|
+
summary: parse(r.summary_json) ?? null,
|
|
94
|
+
});
|
|
95
|
+
const chunk = (r) => ({
|
|
96
|
+
id: Number(r.id),
|
|
97
|
+
sourceId: r.source_id,
|
|
98
|
+
ordinal: Number(r.ordinal),
|
|
99
|
+
text: r.text,
|
|
100
|
+
page: r.page === null || r.page === undefined ? null : Number(r.page),
|
|
101
|
+
charStart: Number(r.char_start),
|
|
102
|
+
charEnd: Number(r.char_end),
|
|
103
|
+
});
|
|
104
|
+
const parse = (text) => {
|
|
105
|
+
if (typeof text !== "string")
|
|
106
|
+
return undefined;
|
|
107
|
+
try {
|
|
108
|
+
return JSON.parse(text);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
const job = (r) => ({
|
|
115
|
+
id: r.id,
|
|
116
|
+
notebookId: r.notebook_id,
|
|
117
|
+
kind: r.kind,
|
|
118
|
+
state: r.state,
|
|
119
|
+
progress: Number(r.progress),
|
|
120
|
+
error: r.error ?? null,
|
|
121
|
+
result: parse(r.result_json) ?? null,
|
|
122
|
+
sourceId: r.source_id ?? null,
|
|
123
|
+
createdAt: Number(r.created_at),
|
|
124
|
+
updatedAt: Number(r.updated_at),
|
|
125
|
+
});
|
|
126
|
+
/** The Notebook's own SQLite store (D10): sources, chunks + FTS5, chat and jobs. */
|
|
127
|
+
export class NotebookDb {
|
|
128
|
+
sql;
|
|
129
|
+
now;
|
|
130
|
+
constructor(sql, now = Date.now) {
|
|
131
|
+
this.sql = sql;
|
|
132
|
+
this.now = now;
|
|
133
|
+
this.migrate();
|
|
134
|
+
// Jobs that were running when the process stopped cannot resume.
|
|
135
|
+
this.sql
|
|
136
|
+
.prepare("UPDATE jobs SET state = 'failed', error = 'operation_failed', updated_at = ? WHERE state IN ('queued', 'running')")
|
|
137
|
+
.run(this.now());
|
|
138
|
+
this.sql
|
|
139
|
+
.prepare("UPDATE sources SET status = 'failed', error = 'operation_failed' WHERE status = 'indexing'")
|
|
140
|
+
.run();
|
|
141
|
+
}
|
|
142
|
+
migrate() {
|
|
143
|
+
const version = Number(this.sql.prepare("PRAGMA user_version").get()?.user_version ?? 0);
|
|
144
|
+
for (let v = version; v < MIGRATIONS.length; v++)
|
|
145
|
+
this.sql.transaction(() => {
|
|
146
|
+
this.sql.exec(MIGRATIONS[v]);
|
|
147
|
+
this.sql.exec(`PRAGMA user_version = ${v + 1}`);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
close() {
|
|
151
|
+
this.sql.close();
|
|
152
|
+
}
|
|
153
|
+
// Notebooks
|
|
154
|
+
createNotebook(id, title) {
|
|
155
|
+
const at = this.now();
|
|
156
|
+
this.sql
|
|
157
|
+
.prepare("INSERT INTO notebooks (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)")
|
|
158
|
+
.run(id, title, at, at);
|
|
159
|
+
}
|
|
160
|
+
notebook(id) {
|
|
161
|
+
const row = this.sql.prepare("SELECT * FROM notebooks WHERE id = ?").get(id);
|
|
162
|
+
return row ? notebook(row) : undefined;
|
|
163
|
+
}
|
|
164
|
+
notebooks() {
|
|
165
|
+
return this.sql
|
|
166
|
+
.prepare("SELECT n.*, (SELECT COUNT(*) FROM sources s WHERE s.notebook_id = n.id) AS source_count FROM notebooks n ORDER BY n.updated_at DESC, n.id")
|
|
167
|
+
.all()
|
|
168
|
+
.map((r) => ({ ...notebook(r), sourceCount: Number(r.source_count) }));
|
|
169
|
+
}
|
|
170
|
+
findNotebook(reference) {
|
|
171
|
+
const byId = this.notebook(reference);
|
|
172
|
+
if (byId)
|
|
173
|
+
return byId;
|
|
174
|
+
const row = this.sql
|
|
175
|
+
.prepare("SELECT * FROM notebooks WHERE title = ? COLLATE NOCASE ORDER BY updated_at DESC LIMIT 1")
|
|
176
|
+
.get(reference);
|
|
177
|
+
return row ? notebook(row) : undefined;
|
|
178
|
+
}
|
|
179
|
+
renameNotebook(id, title) {
|
|
180
|
+
this.sql
|
|
181
|
+
.prepare("UPDATE notebooks SET title = ?, updated_at = ? WHERE id = ?")
|
|
182
|
+
.run(title, this.now(), id);
|
|
183
|
+
}
|
|
184
|
+
touchNotebook(id) {
|
|
185
|
+
this.sql.prepare("UPDATE notebooks SET updated_at = ? WHERE id = ?").run(this.now(), id);
|
|
186
|
+
}
|
|
187
|
+
setModel(id, model) {
|
|
188
|
+
this.sql.prepare("UPDATE notebooks SET model = ? WHERE id = ?").run(model, id);
|
|
189
|
+
}
|
|
190
|
+
ackNotice(id) {
|
|
191
|
+
this.sql.prepare("UPDATE notebooks SET remote_notice_ack = 1 WHERE id = ?").run(id);
|
|
192
|
+
}
|
|
193
|
+
deleteNotebook(id) {
|
|
194
|
+
this.sql.transaction(() => {
|
|
195
|
+
this.sql
|
|
196
|
+
.prepare("DELETE FROM chunks WHERE source_id IN (SELECT id FROM sources WHERE notebook_id = ?)")
|
|
197
|
+
.run(id);
|
|
198
|
+
this.sql.prepare("DELETE FROM sources WHERE notebook_id = ?").run(id);
|
|
199
|
+
this.sql.prepare("DELETE FROM messages WHERE notebook_id = ?").run(id);
|
|
200
|
+
this.sql.prepare("DELETE FROM jobs WHERE notebook_id = ?").run(id);
|
|
201
|
+
this.sql.prepare("DELETE FROM notebooks WHERE id = ?").run(id);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
countNotebooks() {
|
|
205
|
+
return Number(this.sql.prepare("SELECT COUNT(*) AS n FROM notebooks").get()?.n ?? 0);
|
|
206
|
+
}
|
|
207
|
+
// Sources
|
|
208
|
+
setGuide(id, guide) {
|
|
209
|
+
this.sql
|
|
210
|
+
.prepare("UPDATE notebooks SET guide_json = ? WHERE id = ?")
|
|
211
|
+
.run(guide ? JSON.stringify(guide) : null, id);
|
|
212
|
+
}
|
|
213
|
+
setExcluded(id, excluded) {
|
|
214
|
+
this.sql
|
|
215
|
+
.prepare("UPDATE notebooks SET excluded_json = ? WHERE id = ?")
|
|
216
|
+
.run(JSON.stringify(excluded), id);
|
|
217
|
+
}
|
|
218
|
+
setSourceSummary(id, summary) {
|
|
219
|
+
this.sql
|
|
220
|
+
.prepare("UPDATE sources SET summary_json = ? WHERE id = ?")
|
|
221
|
+
.run(summary ? JSON.stringify(summary) : null, id);
|
|
222
|
+
}
|
|
223
|
+
insertSource(row) {
|
|
224
|
+
this.sql
|
|
225
|
+
.prepare("INSERT INTO sources (id, notebook_id, name, kind, mime, sha256, bytes, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
|
226
|
+
.run(row.id, row.notebookId, row.name, row.kind, row.mime, row.sha256, row.bytes, row.status, this.now());
|
|
227
|
+
}
|
|
228
|
+
source(id) {
|
|
229
|
+
const row = this.sql.prepare("SELECT * FROM sources WHERE id = ?").get(id);
|
|
230
|
+
return row ? source(row) : undefined;
|
|
231
|
+
}
|
|
232
|
+
sources(notebookId) {
|
|
233
|
+
return this.sql
|
|
234
|
+
.prepare("SELECT * FROM sources WHERE notebook_id = ? ORDER BY created_at, id")
|
|
235
|
+
.all(notebookId)
|
|
236
|
+
.map(source);
|
|
237
|
+
}
|
|
238
|
+
sourceBySha(notebookId, sha256) {
|
|
239
|
+
const row = this.sql
|
|
240
|
+
.prepare("SELECT * FROM sources WHERE notebook_id = ? AND sha256 = ?")
|
|
241
|
+
.get(notebookId, sha256);
|
|
242
|
+
return row ? source(row) : undefined;
|
|
243
|
+
}
|
|
244
|
+
setSourceStatus(id, status, error = null) {
|
|
245
|
+
this.sql
|
|
246
|
+
.prepare("UPDATE sources SET status = ?, error = ? WHERE id = ?")
|
|
247
|
+
.run(status, error, id);
|
|
248
|
+
}
|
|
249
|
+
deleteSource(id) {
|
|
250
|
+
this.sql.transaction(() => {
|
|
251
|
+
this.sql.prepare("DELETE FROM chunks WHERE source_id = ?").run(id);
|
|
252
|
+
this.sql.prepare("DELETE FROM sources WHERE id = ?").run(id);
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
// Chunks (one transaction per source, D10)
|
|
256
|
+
replaceChunks(sourceId, chunks) {
|
|
257
|
+
this.sql.transaction(() => {
|
|
258
|
+
this.sql.prepare("DELETE FROM chunks WHERE source_id = ?").run(sourceId);
|
|
259
|
+
const insert = this.sql.prepare("INSERT INTO chunks (source_id, ordinal, text, page, char_start, char_end) VALUES (?, ?, ?, ?, ?, ?)");
|
|
260
|
+
for (const c of chunks)
|
|
261
|
+
insert.run(sourceId, c.ordinal, c.text, c.page, c.charStart, c.charEnd);
|
|
262
|
+
this.sql
|
|
263
|
+
.prepare("UPDATE sources SET chunk_count = ? WHERE id = ?")
|
|
264
|
+
.run(chunks.length, sourceId);
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
clearChunks(sourceId) {
|
|
268
|
+
this.replaceChunks(sourceId, []);
|
|
269
|
+
}
|
|
270
|
+
chunks(sourceId) {
|
|
271
|
+
return this.sql
|
|
272
|
+
.prepare("SELECT * FROM chunks WHERE source_id = ? ORDER BY ordinal")
|
|
273
|
+
.all(sourceId)
|
|
274
|
+
.map(chunk);
|
|
275
|
+
}
|
|
276
|
+
chunk(sourceId, ordinal) {
|
|
277
|
+
const row = this.sql
|
|
278
|
+
.prepare("SELECT * FROM chunks WHERE source_id = ? AND ordinal = ?")
|
|
279
|
+
.get(sourceId, ordinal);
|
|
280
|
+
return row ? chunk(row) : undefined;
|
|
281
|
+
}
|
|
282
|
+
notebookChunkCount(notebookId) {
|
|
283
|
+
return Number(this.sql
|
|
284
|
+
.prepare("SELECT COUNT(*) AS n FROM chunks c JOIN sources s ON s.id = c.source_id WHERE s.notebook_id = ?")
|
|
285
|
+
.get(notebookId)?.n ?? 0);
|
|
286
|
+
}
|
|
287
|
+
/** BM25 ranking over the notebook's chunks, as `<sourceId>:<ordinal>` keys. */
|
|
288
|
+
keywordSearch(notebookId, match, limit, sourceIds) {
|
|
289
|
+
if (!match)
|
|
290
|
+
return [];
|
|
291
|
+
const scope = sourceIds?.length
|
|
292
|
+
? ` AND c.source_id IN (${sourceIds.map(() => "?").join(", ")})`
|
|
293
|
+
: "";
|
|
294
|
+
try {
|
|
295
|
+
return this.sql
|
|
296
|
+
.prepare(`SELECT c.source_id, c.ordinal FROM chunks_fts f JOIN chunks c ON c.id = f.rowid
|
|
297
|
+
JOIN sources s ON s.id = c.source_id
|
|
298
|
+
WHERE chunks_fts MATCH ? AND s.notebook_id = ?${scope}
|
|
299
|
+
ORDER BY bm25(chunks_fts), c.id LIMIT ?`)
|
|
300
|
+
.all(match, notebookId, ...(sourceIds ?? []), limit)
|
|
301
|
+
.map((r) => `${r.source_id}:${Number(r.ordinal)}`);
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
return [];
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
// Messages
|
|
308
|
+
addMessage(notebookId, role, text, meta = {}) {
|
|
309
|
+
const result = this.sql
|
|
310
|
+
.prepare("INSERT INTO messages (notebook_id, role, text, citations_json, created_at) VALUES (?, ?, ?, ?, ?)")
|
|
311
|
+
.run(notebookId, role, text, JSON.stringify(meta), this.now());
|
|
312
|
+
return Number(result.lastInsertRowid);
|
|
313
|
+
}
|
|
314
|
+
messages(notebookId, limit, before) {
|
|
315
|
+
const rows = this.sql
|
|
316
|
+
.prepare(`SELECT * FROM messages WHERE notebook_id = ?${before ? " AND id < ?" : ""} ORDER BY id DESC LIMIT ?`)
|
|
317
|
+
.all(...(before ? [notebookId, before, limit] : [notebookId, limit]));
|
|
318
|
+
return rows.reverse().map((r) => ({
|
|
319
|
+
id: Number(r.id),
|
|
320
|
+
notebookId: r.notebook_id,
|
|
321
|
+
role: r.role,
|
|
322
|
+
text: r.text,
|
|
323
|
+
meta: parse(r.citations_json) ?? {},
|
|
324
|
+
createdAt: Number(r.created_at),
|
|
325
|
+
}));
|
|
326
|
+
}
|
|
327
|
+
clearMessages(notebookId) {
|
|
328
|
+
this.sql.prepare("DELETE FROM messages WHERE notebook_id = ?").run(notebookId);
|
|
329
|
+
}
|
|
330
|
+
// Jobs
|
|
331
|
+
insertJob(id, notebookId, kind, sourceId) {
|
|
332
|
+
const at = this.now();
|
|
333
|
+
this.sql
|
|
334
|
+
.prepare("INSERT INTO jobs (id, notebook_id, kind, state, source_id, created_at, updated_at) VALUES (?, ?, ?, 'queued', ?, ?, ?)")
|
|
335
|
+
.run(id, notebookId, kind, sourceId, at, at);
|
|
336
|
+
}
|
|
337
|
+
updateJob(id, patch) {
|
|
338
|
+
const current = this.job(id);
|
|
339
|
+
if (!current)
|
|
340
|
+
return;
|
|
341
|
+
this.sql
|
|
342
|
+
.prepare("UPDATE jobs SET state = ?, progress = ?, error = ?, result_json = ?, updated_at = ? WHERE id = ?")
|
|
343
|
+
.run(patch.state ?? current.state, patch.progress ?? current.progress, patch.error === undefined ? current.error : patch.error, JSON.stringify(patch.result === undefined ? current.result : patch.result), this.now(), id);
|
|
344
|
+
}
|
|
345
|
+
job(id) {
|
|
346
|
+
const row = this.sql.prepare("SELECT * FROM jobs WHERE id = ?").get(id);
|
|
347
|
+
return row ? job(row) : undefined;
|
|
348
|
+
}
|
|
349
|
+
activeJobs(notebookId) {
|
|
350
|
+
return this.sql
|
|
351
|
+
.prepare("SELECT * FROM jobs WHERE notebook_id = ? AND state IN ('queued', 'running') ORDER BY created_at")
|
|
352
|
+
.all(notebookId)
|
|
353
|
+
.map(job);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
//# sourceMappingURL=db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/notebook/db.ts"],"names":[],"mappings":"AAGA,sFAAsF;AACtF,MAAM,UAAU,GAAa;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MA4DI;IACJ,0EAA0E;IAC1E,8CAA8C;IAC9C,8FAA8F;IAC9F;;qDAEmD;CACpD,CAAC;AA2DF,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAe,EAAE,CAAC,CAAC;IAC5C,EAAE,EAAE,CAAC,CAAC,EAAE;IACR,KAAK,EAAE,CAAC,CAAC,KAAK;IACd,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/B,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAClD,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;IAC9D,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,IAAI;IAClC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAc,CAAC,CAAC,CAAC,EAAE;CAC5F,CAAC,CAAC;AACH,MAAM,MAAM,GAAG,CAAC,CAAS,EAAa,EAAE,CAAC,CAAC;IACxC,EAAE,EAAE,CAAC,CAAC,EAAE;IACR,UAAU,EAAE,CAAC,CAAC,WAAW;IACzB,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;IACtB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI;IACtB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/B,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,IAAI;CACvC,CAAC,CAAC;AACH,MAAM,KAAK,GAAG,CAAC,CAAS,EAAY,EAAE,CAAC,CAAC;IACtC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAChB,QAAQ,EAAE,CAAC,CAAC,SAAS;IACrB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;CAC5B,CAAC,CAAC;AACH,MAAM,KAAK,GAAG,CAAC,IAAa,EAAE,EAAE;IAC9B,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC/C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC,CAAC;AACF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC;IAClC,EAAE,EAAE,CAAC,CAAC,EAAE;IACR,UAAU,EAAE,CAAC,CAAC,WAAW;IACzB,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,KAAK,EAAE,CAAC,CAAC,KAAK;IACd,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5B,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI;IACtB,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,IAAI;IACpC,QAAQ,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI;IAC7B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;CAChC,CAAC,CAAC;AAEH,oFAAoF;AACpF,MAAM,OAAO,UAAU;IACZ,GAAG,CAAc;IACT,GAAG,CAAe;IAEnC,YAAY,GAAgB,EAAE,GAAG,GAAiB,IAAI,CAAC,GAAG;QACxD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,iEAAiE;QACjE,IAAI,CAAC,GAAG;aACL,OAAO,CACN,mHAAmH,CACpH;aACA,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,GAAG;aACL,OAAO,CACN,4FAA4F,CAC7F;aACA,GAAG,EAAE,CAAC;IACX,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAE,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QACzF,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE;YAC9C,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAW,CAAC,CAAC;gBACvC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK;QACH,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;IAED,YAAY;IACZ,cAAc,CAAC,EAAU,EAAE,KAAa;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,+EAA+E,CAAC;aACxF,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,QAAQ,CAAC,EAAU;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzC,CAAC;IACD,SAAS;QACP,OAAO,IAAI,CAAC,GAAG;aACZ,OAAO,CACN,2IAA2I,CAC5I;aACA,GAAG,EAAE;aACL,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,YAAY,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG;aACjB,OAAO,CACN,yFAAyF,CAC1F;aACA,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzC,CAAC;IACD,cAAc,CAAC,EAAU,EAAE,KAAa;QACtC,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,6DAA6D,CAAC;aACtE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,aAAa,CAAC,EAAU;QACtB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kDAAkD,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,QAAQ,CAAC,EAAU,EAAE,KAAoB;QACvC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,SAAS,CAAC,EAAU;QAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,cAAc,CAAC,EAAU;QACvB,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;YACxB,IAAI,CAAC,GAAG;iBACL,OAAO,CACN,sFAAsF,CACvF;iBACA,GAAG,CAAC,EAAE,CAAC,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IACD,cAAc;QACZ,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IAED,UAAU;IACV,QAAQ,CAAC,EAAU,EAAE,KAAqC;QACxD,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,kDAAkD,CAAC;aAC3D,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,WAAW,CAAC,EAAU,EAAE,QAAkB;QACxC,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,qDAAqD,CAAC;aAC9D,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,gBAAgB,CAAC,EAAU,EAAE,OAAuC;QAClE,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,kDAAkD,CAAC;aAC3D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,YAAY,CAAC,GAAsE;QACjF,IAAI,CAAC,GAAG;aACL,OAAO,CACN,+HAA+H,CAChI;aACA,GAAG,CACF,GAAG,CAAC,EAAE,EACN,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,MAAM,EACV,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;IACN,CAAC;IACD,MAAM,CAAC,EAAU;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3E,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,CAAC;IACD,OAAO,CAAC,UAAkB;QACxB,OAAO,IAAI,CAAC,GAAG;aACZ,OAAO,CAAC,qEAAqE,CAAC;aAC9E,GAAG,CAAC,UAAU,CAAC;aACf,GAAG,CAAC,MAAM,CAAC,CAAC;IACjB,CAAC;IACD,WAAW,CAAC,UAAkB,EAAE,MAAc;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG;aACjB,OAAO,CAAC,4DAA4D,CAAC;aACrE,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,CAAC;IACD,eAAe,CAAC,EAAU,EAAE,MAAoB,EAAE,KAAK,GAAkB,IAAI;QAC3E,IAAI,CAAC,GAAG;aACL,OAAO,CAAC,uDAAuD,CAAC;aAChE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,YAAY,CAAC,EAAU;QACrB,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;YACxB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,2CAA2C;IAC3C,aAAa,CAAC,QAAgB,EAAE,MAAe;QAC7C,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;YACxB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAC7B,qGAAqG,CACtG,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,MAAM;gBACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAG;iBACL,OAAO,CAAC,iDAAiD,CAAC;iBAC1D,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC;IACD,WAAW,CAAC,QAAgB;QAC1B,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,CAAC,QAAgB;QACrB,OAAO,IAAI,CAAC,GAAG;aACZ,OAAO,CAAC,2DAA2D,CAAC;aACpE,GAAG,CAAC,QAAQ,CAAC;aACb,GAAG,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC;IACD,KAAK,CAAC,QAAgB,EAAE,OAAe;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG;aACjB,OAAO,CAAC,0DAA0D,CAAC;aACnE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtC,CAAC;IACD,kBAAkB,CAAC,UAAkB;QACnC,OAAO,MAAM,CACX,IAAI,CAAC,GAAG;aACL,OAAO,CACN,iGAAiG,CAClG;aACA,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAC3B,CAAC;IACJ,CAAC;IACD,+EAA+E;IAC/E,aAAa,CAAC,UAAkB,EAAE,KAAa,EAAE,KAAa,EAAE,SAAoB;QAClF,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,SAAS,EAAE,MAAM;YAC7B,CAAC,CAAC,wBAAwB,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAChE,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,GAAG;iBACZ,OAAO,CACN;;2DAEiD,KAAK;mDACb,CAC1C;iBACA,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;iBACnD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,WAAW;IACX,UAAU,CACR,UAAkB,EAClB,IAA0B,EAC1B,IAAY,EACZ,IAAI,GAA4B,EAAE;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG;aACpB,OAAO,CACN,mGAAmG,CACpG;aACA,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACjE,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IACD,QAAQ,CAAC,UAAkB,EAAE,KAAa,EAAE,MAAe;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG;aAClB,OAAO,CACN,+CAA+C,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,2BAA2B,CACtG;aACA,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC,CAAC,WAAW;YACzB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE;YACnC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;SAChC,CAAC,CAAC,CAAC;IACN,CAAC;IACD,aAAa,CAAC,UAAkB;QAC9B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACjF,CAAC;IAED,OAAO;IACP,SAAS,CAAC,EAAU,EAAE,UAAkB,EAAE,IAAY,EAAE,QAAuB;QAC7E,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG;aACL,OAAO,CACN,wHAAwH,CACzH;aACA,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,SAAS,CACP,EAAU,EACV,KAAuF;QAEvF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,IAAI,CAAC,GAAG;aACL,OAAO,CACN,kGAAkG,CACnG;aACA,GAAG,CACF,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAC5B,KAAK,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAClC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EACvD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAC1E,IAAI,CAAC,GAAG,EAAE,EACV,EAAE,CACH,CAAC;IACN,CAAC;IACD,GAAG,CAAC,EAAU;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxE,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpC,CAAC;IACD,UAAU,CAAC,UAAkB;QAC3B,OAAO,IAAI,CAAC,GAAG;aACZ,OAAO,CACN,iGAAiG,CAClG;aACA,GAAG,CAAC,UAAU,CAAC;aACf,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docx.d.ts","sourceRoot":"","sources":["../../src/notebook/docx.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAU9C"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// .docx text (§7.3): `word/document.xml`, `<w:p>` per paragraph, `<w:t>` runs, tabs and breaks.
|
|
2
|
+
import { DOCX_LIMITS, decodeXml, ZipArchive } from "./zip.js";
|
|
3
|
+
export { decodeXml } from "./zip.js";
|
|
4
|
+
export function docxText(bytes) {
|
|
5
|
+
const xml = new ZipArchive(bytes, DOCX_LIMITS).xml("word/document.xml");
|
|
6
|
+
const paragraphs = [];
|
|
7
|
+
for (const match of xml.matchAll(/<w:p[\s>][\s\S]*?<\/w:p>|<w:p\/>/g)) {
|
|
8
|
+
let text = "";
|
|
9
|
+
for (const token of match[0].matchAll(/<w:t(?:\s[^>]*)?>([\s\S]*?)<\/w:t>|<w:tab\/>|<w:br\/>/g))
|
|
10
|
+
text += token[1] !== undefined ? decodeXml(token[1]) : token[0] === "<w:tab/>" ? "\t" : "\n";
|
|
11
|
+
if (text.trim())
|
|
12
|
+
paragraphs.push(text);
|
|
13
|
+
}
|
|
14
|
+
return paragraphs.join("\n\n");
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=docx.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docx.js","sourceRoot":"","sources":["../../src/notebook/docx.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACxE,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,mCAAmC,CAAC,EAAE,CAAC;QACtE,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC;YAC7F,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/F,IAAI,IAAI,CAAC,IAAI,EAAE;YAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC"}
|