@okhay-p/mivra 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +30 -2
  3. package/THIRD_PARTY_NOTICES.md +7297 -0
  4. package/bin/mivra.js +16 -0
  5. package/dist/cli.js +11633 -0
  6. package/dist/cli.js.map +7 -0
  7. package/dist/extract-child.js +209 -0
  8. package/dist/extract-child.js.map +7 -0
  9. package/drizzle/0000_init.sql +33 -0
  10. package/drizzle/0001_artifacts_attempts.sql +30 -0
  11. package/drizzle/0002_cloudy_supreme_intelligence.sql +1 -0
  12. package/drizzle/0003_dark_shooting_star.sql +20 -0
  13. package/drizzle/0004_ordinary_phantom_reporter.sql +40 -0
  14. package/drizzle/0005_simple_wolf_cub.sql +1 -0
  15. package/drizzle/0006_closed_karen_page.sql +2 -0
  16. package/drizzle/0007_regular_white_tiger.sql +24 -0
  17. package/drizzle/0008_long_millenium_guard.sql +42 -0
  18. package/drizzle/0009_legal_spirit.sql +5 -0
  19. package/drizzle/0010_goofy_thanos.sql +2 -0
  20. package/drizzle/0011_known_katie_power.sql +8 -0
  21. package/drizzle/meta/0000_snapshot.json +254 -0
  22. package/drizzle/meta/0001_snapshot.json +473 -0
  23. package/drizzle/meta/0002_snapshot.json +480 -0
  24. package/drizzle/meta/0003_snapshot.json +630 -0
  25. package/drizzle/meta/0004_snapshot.json +801 -0
  26. package/drizzle/meta/0005_snapshot.json +821 -0
  27. package/drizzle/meta/0006_snapshot.json +835 -0
  28. package/drizzle/meta/0007_snapshot.json +1004 -0
  29. package/drizzle/meta/0008_snapshot.json +1303 -0
  30. package/drizzle/meta/0009_snapshot.json +1341 -0
  31. package/drizzle/meta/0010_snapshot.json +1356 -0
  32. package/drizzle/meta/0011_snapshot.json +1401 -0
  33. package/drizzle/meta/_journal.json +90 -0
  34. package/package.json +60 -4
  35. package/web/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2 +0 -0
  36. package/web/assets/KaTeX_AMS-Regular-DMm9YOAa.woff +0 -0
  37. package/web/assets/KaTeX_AMS-Regular-DRggAlZN.ttf +0 -0
  38. package/web/assets/KaTeX_Caligraphic-Bold-ATXxdsX0.ttf +0 -0
  39. package/web/assets/KaTeX_Caligraphic-Bold-BEiXGLvX.woff +0 -0
  40. package/web/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2 +0 -0
  41. package/web/assets/KaTeX_Caligraphic-Regular-CTRA-rTL.woff +0 -0
  42. package/web/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2 +0 -0
  43. package/web/assets/KaTeX_Caligraphic-Regular-wX97UBjC.ttf +0 -0
  44. package/web/assets/KaTeX_Fraktur-Bold-BdnERNNW.ttf +0 -0
  45. package/web/assets/KaTeX_Fraktur-Bold-BsDP51OF.woff +0 -0
  46. package/web/assets/KaTeX_Fraktur-Bold-CL6g_b3V.woff2 +0 -0
  47. package/web/assets/KaTeX_Fraktur-Regular-CB_wures.ttf +0 -0
  48. package/web/assets/KaTeX_Fraktur-Regular-CTYiF6lA.woff2 +0 -0
  49. package/web/assets/KaTeX_Fraktur-Regular-Dxdc4cR9.woff +0 -0
  50. package/web/assets/KaTeX_Main-Bold-Cx986IdX.woff2 +0 -0
  51. package/web/assets/KaTeX_Main-Bold-Jm3AIy58.woff +0 -0
  52. package/web/assets/KaTeX_Main-Bold-waoOVXN0.ttf +0 -0
  53. package/web/assets/KaTeX_Main-BoldItalic-DxDJ3AOS.woff2 +0 -0
  54. package/web/assets/KaTeX_Main-BoldItalic-DzxPMmG6.ttf +0 -0
  55. package/web/assets/KaTeX_Main-BoldItalic-SpSLRI95.woff +0 -0
  56. package/web/assets/KaTeX_Main-Italic-3WenGoN9.ttf +0 -0
  57. package/web/assets/KaTeX_Main-Italic-BMLOBm91.woff +0 -0
  58. package/web/assets/KaTeX_Main-Italic-NWA7e6Wa.woff2 +0 -0
  59. package/web/assets/KaTeX_Main-Regular-B22Nviop.woff2 +0 -0
  60. package/web/assets/KaTeX_Main-Regular-Dr94JaBh.woff +0 -0
  61. package/web/assets/KaTeX_Main-Regular-ypZvNtVU.ttf +0 -0
  62. package/web/assets/KaTeX_Math-BoldItalic-B3XSjfu4.ttf +0 -0
  63. package/web/assets/KaTeX_Math-BoldItalic-CZnvNsCZ.woff2 +0 -0
  64. package/web/assets/KaTeX_Math-BoldItalic-iY-2wyZ7.woff +0 -0
  65. package/web/assets/KaTeX_Math-Italic-DA0__PXp.woff +0 -0
  66. package/web/assets/KaTeX_Math-Italic-flOr_0UB.ttf +0 -0
  67. package/web/assets/KaTeX_Math-Italic-t53AETM-.woff2 +0 -0
  68. package/web/assets/KaTeX_SansSerif-Bold-CFMepnvq.ttf +0 -0
  69. package/web/assets/KaTeX_SansSerif-Bold-D1sUS0GD.woff2 +0 -0
  70. package/web/assets/KaTeX_SansSerif-Bold-DbIhKOiC.woff +0 -0
  71. package/web/assets/KaTeX_SansSerif-Italic-C3H0VqGB.woff2 +0 -0
  72. package/web/assets/KaTeX_SansSerif-Italic-DN2j7dab.woff +0 -0
  73. package/web/assets/KaTeX_SansSerif-Italic-YYjJ1zSn.ttf +0 -0
  74. package/web/assets/KaTeX_SansSerif-Regular-BNo7hRIc.ttf +0 -0
  75. package/web/assets/KaTeX_SansSerif-Regular-CS6fqUqJ.woff +0 -0
  76. package/web/assets/KaTeX_SansSerif-Regular-DDBCnlJ7.woff2 +0 -0
  77. package/web/assets/KaTeX_Script-Regular-C5JkGWo-.ttf +0 -0
  78. package/web/assets/KaTeX_Script-Regular-D3wIWfF6.woff2 +0 -0
  79. package/web/assets/KaTeX_Script-Regular-D5yQViql.woff +0 -0
  80. package/web/assets/KaTeX_Size1-Regular-C195tn64.woff +0 -0
  81. package/web/assets/KaTeX_Size1-Regular-Dbsnue_I.ttf +0 -0
  82. package/web/assets/KaTeX_Size1-Regular-mCD8mA8B.woff2 +0 -0
  83. package/web/assets/KaTeX_Size2-Regular-B7gKUWhC.ttf +0 -0
  84. package/web/assets/KaTeX_Size2-Regular-Dy4dx90m.woff2 +0 -0
  85. package/web/assets/KaTeX_Size2-Regular-oD1tc_U0.woff +0 -0
  86. package/web/assets/KaTeX_Size3-Regular-CTq5MqoE.woff +0 -0
  87. package/web/assets/KaTeX_Size3-Regular-DgpXs0kz.ttf +0 -0
  88. package/web/assets/KaTeX_Size4-Regular-BF-4gkZK.woff +0 -0
  89. package/web/assets/KaTeX_Size4-Regular-DWFBv043.ttf +0 -0
  90. package/web/assets/KaTeX_Size4-Regular-Dl5lxZxV.woff2 +0 -0
  91. package/web/assets/KaTeX_Typewriter-Regular-C0xS9mPB.woff +0 -0
  92. package/web/assets/KaTeX_Typewriter-Regular-CO6r4hn1.woff2 +0 -0
  93. package/web/assets/KaTeX_Typewriter-Regular-D3Ib7_Hf.ttf +0 -0
  94. package/web/assets/fraunces-latin-ext-wght-normal-Ca2vKHc0.woff2 +0 -0
  95. package/web/assets/fraunces-latin-wght-normal-ukD16Tqj.woff2 +0 -0
  96. package/web/assets/fraunces-vietnamese-wght-normal-CnvboYUG.woff2 +0 -0
  97. package/web/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
  98. package/web/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
  99. package/web/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
  100. package/web/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
  101. package/web/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
  102. package/web/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
  103. package/web/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
  104. package/web/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
  105. package/web/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
  106. package/web/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
  107. package/web/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
  108. package/web/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
  109. package/web/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
  110. package/web/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
  111. package/web/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
  112. package/web/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
  113. package/web/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
  114. package/web/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
  115. package/web/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
  116. package/web/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
  117. package/web/assets/index-Cv4wPRY7.css +1 -0
  118. package/web/assets/index-aRUQfiYR.js +406 -0
  119. package/web/assets/render.worker-Bw6rO7Qu.js +413 -0
  120. package/web/assets/source-sans-3-cyrillic-ext-wght-normal-c-5QAGAI.woff2 +0 -0
  121. package/web/assets/source-sans-3-cyrillic-wght-normal-BMDVbyM7.woff2 +0 -0
  122. package/web/assets/source-sans-3-greek-ext-wght-normal-BXZfsJY4.woff2 +0 -0
  123. package/web/assets/source-sans-3-greek-wght-normal-Cc-u-6OI.woff2 +0 -0
  124. package/web/assets/source-sans-3-latin-ext-wght-normal-C8iNium2.woff2 +0 -0
  125. package/web/assets/source-sans-3-latin-wght-normal-Bo_j9asI.woff2 +0 -0
  126. package/web/assets/source-sans-3-vietnamese-wght-normal-BDs68eHQ.woff2 +0 -0
  127. package/web/index.html +24 -0
@@ -0,0 +1,209 @@
1
+ // ../server/src/documents/extract.ts
2
+ import { posix } from "node:path";
3
+ import { getResolvedPDFJS } from "unpdf";
4
+ import yauzl from "yauzl";
5
+ var DocumentError = class extends Error {
6
+ constructor(code, message) {
7
+ super(message);
8
+ this.code = code;
9
+ }
10
+ code;
11
+ };
12
+ var MAX_PAGES = 5e3;
13
+ var MAX_TEXT_CHARS = 1e7;
14
+ var MIN_TEXT_CHARS = 20;
15
+ var MAX_ZIP_ENTRIES = 2e4;
16
+ var MAX_ZIP_DECLARED_BYTES = 1024 * 1024 * 1024;
17
+ var MAX_XML_PART_BYTES = 32 * 1024 * 1024;
18
+ var MAX_XML_TOTAL_BYTES = 256 * 1024 * 1024;
19
+ var WRONG_TYPE = "Only PDF and PowerPoint (.pptx) files can be attached.";
20
+ var CORRUPT = "This file looks damaged and can't be read.";
21
+ var ENCRYPTED = "This file is password-protected. Remove the password and upload it again.";
22
+ var TOO_MUCH_TEXT = "This file has too much text to attach. Try uploading the part you need.";
23
+ function cleanText(text) {
24
+ return text.replace(/[^\P{C}\n\t]/gu, "").replace(/[ff-st\u{1D400}-\u{1D7FF}]/gu, (c) => c.normalize("NFKC")).replace(/[ \t]+/g, " ").replace(/ ?\n ?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
25
+ }
26
+ function finish(kind, pageCount, raw) {
27
+ const pages = raw.map(({ page, text }) => ({ page, text: cleanText(text) })).filter((page) => page.text);
28
+ if (pages.reduce((sum, page) => sum + page.text.replace(/\s/g, "").length, 0) < MIN_TEXT_CHARS) {
29
+ throw new DocumentError(
30
+ "no_text",
31
+ kind === "pdf" ? "This PDF has no readable text; it's probably scanned. Mivra can't read images yet, so try a PDF with selectable text." : "These slides have no readable text; they're probably images. Mivra can't read images yet."
32
+ );
33
+ }
34
+ return { kind, pageCount, pages };
35
+ }
36
+ async function extractPdf(bytes) {
37
+ const { getDocument } = await getResolvedPDFJS();
38
+ const task = getDocument({ data: new Uint8Array(bytes), useSystemFonts: false, disableFontFace: true, verbosity: 0 });
39
+ try {
40
+ const pdf = await task.promise.catch((error) => {
41
+ throw error?.name === "PasswordException" ? new DocumentError("encrypted", ENCRYPTED) : new DocumentError("corrupt", CORRUPT);
42
+ });
43
+ if (pdf.numPages > MAX_PAGES) throw new DocumentError("too_complex", `This PDF has more than ${MAX_PAGES} pages.`);
44
+ const pages = [];
45
+ let chars = 0;
46
+ for (let page = 1; page <= pdf.numPages; page++) {
47
+ const text = await pdf.getPage(page).then(async (handle) => {
48
+ const content = await handle.getTextContent();
49
+ handle.cleanup();
50
+ return content.items.map((item) => "str" in item ? item.str + (item.hasEOL ? "\n" : "") : "").join("");
51
+ }).catch(() => "");
52
+ chars += text.length;
53
+ if (chars > MAX_TEXT_CHARS) throw new DocumentError("too_complex", TOO_MUCH_TEXT);
54
+ pages.push({ page, text });
55
+ }
56
+ return finish("pdf", pdf.numPages, pages);
57
+ } finally {
58
+ await task.destroy().catch(() => {
59
+ });
60
+ }
61
+ }
62
+ function openZip(bytes) {
63
+ const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
64
+ return new Promise((resolve, reject) => {
65
+ yauzl.fromBuffer(buffer, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (openError, zip) => {
66
+ if (openError) return reject(new DocumentError("corrupt", CORRUPT));
67
+ if (zip.entryCount > MAX_ZIP_ENTRIES) return reject(new DocumentError("too_complex", "This file contains too many parts to read safely."));
68
+ const entries = /* @__PURE__ */ new Map();
69
+ let declared = 0;
70
+ let unpacked = 0;
71
+ zip.on("error", () => reject(new DocumentError("corrupt", CORRUPT)));
72
+ zip.on("entry", (entry) => {
73
+ if (entry.isEncrypted()) return reject(new DocumentError("encrypted", ENCRYPTED));
74
+ declared += entry.uncompressedSize;
75
+ if (declared > MAX_ZIP_DECLARED_BYTES) return reject(new DocumentError("too_complex", "This file unpacks to too much data to read safely."));
76
+ if (entries.has(entry.fileName)) return reject(new DocumentError("corrupt", CORRUPT));
77
+ entries.set(entry.fileName, entry);
78
+ zip.readEntry();
79
+ });
80
+ const read = (name) => new Promise((done, fail) => {
81
+ const entry = entries.get(name);
82
+ if (!entry) return fail(new DocumentError("corrupt", CORRUPT));
83
+ const tooMuch = new DocumentError("too_complex", "This file unpacks to too much data to read safely.");
84
+ if (entry.uncompressedSize > MAX_XML_PART_BYTES) return fail(tooMuch);
85
+ zip.openReadStream(entry, (streamError, stream) => {
86
+ if (streamError) return fail(new DocumentError("corrupt", CORRUPT));
87
+ const chunks = [];
88
+ stream.on("data", (chunk) => {
89
+ unpacked += chunk.length;
90
+ if (unpacked > MAX_XML_TOTAL_BYTES) stream.destroy(tooMuch);
91
+ else chunks.push(chunk);
92
+ });
93
+ stream.on("error", (error) => fail(error instanceof DocumentError ? error : new DocumentError("corrupt", CORRUPT)));
94
+ stream.on("end", () => done(Buffer.concat(chunks)));
95
+ });
96
+ });
97
+ zip.on("end", () => resolve({ names: new Set(entries.keys()), read, close: () => zip.close() }));
98
+ zip.readEntry();
99
+ });
100
+ });
101
+ }
102
+ function decodeXml(bytes) {
103
+ const [a, b] = bytes;
104
+ const encoding = a === 255 && b === 254 || a === 60 && b === 0 ? "utf-16le" : a === 254 && b === 255 || a === 0 && b === 60 ? "utf-16be" : "utf-8";
105
+ const xml = new TextDecoder(encoding).decode(bytes);
106
+ if (/<!(DOCTYPE|ENTITY)/i.test(xml)) throw new DocumentError("corrupt", "This file contains unsafe XML and can't be read.");
107
+ return xml;
108
+ }
109
+ var NAMED_ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
110
+ function decodeEntities(text) {
111
+ return text.replace(/&(?:#(\d{1,7})|#x([\da-f]{1,6})|(amp|lt|gt|quot|apos));/gi, (whole, dec, hex, name) => {
112
+ if (name) return NAMED_ENTITIES[name.toLowerCase()] ?? whole;
113
+ const code = dec ? Number(dec) : parseInt(hex, 16);
114
+ return code <= 1114111 ? String.fromCodePoint(code) : whole;
115
+ });
116
+ }
117
+ function* xmlTokens(xml) {
118
+ for (const [token] of xml.matchAll(/<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<[^>]*>|[^<]+/g)) {
119
+ if (token.startsWith("<!--") || token.startsWith("<?")) continue;
120
+ if (token.startsWith("<![CDATA[")) yield { text: token.slice(9, -3) };
121
+ else if (token.startsWith("<")) {
122
+ const name = /^<\/?(?:[\w.-]+:)?([\w.-]+)/.exec(token)?.[1] ?? "";
123
+ yield { tag: token, name, closing: token.startsWith("</"), selfClosing: token.endsWith("/>") };
124
+ } else yield { text: decodeEntities(token) };
125
+ }
126
+ }
127
+ function slideText(xml) {
128
+ let out = "";
129
+ let inText = 0;
130
+ let inFallback = 0;
131
+ for (const token of xmlTokens(xml)) {
132
+ if ("text" in token) {
133
+ if (inText > 0 && inFallback === 0) out += token.text;
134
+ continue;
135
+ }
136
+ const delta = token.selfClosing ? 0 : token.closing ? -1 : 1;
137
+ if (token.name === "Fallback") inFallback = Math.max(0, inFallback + delta);
138
+ else if (token.name === "t") inText = Math.max(0, inText + delta);
139
+ else if (inFallback === 0 && (token.name === "br" || token.name === "p" && token.closing)) out += "\n";
140
+ }
141
+ return out;
142
+ }
143
+ var attribute = (tag, name) => new RegExp(`\\s${name.source}\\s*=\\s*["']([^"']*)["']`).exec(tag)?.[1];
144
+ function slideOrder(presentationXml, relsXml) {
145
+ const targets = /* @__PURE__ */ new Map();
146
+ for (const token of xmlTokens(relsXml)) {
147
+ if ("tag" in token && token.name === "Relationship" && !token.closing) {
148
+ const id = attribute(token.tag, /Id/);
149
+ const target = attribute(token.tag, /Target/);
150
+ if (id && target) targets.set(id, target);
151
+ }
152
+ }
153
+ const slides = [];
154
+ for (const token of xmlTokens(presentationXml)) {
155
+ if ("tag" in token && token.name === "sldId" && !token.closing) {
156
+ const target = targets.get(attribute(token.tag, /[\w.-]+:id/) ?? "");
157
+ const part = target && (target.startsWith("/") ? posix.normalize(target).slice(1) : posix.join("ppt", target));
158
+ slides.push(part && /^ppt\/slides\/[^/]+\.xml$/.test(part) ? part : void 0);
159
+ }
160
+ }
161
+ return slides;
162
+ }
163
+ async function extractPptx(bytes) {
164
+ const zip = await openZip(bytes);
165
+ try {
166
+ if (!zip.names.has("[Content_Types].xml")) throw new DocumentError("wrong_type", WRONG_TYPE);
167
+ const types = decodeXml(await zip.read("[Content_Types].xml"));
168
+ if (!/officedocument\.presentationml\.(presentation|slideshow|template)\.main\+xml/i.test(types)) throw new DocumentError("wrong_type", WRONG_TYPE);
169
+ if (!zip.names.has("ppt/presentation.xml") || !zip.names.has("ppt/_rels/presentation.xml.rels")) throw new DocumentError("corrupt", CORRUPT);
170
+ const slides = slideOrder(decodeXml(await zip.read("ppt/presentation.xml")), decodeXml(await zip.read("ppt/_rels/presentation.xml.rels")));
171
+ if (slides.length > MAX_PAGES) throw new DocumentError("too_complex", `This file has more than ${MAX_PAGES} slides.`);
172
+ const pages = [];
173
+ let chars = 0;
174
+ for (const [index, part] of slides.entries()) {
175
+ if (!part || !zip.names.has(part)) continue;
176
+ const text = slideText(decodeXml(await zip.read(part)));
177
+ chars += text.length;
178
+ if (chars > MAX_TEXT_CHARS) throw new DocumentError("too_complex", TOO_MUCH_TEXT);
179
+ pages.push({ page: index + 1, text });
180
+ }
181
+ return finish("pptx", slides.length, pages);
182
+ } finally {
183
+ zip.close();
184
+ }
185
+ }
186
+ var startsWith = (bytes, magic) => magic.every((byte, i) => bytes[i] === byte);
187
+ async function extractDocument(bytes) {
188
+ const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
189
+ if (buffer.subarray(0, 1024).includes("%PDF-")) return extractPdf(bytes);
190
+ if (startsWith(bytes, [80, 75, 3, 4])) return extractPptx(bytes);
191
+ if (startsWith(bytes, [208, 207, 17, 224]) && buffer.includes(Buffer.from("EncryptedPackage", "utf16le"))) throw new DocumentError("encrypted", ENCRYPTED);
192
+ throw new DocumentError("wrong_type", WRONG_TYPE);
193
+ }
194
+
195
+ // ../server/src/documents/extract-child.ts
196
+ var send = (message) => process.send(message, () => process.exit(0));
197
+ process.once("disconnect", () => process.exit(0));
198
+ process.once("message", async (message) => {
199
+ if (message?.reportRss === true) setInterval(() => process.send({ rss: process.memoryUsage.rss() }), 100).unref();
200
+ try {
201
+ if (!(message?.bytes instanceof Uint8Array)) throw new DocumentError("failed", "Couldn't read this file.");
202
+ send({ ok: true, document: await extractDocument(message.bytes) });
203
+ } catch (error) {
204
+ send(
205
+ error instanceof DocumentError ? { ok: false, code: error.code, message: error.message } : { ok: false, code: "corrupt", message: "This file looks damaged and can't be read." }
206
+ );
207
+ }
208
+ });
209
+ //# sourceMappingURL=extract-child.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../server/src/documents/extract.ts", "../../server/src/documents/extract-child.ts"],
4
+ "sourcesContent": ["import { posix } from \"node:path\";\nimport { getResolvedPDFJS } from \"unpdf\";\nimport yauzl from \"yauzl\";\n\n/**\n * Text extraction for untrusted PDF and PPTX files (ADR-010). This runs inside a subprocess (see\n * extract-in-child.ts), which supplies the timeout and memory limits this code can't enforce itself.\n */\n\nexport type DocumentErrorCode = \"too_large\" | \"wrong_type\" | \"no_text\" | \"encrypted\" | \"corrupt\" | \"too_complex\" | \"timeout\" | \"failed\";\n\n/** A failure whose message is safe and useful to show the learner. */\nexport class DocumentError extends Error {\n constructor(\n readonly code: DocumentErrorCode,\n message: string,\n ) {\n super(message);\n }\n}\n\nexport interface ExtractedPage {\n /** 1-based page or slide number. */\n page: number;\n text: string;\n}\n\nexport interface ExtractedDocument {\n kind: \"pdf\" | \"pptx\";\n pageCount: number;\n /** Only the pages that have text. */\n pages: ExtractedPage[];\n}\n\nconst MAX_PAGES = 5000;\nconst MAX_TEXT_CHARS = 10_000_000;\n/** Below this across the whole file there is nothing useful to read: it's images, or at most a stray page number. */\nconst MIN_TEXT_CHARS = 20;\n\nconst MAX_ZIP_ENTRIES = 20_000;\n/** What all entries claim to unpack to, including media that is never unpacked. */\nconst MAX_ZIP_DECLARED_BYTES = 1024 * 1024 * 1024;\n/** What may actually be unpacked: only XML parts are read. */\nconst MAX_XML_PART_BYTES = 32 * 1024 * 1024;\nconst MAX_XML_TOTAL_BYTES = 256 * 1024 * 1024;\n\nconst WRONG_TYPE = \"Only PDF and PowerPoint (.pptx) files can be attached.\";\nconst CORRUPT = \"This file looks damaged and can't be read.\";\nconst ENCRYPTED = \"This file is password-protected. Remove the password and upload it again.\";\nconst TOO_MUCH_TEXT = \"This file has too much text to attach. Try uploading the part you need.\";\n\n/**\n * Keeps text that can be read and searched. Invisible formatting characters go too, since they can\n * hide text from the learner while the model still reads it. Ligatures and maths letters (\uD835\uDC65) become\n * plain ones so a search finds them, while the rest of Unicode, such as \u00B2, is left alone.\n */\nexport function cleanText(text: string): string {\n return text\n .replace(/[^\\P{C}\\n\\t]/gu, \"\")\n .replace(/[\uFB00-\uFB06\\u{1D400}-\\u{1D7FF}]/gu, (c: string) => c.normalize(\"NFKC\"))\n .replace(/[ \\t]+/g, \" \")\n .replace(/ ?\\n ?/g, \"\\n\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n}\n\nfunction finish(kind: ExtractedDocument[\"kind\"], pageCount: number, raw: ExtractedPage[]): ExtractedDocument {\n const pages = raw.map(({ page, text }) => ({ page, text: cleanText(text) })).filter((page) => page.text);\n if (pages.reduce((sum, page) => sum + page.text.replace(/\\s/g, \"\").length, 0) < MIN_TEXT_CHARS) {\n throw new DocumentError(\n \"no_text\",\n kind === \"pdf\"\n ? \"This PDF has no readable text; it's probably scanned. Mivra can't read images yet, so try a PDF with selectable text.\"\n : \"These slides have no readable text; they're probably images. Mivra can't read images yet.\",\n );\n }\n return { kind, pageCount, pages };\n}\n\nasync function extractPdf(bytes: Uint8Array): Promise<ExtractedDocument> {\n const { getDocument } = await getResolvedPDFJS();\n // This pdf.js build has no eval path for fonts (the CVE-2024-4367 route); text extraction never renders. It only warns on recoverable damage.\n // pdf.js refuses a Buffer, and takes ownership of what it's given.\n const task = getDocument({ data: new Uint8Array(bytes), useSystemFonts: false, disableFontFace: true, verbosity: 0 });\n try {\n const pdf = await task.promise.catch((error: unknown) => {\n throw (error as Error)?.name === \"PasswordException\" ? new DocumentError(\"encrypted\", ENCRYPTED) : new DocumentError(\"corrupt\", CORRUPT);\n });\n if (pdf.numPages > MAX_PAGES) throw new DocumentError(\"too_complex\", `This PDF has more than ${MAX_PAGES} pages.`);\n const pages: ExtractedPage[] = [];\n let chars = 0;\n for (let page = 1; page <= pdf.numPages; page++) {\n // One bad page shouldn't lose the rest; it counts as a page without text.\n const text = await pdf\n .getPage(page)\n .then(async (handle) => {\n const content = await handle.getTextContent();\n handle.cleanup();\n // Reading order is the file's own, which is usually right for columns but isn't guaranteed.\n return content.items.map((item) => (\"str\" in item ? item.str + (item.hasEOL ? \"\\n\" : \"\") : \"\")).join(\"\");\n })\n .catch(() => \"\");\n chars += text.length;\n if (chars > MAX_TEXT_CHARS) throw new DocumentError(\"too_complex\", TOO_MUCH_TEXT);\n pages.push({ page, text });\n }\n return finish(\"pdf\", pdf.numPages, pages);\n } finally {\n await task.destroy().catch(() => {});\n }\n}\n\n/** The zip's entries by name, with a reader for each that enforces the unpacked-size limits. */\nfunction openZip(bytes: Uint8Array): Promise<{ names: Set<string>; read: (name: string) => Promise<Buffer>; close: () => void }> {\n const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n return new Promise((resolve, reject) => {\n // yauzl rejects absolute paths, \"..\" and backslashes in names, and checks entries against their declared sizes.\n yauzl.fromBuffer(buffer, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (openError, zip) => {\n if (openError) return reject(new DocumentError(\"corrupt\", CORRUPT));\n if (zip.entryCount > MAX_ZIP_ENTRIES) return reject(new DocumentError(\"too_complex\", \"This file contains too many parts to read safely.\"));\n const entries = new Map<string, yauzl.Entry>();\n let declared = 0;\n let unpacked = 0;\n zip.on(\"error\", () => reject(new DocumentError(\"corrupt\", CORRUPT)));\n zip.on(\"entry\", (entry: yauzl.Entry) => {\n if (entry.isEncrypted()) return reject(new DocumentError(\"encrypted\", ENCRYPTED));\n declared += entry.uncompressedSize;\n if (declared > MAX_ZIP_DECLARED_BYTES) return reject(new DocumentError(\"too_complex\", \"This file unpacks to too much data to read safely.\"));\n // Two entries with one name could show different content to different readers.\n if (entries.has(entry.fileName)) return reject(new DocumentError(\"corrupt\", CORRUPT));\n entries.set(entry.fileName, entry);\n zip.readEntry();\n });\n const read = (name: string) =>\n new Promise<Buffer>((done, fail) => {\n const entry = entries.get(name);\n if (!entry) return fail(new DocumentError(\"corrupt\", CORRUPT));\n const tooMuch = new DocumentError(\"too_complex\", \"This file unpacks to too much data to read safely.\");\n if (entry.uncompressedSize > MAX_XML_PART_BYTES) return fail(tooMuch);\n zip.openReadStream(entry, (streamError, stream) => {\n if (streamError) return fail(new DocumentError(\"corrupt\", CORRUPT));\n const chunks: Buffer[] = [];\n stream.on(\"data\", (chunk: Buffer) => {\n unpacked += chunk.length;\n if (unpacked > MAX_XML_TOTAL_BYTES) stream.destroy(tooMuch);\n else chunks.push(chunk);\n });\n stream.on(\"error\", (error) => fail(error instanceof DocumentError ? error : new DocumentError(\"corrupt\", CORRUPT)));\n stream.on(\"end\", () => done(Buffer.concat(chunks)));\n });\n });\n zip.on(\"end\", () => resolve({ names: new Set(entries.keys()), read, close: () => zip.close() }));\n zip.readEntry();\n });\n });\n}\n\n/** Office XML is UTF-8 or UTF-16. Any other declared encoding reads as damaged text, not as a risk. */\nfunction decodeXml(bytes: Buffer): string {\n const [a, b] = bytes;\n const encoding = (a === 0xff && b === 0xfe) || (a === 0x3c && b === 0) ? \"utf-16le\" : (a === 0xfe && b === 0xff) || (a === 0 && b === 0x3c) ? \"utf-16be\" : \"utf-8\";\n const xml = new TextDecoder(encoding).decode(bytes);\n // Office never writes a DTD, so one means a crafted file. Nothing here expands entities anyway: see xmlTokens.\n if (/<!(DOCTYPE|ENTITY)/i.test(xml)) throw new DocumentError(\"corrupt\", \"This file contains unsafe XML and can't be read.\");\n return xml;\n}\n\nconst NAMED_ENTITIES: Record<string, string> = { amp: \"&\", lt: \"<\", gt: \">\", quot: '\"', apos: \"'\" };\n\n/** The five predefined entities and character references. Any other entity stays as literal text. */\nfunction decodeEntities(text: string): string {\n return text.replace(/&(?:#(\\d{1,7})|#x([\\da-f]{1,6})|(amp|lt|gt|quot|apos));/gi, (whole, dec?: string, hex?: string, name?: string) => {\n if (name) return NAMED_ENTITIES[name.toLowerCase()] ?? whole;\n const code = dec ? Number(dec) : parseInt(hex!, 16);\n return code <= 0x10ffff ? String.fromCodePoint(code) : whole;\n });\n}\n\n/** Splits XML into tags and text without interpreting it, which is all these few element types need. */\nfunction* xmlTokens(xml: string) {\n for (const [token] of xml.matchAll(/<!--[\\s\\S]*?-->|<!\\[CDATA\\[[\\s\\S]*?\\]\\]>|<[^>]*>|[^<]+/g)) {\n if (token.startsWith(\"<!--\") || token.startsWith(\"<?\")) continue;\n if (token.startsWith(\"<![CDATA[\")) yield { text: token.slice(9, -3) };\n else if (token.startsWith(\"<\")) {\n const name = /^<\\/?(?:[\\w.-]+:)?([\\w.-]+)/.exec(token)?.[1] ?? \"\";\n yield { tag: token, name, closing: token.startsWith(\"</\"), selfClosing: token.endsWith(\"/>\") };\n } else yield { text: decodeEntities(token) };\n }\n}\n\n/** Slide text: `a:t` runs and `m:t` equation runs, without `mc:Fallback`, which repeats content for old readers. */\nfunction slideText(xml: string): string {\n let out = \"\";\n let inText = 0;\n let inFallback = 0;\n for (const token of xmlTokens(xml)) {\n if (\"text\" in token) {\n if (inText > 0 && inFallback === 0) out += token.text;\n continue;\n }\n const delta = token.selfClosing ? 0 : token.closing ? -1 : 1;\n if (token.name === \"Fallback\") inFallback = Math.max(0, inFallback + delta);\n else if (token.name === \"t\") inText = Math.max(0, inText + delta);\n else if (inFallback === 0 && (token.name === \"br\" || (token.name === \"p\" && token.closing))) out += \"\\n\";\n }\n return out;\n}\n\nconst attribute = (tag: string, name: RegExp) => new RegExp(`\\\\s${name.source}\\\\s*=\\\\s*[\"']([^\"']*)[\"']`).exec(tag)?.[1];\n\n/** The presentation's slide parts in the order it shows them, which needn't match their file names. */\nfunction slideOrder(presentationXml: string, relsXml: string): (string | undefined)[] {\n const targets = new Map<string, string>();\n for (const token of xmlTokens(relsXml)) {\n if (\"tag\" in token && token.name === \"Relationship\" && !token.closing) {\n const id = attribute(token.tag, /Id/);\n const target = attribute(token.tag, /Target/);\n if (id && target) targets.set(id, target);\n }\n }\n const slides: (string | undefined)[] = [];\n for (const token of xmlTokens(presentationXml)) {\n if (\"tag\" in token && token.name === \"sldId\" && !token.closing) {\n const target = targets.get(attribute(token.tag, /[\\w.-]+:id/) ?? \"\");\n // Relationship targets come from the file, so only a part directly under ppt/slides is ever opened.\n const part = target && (target.startsWith(\"/\") ? posix.normalize(target).slice(1) : posix.join(\"ppt\", target));\n slides.push(part && /^ppt\\/slides\\/[^/]+\\.xml$/.test(part) ? part : undefined);\n }\n }\n return slides;\n}\n\nasync function extractPptx(bytes: Uint8Array): Promise<ExtractedDocument> {\n const zip = await openZip(bytes);\n try {\n if (!zip.names.has(\"[Content_Types].xml\")) throw new DocumentError(\"wrong_type\", WRONG_TYPE);\n const types = decodeXml(await zip.read(\"[Content_Types].xml\"));\n if (!/officedocument\\.presentationml\\.(presentation|slideshow|template)\\.main\\+xml/i.test(types)) throw new DocumentError(\"wrong_type\", WRONG_TYPE);\n if (!zip.names.has(\"ppt/presentation.xml\") || !zip.names.has(\"ppt/_rels/presentation.xml.rels\")) throw new DocumentError(\"corrupt\", CORRUPT);\n\n const slides = slideOrder(decodeXml(await zip.read(\"ppt/presentation.xml\")), decodeXml(await zip.read(\"ppt/_rels/presentation.xml.rels\")));\n if (slides.length > MAX_PAGES) throw new DocumentError(\"too_complex\", `This file has more than ${MAX_PAGES} slides.`);\n const pages: ExtractedPage[] = [];\n let chars = 0;\n for (const [index, part] of slides.entries()) {\n if (!part || !zip.names.has(part)) continue;\n const text = slideText(decodeXml(await zip.read(part)));\n chars += text.length;\n if (chars > MAX_TEXT_CHARS) throw new DocumentError(\"too_complex\", TOO_MUCH_TEXT);\n pages.push({ page: index + 1, text });\n }\n return finish(\"pptx\", slides.length, pages);\n } finally {\n zip.close();\n }\n}\n\nconst startsWith = (bytes: Uint8Array, magic: number[]) => magic.every((byte, i) => bytes[i] === byte);\n\n/** Dispatches on what the file contains, whatever it is called. */\nexport async function extractDocument(bytes: Uint8Array): Promise<ExtractedDocument> {\n const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n // The PDF spec lets a header sit a little way into the file.\n if (buffer.subarray(0, 1024).includes(\"%PDF-\")) return extractPdf(bytes);\n if (startsWith(bytes, [0x50, 0x4b, 0x03, 0x04])) return extractPptx(bytes);\n // Office stores a password-protected .pptx in the same container as an old .ppt.\n if (startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0]) && buffer.includes(Buffer.from(\"EncryptedPackage\", \"utf16le\"))) throw new DocumentError(\"encrypted\", ENCRYPTED);\n throw new DocumentError(\"wrong_type\", WRONG_TYPE);\n}\n", "import { DocumentError, extractDocument } from \"./extract.js\";\n\n/**\n * The entry of an extraction subprocess (see extract-in-child.ts): reads one file sent over IPC, reports\n * its text or a DocumentError, and exits. The parent enforces the memory limits and kills this process if\n * it runs too long, so nothing here needs to clean up.\n */\nconst send = (message: unknown) => process.send!(message, () => process.exit(0));\n\n// The parent is gone, so nobody is waiting for the result.\nprocess.once(\"disconnect\", () => process.exit(0));\nprocess.once(\"message\", async (message: { bytes?: unknown; reportRss?: unknown }) => {\n // Where the parent can't read this process's memory from /proc, it relies on these reports.\n if (message?.reportRss === true) setInterval(() => process.send!({ rss: process.memoryUsage.rss() }), 100).unref();\n try {\n if (!(message?.bytes instanceof Uint8Array)) throw new DocumentError(\"failed\", \"Couldn't read this file.\");\n send({ ok: true, document: await extractDocument(message.bytes) });\n } catch (error) {\n send(\n error instanceof DocumentError\n ? { ok: false, code: error.code, message: error.message }\n : { ok: false, code: \"corrupt\", message: \"This file looks damaged and can't be read.\" },\n );\n }\n});\n"],
5
+ "mappings": ";AAAA,SAAS,aAAa;AACtB,SAAS,wBAAwB;AACjC,OAAO,WAAW;AAUX,IAAM,gBAAN,cAA4B,MAAM;AAAA,EACvC,YACW,MACT,SACA;AACA,UAAM,OAAO;AAHJ;AAAA,EAIX;AAAA,EAJW;AAKb;AAeA,IAAM,YAAY;AAClB,IAAM,iBAAiB;AAEvB,IAAM,iBAAiB;AAEvB,IAAM,kBAAkB;AAExB,IAAM,yBAAyB,OAAO,OAAO;AAE7C,IAAM,qBAAqB,KAAK,OAAO;AACvC,IAAM,sBAAsB,MAAM,OAAO;AAEzC,IAAM,aAAa;AACnB,IAAM,UAAU;AAChB,IAAM,YAAY;AAClB,IAAM,gBAAgB;AAOf,SAAS,UAAU,MAAsB;AAC9C,SAAO,KACJ,QAAQ,kBAAkB,EAAE,EAC5B,QAAQ,8BAA8B,CAAC,MAAc,EAAE,UAAU,MAAM,CAAC,EACxE,QAAQ,WAAW,GAAG,EACtB,QAAQ,WAAW,IAAI,EACvB,QAAQ,WAAW,MAAM,EACzB,KAAK;AACV;AAEA,SAAS,OAAO,MAAiC,WAAmB,KAAyC;AAC3G,QAAM,QAAQ,IAAI,IAAI,CAAC,EAAE,MAAM,KAAK,OAAO,EAAE,MAAM,MAAM,UAAU,IAAI,EAAE,EAAE,EAAE,OAAO,CAAC,SAAS,KAAK,IAAI;AACvG,MAAI,MAAM,OAAO,CAAC,KAAK,SAAS,MAAM,KAAK,KAAK,QAAQ,OAAO,EAAE,EAAE,QAAQ,CAAC,IAAI,gBAAgB;AAC9F,UAAM,IAAI;AAAA,MACR;AAAA,MACA,SAAS,QACL,0HACA;AAAA,IACN;AAAA,EACF;AACA,SAAO,EAAE,MAAM,WAAW,MAAM;AAClC;AAEA,eAAe,WAAW,OAA+C;AACvE,QAAM,EAAE,YAAY,IAAI,MAAM,iBAAiB;AAG/C,QAAM,OAAO,YAAY,EAAE,MAAM,IAAI,WAAW,KAAK,GAAG,gBAAgB,OAAO,iBAAiB,MAAM,WAAW,EAAE,CAAC;AACpH,MAAI;AACF,UAAM,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC,UAAmB;AACvD,YAAO,OAAiB,SAAS,sBAAsB,IAAI,cAAc,aAAa,SAAS,IAAI,IAAI,cAAc,WAAW,OAAO;AAAA,IACzI,CAAC;AACD,QAAI,IAAI,WAAW,UAAW,OAAM,IAAI,cAAc,eAAe,0BAA0B,SAAS,SAAS;AACjH,UAAM,QAAyB,CAAC;AAChC,QAAI,QAAQ;AACZ,aAAS,OAAO,GAAG,QAAQ,IAAI,UAAU,QAAQ;AAE/C,YAAM,OAAO,MAAM,IAChB,QAAQ,IAAI,EACZ,KAAK,OAAO,WAAW;AACtB,cAAM,UAAU,MAAM,OAAO,eAAe;AAC5C,eAAO,QAAQ;AAEf,eAAO,QAAQ,MAAM,IAAI,CAAC,SAAU,SAAS,OAAO,KAAK,OAAO,KAAK,SAAS,OAAO,MAAM,EAAG,EAAE,KAAK,EAAE;AAAA,MACzG,CAAC,EACA,MAAM,MAAM,EAAE;AACjB,eAAS,KAAK;AACd,UAAI,QAAQ,eAAgB,OAAM,IAAI,cAAc,eAAe,aAAa;AAChF,YAAM,KAAK,EAAE,MAAM,KAAK,CAAC;AAAA,IAC3B;AACA,WAAO,OAAO,OAAO,IAAI,UAAU,KAAK;AAAA,EAC1C,UAAE;AACA,UAAM,KAAK,QAAQ,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACrC;AACF;AAGA,SAAS,QAAQ,OAAgH;AAC/H,QAAM,SAAS,OAAO,KAAK,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AAC3E,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AAEtC,UAAM,WAAW,QAAQ,EAAE,aAAa,MAAM,iBAAiB,MAAM,oBAAoB,KAAK,GAAG,CAAC,WAAW,QAAQ;AACnH,UAAI,UAAW,QAAO,OAAO,IAAI,cAAc,WAAW,OAAO,CAAC;AAClE,UAAI,IAAI,aAAa,gBAAiB,QAAO,OAAO,IAAI,cAAc,eAAe,mDAAmD,CAAC;AACzI,YAAM,UAAU,oBAAI,IAAyB;AAC7C,UAAI,WAAW;AACf,UAAI,WAAW;AACf,UAAI,GAAG,SAAS,MAAM,OAAO,IAAI,cAAc,WAAW,OAAO,CAAC,CAAC;AACnE,UAAI,GAAG,SAAS,CAAC,UAAuB;AACtC,YAAI,MAAM,YAAY,EAAG,QAAO,OAAO,IAAI,cAAc,aAAa,SAAS,CAAC;AAChF,oBAAY,MAAM;AAClB,YAAI,WAAW,uBAAwB,QAAO,OAAO,IAAI,cAAc,eAAe,oDAAoD,CAAC;AAE3I,YAAI,QAAQ,IAAI,MAAM,QAAQ,EAAG,QAAO,OAAO,IAAI,cAAc,WAAW,OAAO,CAAC;AACpF,gBAAQ,IAAI,MAAM,UAAU,KAAK;AACjC,YAAI,UAAU;AAAA,MAChB,CAAC;AACD,YAAM,OAAO,CAAC,SACZ,IAAI,QAAgB,CAAC,MAAM,SAAS;AAClC,cAAM,QAAQ,QAAQ,IAAI,IAAI;AAC9B,YAAI,CAAC,MAAO,QAAO,KAAK,IAAI,cAAc,WAAW,OAAO,CAAC;AAC7D,cAAM,UAAU,IAAI,cAAc,eAAe,oDAAoD;AACrG,YAAI,MAAM,mBAAmB,mBAAoB,QAAO,KAAK,OAAO;AACpE,YAAI,eAAe,OAAO,CAAC,aAAa,WAAW;AACjD,cAAI,YAAa,QAAO,KAAK,IAAI,cAAc,WAAW,OAAO,CAAC;AAClE,gBAAM,SAAmB,CAAC;AAC1B,iBAAO,GAAG,QAAQ,CAAC,UAAkB;AACnC,wBAAY,MAAM;AAClB,gBAAI,WAAW,oBAAqB,QAAO,QAAQ,OAAO;AAAA,gBACrD,QAAO,KAAK,KAAK;AAAA,UACxB,CAAC;AACD,iBAAO,GAAG,SAAS,CAAC,UAAU,KAAK,iBAAiB,gBAAgB,QAAQ,IAAI,cAAc,WAAW,OAAO,CAAC,CAAC;AAClH,iBAAO,GAAG,OAAO,MAAM,KAAK,OAAO,OAAO,MAAM,CAAC,CAAC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AACH,UAAI,GAAG,OAAO,MAAM,QAAQ,EAAE,OAAO,IAAI,IAAI,QAAQ,KAAK,CAAC,GAAG,MAAM,OAAO,MAAM,IAAI,MAAM,EAAE,CAAC,CAAC;AAC/F,UAAI,UAAU;AAAA,IAChB,CAAC;AAAA,EACH,CAAC;AACH;AAGA,SAAS,UAAU,OAAuB;AACxC,QAAM,CAAC,GAAG,CAAC,IAAI;AACf,QAAM,WAAY,MAAM,OAAQ,MAAM,OAAU,MAAM,MAAQ,MAAM,IAAK,aAAc,MAAM,OAAQ,MAAM,OAAU,MAAM,KAAK,MAAM,KAAQ,aAAa;AAC3J,QAAM,MAAM,IAAI,YAAY,QAAQ,EAAE,OAAO,KAAK;AAElD,MAAI,sBAAsB,KAAK,GAAG,EAAG,OAAM,IAAI,cAAc,WAAW,kDAAkD;AAC1H,SAAO;AACT;AAEA,IAAM,iBAAyC,EAAE,KAAK,KAAK,IAAI,KAAK,IAAI,KAAK,MAAM,KAAK,MAAM,IAAI;AAGlG,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,QAAQ,6DAA6D,CAAC,OAAO,KAAc,KAAc,SAAkB;AACrI,QAAI,KAAM,QAAO,eAAe,KAAK,YAAY,CAAC,KAAK;AACvD,UAAM,OAAO,MAAM,OAAO,GAAG,IAAI,SAAS,KAAM,EAAE;AAClD,WAAO,QAAQ,UAAW,OAAO,cAAc,IAAI,IAAI;AAAA,EACzD,CAAC;AACH;AAGA,UAAU,UAAU,KAAa;AAC/B,aAAW,CAAC,KAAK,KAAK,IAAI,SAAS,yDAAyD,GAAG;AAC7F,QAAI,MAAM,WAAW,MAAM,KAAK,MAAM,WAAW,IAAI,EAAG;AACxD,QAAI,MAAM,WAAW,WAAW,EAAG,OAAM,EAAE,MAAM,MAAM,MAAM,GAAG,EAAE,EAAE;AAAA,aAC3D,MAAM,WAAW,GAAG,GAAG;AAC9B,YAAM,OAAO,8BAA8B,KAAK,KAAK,IAAI,CAAC,KAAK;AAC/D,YAAM,EAAE,KAAK,OAAO,MAAM,SAAS,MAAM,WAAW,IAAI,GAAG,aAAa,MAAM,SAAS,IAAI,EAAE;AAAA,IAC/F,MAAO,OAAM,EAAE,MAAM,eAAe,KAAK,EAAE;AAAA,EAC7C;AACF;AAGA,SAAS,UAAU,KAAqB;AACtC,MAAI,MAAM;AACV,MAAI,SAAS;AACb,MAAI,aAAa;AACjB,aAAW,SAAS,UAAU,GAAG,GAAG;AAClC,QAAI,UAAU,OAAO;AACnB,UAAI,SAAS,KAAK,eAAe,EAAG,QAAO,MAAM;AACjD;AAAA,IACF;AACA,UAAM,QAAQ,MAAM,cAAc,IAAI,MAAM,UAAU,KAAK;AAC3D,QAAI,MAAM,SAAS,WAAY,cAAa,KAAK,IAAI,GAAG,aAAa,KAAK;AAAA,aACjE,MAAM,SAAS,IAAK,UAAS,KAAK,IAAI,GAAG,SAAS,KAAK;AAAA,aACvD,eAAe,MAAM,MAAM,SAAS,QAAS,MAAM,SAAS,OAAO,MAAM,SAAW,QAAO;AAAA,EACtG;AACA,SAAO;AACT;AAEA,IAAM,YAAY,CAAC,KAAa,SAAiB,IAAI,OAAO,MAAM,KAAK,MAAM,2BAA2B,EAAE,KAAK,GAAG,IAAI,CAAC;AAGvH,SAAS,WAAW,iBAAyB,SAAyC;AACpF,QAAM,UAAU,oBAAI,IAAoB;AACxC,aAAW,SAAS,UAAU,OAAO,GAAG;AACtC,QAAI,SAAS,SAAS,MAAM,SAAS,kBAAkB,CAAC,MAAM,SAAS;AACrE,YAAM,KAAK,UAAU,MAAM,KAAK,IAAI;AACpC,YAAM,SAAS,UAAU,MAAM,KAAK,QAAQ;AAC5C,UAAI,MAAM,OAAQ,SAAQ,IAAI,IAAI,MAAM;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,SAAiC,CAAC;AACxC,aAAW,SAAS,UAAU,eAAe,GAAG;AAC9C,QAAI,SAAS,SAAS,MAAM,SAAS,WAAW,CAAC,MAAM,SAAS;AAC9D,YAAM,SAAS,QAAQ,IAAI,UAAU,MAAM,KAAK,YAAY,KAAK,EAAE;AAEnE,YAAM,OAAO,WAAW,OAAO,WAAW,GAAG,IAAI,MAAM,UAAU,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,KAAK,OAAO,MAAM;AAC5G,aAAO,KAAK,QAAQ,4BAA4B,KAAK,IAAI,IAAI,OAAO,MAAS;AAAA,IAC/E;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAe,YAAY,OAA+C;AACxE,QAAM,MAAM,MAAM,QAAQ,KAAK;AAC/B,MAAI;AACF,QAAI,CAAC,IAAI,MAAM,IAAI,qBAAqB,EAAG,OAAM,IAAI,cAAc,cAAc,UAAU;AAC3F,UAAM,QAAQ,UAAU,MAAM,IAAI,KAAK,qBAAqB,CAAC;AAC7D,QAAI,CAAC,gFAAgF,KAAK,KAAK,EAAG,OAAM,IAAI,cAAc,cAAc,UAAU;AAClJ,QAAI,CAAC,IAAI,MAAM,IAAI,sBAAsB,KAAK,CAAC,IAAI,MAAM,IAAI,iCAAiC,EAAG,OAAM,IAAI,cAAc,WAAW,OAAO;AAE3I,UAAM,SAAS,WAAW,UAAU,MAAM,IAAI,KAAK,sBAAsB,CAAC,GAAG,UAAU,MAAM,IAAI,KAAK,iCAAiC,CAAC,CAAC;AACzI,QAAI,OAAO,SAAS,UAAW,OAAM,IAAI,cAAc,eAAe,2BAA2B,SAAS,UAAU;AACpH,UAAM,QAAyB,CAAC;AAChC,QAAI,QAAQ;AACZ,eAAW,CAAC,OAAO,IAAI,KAAK,OAAO,QAAQ,GAAG;AAC5C,UAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,IAAI,IAAI,EAAG;AACnC,YAAM,OAAO,UAAU,UAAU,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC;AACtD,eAAS,KAAK;AACd,UAAI,QAAQ,eAAgB,OAAM,IAAI,cAAc,eAAe,aAAa;AAChF,YAAM,KAAK,EAAE,MAAM,QAAQ,GAAG,KAAK,CAAC;AAAA,IACtC;AACA,WAAO,OAAO,QAAQ,OAAO,QAAQ,KAAK;AAAA,EAC5C,UAAE;AACA,QAAI,MAAM;AAAA,EACZ;AACF;AAEA,IAAM,aAAa,CAAC,OAAmB,UAAoB,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,IAAI;AAGrG,eAAsB,gBAAgB,OAA+C;AACnF,QAAM,SAAS,OAAO,KAAK,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AAE3E,MAAI,OAAO,SAAS,GAAG,IAAI,EAAE,SAAS,OAAO,EAAG,QAAO,WAAW,KAAK;AACvE,MAAI,WAAW,OAAO,CAAC,IAAM,IAAM,GAAM,CAAI,CAAC,EAAG,QAAO,YAAY,KAAK;AAEzE,MAAI,WAAW,OAAO,CAAC,KAAM,KAAM,IAAM,GAAI,CAAC,KAAK,OAAO,SAAS,OAAO,KAAK,oBAAoB,SAAS,CAAC,EAAG,OAAM,IAAI,cAAc,aAAa,SAAS;AAC9J,QAAM,IAAI,cAAc,cAAc,UAAU;AAClD;;;ACrQA,IAAM,OAAO,CAAC,YAAqB,QAAQ,KAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,CAAC;AAG/E,QAAQ,KAAK,cAAc,MAAM,QAAQ,KAAK,CAAC,CAAC;AAChD,QAAQ,KAAK,WAAW,OAAO,YAAsD;AAEnF,MAAI,SAAS,cAAc,KAAM,aAAY,MAAM,QAAQ,KAAM,EAAE,KAAK,QAAQ,YAAY,IAAI,EAAE,CAAC,GAAG,GAAG,EAAE,MAAM;AACjH,MAAI;AACF,QAAI,EAAE,SAAS,iBAAiB,YAAa,OAAM,IAAI,cAAc,UAAU,0BAA0B;AACzG,SAAK,EAAE,IAAI,MAAM,UAAU,MAAM,gBAAgB,QAAQ,KAAK,EAAE,CAAC;AAAA,EACnE,SAAS,OAAO;AACd;AAAA,MACE,iBAAiB,gBACb,EAAE,IAAI,OAAO,MAAM,MAAM,MAAM,SAAS,MAAM,QAAQ,IACtD,EAAE,IAAI,OAAO,MAAM,WAAW,SAAS,6CAA6C;AAAA,IAC1F;AAAA,EACF;AACF,CAAC;",
6
+ "names": []
7
+ }
@@ -0,0 +1,33 @@
1
+ CREATE TABLE `messages` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `session_id` text NOT NULL,
4
+ `seq` integer NOT NULL,
5
+ `role` text NOT NULL,
6
+ `parts` text NOT NULL,
7
+ `created_at` integer NOT NULL,
8
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade
9
+ );
10
+ --> statement-breakpoint
11
+ CREATE UNIQUE INDEX `messages_session_seq_idx` ON `messages` (`session_id`,`seq`);--> statement-breakpoint
12
+ CREATE TABLE `runs` (
13
+ `id` text PRIMARY KEY NOT NULL,
14
+ `session_id` text NOT NULL,
15
+ `user_message_id` text NOT NULL,
16
+ `assistant_message_id` text,
17
+ `status` text NOT NULL,
18
+ `error` text,
19
+ `started_at` integer NOT NULL,
20
+ `ended_at` integer,
21
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade,
22
+ FOREIGN KEY (`user_message_id`) REFERENCES `messages`(`id`) ON UPDATE no action ON DELETE cascade,
23
+ FOREIGN KEY (`assistant_message_id`) REFERENCES `messages`(`id`) ON UPDATE no action ON DELETE set null
24
+ );
25
+ --> statement-breakpoint
26
+ CREATE INDEX `runs_session_idx` ON `runs` (`session_id`,`started_at`);--> statement-breakpoint
27
+ CREATE UNIQUE INDEX `runs_one_running_per_session_idx` ON `runs` (`session_id`) WHERE "runs"."status" = 'running';--> statement-breakpoint
28
+ CREATE TABLE `sessions` (
29
+ `id` text PRIMARY KEY NOT NULL,
30
+ `title` text NOT NULL,
31
+ `created_at` integer NOT NULL,
32
+ `updated_at` integer NOT NULL
33
+ );
@@ -0,0 +1,30 @@
1
+ CREATE TABLE `artifacts` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `session_id` text NOT NULL,
4
+ `run_id` text NOT NULL,
5
+ `tool_call_id` text NOT NULL,
6
+ `kind` text NOT NULL,
7
+ `version` integer NOT NULL,
8
+ `spec` text NOT NULL,
9
+ `created_at` integer NOT NULL,
10
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade,
11
+ FOREIGN KEY (`run_id`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE cascade
12
+ );
13
+ --> statement-breakpoint
14
+ CREATE INDEX `artifacts_session_idx` ON `artifacts` (`session_id`,`created_at`);--> statement-breakpoint
15
+ CREATE UNIQUE INDEX `artifacts_tool_call_idx` ON `artifacts` (`run_id`,`tool_call_id`);--> statement-breakpoint
16
+ CREATE TABLE `attempts` (
17
+ `id` text PRIMARY KEY NOT NULL,
18
+ `artifact_id` text NOT NULL,
19
+ `session_id` text NOT NULL,
20
+ `answer` text NOT NULL,
21
+ `grade` text NOT NULL,
22
+ `correct` integer NOT NULL,
23
+ `concept` text,
24
+ `created_at` integer NOT NULL,
25
+ FOREIGN KEY (`artifact_id`) REFERENCES `artifacts`(`id`) ON UPDATE no action ON DELETE cascade,
26
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade
27
+ );
28
+ --> statement-breakpoint
29
+ CREATE UNIQUE INDEX `attempts_one_per_artifact_idx` ON `attempts` (`artifact_id`);--> statement-breakpoint
30
+ CREATE INDEX `attempts_session_idx` ON `attempts` (`session_id`,`created_at`);
@@ -0,0 +1 @@
1
+ CREATE INDEX `attempts_created_idx` ON `attempts` (`created_at`);
@@ -0,0 +1,20 @@
1
+ CREATE TABLE `memories` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `kind` text NOT NULL,
4
+ `text` text NOT NULL,
5
+ `concept` text,
6
+ `basis` text NOT NULL,
7
+ `attempt_id` text,
8
+ `session_id` text,
9
+ `run_id` text,
10
+ `status` text NOT NULL,
11
+ `replaces_id` text,
12
+ `created_at` integer NOT NULL,
13
+ `edited_at` integer,
14
+ FOREIGN KEY (`attempt_id`) REFERENCES `attempts`(`id`) ON UPDATE no action ON DELETE set null,
15
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE set null,
16
+ FOREIGN KEY (`run_id`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE set null
17
+ );
18
+ --> statement-breakpoint
19
+ CREATE INDEX `memories_status_idx` ON `memories` (`status`,`created_at`);--> statement-breakpoint
20
+ CREATE INDEX `memories_run_idx` ON `memories` (`run_id`);
@@ -0,0 +1,40 @@
1
+ CREATE TABLE `document_chunks` (
2
+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
3
+ `document_id` text NOT NULL,
4
+ `page` integer NOT NULL,
5
+ `seq` integer NOT NULL,
6
+ `text` text NOT NULL,
7
+ FOREIGN KEY (`document_id`) REFERENCES `documents`(`id`) ON UPDATE no action ON DELETE cascade
8
+ );
9
+ --> statement-breakpoint
10
+ CREATE UNIQUE INDEX `document_chunks_document_seq_idx` ON `document_chunks` (`document_id`,`seq`);--> statement-breakpoint
11
+ CREATE TABLE `documents` (
12
+ `id` text PRIMARY KEY NOT NULL,
13
+ `session_id` text NOT NULL,
14
+ `filename` text NOT NULL,
15
+ `kind` text NOT NULL,
16
+ `byte_size` integer NOT NULL,
17
+ `text_chars` integer NOT NULL,
18
+ `page_count` integer NOT NULL,
19
+ `pages_with_text` integer NOT NULL,
20
+ `status` text NOT NULL,
21
+ `created_at` integer NOT NULL,
22
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade
23
+ );
24
+ --> statement-breakpoint
25
+ CREATE INDEX `documents_session_idx` ON `documents` (`session_id`,`created_at`);--> statement-breakpoint
26
+ -- Hand-written: Drizzle doesn't model FTS5. An external-content index over document_chunks, kept in sync by triggers, which also fire on foreign-key cascades.
27
+ CREATE VIRTUAL TABLE `document_chunks_fts` USING fts5(`text`, content=`document_chunks`, content_rowid=`id`, tokenize='porter unicode61 remove_diacritics 2');
28
+ --> statement-breakpoint
29
+ CREATE TRIGGER `document_chunks_fts_insert` AFTER INSERT ON `document_chunks` BEGIN
30
+ INSERT INTO `document_chunks_fts`(rowid, `text`) VALUES (new.`id`, new.`text`);
31
+ END;
32
+ --> statement-breakpoint
33
+ CREATE TRIGGER `document_chunks_fts_delete` AFTER DELETE ON `document_chunks` BEGIN
34
+ INSERT INTO `document_chunks_fts`(`document_chunks_fts`, rowid, `text`) VALUES ('delete', old.`id`, old.`text`);
35
+ END;
36
+ --> statement-breakpoint
37
+ CREATE TRIGGER `document_chunks_fts_update` AFTER UPDATE ON `document_chunks` BEGIN
38
+ INSERT INTO `document_chunks_fts`(`document_chunks_fts`, rowid, `text`) VALUES ('delete', old.`id`, old.`text`);
39
+ INSERT INTO `document_chunks_fts`(rowid, `text`) VALUES (new.`id`, new.`text`);
40
+ END;
@@ -0,0 +1 @@
1
+ ALTER TABLE `documents` ADD `message_id` text REFERENCES messages(id) ON DELETE SET NULL;
@@ -0,0 +1,2 @@
1
+ ALTER TABLE `runs` ADD `continuation_of` text;--> statement-breakpoint
2
+ CREATE UNIQUE INDEX `runs_continuation_once_idx` ON `runs` (`continuation_of`);
@@ -0,0 +1,24 @@
1
+ CREATE TABLE `courses` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `session_id` text NOT NULL,
4
+ `goal` text NOT NULL,
5
+ `status` text NOT NULL,
6
+ `created_at` integer NOT NULL,
7
+ `updated_at` integer NOT NULL,
8
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade
9
+ );
10
+ --> statement-breakpoint
11
+ CREATE UNIQUE INDEX `courses_session_idx` ON `courses` (`session_id`);--> statement-breakpoint
12
+ CREATE TABLE `syllabus_versions` (
13
+ `id` text PRIMARY KEY NOT NULL,
14
+ `course_id` text NOT NULL,
15
+ `version` integer NOT NULL,
16
+ `topics` text NOT NULL,
17
+ `known` text NOT NULL,
18
+ `run_id` text,
19
+ `created_at` integer NOT NULL,
20
+ FOREIGN KEY (`course_id`) REFERENCES `courses`(`id`) ON UPDATE no action ON DELETE cascade,
21
+ FOREIGN KEY (`run_id`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE set null
22
+ );
23
+ --> statement-breakpoint
24
+ CREATE UNIQUE INDEX `syllabus_versions_course_version_idx` ON `syllabus_versions` (`course_id`,`version`);
@@ -0,0 +1,42 @@
1
+ CREATE TABLE `lesson_sections` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `lesson_id` text NOT NULL,
4
+ `kind` text NOT NULL,
5
+ `title` text NOT NULL,
6
+ `markdown` text,
7
+ `artifact_id` text,
8
+ `created_at` integer NOT NULL,
9
+ FOREIGN KEY (`lesson_id`) REFERENCES `lessons`(`id`) ON UPDATE no action ON DELETE cascade,
10
+ FOREIGN KEY (`artifact_id`) REFERENCES `artifacts`(`id`) ON UPDATE no action ON DELETE cascade
11
+ );
12
+ --> statement-breakpoint
13
+ CREATE INDEX `lesson_sections_lesson_idx` ON `lesson_sections` (`lesson_id`);--> statement-breakpoint
14
+ CREATE UNIQUE INDEX `lesson_sections_artifact_idx` ON `lesson_sections` (`artifact_id`);--> statement-breakpoint
15
+ CREATE TABLE `lesson_versions` (
16
+ `id` text PRIMARY KEY NOT NULL,
17
+ `lesson_id` text NOT NULL,
18
+ `version` integer NOT NULL,
19
+ `outline` text NOT NULL,
20
+ `section_ids` text NOT NULL,
21
+ `created_at` integer NOT NULL,
22
+ FOREIGN KEY (`lesson_id`) REFERENCES `lessons`(`id`) ON UPDATE no action ON DELETE cascade
23
+ );
24
+ --> statement-breakpoint
25
+ CREATE UNIQUE INDEX `lesson_versions_lesson_version_idx` ON `lesson_versions` (`lesson_id`,`version`);--> statement-breakpoint
26
+ CREATE TABLE `lessons` (
27
+ `id` text PRIMARY KEY NOT NULL,
28
+ `course_id` text NOT NULL,
29
+ `topic_id` text NOT NULL,
30
+ `topic_title` text NOT NULL,
31
+ `position` integer NOT NULL,
32
+ `session_id` text NOT NULL,
33
+ `status` text NOT NULL,
34
+ `error` text,
35
+ `created_at` integer NOT NULL,
36
+ `finished_at` integer,
37
+ FOREIGN KEY (`course_id`) REFERENCES `courses`(`id`) ON UPDATE no action ON DELETE cascade,
38
+ FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE cascade
39
+ );
40
+ --> statement-breakpoint
41
+ CREATE UNIQUE INDEX `lessons_course_topic_idx` ON `lessons` (`course_id`,`topic_id`);--> statement-breakpoint
42
+ CREATE UNIQUE INDEX `lessons_session_idx` ON `lessons` (`session_id`);
@@ -0,0 +1,5 @@
1
+ ALTER TABLE `lesson_versions` ADD `changed_section_ids` text DEFAULT '[]' NOT NULL;--> statement-breakpoint
2
+ ALTER TABLE `lesson_versions` ADD `reason` text;--> statement-breakpoint
3
+ ALTER TABLE `syllabus_versions` ADD `source` text DEFAULT 'interview' NOT NULL;--> statement-breakpoint
4
+ ALTER TABLE `syllabus_versions` ADD `reason` text;--> statement-breakpoint
5
+ ALTER TABLE `syllabus_versions` ADD `changed_topic_ids` text DEFAULT '[]' NOT NULL;
@@ -0,0 +1,2 @@
1
+ ALTER TABLE `lesson_sections` ADD `revisits` text DEFAULT '[]' NOT NULL;--> statement-breakpoint
2
+ ALTER TABLE `syllabus_versions` ADD `trigger_lesson_id` text;
@@ -0,0 +1,8 @@
1
+ CREATE TABLE `settings` (
2
+ `key` text PRIMARY KEY NOT NULL,
3
+ `value` text NOT NULL,
4
+ `updated_at` integer NOT NULL
5
+ );
6
+ --> statement-breakpoint
7
+ ALTER TABLE `runs` ADD `model` text;--> statement-breakpoint
8
+ ALTER TABLE `runs` ADD `effort` text;