dsh-aha-tutor 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.en.md +3 -3
- package/README.md +3 -3
- package/compatibility.json +5 -3
- package/cordis.patch.yml +4 -0
- package/docs/guide.zh-CN.md +9 -7
- package/lib/agent.js +1 -1
- package/lib/client.js +142 -115
- package/lib/index.js +768 -44
- package/lib/{runtime-DfoeKFPj.js → runtime-C3yJfJTT.js} +43 -9
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as documentParts, A as shortPrompt, At as safeRelativePath, B as emptyProgress, C as questionnairePrompt, Ct as normalizeHandout, D as researchQueriesPrompt, Dt as normalizeProject, E as reexplainPrompt, Et as normalizePractice, F as generateJson, G as readMaterials, H as materialImages, I as generateSteps, J as applyEvidence, K as readProfile, L as generateText, M as verifyItemPrompt, Mt as plotToText, N as verifyStepPrompt, Nt as assertId, O as revisePrompt, Ot as normalizeQuestionnaire, P as TutorError, Pt as newId, Q as buildIndex, R as streamCounter, S as projectsPrompt, St as normalizeCheatsheet, T as recognizePrompt, Tt as normalizeLesson, U as materialsSummary, V as masteryFor, W as passagesFor, X as masteryPercent, Y as createConceptState, Z as recommendNext, _ as materialBriefPrompt, _t as intakeSummary, a as assistPrompt, at as extractJson, b as profilePrompt, bt as normalizeBlueprint, c as cheatsheetPrompt, ct as estimateSpeechMs, d as digestPartPrompt, dt as blueprintOutline, et as formatPassages, f as digestPrompt, ft as conceptIndex, g as lessonPrompt, gt as insertLesson, h as handoutPrompt, ht as flattenLessons, i as askPrompt, it as spreadPassages, j as summaryPrompt, jt as text, k as reworkPrompt, kt as outlineForPrompt, l as correctionPrompt, lt as normalizeBoard, m as gradePrompt, mt as findLesson, n as publishRuntime, nt as search, o as blueprintPrompt, ot as BOARD_LIMITS, p as flashcardsPrompt, pt as courseProgress, q as readProgress, r as RECOGNIZE_EMPTY, rt as snippetOf, s as challengePrompt, st as boardToPlainText, tt as passageRange, u as detectLanguage, ut as normalizeOp, v as materialsRevisePrompt, vt as lessonAfter, w as quizPrompt, wt as normalizeItem, x as projectReviewPrompt, xt as normalizeCards, y as practicePrompt, yt as nextLesson, z as EMPTY_PROFILE } from "./runtime-C3yJfJTT.js";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { dirname, join, relative, sep } from "node:path";
|
|
4
4
|
import { credentialKey } from "@deepseek-ai/dsh-credentials";
|
|
5
5
|
import { lstat, mkdir, readFile, readdir, realpath, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
6
|
+
import zlib, { deflateSync } from "node:zlib";
|
|
6
7
|
import { lookup } from "node:dns";
|
|
7
8
|
import http from "node:http";
|
|
8
9
|
import https from "node:https";
|
|
9
10
|
import { isIP } from "node:net";
|
|
10
11
|
import { pipeline } from "node:stream";
|
|
11
|
-
import zlib from "node:zlib";
|
|
12
12
|
import { spawn } from "node:child_process";
|
|
13
13
|
import { existsSync } from "node:fs";
|
|
14
14
|
import { fileURLToPath } from "node:url";
|
|
@@ -1081,6 +1081,484 @@ const USER_ERRORS$1 = [
|
|
|
1081
1081
|
"TutorError"
|
|
1082
1082
|
];
|
|
1083
1083
|
//#endregion
|
|
1084
|
+
//#region src/host/page-image.js
|
|
1085
|
+
const MAX_PIXELS = 4e6;
|
|
1086
|
+
const MAX_EDGE = 2400;
|
|
1087
|
+
const OBJECT_WAIT_MS = 2e4;
|
|
1088
|
+
const IDENTITY = [
|
|
1089
|
+
1,
|
|
1090
|
+
0,
|
|
1091
|
+
0,
|
|
1092
|
+
1,
|
|
1093
|
+
0,
|
|
1094
|
+
0
|
|
1095
|
+
];
|
|
1096
|
+
function multiply(m, n) {
|
|
1097
|
+
return [
|
|
1098
|
+
m[0] * n[0] + m[2] * n[1],
|
|
1099
|
+
m[1] * n[0] + m[3] * n[1],
|
|
1100
|
+
m[0] * n[2] + m[2] * n[3],
|
|
1101
|
+
m[1] * n[2] + m[3] * n[3],
|
|
1102
|
+
m[0] * n[4] + m[2] * n[5] + m[4],
|
|
1103
|
+
m[1] * n[4] + m[3] * n[5] + m[5]
|
|
1104
|
+
];
|
|
1105
|
+
}
|
|
1106
|
+
function invert(m) {
|
|
1107
|
+
const det = m[0] * m[3] - m[1] * m[2];
|
|
1108
|
+
if (!det) return null;
|
|
1109
|
+
return [
|
|
1110
|
+
m[3] / det,
|
|
1111
|
+
-m[1] / det,
|
|
1112
|
+
-m[2] / det,
|
|
1113
|
+
m[0] / det,
|
|
1114
|
+
(m[2] * m[5] - m[3] * m[4]) / det,
|
|
1115
|
+
(m[1] * m[4] - m[0] * m[5]) / det
|
|
1116
|
+
];
|
|
1117
|
+
}
|
|
1118
|
+
const apply$1 = (m, x, y) => [m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5]];
|
|
1119
|
+
function grayOf(color) {
|
|
1120
|
+
const hex = /^#([0-9a-f]{6})$/iu.exec(String(color ?? ""))?.[1];
|
|
1121
|
+
if (!hex) return 0;
|
|
1122
|
+
const value = Number.parseInt(hex, 16);
|
|
1123
|
+
return Math.round(((value >> 16) * 299 + (value >> 8 & 255) * 587 + (value & 255) * 114) / 1e3);
|
|
1124
|
+
}
|
|
1125
|
+
function placements(OPS, operatorList) {
|
|
1126
|
+
const draws = [];
|
|
1127
|
+
let ctm = IDENTITY;
|
|
1128
|
+
let fill = 0;
|
|
1129
|
+
const stack = [];
|
|
1130
|
+
const push = () => stack.push({
|
|
1131
|
+
ctm,
|
|
1132
|
+
fill
|
|
1133
|
+
});
|
|
1134
|
+
const pop = () => {
|
|
1135
|
+
const saved = stack.pop();
|
|
1136
|
+
if (saved) ({ctm, fill} = saved);
|
|
1137
|
+
};
|
|
1138
|
+
const { fnArray, argsArray } = operatorList;
|
|
1139
|
+
for (let index = 0; index < fnArray.length; index += 1) {
|
|
1140
|
+
const args = argsArray[index] ?? [];
|
|
1141
|
+
switch (fnArray[index]) {
|
|
1142
|
+
case OPS.save:
|
|
1143
|
+
push();
|
|
1144
|
+
break;
|
|
1145
|
+
case OPS.restore:
|
|
1146
|
+
case OPS.paintFormXObjectEnd:
|
|
1147
|
+
case OPS.endGroup:
|
|
1148
|
+
pop();
|
|
1149
|
+
break;
|
|
1150
|
+
case OPS.transform:
|
|
1151
|
+
ctm = multiply(ctm, args);
|
|
1152
|
+
break;
|
|
1153
|
+
case OPS.paintFormXObjectBegin:
|
|
1154
|
+
push();
|
|
1155
|
+
if (Array.isArray(args[0])) ctm = multiply(ctm, args[0]);
|
|
1156
|
+
break;
|
|
1157
|
+
case OPS.beginGroup:
|
|
1158
|
+
push();
|
|
1159
|
+
if (Array.isArray(args[0]?.matrix)) ctm = multiply(ctm, args[0].matrix);
|
|
1160
|
+
break;
|
|
1161
|
+
case OPS.setFillRGBColor:
|
|
1162
|
+
fill = grayOf(args[0]);
|
|
1163
|
+
break;
|
|
1164
|
+
case OPS.paintImageXObject:
|
|
1165
|
+
draws.push({
|
|
1166
|
+
matrix: ctm,
|
|
1167
|
+
image: args[0]
|
|
1168
|
+
});
|
|
1169
|
+
break;
|
|
1170
|
+
case OPS.paintInlineImageXObject:
|
|
1171
|
+
draws.push({
|
|
1172
|
+
matrix: ctm,
|
|
1173
|
+
image: args[0]
|
|
1174
|
+
});
|
|
1175
|
+
break;
|
|
1176
|
+
case OPS.paintImageXObjectRepeat: {
|
|
1177
|
+
const [image, scaleX, scaleY, positions = []] = args;
|
|
1178
|
+
for (let at = 0; at + 1 < positions.length; at += 2) draws.push({
|
|
1179
|
+
matrix: multiply(ctm, [
|
|
1180
|
+
scaleX,
|
|
1181
|
+
0,
|
|
1182
|
+
0,
|
|
1183
|
+
scaleY,
|
|
1184
|
+
positions[at],
|
|
1185
|
+
positions[at + 1]
|
|
1186
|
+
]),
|
|
1187
|
+
image
|
|
1188
|
+
});
|
|
1189
|
+
break;
|
|
1190
|
+
}
|
|
1191
|
+
case OPS.paintInlineImageXObjectGroup: {
|
|
1192
|
+
const [image, map = []] = args;
|
|
1193
|
+
for (const entry of map) draws.push({
|
|
1194
|
+
matrix: multiply(ctm, entry.transform),
|
|
1195
|
+
image,
|
|
1196
|
+
region: {
|
|
1197
|
+
x: entry.x,
|
|
1198
|
+
y: entry.y,
|
|
1199
|
+
w: entry.w,
|
|
1200
|
+
h: entry.h
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
break;
|
|
1204
|
+
}
|
|
1205
|
+
case OPS.paintImageMaskXObject:
|
|
1206
|
+
draws.push({
|
|
1207
|
+
matrix: ctm,
|
|
1208
|
+
mask: args[0],
|
|
1209
|
+
fill
|
|
1210
|
+
});
|
|
1211
|
+
break;
|
|
1212
|
+
case OPS.paintImageMaskXObjectGroup:
|
|
1213
|
+
for (const mask of args[0] ?? []) draws.push({
|
|
1214
|
+
matrix: multiply(ctm, mask.transform),
|
|
1215
|
+
mask,
|
|
1216
|
+
fill
|
|
1217
|
+
});
|
|
1218
|
+
break;
|
|
1219
|
+
case OPS.paintImageMaskXObjectRepeat: {
|
|
1220
|
+
const [mask, scaleX, skewX = 0, skewY = 0, scaleY, positions = []] = args;
|
|
1221
|
+
for (let at = 0; at + 1 < positions.length; at += 2) draws.push({
|
|
1222
|
+
matrix: multiply(ctm, [
|
|
1223
|
+
scaleX,
|
|
1224
|
+
skewX,
|
|
1225
|
+
skewY,
|
|
1226
|
+
scaleY,
|
|
1227
|
+
positions[at],
|
|
1228
|
+
positions[at + 1]
|
|
1229
|
+
]),
|
|
1230
|
+
mask,
|
|
1231
|
+
fill
|
|
1232
|
+
});
|
|
1233
|
+
break;
|
|
1234
|
+
}
|
|
1235
|
+
case OPS.paintSolidColorImageMask:
|
|
1236
|
+
draws.push({
|
|
1237
|
+
matrix: ctm,
|
|
1238
|
+
solid: fill
|
|
1239
|
+
});
|
|
1240
|
+
break;
|
|
1241
|
+
default: break;
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
return draws;
|
|
1245
|
+
}
|
|
1246
|
+
async function resolve(page, value) {
|
|
1247
|
+
if (typeof value !== "string") return value ?? null;
|
|
1248
|
+
const objects = value.startsWith("g_") ? page.commonObjs : page.objs;
|
|
1249
|
+
let timer;
|
|
1250
|
+
try {
|
|
1251
|
+
return await Promise.race([new Promise((done) => {
|
|
1252
|
+
objects.get(value, done);
|
|
1253
|
+
}), new Promise((done) => {
|
|
1254
|
+
timer = setTimeout(() => done(null), OBJECT_WAIT_MS);
|
|
1255
|
+
})]);
|
|
1256
|
+
} finally {
|
|
1257
|
+
clearTimeout(timer);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
function shrink(image, { factor = 1, mask = false, region } = {}) {
|
|
1261
|
+
const { width, height, kind, data } = image;
|
|
1262
|
+
const x0 = region?.x ?? 0;
|
|
1263
|
+
const y0 = region?.y ?? 0;
|
|
1264
|
+
const w = Math.max(1, Math.min(region?.w ?? width, width - x0));
|
|
1265
|
+
const h = Math.max(1, Math.min(region?.h ?? height, height - y0));
|
|
1266
|
+
const outW = Math.max(1, Math.ceil(w / factor));
|
|
1267
|
+
const outH = Math.max(1, Math.ceil(h / factor));
|
|
1268
|
+
const gray = new Uint8Array(outW * outH);
|
|
1269
|
+
const alpha = new Uint8Array(outW * outH);
|
|
1270
|
+
let opaque = true;
|
|
1271
|
+
const bits = mask || kind === 1;
|
|
1272
|
+
const stride = bits ? width + 7 >> 3 : width * (kind === 3 ? 4 : 3);
|
|
1273
|
+
for (let oy = 0; oy < outH; oy += 1) {
|
|
1274
|
+
const yStart = y0 + oy * factor;
|
|
1275
|
+
const yEnd = Math.min(y0 + h, yStart + factor);
|
|
1276
|
+
for (let ox = 0; ox < outW; ox += 1) {
|
|
1277
|
+
const xStart = x0 + ox * factor;
|
|
1278
|
+
const xEnd = Math.min(x0 + w, xStart + factor);
|
|
1279
|
+
let level = 0;
|
|
1280
|
+
let cover = 0;
|
|
1281
|
+
let count = 0;
|
|
1282
|
+
for (let y = yStart; y < yEnd; y += 1) {
|
|
1283
|
+
const row = y * stride;
|
|
1284
|
+
for (let x = xStart; x < xEnd; x += 1) {
|
|
1285
|
+
count += 1;
|
|
1286
|
+
if (bits) {
|
|
1287
|
+
const set = data[row + (x >> 3)] >> 7 - (x & 7) & 1;
|
|
1288
|
+
if (mask) cover += set ? 0 : 255;
|
|
1289
|
+
else {
|
|
1290
|
+
level += set ? 255 : 0;
|
|
1291
|
+
cover += 255;
|
|
1292
|
+
}
|
|
1293
|
+
} else {
|
|
1294
|
+
const at = row + x * (kind === 3 ? 4 : 3);
|
|
1295
|
+
const value = (data[at] * 299 + data[at + 1] * 587 + data[at + 2] * 114) / 1e3;
|
|
1296
|
+
const opacity = kind === 3 ? data[at + 3] : 255;
|
|
1297
|
+
level += value * opacity;
|
|
1298
|
+
cover += opacity;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
const at = oy * outW + ox;
|
|
1303
|
+
alpha[at] = count ? Math.round(cover / count) : 0;
|
|
1304
|
+
if (alpha[at] < 255) opaque = false;
|
|
1305
|
+
gray[at] = bits ? mask ? 0 : Math.round(level / count) : cover ? Math.round(level / cover) : 255;
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
return {
|
|
1309
|
+
width: outW,
|
|
1310
|
+
height: outH,
|
|
1311
|
+
gray,
|
|
1312
|
+
alpha,
|
|
1313
|
+
opaque
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
function paint(page, full, picture, { fill = null } = {}) {
|
|
1317
|
+
const inverse = invert(full);
|
|
1318
|
+
if (!inverse) return;
|
|
1319
|
+
const corners = [
|
|
1320
|
+
[0, 0],
|
|
1321
|
+
[1, 0],
|
|
1322
|
+
[0, 1],
|
|
1323
|
+
[1, 1]
|
|
1324
|
+
].map(([x, y]) => apply$1(full, x, y));
|
|
1325
|
+
const left = Math.max(0, Math.floor(Math.min(...corners.map((point) => point[0]))));
|
|
1326
|
+
const right = Math.min(page.width, Math.ceil(Math.max(...corners.map((point) => point[0]))));
|
|
1327
|
+
const top = Math.max(0, Math.floor(Math.min(...corners.map((point) => point[1]))));
|
|
1328
|
+
const bottom = Math.min(page.height, Math.ceil(Math.max(...corners.map((point) => point[1]))));
|
|
1329
|
+
const [ia, ib, ic, id, ie, iff] = inverse;
|
|
1330
|
+
if (picture && Math.abs(ib) < 1e-9 && Math.abs(ic) < 1e-9) {
|
|
1331
|
+
paintAligned(page, picture, {
|
|
1332
|
+
left,
|
|
1333
|
+
right,
|
|
1334
|
+
top,
|
|
1335
|
+
bottom,
|
|
1336
|
+
ia,
|
|
1337
|
+
id,
|
|
1338
|
+
ie,
|
|
1339
|
+
iff,
|
|
1340
|
+
fill
|
|
1341
|
+
});
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
const { width, height, gray, alpha } = picture ?? {
|
|
1345
|
+
width: 1,
|
|
1346
|
+
height: 1
|
|
1347
|
+
};
|
|
1348
|
+
for (let y = top; y < bottom; y += 1) {
|
|
1349
|
+
const py = y + .5;
|
|
1350
|
+
for (let x = left; x < right; x += 1) {
|
|
1351
|
+
const px = x + .5;
|
|
1352
|
+
const u = ia * px + ic * py + ie;
|
|
1353
|
+
const v = ib * px + id * py + iff;
|
|
1354
|
+
if (u < 0 || u >= 1 || v < 0 || v >= 1) continue;
|
|
1355
|
+
const at = y * page.width + x;
|
|
1356
|
+
if (!picture) {
|
|
1357
|
+
page.gray[at] = fill;
|
|
1358
|
+
continue;
|
|
1359
|
+
}
|
|
1360
|
+
const fx = Math.min(width - 1, Math.max(0, u * width - .5));
|
|
1361
|
+
const fy = Math.min(height - 1, Math.max(0, (1 - v) * height - .5));
|
|
1362
|
+
const x1 = Math.floor(fx);
|
|
1363
|
+
const y1 = Math.floor(fy);
|
|
1364
|
+
const x2 = Math.min(width - 1, x1 + 1);
|
|
1365
|
+
const y2 = Math.min(height - 1, y1 + 1);
|
|
1366
|
+
const tx = fx - x1;
|
|
1367
|
+
const ty = fy - y1;
|
|
1368
|
+
const w1 = (1 - tx) * (1 - ty);
|
|
1369
|
+
const w2 = tx * (1 - ty);
|
|
1370
|
+
const w3 = (1 - tx) * ty;
|
|
1371
|
+
const w4 = tx * ty;
|
|
1372
|
+
const p1 = y1 * width + x1;
|
|
1373
|
+
const p2 = y1 * width + x2;
|
|
1374
|
+
const p3 = y2 * width + x1;
|
|
1375
|
+
const p4 = y2 * width + x2;
|
|
1376
|
+
const cover = alpha[p1] * w1 + alpha[p2] * w2 + alpha[p3] * w3 + alpha[p4] * w4;
|
|
1377
|
+
if (!cover) continue;
|
|
1378
|
+
const ink = fill ?? (gray[p1] * alpha[p1] * w1 + gray[p2] * alpha[p2] * w2 + gray[p3] * alpha[p3] * w3 + gray[p4] * alpha[p4] * w4) / cover;
|
|
1379
|
+
const share = cover / 255;
|
|
1380
|
+
page.gray[at] = Math.round(page.gray[at] * (1 - share) + ink * share);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
function axisSamples(from, to, scale, offset, size, flip) {
|
|
1385
|
+
const first = new Int32Array(to - from);
|
|
1386
|
+
const second = new Int32Array(to - from);
|
|
1387
|
+
const weight = new Float32Array(to - from);
|
|
1388
|
+
for (let index = 0; index < to - from; index += 1) {
|
|
1389
|
+
const unit = scale * (from + index + .5) + offset;
|
|
1390
|
+
if (unit < 0 || unit >= 1) {
|
|
1391
|
+
first[index] = -1;
|
|
1392
|
+
continue;
|
|
1393
|
+
}
|
|
1394
|
+
const position = Math.min(size - 1, Math.max(0, (flip ? 1 - unit : unit) * size - .5));
|
|
1395
|
+
first[index] = Math.floor(position);
|
|
1396
|
+
second[index] = Math.min(size - 1, first[index] + 1);
|
|
1397
|
+
weight[index] = position - first[index];
|
|
1398
|
+
}
|
|
1399
|
+
return {
|
|
1400
|
+
first,
|
|
1401
|
+
second,
|
|
1402
|
+
weight
|
|
1403
|
+
};
|
|
1404
|
+
}
|
|
1405
|
+
function paintAligned(page, picture, { left, right, top, bottom, ia, id, ie, iff, fill }) {
|
|
1406
|
+
const { width, height, gray, alpha, opaque } = picture;
|
|
1407
|
+
const columns = axisSamples(left, right, ia, ie, width, false);
|
|
1408
|
+
const rows = axisSamples(top, bottom, id, iff, height, true);
|
|
1409
|
+
for (let y = top; y < bottom; y += 1) {
|
|
1410
|
+
const row = y - top;
|
|
1411
|
+
if (rows.first[row] < 0) continue;
|
|
1412
|
+
const r1 = rows.first[row] * width;
|
|
1413
|
+
const r2 = rows.second[row] * width;
|
|
1414
|
+
const ty = rows.weight[row];
|
|
1415
|
+
const out = y * page.width;
|
|
1416
|
+
for (let x = left; x < right; x += 1) {
|
|
1417
|
+
const column = x - left;
|
|
1418
|
+
const c1 = columns.first[column];
|
|
1419
|
+
if (c1 < 0) continue;
|
|
1420
|
+
const c2 = columns.second[column];
|
|
1421
|
+
const tx = columns.weight[column];
|
|
1422
|
+
const p1 = r1 + c1;
|
|
1423
|
+
const p2 = r1 + c2;
|
|
1424
|
+
const p3 = r2 + c1;
|
|
1425
|
+
const p4 = r2 + c2;
|
|
1426
|
+
if (opaque && fill === null) {
|
|
1427
|
+
const above = gray[p1] + (gray[p2] - gray[p1]) * tx;
|
|
1428
|
+
const below = gray[p3] + (gray[p4] - gray[p3]) * tx;
|
|
1429
|
+
page.gray[out + x] = Math.round(above + (below - above) * ty);
|
|
1430
|
+
continue;
|
|
1431
|
+
}
|
|
1432
|
+
const w1 = (1 - tx) * (1 - ty);
|
|
1433
|
+
const w2 = tx * (1 - ty);
|
|
1434
|
+
const w3 = (1 - tx) * ty;
|
|
1435
|
+
const w4 = tx * ty;
|
|
1436
|
+
const cover = alpha[p1] * w1 + alpha[p2] * w2 + alpha[p3] * w3 + alpha[p4] * w4;
|
|
1437
|
+
if (!cover) continue;
|
|
1438
|
+
const ink = fill ?? (gray[p1] * alpha[p1] * w1 + gray[p2] * alpha[p2] * w2 + gray[p3] * alpha[p3] * w3 + gray[p4] * alpha[p4] * w4) / cover;
|
|
1439
|
+
const share = cover / 255;
|
|
1440
|
+
page.gray[out + x] = Math.round(page.gray[out + x] * (1 - share) + ink * share);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
function extent(full) {
|
|
1445
|
+
const [x0, y0] = apply$1(full, 0, 0);
|
|
1446
|
+
const [x1, y1] = apply$1(full, 1, 0);
|
|
1447
|
+
const [x2, y2] = apply$1(full, 0, 1);
|
|
1448
|
+
return {
|
|
1449
|
+
across: Math.hypot(x1 - x0, y1 - y0),
|
|
1450
|
+
down: Math.hypot(x2 - x0, y2 - y0)
|
|
1451
|
+
};
|
|
1452
|
+
}
|
|
1453
|
+
async function pageImage(pdfjs, page) {
|
|
1454
|
+
const operatorList = await page.getOperatorList({ annotationMode: pdfjs.AnnotationMode?.DISABLE });
|
|
1455
|
+
const draws = placements(pdfjs.OPS, operatorList);
|
|
1456
|
+
if (!draws.length) return null;
|
|
1457
|
+
const base = page.getViewport({ scale: 1 });
|
|
1458
|
+
const pictures = await Promise.all(draws.map((draw) => draw.solid !== void 0 ? null : resolve(page, draw.image ?? (typeof draw.mask?.data === "string" ? draw.mask.data : draw.mask))));
|
|
1459
|
+
let density = 1;
|
|
1460
|
+
for (const [index, draw] of draws.entries()) {
|
|
1461
|
+
const decoded = pictures[index];
|
|
1462
|
+
if (!decoded?.width) continue;
|
|
1463
|
+
const size = extent(multiply(base.transform, draw.matrix));
|
|
1464
|
+
if (size.across * size.down < base.width * base.height * .05) continue;
|
|
1465
|
+
density = Math.max(density, Math.min((draw.region?.w ?? decoded.width) / Math.max(1, size.across), (draw.region?.h ?? decoded.height) / Math.max(1, size.down)));
|
|
1466
|
+
}
|
|
1467
|
+
const scale = Math.min(density, Math.sqrt(MAX_PIXELS / (base.width * base.height)), MAX_EDGE / Math.max(base.width, base.height));
|
|
1468
|
+
const viewport = page.getViewport({ scale });
|
|
1469
|
+
const bitmap = {
|
|
1470
|
+
width: Math.max(1, Math.ceil(viewport.width)),
|
|
1471
|
+
height: Math.max(1, Math.ceil(viewport.height))
|
|
1472
|
+
};
|
|
1473
|
+
bitmap.gray = new Uint8Array(bitmap.width * bitmap.height).fill(255);
|
|
1474
|
+
let painted = false;
|
|
1475
|
+
for (const [index, draw] of draws.entries()) {
|
|
1476
|
+
const full = multiply(viewport.transform, draw.matrix);
|
|
1477
|
+
if (draw.solid !== void 0) {
|
|
1478
|
+
paint(bitmap, full, null, { fill: draw.solid });
|
|
1479
|
+
painted = true;
|
|
1480
|
+
continue;
|
|
1481
|
+
}
|
|
1482
|
+
const decoded = pictures[index];
|
|
1483
|
+
const mask = Boolean(draw.mask);
|
|
1484
|
+
const image = mask ? {
|
|
1485
|
+
...draw.mask,
|
|
1486
|
+
...decoded ?? {},
|
|
1487
|
+
kind: 1
|
|
1488
|
+
} : decoded;
|
|
1489
|
+
if (!image?.data || !image.width || !image.height) continue;
|
|
1490
|
+
if (!mask && ![
|
|
1491
|
+
1,
|
|
1492
|
+
2,
|
|
1493
|
+
3
|
|
1494
|
+
].includes(image.kind)) continue;
|
|
1495
|
+
const size = extent(full);
|
|
1496
|
+
paint(bitmap, full, shrink(image, {
|
|
1497
|
+
factor: Math.max(1, Math.floor(Math.min((draw.region?.w ?? image.width) / Math.max(1, size.across), (draw.region?.h ?? image.height) / Math.max(1, size.down)))),
|
|
1498
|
+
mask,
|
|
1499
|
+
region: draw.region
|
|
1500
|
+
}), { fill: mask ? draw.fill : null });
|
|
1501
|
+
painted = true;
|
|
1502
|
+
}
|
|
1503
|
+
if (!painted) return null;
|
|
1504
|
+
return {
|
|
1505
|
+
mime: "image/png",
|
|
1506
|
+
data: encodeGrayPng(bitmap.width, bitmap.height, bitmap.gray),
|
|
1507
|
+
width: bitmap.width,
|
|
1508
|
+
height: bitmap.height
|
|
1509
|
+
};
|
|
1510
|
+
}
|
|
1511
|
+
const CRC_TABLE = (/* @__PURE__ */ new Uint32Array(256)).map((_, index) => {
|
|
1512
|
+
let value = index;
|
|
1513
|
+
for (let bit = 0; bit < 8; bit += 1) value = value & 1 ? 3988292384 ^ value >>> 1 : value >>> 1;
|
|
1514
|
+
return value >>> 0;
|
|
1515
|
+
});
|
|
1516
|
+
function crc32(bytes) {
|
|
1517
|
+
let value = 4294967295;
|
|
1518
|
+
for (const byte of bytes) value = CRC_TABLE[(value ^ byte) & 255] ^ value >>> 8;
|
|
1519
|
+
return (value ^ 4294967295) >>> 0;
|
|
1520
|
+
}
|
|
1521
|
+
function chunk(type, body) {
|
|
1522
|
+
const head = Buffer.alloc(8);
|
|
1523
|
+
head.writeUInt32BE(body.length, 0);
|
|
1524
|
+
head.write(type, 4, "latin1");
|
|
1525
|
+
const tail = Buffer.alloc(4);
|
|
1526
|
+
tail.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), body])), 0);
|
|
1527
|
+
return Buffer.concat([
|
|
1528
|
+
head,
|
|
1529
|
+
body,
|
|
1530
|
+
tail
|
|
1531
|
+
]);
|
|
1532
|
+
}
|
|
1533
|
+
function encodeGrayPng(width, height, gray) {
|
|
1534
|
+
const raw = Buffer.alloc((width + 1) * height);
|
|
1535
|
+
for (let y = 0; y < height; y += 1) {
|
|
1536
|
+
const out = y * (width + 1);
|
|
1537
|
+
raw[out] = 2;
|
|
1538
|
+
for (let x = 0; x < width; x += 1) raw[out + 1 + x] = gray[y * width + x] - (y ? gray[(y - 1) * width + x] : 0) & 255;
|
|
1539
|
+
}
|
|
1540
|
+
const header = Buffer.alloc(13);
|
|
1541
|
+
header.writeUInt32BE(width, 0);
|
|
1542
|
+
header.writeUInt32BE(height, 4);
|
|
1543
|
+
header[8] = 8;
|
|
1544
|
+
header[9] = 0;
|
|
1545
|
+
return Buffer.concat([
|
|
1546
|
+
Buffer.from([
|
|
1547
|
+
137,
|
|
1548
|
+
80,
|
|
1549
|
+
78,
|
|
1550
|
+
71,
|
|
1551
|
+
13,
|
|
1552
|
+
10,
|
|
1553
|
+
26,
|
|
1554
|
+
10
|
|
1555
|
+
]),
|
|
1556
|
+
chunk("IHDR", header),
|
|
1557
|
+
chunk("IDAT", deflateSync(raw)),
|
|
1558
|
+
chunk("IEND", Buffer.alloc(0))
|
|
1559
|
+
]);
|
|
1560
|
+
}
|
|
1561
|
+
//#endregion
|
|
1084
1562
|
//#region src/host/web-fetch.js
|
|
1085
1563
|
const LOCAL_ADDRESS_MESSAGE = "Local network addresses are not supported";
|
|
1086
1564
|
var AddressError = class extends Error {
|
|
@@ -1319,6 +1797,16 @@ const MAX_TEXT_CHARS = 2e6;
|
|
|
1319
1797
|
const CHUNK_CHARS = 1600;
|
|
1320
1798
|
const FETCH_TIMEOUT_MS = 2e4;
|
|
1321
1799
|
const MAX_PAGE_BYTES = 6 * 1024 * 1024;
|
|
1800
|
+
const MIN_PAGE_TEXT = 30;
|
|
1801
|
+
const TEXT_PER_PICTURE = 150;
|
|
1802
|
+
const MIN_PICTURE_BYTES = 2048;
|
|
1803
|
+
const PICTURE_TYPES = Object.freeze({
|
|
1804
|
+
png: "image/png",
|
|
1805
|
+
jpg: "image/jpeg",
|
|
1806
|
+
jpeg: "image/jpeg",
|
|
1807
|
+
gif: "image/gif",
|
|
1808
|
+
webp: "image/webp"
|
|
1809
|
+
});
|
|
1322
1810
|
const pageFetch = createGuardedFetch({ maxBytes: MAX_UPLOAD_BYTES });
|
|
1323
1811
|
var MaterialError = class extends Error {
|
|
1324
1812
|
constructor(code, message) {
|
|
@@ -1459,6 +1947,71 @@ function finish(kind, title, chunks, extra = {}) {
|
|
|
1459
1947
|
...extra
|
|
1460
1948
|
};
|
|
1461
1949
|
}
|
|
1950
|
+
const visibleLength = (value) => String(value ?? "").replace(/\s+/gu, "").length;
|
|
1951
|
+
async function readPictures(recognize, sources) {
|
|
1952
|
+
if (!sources.length) return {
|
|
1953
|
+
texts: [],
|
|
1954
|
+
note: null,
|
|
1955
|
+
available: true,
|
|
1956
|
+
error: null
|
|
1957
|
+
};
|
|
1958
|
+
const tried = sources.slice(0, 200);
|
|
1959
|
+
const results = recognize ? await recognize(tried) : null;
|
|
1960
|
+
if (!results) return {
|
|
1961
|
+
texts: [],
|
|
1962
|
+
note: { unread: sources.length },
|
|
1963
|
+
available: false,
|
|
1964
|
+
error: null
|
|
1965
|
+
};
|
|
1966
|
+
const error = results.find((result) => result?.error)?.error ?? null;
|
|
1967
|
+
const failed = tried.filter((source, index) => results[index]?.error).map((source) => source.where);
|
|
1968
|
+
return {
|
|
1969
|
+
texts: tried.map((source, index) => results[index]?.text ?? ""),
|
|
1970
|
+
note: {
|
|
1971
|
+
read: results.filter((result) => result && !result.error).length,
|
|
1972
|
+
failed,
|
|
1973
|
+
reason: error?.code ?? null,
|
|
1974
|
+
left: sources.length - tried.length
|
|
1975
|
+
},
|
|
1976
|
+
available: true,
|
|
1977
|
+
error
|
|
1978
|
+
};
|
|
1979
|
+
}
|
|
1980
|
+
function resolvePart(base, target) {
|
|
1981
|
+
const parts = target.startsWith("/") ? [] : base.split("/").filter(Boolean);
|
|
1982
|
+
for (const piece of target.split("/")) if (piece === "..") parts.pop();
|
|
1983
|
+
else if (piece && piece !== ".") parts.push(piece);
|
|
1984
|
+
return parts.join("/");
|
|
1985
|
+
}
|
|
1986
|
+
async function partPictures(zip, part) {
|
|
1987
|
+
const slash = part.lastIndexOf("/");
|
|
1988
|
+
const folder = part.slice(0, slash);
|
|
1989
|
+
const xml = await zip.file(part)?.async("string") ?? "";
|
|
1990
|
+
const rels = await zip.file(`${folder}/_rels/${part.slice(slash + 1)}.rels`)?.async("string") ?? "";
|
|
1991
|
+
const targets = /* @__PURE__ */ new Map();
|
|
1992
|
+
for (const [, attributes] of rels.matchAll(/<Relationship\b([^>]*)>/gu)) {
|
|
1993
|
+
const id = /\bId="([^"]+)"/u.exec(attributes)?.[1];
|
|
1994
|
+
const target = /\bTarget="([^"]+)"/u.exec(attributes)?.[1];
|
|
1995
|
+
if (!id || !target || !/\bType="[^"]*\/image"/u.test(attributes) || /\bTargetMode="External"/u.test(attributes)) continue;
|
|
1996
|
+
targets.set(id, resolvePart(folder, decodeXml(target)));
|
|
1997
|
+
}
|
|
1998
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1999
|
+
const pictures = [];
|
|
2000
|
+
for (const [, id] of xml.matchAll(/\br:(?:embed|id)="([^"]+)"/gu)) {
|
|
2001
|
+
const path = targets.get(id);
|
|
2002
|
+
if (!path || seen.has(path)) continue;
|
|
2003
|
+
seen.add(path);
|
|
2004
|
+
const mime = PICTURE_TYPES[path.split(".").pop().toLowerCase()];
|
|
2005
|
+
const file = zip.file(path);
|
|
2006
|
+
if (!mime || !file) continue;
|
|
2007
|
+
const data = await file.async("nodebuffer");
|
|
2008
|
+
if (data.length >= MIN_PICTURE_BYTES && data.length <= 10485760) pictures.push({
|
|
2009
|
+
mime,
|
|
2010
|
+
data
|
|
2011
|
+
});
|
|
2012
|
+
}
|
|
2013
|
+
return pictures;
|
|
2014
|
+
}
|
|
1462
2015
|
let pdfjsModule;
|
|
1463
2016
|
async function loadPdfjs() {
|
|
1464
2017
|
pdfjsModule ??= import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
@@ -1467,20 +2020,23 @@ async function loadPdfjs() {
|
|
|
1467
2020
|
function pdfjsAssetUrls(root, { join: joinPath = join } = {}) {
|
|
1468
2021
|
return {
|
|
1469
2022
|
cMapUrl: `${joinPath(root, "cmaps")}/`,
|
|
1470
|
-
standardFontDataUrl: `${joinPath(root, "standard_fonts")}
|
|
2023
|
+
standardFontDataUrl: `${joinPath(root, "standard_fonts")}/`,
|
|
2024
|
+
wasmUrl: `${joinPath(root, "wasm")}/`
|
|
1471
2025
|
};
|
|
1472
2026
|
}
|
|
1473
2027
|
function pdfjsAssets() {
|
|
1474
2028
|
return pdfjsAssetUrls(dirname(createRequire(import.meta.url).resolve("pdfjs-dist/package.json")));
|
|
1475
2029
|
}
|
|
1476
|
-
async function extractPdf(bytes, { signal } = {}) {
|
|
2030
|
+
async function extractPdf(bytes, { signal, recognize } = {}) {
|
|
1477
2031
|
const pdfjs = await loadPdfjs();
|
|
1478
|
-
const { cMapUrl, standardFontDataUrl } = pdfjsAssets();
|
|
2032
|
+
const { cMapUrl, standardFontDataUrl, wasmUrl } = pdfjsAssets();
|
|
1479
2033
|
const task = pdfjs.getDocument({
|
|
1480
2034
|
data: new Uint8Array(bytes),
|
|
1481
2035
|
cMapUrl,
|
|
1482
2036
|
cMapPacked: true,
|
|
1483
2037
|
standardFontDataUrl,
|
|
2038
|
+
wasmUrl,
|
|
2039
|
+
maxImageSize: 1e8,
|
|
1484
2040
|
isEvalSupported: false,
|
|
1485
2041
|
disableFontFace: true,
|
|
1486
2042
|
useSystemFonts: false,
|
|
@@ -1494,12 +2050,12 @@ async function extractPdf(bytes, { signal } = {}) {
|
|
|
1494
2050
|
throw new MaterialError("unreadable", "This PDF could not be read");
|
|
1495
2051
|
}
|
|
1496
2052
|
try {
|
|
1497
|
-
const chunks = [];
|
|
1498
2053
|
let title = "";
|
|
1499
2054
|
try {
|
|
1500
2055
|
const meta = await doc.getMetadata();
|
|
1501
2056
|
title = String(meta?.info?.Title ?? "").trim();
|
|
1502
2057
|
} catch {}
|
|
2058
|
+
const texts = [];
|
|
1503
2059
|
for (let pageNumber = 1; pageNumber <= doc.numPages; pageNumber += 1) {
|
|
1504
2060
|
signal?.throwIfAborted();
|
|
1505
2061
|
const page = await doc.getPage(pageNumber);
|
|
@@ -1511,19 +2067,39 @@ async function extractPdf(bytes, { signal } = {}) {
|
|
|
1511
2067
|
pageText += item.hasEOL ? "\n" : "";
|
|
1512
2068
|
}
|
|
1513
2069
|
page.cleanup();
|
|
1514
|
-
|
|
2070
|
+
texts.push(pageText);
|
|
2071
|
+
}
|
|
2072
|
+
const pictured = texts.flatMap((pageText, index) => visibleLength(pageText) < MIN_PAGE_TEXT ? [index + 1] : []);
|
|
2073
|
+
const read = await readPictures(recognize, pictured.map((pageNumber) => ({
|
|
2074
|
+
where: { page: pageNumber },
|
|
2075
|
+
image: async () => {
|
|
2076
|
+
signal?.throwIfAborted();
|
|
2077
|
+
const page = await doc.getPage(pageNumber);
|
|
2078
|
+
try {
|
|
2079
|
+
return await pageImage(pdfjs, page);
|
|
2080
|
+
} finally {
|
|
2081
|
+
page.cleanup();
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
})));
|
|
2085
|
+
if (pictured.length === doc.numPages && !read.note?.read) {
|
|
2086
|
+
if (!read.available) throw new MaterialError("scan-needs-vision", "This PDF is a scan; reading it needs a model that can see images");
|
|
2087
|
+
throw read.error ?? new MaterialError("no-text", "No text found in this PDF");
|
|
1515
2088
|
}
|
|
1516
|
-
|
|
2089
|
+
const recognized = new Map(pictured.slice(0, read.texts.length).map((pageNumber, index) => [pageNumber, read.texts[index]]));
|
|
2090
|
+
const chunks = texts.flatMap((pageText, index) => chunkText([pageText, recognized.get(index + 1)].filter(Boolean).join("\n\n"), { page: index + 1 }));
|
|
2091
|
+
if (!chunks.length) throw read.error ?? new MaterialError("no-text", "No text found in this PDF");
|
|
1517
2092
|
return {
|
|
1518
2093
|
title,
|
|
1519
2094
|
pages: doc.numPages,
|
|
1520
|
-
chunks
|
|
2095
|
+
chunks,
|
|
2096
|
+
ocr: read.note
|
|
1521
2097
|
};
|
|
1522
2098
|
} finally {
|
|
1523
2099
|
await Promise.resolve(task.destroy?.()).catch(() => void 0);
|
|
1524
2100
|
}
|
|
1525
2101
|
}
|
|
1526
|
-
async function extractDocx(bytes) {
|
|
2102
|
+
async function extractDocx(bytes, { recognize } = {}) {
|
|
1527
2103
|
const mammoth = await import("mammoth");
|
|
1528
2104
|
const extract = mammoth.extractRawText ?? mammoth.default?.extractRawText;
|
|
1529
2105
|
let result;
|
|
@@ -1532,7 +2108,8 @@ async function extractDocx(bytes) {
|
|
|
1532
2108
|
} catch {
|
|
1533
2109
|
throw new MaterialError("unreadable", "This Word document could not be read");
|
|
1534
2110
|
}
|
|
1535
|
-
const
|
|
2111
|
+
const own = String(result?.value ?? "");
|
|
2112
|
+
const sections = own.split(/\n{2,}/u);
|
|
1536
2113
|
const chunks = [];
|
|
1537
2114
|
let buffer = "";
|
|
1538
2115
|
let section = 1;
|
|
@@ -1544,9 +2121,31 @@ async function extractDocx(bytes) {
|
|
|
1544
2121
|
}
|
|
1545
2122
|
buffer += `${paragraph}\n\n`;
|
|
1546
2123
|
}
|
|
1547
|
-
if (buffer.trim())
|
|
2124
|
+
if (buffer.trim()) {
|
|
2125
|
+
chunks.push(...chunkText(buffer, { section: `§${section}` }));
|
|
2126
|
+
section += 1;
|
|
2127
|
+
}
|
|
2128
|
+
const JSZip = (await import("jszip")).default;
|
|
2129
|
+
const pictures = await partPictures(await JSZip.loadAsync(bytes), "word/document.xml");
|
|
2130
|
+
let note = null;
|
|
2131
|
+
if (pictures.length && visibleLength(own) < TEXT_PER_PICTURE * pictures.length) {
|
|
2132
|
+
const read = await readPictures(recognize, pictures.map((picture, index) => ({
|
|
2133
|
+
where: { picture: index + 1 },
|
|
2134
|
+
image: async () => picture
|
|
2135
|
+
})));
|
|
2136
|
+
if (!read.note?.read && visibleLength(own) < MIN_PAGE_TEXT) {
|
|
2137
|
+
if (!read.available) throw new MaterialError("pictures-need-vision", "This document is pictures; reading them needs a model that can see images");
|
|
2138
|
+
throw read.error ?? new MaterialError("no-text", "No text found in this document");
|
|
2139
|
+
}
|
|
2140
|
+
for (const [index, pictureText] of read.texts.entries()) chunks.push(...chunkText(pictureText, { section: `§${section + index}` }));
|
|
2141
|
+
if (!chunks.length) throw read.error ?? new MaterialError("no-text", "No text found in this document");
|
|
2142
|
+
note = read.note;
|
|
2143
|
+
}
|
|
1548
2144
|
if (!chunks.length) throw new MaterialError("no-text", "No text found in this document");
|
|
1549
|
-
return {
|
|
2145
|
+
return {
|
|
2146
|
+
chunks,
|
|
2147
|
+
ocr: note
|
|
2148
|
+
};
|
|
1550
2149
|
}
|
|
1551
2150
|
const XML_ENTITIES = {
|
|
1552
2151
|
amp: "&",
|
|
@@ -1566,7 +2165,7 @@ function slideText(xml) {
|
|
|
1566
2165
|
return [...paragraph.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/gu)].map((match) => decodeXml(match[1])).join("");
|
|
1567
2166
|
}).filter((line) => line.trim()).join("\n");
|
|
1568
2167
|
}
|
|
1569
|
-
async function extractPptx(bytes) {
|
|
2168
|
+
async function extractPptx(bytes, { recognize } = {}) {
|
|
1570
2169
|
const JSZip = (await import("jszip")).default;
|
|
1571
2170
|
let zip;
|
|
1572
2171
|
try {
|
|
@@ -1576,19 +2175,40 @@ async function extractPptx(bytes) {
|
|
|
1576
2175
|
}
|
|
1577
2176
|
const slideNumber = (name) => Number(/(\d+)\.xml$/u.exec(name)?.[1] ?? 0);
|
|
1578
2177
|
const slides = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/u.test(name)).sort((a, b) => slideNumber(a) - slideNumber(b));
|
|
1579
|
-
const
|
|
2178
|
+
const texts = [];
|
|
2179
|
+
const sources = [];
|
|
2180
|
+
let written = 0;
|
|
1580
2181
|
for (const name of slides) {
|
|
1581
2182
|
const number = slideNumber(name);
|
|
1582
2183
|
const body = slideText(await zip.file(name).async("string"));
|
|
1583
2184
|
const notesFile = zip.file(`ppt/notesSlides/notesSlide${number}.xml`);
|
|
1584
2185
|
const notes = notesFile ? slideText(await notesFile.async("string")) : "";
|
|
1585
|
-
|
|
1586
|
-
|
|
2186
|
+
texts.push({
|
|
2187
|
+
number,
|
|
2188
|
+
text: notes ? `${body}\n[notes] ${notes}` : body
|
|
2189
|
+
});
|
|
2190
|
+
if (visibleLength(body) < MIN_PAGE_TEXT) for (const picture of await partPictures(zip, name)) sources.push({
|
|
2191
|
+
where: { slide: number },
|
|
2192
|
+
image: async () => picture
|
|
2193
|
+
});
|
|
2194
|
+
else written += 1;
|
|
1587
2195
|
}
|
|
1588
|
-
|
|
2196
|
+
const read = await readPictures(recognize, sources);
|
|
2197
|
+
if (sources.length && !written && !read.note?.read) {
|
|
2198
|
+
if (!read.available) throw new MaterialError("pictures-need-vision", "These slides are pictures; reading them needs a model that can see images");
|
|
2199
|
+
throw read.error ?? new MaterialError("no-text", "No text found in these slides");
|
|
2200
|
+
}
|
|
2201
|
+
const pictured = /* @__PURE__ */ new Map();
|
|
2202
|
+
for (const [index, pictureText] of read.texts.entries()) {
|
|
2203
|
+
const { slide } = sources[index].where;
|
|
2204
|
+
pictured.set(slide, [pictured.get(slide), pictureText].filter(Boolean).join("\n\n"));
|
|
2205
|
+
}
|
|
2206
|
+
const chunks = texts.flatMap(({ number, text }) => chunkText([text, pictured.get(number)].filter(Boolean).join("\n\n"), { slide: number }));
|
|
2207
|
+
if (!chunks.length) throw read.error ?? new MaterialError("no-text", "No text found in these slides");
|
|
1589
2208
|
return {
|
|
1590
2209
|
slides: slides.length,
|
|
1591
|
-
chunks
|
|
2210
|
+
chunks,
|
|
2211
|
+
ocr: read.note
|
|
1592
2212
|
};
|
|
1593
2213
|
}
|
|
1594
2214
|
function extractPlain(bytes, kind) {
|
|
@@ -1607,30 +2227,46 @@ function extractPlain(bytes, kind) {
|
|
|
1607
2227
|
chunks
|
|
1608
2228
|
};
|
|
1609
2229
|
}
|
|
1610
|
-
async function ingestFile({ name, bytes, signal }) {
|
|
2230
|
+
async function ingestFile({ name, bytes, signal, recognize }) {
|
|
1611
2231
|
if (!bytes?.length) throw new MaterialError("empty", "The file is empty");
|
|
1612
2232
|
if (bytes.length > 62914560) throw new MaterialError("too-large", "The file is larger than 60 MB");
|
|
1613
2233
|
const kind = detectKind(name, bytes);
|
|
1614
2234
|
const baseTitle = String(name ?? "").replace(/\.[^.]+$/u, "").slice(0, 120) || "Untitled";
|
|
1615
2235
|
if (kind === "image") {
|
|
1616
2236
|
if (bytes.length > 10485760) throw new MaterialError("image-too-large", "Images must be smaller than 10 MB");
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
};
|
|
2237
|
+
const mime = imageMime(name, bytes);
|
|
2238
|
+
const read = await readPictures(recognize, [{
|
|
2239
|
+
where: { picture: 1 },
|
|
2240
|
+
image: async () => ({
|
|
2241
|
+
mime,
|
|
2242
|
+
data: Buffer.from(bytes)
|
|
2243
|
+
})
|
|
2244
|
+
}]);
|
|
2245
|
+
return finish(kind, baseTitle, chunkText(read.texts[0] ?? "", {}), {
|
|
2246
|
+
mime,
|
|
2247
|
+
...read.available ? { ocr: read.note } : {}
|
|
2248
|
+
});
|
|
1625
2249
|
}
|
|
1626
2250
|
if (kind === "pdf") {
|
|
1627
|
-
const result = await extractPdf(bytes, {
|
|
1628
|
-
|
|
2251
|
+
const result = await extractPdf(bytes, {
|
|
2252
|
+
signal,
|
|
2253
|
+
recognize
|
|
2254
|
+
});
|
|
2255
|
+
return finish(kind, result.title || baseTitle, result.chunks, {
|
|
2256
|
+
pages: result.pages,
|
|
2257
|
+
ocr: result.ocr
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
if (kind === "docx") {
|
|
2261
|
+
const result = await extractDocx(bytes, { recognize });
|
|
2262
|
+
return finish(kind, baseTitle, result.chunks, { ocr: result.ocr });
|
|
1629
2263
|
}
|
|
1630
|
-
if (kind === "docx") return finish(kind, baseTitle, (await extractDocx(bytes)).chunks);
|
|
1631
2264
|
if (kind === "pptx") {
|
|
1632
|
-
const result = await extractPptx(bytes);
|
|
1633
|
-
return finish(kind, baseTitle, result.chunks, {
|
|
2265
|
+
const result = await extractPptx(bytes, { recognize });
|
|
2266
|
+
return finish(kind, baseTitle, result.chunks, {
|
|
2267
|
+
slides: result.slides,
|
|
2268
|
+
ocr: result.ocr
|
|
2269
|
+
});
|
|
1634
2270
|
}
|
|
1635
2271
|
const result = extractPlain(bytes, kind);
|
|
1636
2272
|
return finish(kind, result.title || baseTitle, result.chunks);
|
|
@@ -1651,7 +2287,7 @@ function checkPublicUrl(value) {
|
|
|
1651
2287
|
}
|
|
1652
2288
|
return url;
|
|
1653
2289
|
}
|
|
1654
|
-
async function ingestUrl(address, { fetchImpl = pageFetch, signal } = {}) {
|
|
2290
|
+
async function ingestUrl(address, { fetchImpl = pageFetch, signal, recognize } = {}) {
|
|
1655
2291
|
let url = checkPublicUrl(address);
|
|
1656
2292
|
let response;
|
|
1657
2293
|
for (let hop = 0; hop < 6; hop += 1) {
|
|
@@ -1685,7 +2321,8 @@ async function ingestUrl(address, { fetchImpl = pageFetch, signal } = {}) {
|
|
|
1685
2321
|
...await ingestFile({
|
|
1686
2322
|
name: `${url.pathname.split("/").pop() || "page"}.pdf`,
|
|
1687
2323
|
bytes,
|
|
1688
|
-
signal
|
|
2324
|
+
signal,
|
|
2325
|
+
recognize
|
|
1689
2326
|
}),
|
|
1690
2327
|
kind: "url",
|
|
1691
2328
|
url: url.href
|
|
@@ -3071,7 +3708,7 @@ function correctedItem(old, raw) {
|
|
|
3071
3708
|
...raw,
|
|
3072
3709
|
kind: old.kind
|
|
3073
3710
|
}, 0);
|
|
3074
|
-
if (!next || (old.options?.length ?? 0) !== (next.options?.length ?? 0)) return null;
|
|
3711
|
+
if (!next || next.kind !== old.kind || (old.options?.length ?? 0) !== (next.options?.length ?? 0)) return null;
|
|
3075
3712
|
const redrawn = [
|
|
3076
3713
|
"svg",
|
|
3077
3714
|
"figure",
|
|
@@ -4454,14 +5091,17 @@ function createLlmPort({ getLlm, getDefault, getAttachments, prefs, logger }) {
|
|
|
4454
5091
|
return null;
|
|
4455
5092
|
}
|
|
4456
5093
|
}
|
|
4457
|
-
async function content(user, images, prepared) {
|
|
5094
|
+
async function content(user, images, prepared, required) {
|
|
4458
5095
|
const blocks = [{
|
|
4459
5096
|
type: "text",
|
|
4460
5097
|
text: String(user ?? "")
|
|
4461
5098
|
}];
|
|
4462
|
-
if (!images?.length
|
|
5099
|
+
if (!images?.length) return blocks;
|
|
4463
5100
|
const attachments = getAttachments();
|
|
4464
|
-
if (typeof attachments?.saveImage !== "function")
|
|
5101
|
+
if (prepared.inputModalities?.includes("image") !== true || typeof attachments?.saveImage !== "function") {
|
|
5102
|
+
if (required) throw new TutorError("llm-no-vision", "The selected model cannot see images. Pick one that can in Aha settings.");
|
|
5103
|
+
return blocks;
|
|
5104
|
+
}
|
|
4465
5105
|
for (const [index, image] of images.slice(0, 4).entries()) try {
|
|
4466
5106
|
const ref = await attachments.saveImage({
|
|
4467
5107
|
data: Buffer.from(image.base64, "base64"),
|
|
@@ -4473,11 +5113,12 @@ function createLlmPort({ getLlm, getDefault, getAttachments, prefs, logger }) {
|
|
|
4473
5113
|
attachment: ref
|
|
4474
5114
|
});
|
|
4475
5115
|
} catch (error) {
|
|
5116
|
+
if (required) throw new TutorError("image-unusable", "This picture could not be given to the model.");
|
|
4476
5117
|
logger?.debug?.("aha-tutor could not attach an image: %s", error?.message);
|
|
4477
5118
|
}
|
|
4478
5119
|
return blocks;
|
|
4479
5120
|
}
|
|
4480
|
-
async function attempt({ purpose, effort, system, user, images, maxTokens, signal, onText, onReasoning }) {
|
|
5121
|
+
async function attempt({ purpose, effort, system, user, images, requireImages = false, maxTokens, signal, onText, onReasoning }) {
|
|
4481
5122
|
const llm = getLlm();
|
|
4482
5123
|
if (!llm) throw new TutorError("llm-missing", "DSH has no model service in this profile.");
|
|
4483
5124
|
const target = route(purpose);
|
|
@@ -4504,7 +5145,7 @@ function createLlmPort({ getLlm, getDefault, getAttachments, prefs, logger }) {
|
|
|
4504
5145
|
let finish = null;
|
|
4505
5146
|
const messages = [{
|
|
4506
5147
|
role: "user",
|
|
4507
|
-
content: await content(user, images, prepared)
|
|
5148
|
+
content: await content(user, images, prepared, requireImages)
|
|
4508
5149
|
}];
|
|
4509
5150
|
try {
|
|
4510
5151
|
for await (const chunk of prepared.stream({
|
|
@@ -6379,6 +7020,7 @@ const PREF_DEFAULTS = Object.freeze({
|
|
|
6379
7020
|
boardTheme: "auto",
|
|
6380
7021
|
boardFont: "hand",
|
|
6381
7022
|
boardView: "canvas",
|
|
7023
|
+
textSize: "standard",
|
|
6382
7024
|
ttsProvider: "openai",
|
|
6383
7025
|
tts: Object.fromEntries(TTS_PROVIDERS.map((provider) => [provider, providerDefaults(provider)]))
|
|
6384
7026
|
});
|
|
@@ -6410,6 +7052,11 @@ function cleanPrefs(input = {}, base = PREF_DEFAULTS) {
|
|
|
6410
7052
|
].includes(input.boardTheme)) next.boardTheme = input.boardTheme;
|
|
6411
7053
|
if (["hand", "standard"].includes(input.boardFont)) next.boardFont = input.boardFont;
|
|
6412
7054
|
if (["canvas", "pages"].includes(input.boardView)) next.boardView = input.boardView;
|
|
7055
|
+
if ([
|
|
7056
|
+
"standard",
|
|
7057
|
+
"large",
|
|
7058
|
+
"larger"
|
|
7059
|
+
].includes(input.textSize)) next.textSize = input.textSize;
|
|
6413
7060
|
if (TTS_PROVIDERS.includes(input.ttsProvider)) next.ttsProvider = input.ttsProvider;
|
|
6414
7061
|
for (const provider of TTS_PROVIDERS) {
|
|
6415
7062
|
const patch = input.tts?.[provider];
|
|
@@ -6513,6 +7160,73 @@ function createSettingsService({ prefs, keys, llmPort }) {
|
|
|
6513
7160
|
};
|
|
6514
7161
|
}
|
|
6515
7162
|
//#endregion
|
|
7163
|
+
//#region src/host/recognize.js
|
|
7164
|
+
const AT_ONCE = 3;
|
|
7165
|
+
const STOPS = /* @__PURE__ */ new Set([
|
|
7166
|
+
"llm-auth",
|
|
7167
|
+
"llm-quota",
|
|
7168
|
+
"llm-missing",
|
|
7169
|
+
"llm-unavailable",
|
|
7170
|
+
"llm-no-vision"
|
|
7171
|
+
]);
|
|
7172
|
+
const MAX_TOKENS = 6e3;
|
|
7173
|
+
async function visionPurpose(llm, signal) {
|
|
7174
|
+
await llm.refreshCapabilities?.(signal);
|
|
7175
|
+
return ["quick", "teach"].find((purpose) => llm.supportsImages?.(purpose)) ?? null;
|
|
7176
|
+
}
|
|
7177
|
+
function cleanTranscript(value) {
|
|
7178
|
+
return String(value ?? "").trim().replace(/^```[\w-]*\n([\s\S]*?)\n```$/u, "$1").replaceAll(RECOGNIZE_EMPTY, "").trim();
|
|
7179
|
+
}
|
|
7180
|
+
function createRecognizer({ llm, signal, onProgress, atOnce = AT_ONCE }) {
|
|
7181
|
+
let route = null;
|
|
7182
|
+
async function read(purpose, image) {
|
|
7183
|
+
const prompt = recognizePrompt();
|
|
7184
|
+
return cleanTranscript(await llm.complete({
|
|
7185
|
+
purpose,
|
|
7186
|
+
effort: "lowest",
|
|
7187
|
+
system: prompt.system,
|
|
7188
|
+
user: prompt.user,
|
|
7189
|
+
images: [{
|
|
7190
|
+
mime: image.mime,
|
|
7191
|
+
base64: Buffer.from(image.data).toString("base64")
|
|
7192
|
+
}],
|
|
7193
|
+
requireImages: true,
|
|
7194
|
+
maxTokens: MAX_TOKENS,
|
|
7195
|
+
signal
|
|
7196
|
+
}));
|
|
7197
|
+
}
|
|
7198
|
+
return async (sources) => {
|
|
7199
|
+
route ??= visionPurpose(llm, signal);
|
|
7200
|
+
const purpose = await route;
|
|
7201
|
+
if (!purpose) return null;
|
|
7202
|
+
const results = new Array(sources.length).fill(null);
|
|
7203
|
+
let next = 0;
|
|
7204
|
+
let done = 0;
|
|
7205
|
+
let stopped = null;
|
|
7206
|
+
onProgress?.(0, sources.length);
|
|
7207
|
+
const worker = async () => {
|
|
7208
|
+
while (next < sources.length) {
|
|
7209
|
+
const index = next;
|
|
7210
|
+
next += 1;
|
|
7211
|
+
signal?.throwIfAborted();
|
|
7212
|
+
try {
|
|
7213
|
+
if (stopped) throw stopped;
|
|
7214
|
+
const image = await sources[index].image();
|
|
7215
|
+
results[index] = { text: image ? await read(purpose, image) : "" };
|
|
7216
|
+
} catch (error) {
|
|
7217
|
+
if (signal?.aborted) throw signal.reason ?? error;
|
|
7218
|
+
if (STOPS.has(error?.code)) stopped ??= error;
|
|
7219
|
+
results[index] = { error };
|
|
7220
|
+
}
|
|
7221
|
+
done += 1;
|
|
7222
|
+
onProgress?.(done, sources.length);
|
|
7223
|
+
}
|
|
7224
|
+
};
|
|
7225
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(atOnce, sources.length)) }, worker));
|
|
7226
|
+
return results;
|
|
7227
|
+
};
|
|
7228
|
+
}
|
|
7229
|
+
//#endregion
|
|
6516
7230
|
//#region src/host/studio.js
|
|
6517
7231
|
const ASSIST_MODES = [
|
|
6518
7232
|
"explain",
|
|
@@ -6535,6 +7249,7 @@ function createStudio({ store, llm, jobs, now = () => /* @__PURE__ */ new Date()
|
|
|
6535
7249
|
truncated: result.truncated === true,
|
|
6536
7250
|
mime: result.mime ?? null,
|
|
6537
7251
|
url: result.url ?? null,
|
|
7252
|
+
ocr: result.ocr ?? null,
|
|
6538
7253
|
createdAt: stamp(),
|
|
6539
7254
|
brief: null,
|
|
6540
7255
|
...extra
|
|
@@ -6565,6 +7280,11 @@ function createStudio({ store, llm, jobs, now = () => /* @__PURE__ */ new Date()
|
|
|
6565
7280
|
await store.writeJson(store.paths.material(id), meta);
|
|
6566
7281
|
return meta;
|
|
6567
7282
|
}
|
|
7283
|
+
const recognizer = (api) => createRecognizer({
|
|
7284
|
+
llm,
|
|
7285
|
+
signal: api.signal,
|
|
7286
|
+
onProgress: (done, total) => api.progress(done, total, "recognizing")
|
|
7287
|
+
});
|
|
6568
7288
|
function upload({ name, base64, bytes: raw }) {
|
|
6569
7289
|
const fileName = text(name, 200) || "file";
|
|
6570
7290
|
if (!raw && (typeof base64 !== "string" || !base64)) throw new TutorError("invalid", "No file data");
|
|
@@ -6575,7 +7295,8 @@ function createStudio({ store, llm, jobs, now = () => /* @__PURE__ */ new Date()
|
|
|
6575
7295
|
const result = await ingestFile({
|
|
6576
7296
|
name: fileName,
|
|
6577
7297
|
bytes,
|
|
6578
|
-
signal: api.signal
|
|
7298
|
+
signal: api.signal,
|
|
7299
|
+
recognize: recognizer(api)
|
|
6579
7300
|
});
|
|
6580
7301
|
api.progress(1, 2, "summarizing");
|
|
6581
7302
|
return { material: await saveMaterial(result, {
|
|
@@ -6588,7 +7309,10 @@ function createStudio({ store, llm, jobs, now = () => /* @__PURE__ */ new Date()
|
|
|
6588
7309
|
function addUrl({ url }) {
|
|
6589
7310
|
return jobs.start("material/url", async (api) => {
|
|
6590
7311
|
api.progress(0, 2, "fetching");
|
|
6591
|
-
const result = await ingestUrl(url, {
|
|
7312
|
+
const result = await ingestUrl(url, {
|
|
7313
|
+
signal: api.signal,
|
|
7314
|
+
recognize: recognizer(api)
|
|
7315
|
+
});
|
|
6592
7316
|
api.progress(1, 2, "summarizing");
|
|
6593
7317
|
return { material: await saveMaterial(result, { signal: api.signal }) };
|
|
6594
7318
|
});
|
|
@@ -6602,7 +7326,7 @@ function createStudio({ store, llm, jobs, now = () => /* @__PURE__ */ new Date()
|
|
|
6602
7326
|
async function searchMaterials({ query }) {
|
|
6603
7327
|
const wanted = text(query, 200);
|
|
6604
7328
|
if (!wanted) return { results: [] };
|
|
6605
|
-
const metas = (await listMaterials()).filter((meta) => meta.kind !== "image");
|
|
7329
|
+
const metas = (await listMaterials()).filter((meta) => meta.kind !== "image" || meta.chars > 0);
|
|
6606
7330
|
const key = metas.map((meta) => `${meta.id}:${meta.title}:${meta.chars ?? meta.pages ?? ""}`).join("|");
|
|
6607
7331
|
if (libraryIndex?.key !== key) {
|
|
6608
7332
|
const chunks = [];
|