bell-reader 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +75 -2
- package/README.zh-TW.md +55 -0
- package/TRADEMARKS.md +32 -0
- package/bin/bell-reader.mjs +202 -0
- package/dist/assets/caveat-cyrillic-400-normal-9cDH9rLW.woff2 +0 -0
- package/dist/assets/caveat-cyrillic-400-normal-CebvvJET.woff +0 -0
- package/dist/assets/caveat-cyrillic-500-normal-DLSfp5IU.woff +0 -0
- package/dist/assets/caveat-cyrillic-500-normal-DpMZbbjM.woff2 +0 -0
- package/dist/assets/caveat-cyrillic-ext-400-normal-3iEGd-c5.woff2 +0 -0
- package/dist/assets/caveat-cyrillic-ext-400-normal-Cg0RnRQ5.woff +0 -0
- package/dist/assets/caveat-cyrillic-ext-500-normal-DcrB4vVx.woff2 +0 -0
- package/dist/assets/caveat-cyrillic-ext-500-normal-DqTXtm5N.woff +0 -0
- package/dist/assets/caveat-latin-400-normal-BzhAQZkN.woff +0 -0
- package/dist/assets/caveat-latin-400-normal-D6LQsQ_v.woff2 +0 -0
- package/dist/assets/caveat-latin-500-normal-B9SDL8cy.woff2 +0 -0
- package/dist/assets/caveat-latin-500-normal-Do9veZfZ.woff +0 -0
- package/dist/assets/caveat-latin-ext-400-normal-D7hBUiug.woff2 +0 -0
- package/dist/assets/caveat-latin-ext-400-normal-DtiRFvw0.woff +0 -0
- package/dist/assets/caveat-latin-ext-500-normal-jKqJW7bC.woff2 +0 -0
- package/dist/assets/caveat-latin-ext-500-normal-wUUXxlbS.woff +0 -0
- package/dist/assets/index-BO29bkMJ.js +84 -0
- package/dist/assets/index-DtC09ml8.css +1 -0
- package/dist/assets/newsreader-latin-ext-opsz-normal-BQn1nviT.woff2 +0 -0
- package/dist/assets/newsreader-latin-opsz-normal-s-izfB6B.woff2 +0 -0
- package/dist/assets/newsreader-vietnamese-opsz-normal-BjhtXyW2.woff2 +0 -0
- package/dist/fonts/LICENSE-geist-mono.txt +93 -0
- package/dist/fonts/LICENSE-geist.txt +93 -0
- package/dist/fonts/geist-latin-wght.woff2 +0 -0
- package/dist/fonts/geist-mono-latin-wght.woff2 +0 -0
- package/dist/fonts/iansui-tc.ttf +0 -0
- package/dist/fonts/naikai-tc.woff2 +0 -0
- package/dist/index.html +13 -0
- package/package.json +79 -4
- package/server/ai.ts +1391 -0
- package/server/app.ts +644 -0
- package/server/config.ts +140 -0
- package/server/errors.ts +22 -0
- package/server/explain-bridge.ts +320 -0
- package/server/extractor.ts +548 -0
- package/server/index.ts +28 -0
- package/server/security.ts +78 -0
- package/server/store.ts +595 -0
- package/shared/suite.ts +248 -0
- package/shared/types.ts +132 -0
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
import { lookup } from "node:dns/promises";
|
|
2
|
+
import { isIP } from "node:net";
|
|
3
|
+
import { Agent, fetch as safeFetch } from "undici";
|
|
4
|
+
import ipaddr from "ipaddr.js";
|
|
5
|
+
import * as cheerio from "cheerio";
|
|
6
|
+
import type { AnyNode, Element } from "domhandler";
|
|
7
|
+
import type { ReaderBlock } from "../shared/types.js";
|
|
8
|
+
import { AppError } from "./errors.js";
|
|
9
|
+
|
|
10
|
+
const MAX_BYTES = 1_500_000;
|
|
11
|
+
const MAX_BLOCKS = 400;
|
|
12
|
+
const MAX_BLOCK_CHARS = 2400;
|
|
13
|
+
|
|
14
|
+
export function isPublicAddress(address: string): boolean {
|
|
15
|
+
try {
|
|
16
|
+
let parsed = ipaddr.parse(address);
|
|
17
|
+
if (parsed.kind() === "ipv6" && (parsed as ipaddr.IPv6).isIPv4MappedAddress()) {
|
|
18
|
+
parsed = (parsed as ipaddr.IPv6).toIPv4Address();
|
|
19
|
+
}
|
|
20
|
+
return parsed.range() === "unicast";
|
|
21
|
+
} catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function validatePublicUrl(input: string): URL {
|
|
27
|
+
let url: URL;
|
|
28
|
+
try {
|
|
29
|
+
url = new URL(input.trim());
|
|
30
|
+
} catch {
|
|
31
|
+
throw new AppError(400, "請輸入完整的 HTTPS 網址。");
|
|
32
|
+
}
|
|
33
|
+
const host = url.hostname.replace(/^\[|\]$/g, "").toLowerCase();
|
|
34
|
+
if (
|
|
35
|
+
url.protocol !== "https:" ||
|
|
36
|
+
url.username ||
|
|
37
|
+
url.password ||
|
|
38
|
+
(url.port && url.port !== "443") ||
|
|
39
|
+
host === "localhost" ||
|
|
40
|
+
host.endsWith(".localhost") ||
|
|
41
|
+
host.endsWith(".local") ||
|
|
42
|
+
host.endsWith(".ts.net") ||
|
|
43
|
+
(isIP(host) && !isPublicAddress(host))
|
|
44
|
+
) {
|
|
45
|
+
throw new AppError(400, "只支援公開的 HTTPS 網址,無法讀取本機或內部網路。");
|
|
46
|
+
}
|
|
47
|
+
url.hash = "";
|
|
48
|
+
if (url.href.length > 4000) {
|
|
49
|
+
throw new AppError(400, "網址過長,請直接貼上內容。");
|
|
50
|
+
}
|
|
51
|
+
return url;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function safeImageUrl(raw: string | undefined, baseUrl: URL): string | undefined {
|
|
55
|
+
if (!raw) return undefined;
|
|
56
|
+
try {
|
|
57
|
+
const resolved = new URL(raw, baseUrl);
|
|
58
|
+
return validatePublicUrl(resolved.href).href;
|
|
59
|
+
} catch {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function fetchPublicResource(
|
|
65
|
+
input: string,
|
|
66
|
+
signal: AbortSignal,
|
|
67
|
+
opts: {
|
|
68
|
+
accept: string;
|
|
69
|
+
contentTypePattern: RegExp;
|
|
70
|
+
invalidTypeError: string;
|
|
71
|
+
maxBytes: number;
|
|
72
|
+
sizeError: string;
|
|
73
|
+
timeoutMs: number;
|
|
74
|
+
},
|
|
75
|
+
): Promise<{ buffer: Buffer; finalUrl: URL; contentType: string }> {
|
|
76
|
+
let url = validatePublicUrl(input);
|
|
77
|
+
const timeout = AbortSignal.any([signal, AbortSignal.timeout(opts.timeoutMs)]);
|
|
78
|
+
|
|
79
|
+
for (let redirects = 0; redirects <= 3; redirects++) {
|
|
80
|
+
timeout.throwIfAborted();
|
|
81
|
+
const hostname = url.hostname.replace(/^\[|\]$/g, "");
|
|
82
|
+
const addresses = await lookup(hostname, { all: true });
|
|
83
|
+
timeout.throwIfAborted();
|
|
84
|
+
if (!addresses.length || addresses.some((entry) => !isPublicAddress(entry.address))) {
|
|
85
|
+
throw new AppError(400, "此網址解析到非公開網路,無法讀取。");
|
|
86
|
+
}
|
|
87
|
+
const dispatcher = new Agent({
|
|
88
|
+
connect: {
|
|
89
|
+
timeout: 7000,
|
|
90
|
+
lookup: (_host, options, callback) => {
|
|
91
|
+
const matching = options.family
|
|
92
|
+
? addresses.filter((item) => item.family === options.family)
|
|
93
|
+
: addresses;
|
|
94
|
+
const selected = matching[0];
|
|
95
|
+
if (!selected) {
|
|
96
|
+
callback(new Error("No public address"), "", 4);
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (options.all) callback(null, matching);
|
|
100
|
+
else callback(null, selected.address, selected.family);
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
try {
|
|
105
|
+
const response = await safeFetch(url, {
|
|
106
|
+
dispatcher,
|
|
107
|
+
signal: timeout,
|
|
108
|
+
redirect: "manual",
|
|
109
|
+
headers: {
|
|
110
|
+
"user-agent": "BELL-Web-Reader/0.1 (local personal study reader)",
|
|
111
|
+
accept: opts.accept,
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
115
|
+
const location = response.headers.get("location");
|
|
116
|
+
await response.body?.cancel();
|
|
117
|
+
if (!location || redirects === 3) {
|
|
118
|
+
throw new AppError(422, "重新導向次數過多。");
|
|
119
|
+
}
|
|
120
|
+
url = validatePublicUrl(new URL(location, url).href);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (!response.ok) {
|
|
124
|
+
await response.body?.cancel();
|
|
125
|
+
throw new AppError(
|
|
126
|
+
response.status === 404 ? 404 : 422,
|
|
127
|
+
response.status === 404 ? "找不到此網頁,請確認網址。" : "來源網站拒絕讀取,可直接貼上文字內容。",
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
const contentType = response.headers.get("content-type") || "";
|
|
131
|
+
if (!opts.contentTypePattern.test(contentType)) {
|
|
132
|
+
await response.body?.cancel();
|
|
133
|
+
throw new AppError(422, opts.invalidTypeError);
|
|
134
|
+
}
|
|
135
|
+
if (Number(response.headers.get("content-length") || 0) > opts.maxBytes) {
|
|
136
|
+
await response.body?.cancel();
|
|
137
|
+
throw new AppError(413, opts.sizeError);
|
|
138
|
+
}
|
|
139
|
+
const reader = response.body?.getReader();
|
|
140
|
+
if (!reader) throw new AppError(422, "無法讀取遠端內容。");
|
|
141
|
+
const chunks: Uint8Array[] = [];
|
|
142
|
+
let total = 0;
|
|
143
|
+
while (true) {
|
|
144
|
+
const { done, value } = await reader.read();
|
|
145
|
+
if (done) break;
|
|
146
|
+
total += value.byteLength;
|
|
147
|
+
if (total > opts.maxBytes) {
|
|
148
|
+
await reader.cancel();
|
|
149
|
+
throw new AppError(413, opts.sizeError);
|
|
150
|
+
}
|
|
151
|
+
chunks.push(value);
|
|
152
|
+
}
|
|
153
|
+
return { buffer: Buffer.concat(chunks), finalUrl: url, contentType };
|
|
154
|
+
} finally {
|
|
155
|
+
await dispatcher.close();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
throw new AppError(422, "無法讀取遠端內容。");
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export async function fetchPublicHtml(
|
|
162
|
+
input: string,
|
|
163
|
+
signal: AbortSignal,
|
|
164
|
+
): Promise<{ html: string; finalUrl: URL; contentType: string }> {
|
|
165
|
+
const { buffer, finalUrl, contentType } = await fetchPublicResource(input, signal, {
|
|
166
|
+
accept: "text/html,text/plain,text/markdown;q=0.9,application/xhtml+xml;q=0.9",
|
|
167
|
+
contentTypePattern: /text\/|application\/(xhtml\+xml|xml|json)/i,
|
|
168
|
+
invalidTypeError: "此網址不是可閱讀的文字網頁,請直接貼上內容。",
|
|
169
|
+
maxBytes: MAX_BYTES,
|
|
170
|
+
sizeError: "網頁超過 1.5 MB 上限,請直接貼上想讀的段落。",
|
|
171
|
+
timeoutMs: 12_000,
|
|
172
|
+
});
|
|
173
|
+
return { html: buffer.toString("utf8"), finalUrl, contentType };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export interface FetchedImage {
|
|
177
|
+
bytes: Buffer;
|
|
178
|
+
mimeType: "image/png" | "image/jpeg" | "image/webp" | "image/gif";
|
|
179
|
+
ext: "png" | "jpg" | "webp" | "gif";
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export async function fetchPublicImage(input: string, signal: AbortSignal): Promise<FetchedImage> {
|
|
183
|
+
const { buffer, contentType } = await fetchPublicResource(input, signal, {
|
|
184
|
+
accept: "image/png,image/jpeg,image/webp,image/gif;q=0.9,*/*;q=0.5",
|
|
185
|
+
contentTypePattern: /^image\/(png|jpe?g|webp|gif)\b/i,
|
|
186
|
+
invalidTypeError: "不支援的圖片格式。",
|
|
187
|
+
maxBytes: 3_000_000,
|
|
188
|
+
sizeError: "圖片超過 3 MB 上限。",
|
|
189
|
+
timeoutMs: 8_000,
|
|
190
|
+
});
|
|
191
|
+
const lower = contentType.toLowerCase();
|
|
192
|
+
if (lower.includes("png")) return { bytes: buffer, mimeType: "image/png", ext: "png" };
|
|
193
|
+
if (lower.includes("webp")) return { bytes: buffer, mimeType: "image/webp", ext: "webp" };
|
|
194
|
+
if (lower.includes("gif")) return { bytes: buffer, mimeType: "image/gif", ext: "gif" };
|
|
195
|
+
return { bytes: buffer, mimeType: "image/jpeg", ext: "jpg" };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function normalizeInline(text: string): string {
|
|
199
|
+
return text.replace(/\s+/g, " ").trim().slice(0, MAX_BLOCK_CHARS);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function stripInlineMarkdown(text: string): string {
|
|
203
|
+
return text
|
|
204
|
+
.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1")
|
|
205
|
+
.replace(/\[([^\]]+)\]\((https?:\/\/[^)]+)\)/g, "$1")
|
|
206
|
+
.replace(/\*\*([^*]+)\*\*/g, "$1")
|
|
207
|
+
.replace(/__([^_]+)__/g, "$1")
|
|
208
|
+
.replace(/`([^`]+)`/g, "$1");
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const TABLE_DIVIDER_RE = /^\|?\s*:?-{1,}:?\s*(?:\|\s*:?-{1,}:?\s*)+\|?$/;
|
|
212
|
+
|
|
213
|
+
function splitMarkdownTableRow(line: string): string[] {
|
|
214
|
+
const trimmed = line.trim().replace(/^\|/, "").replace(/\|$/, "");
|
|
215
|
+
return trimmed
|
|
216
|
+
.split(/(?<!\\)\|/)
|
|
217
|
+
.map((cell) => normalizeInline(stripInlineMarkdown(cell.replace(/\\\|/g, "|"))));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function normalizeTableRows(rawRows: string[][]): string[][] {
|
|
221
|
+
const colCount = rawRows.reduce((max, r) => Math.max(max, r.length), 0);
|
|
222
|
+
if (colCount === 0) return [];
|
|
223
|
+
return rawRows.map((row) =>
|
|
224
|
+
Array.from({ length: colCount }, (_, idx) => row[idx] ?? ""),
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function isMarkdownTableAt(lines: string[], index: number): boolean {
|
|
229
|
+
if (index + 1 >= lines.length) return false;
|
|
230
|
+
const header = lines[index].trim();
|
|
231
|
+
const divider = lines[index + 1].trim();
|
|
232
|
+
return header.includes("|") && TABLE_DIVIDER_RE.test(divider);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export interface ExtractedPage {
|
|
236
|
+
title: string;
|
|
237
|
+
siteName?: string;
|
|
238
|
+
excerpt: string;
|
|
239
|
+
blocks: ReaderBlock[];
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export function extractBlocksFromHtml(html: string, pageUrl: URL, contentType = "text/html"): ExtractedPage {
|
|
243
|
+
if (/text\/(plain|markdown)/i.test(contentType)) {
|
|
244
|
+
return parsePastedContent(html, pageUrl.hostname);
|
|
245
|
+
}
|
|
246
|
+
const $ = cheerio.load(html);
|
|
247
|
+
const rawTitle =
|
|
248
|
+
normalizeInline($('meta[property="og:title"]').attr("content") || "") ||
|
|
249
|
+
normalizeInline($("article h1, main h1, h1").first().text()) ||
|
|
250
|
+
normalizeInline($("title").first().text()) ||
|
|
251
|
+
pageUrl.hostname;
|
|
252
|
+
const siteName =
|
|
253
|
+
normalizeInline($('meta[property="og:site_name"]').attr("content") || "") || pageUrl.hostname;
|
|
254
|
+
|
|
255
|
+
$(
|
|
256
|
+
"script, style, noscript, iframe, svg, canvas, form, nav, footer, aside, dialog, [role='navigation'], [role='banner'], [role='contentinfo'], [aria-hidden='true']",
|
|
257
|
+
).remove();
|
|
258
|
+
|
|
259
|
+
let root = $("article").first();
|
|
260
|
+
if (!root.length || root.text().trim().length < 160) root = $("main, [role='main']").first();
|
|
261
|
+
if (!root.length || root.text().trim().length < 120) root = $("body").first();
|
|
262
|
+
|
|
263
|
+
const blocks: ReaderBlock[] = [];
|
|
264
|
+
const seen = new Set<string>();
|
|
265
|
+
const pushBlock = (block: Omit<ReaderBlock, "id">) => {
|
|
266
|
+
if (blocks.length >= MAX_BLOCKS) return;
|
|
267
|
+
const dedupeKey = `${block.kind}:${block.text || block.imageUrl || ""}`;
|
|
268
|
+
if (seen.has(dedupeKey)) return;
|
|
269
|
+
seen.add(dedupeKey);
|
|
270
|
+
blocks.push({ id: `b${blocks.length + 1}`, ...block });
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
const walk = (node: AnyNode) => {
|
|
274
|
+
if (blocks.length >= MAX_BLOCKS || node.type !== "tag") return;
|
|
275
|
+
const el = node as Element;
|
|
276
|
+
const tag = el.tagName.toLowerCase();
|
|
277
|
+
|
|
278
|
+
if (tag === "h1" || tag === "h2" || tag === "h3" || tag === "h4") {
|
|
279
|
+
const text = normalizeInline($(el).text());
|
|
280
|
+
if (text && text !== rawTitle) {
|
|
281
|
+
const level = tag === "h1" ? 1 : tag === "h2" ? 2 : 3;
|
|
282
|
+
pushBlock({ kind: "heading", text, level });
|
|
283
|
+
}
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
if (tag === "pre") {
|
|
287
|
+
const codeEl = $(el).find("code").first();
|
|
288
|
+
const rawCode = (codeEl.length ? codeEl.text() : $(el).text())
|
|
289
|
+
.replace(/\r\n?/g, "\n")
|
|
290
|
+
.replace(/^\n+|\n+$/g, "")
|
|
291
|
+
.slice(0, MAX_BLOCK_CHARS);
|
|
292
|
+
if (rawCode.trim()) {
|
|
293
|
+
const cls = codeEl.attr("class") || $(el).attr("class") || "";
|
|
294
|
+
const langMatch = /language-([a-z0-9_+-]+)/i.exec(cls);
|
|
295
|
+
pushBlock({ kind: "code", text: rawCode, ...(langMatch ? { language: langMatch[1] } : {}) });
|
|
296
|
+
}
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
if (tag === "blockquote") {
|
|
300
|
+
const text = normalizeInline($(el).text());
|
|
301
|
+
if (text.length >= 8) pushBlock({ kind: "quote", text });
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
if (tag === "ul" || tag === "ol") {
|
|
305
|
+
$(el)
|
|
306
|
+
.children("li")
|
|
307
|
+
.each((_, li) => {
|
|
308
|
+
const text = normalizeInline($(li).clone().children("ul, ol").remove().end().text());
|
|
309
|
+
if (text.length >= 4) pushBlock({ kind: "list", text });
|
|
310
|
+
});
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
if (tag === "table") {
|
|
314
|
+
const rawRows: string[][] = [];
|
|
315
|
+
$(el)
|
|
316
|
+
.find("tr")
|
|
317
|
+
.each((_, tr) => {
|
|
318
|
+
const cells: string[] = [];
|
|
319
|
+
$(tr)
|
|
320
|
+
.children("th, td")
|
|
321
|
+
.each((__, cell) => {
|
|
322
|
+
cells.push(normalizeInline($(cell).text()));
|
|
323
|
+
});
|
|
324
|
+
if (cells.some((c) => c.length > 0)) {
|
|
325
|
+
rawRows.push(cells);
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
const rows = normalizeTableRows(rawRows);
|
|
329
|
+
if (rows.length > 0) {
|
|
330
|
+
pushBlock({
|
|
331
|
+
kind: "table",
|
|
332
|
+
text: rows.map((r) => r.join(" | ")).join("\n"),
|
|
333
|
+
tableRows: rows,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
if (tag === "img") {
|
|
339
|
+
const src = safeImageUrl($(el).attr("src") || $(el).attr("data-src"), pageUrl);
|
|
340
|
+
const alt = normalizeInline($(el).attr("alt") || "");
|
|
341
|
+
const width = Number($(el).attr("width") || 400);
|
|
342
|
+
if (src && width >= 120) {
|
|
343
|
+
pushBlock({ kind: "image", text: alt || "文章附圖", imageUrl: src, ...(alt ? { alt } : {}) });
|
|
344
|
+
}
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
if (tag === "p") {
|
|
348
|
+
const img = $(el).find("img").first();
|
|
349
|
+
if (img.length && !normalizeInline($(el).text())) {
|
|
350
|
+
walk(img[0]);
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
const text = normalizeInline($(el).text());
|
|
354
|
+
if (text.length >= 12) pushBlock({ kind: "paragraph", text });
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
$(el)
|
|
359
|
+
.contents()
|
|
360
|
+
.each((_, child) => walk(child));
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
root.contents().each((_, child) => walk(child));
|
|
364
|
+
|
|
365
|
+
if (!blocks.length) {
|
|
366
|
+
const rawCandidate = ($("#publish-page, #doc").first().text() || root.text()).replace(/\r\n?/g, "\n").trim();
|
|
367
|
+
if (rawCandidate.includes("\n") && /^(?:#{1,3}\s+|```|!\[[^\]]*\]\(https?:\/\/|[-*+]\s+|\$\$)/m.test(rawCandidate)) {
|
|
368
|
+
const cleanTitle = rawTitle.replace(/\s+-\s+HackMD$/i, "").trim();
|
|
369
|
+
return parsePastedContent(rawCandidate, cleanTitle, siteName.slice(0, 60));
|
|
370
|
+
}
|
|
371
|
+
const fallback = normalizeInline(root.text());
|
|
372
|
+
if (!fallback) throw new AppError(422, "無法從此網頁擷取正文(可能是純動態渲染網站),請直接貼上文字內容。");
|
|
373
|
+
pushBlock({ kind: "paragraph", text: fallback });
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const firstProse = blocks.find((b) => b.kind === "paragraph" || b.kind === "quote" || b.kind === "list");
|
|
377
|
+
const excerpt = (firstProse?.text || blocks[0].text).slice(0, 160);
|
|
378
|
+
return {
|
|
379
|
+
title: rawTitle.slice(0, 120),
|
|
380
|
+
siteName: siteName.slice(0, 60),
|
|
381
|
+
excerpt,
|
|
382
|
+
blocks,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export function parsePastedContent(raw: string, explicitTitle?: string, siteName?: string): ExtractedPage {
|
|
387
|
+
const cleaned = raw.replace(/\r\n?/g, "\n").trim();
|
|
388
|
+
if (!cleaned) throw new AppError(400, "請輸入網址或貼上想閱讀的內容。");
|
|
389
|
+
|
|
390
|
+
const lines = cleaned.split("\n");
|
|
391
|
+
const blocks: ReaderBlock[] = [];
|
|
392
|
+
let inferredTitle = explicitTitle?.trim() || "";
|
|
393
|
+
|
|
394
|
+
const push = (block: Omit<ReaderBlock, "id">) => {
|
|
395
|
+
if (blocks.length >= MAX_BLOCKS || !block.text.trim()) return;
|
|
396
|
+
blocks.push({ id: `b${blocks.length + 1}`, ...block, text: block.text.slice(0, MAX_BLOCK_CHARS) });
|
|
397
|
+
};
|
|
398
|
+
|
|
399
|
+
let i = 0;
|
|
400
|
+
while (i < lines.length && blocks.length < MAX_BLOCKS) {
|
|
401
|
+
const line = lines[i];
|
|
402
|
+
const trimmed = line.trim();
|
|
403
|
+
if (!trimmed) {
|
|
404
|
+
i++;
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
if (isMarkdownTableAt(lines, i)) {
|
|
408
|
+
const rawRows = [splitMarkdownTableRow(lines[i])];
|
|
409
|
+
i += 2; // Skip header and separator line
|
|
410
|
+
while (i < lines.length) {
|
|
411
|
+
const rowLine = lines[i].trim();
|
|
412
|
+
if (!rowLine || /^(```|\$\$|#{1,4}\s+)/.test(rowLine) || !rowLine.includes("|")) break;
|
|
413
|
+
rawRows.push(splitMarkdownTableRow(lines[i]));
|
|
414
|
+
i++;
|
|
415
|
+
}
|
|
416
|
+
const rows = normalizeTableRows(rawRows);
|
|
417
|
+
if (rows.length > 0) {
|
|
418
|
+
push({
|
|
419
|
+
kind: "table",
|
|
420
|
+
text: rows.map((r) => r.join(" | ")).join("\n"),
|
|
421
|
+
tableRows: rows,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
const fence = /^```([a-z0-9_+-]*)\s*$/i.exec(trimmed);
|
|
427
|
+
if (fence) {
|
|
428
|
+
const codeLines: string[] = [];
|
|
429
|
+
i++;
|
|
430
|
+
while (i < lines.length && !lines[i].trim().startsWith("```")) {
|
|
431
|
+
codeLines.push(lines[i]);
|
|
432
|
+
i++;
|
|
433
|
+
}
|
|
434
|
+
if (i < lines.length) i++;
|
|
435
|
+
push({ kind: "code", text: codeLines.join("\n"), ...(fence[1] ? { language: fence[1] } : {}) });
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
if (trimmed === "$$") {
|
|
439
|
+
const mathLines: string[] = [];
|
|
440
|
+
i++;
|
|
441
|
+
while (i < lines.length && lines[i].trim() !== "$$") {
|
|
442
|
+
mathLines.push(lines[i]);
|
|
443
|
+
i++;
|
|
444
|
+
}
|
|
445
|
+
if (i < lines.length) i++;
|
|
446
|
+
push({ kind: "code", language: "math", text: mathLines.join("\n").trim() });
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
const imgMatch = /^!\[([^\]]*)\]\((https?:\/\/[^\s)]+)\)\s*$/.exec(trimmed);
|
|
450
|
+
if (imgMatch) {
|
|
451
|
+
const alt = normalizeInline(imgMatch[1]);
|
|
452
|
+
try {
|
|
453
|
+
const src = validatePublicUrl(imgMatch[2]).href;
|
|
454
|
+
push({ kind: "image", text: alt || "文章附圖", imageUrl: src, ...(alt ? { alt } : {}) });
|
|
455
|
+
} catch {
|
|
456
|
+
// Skip invalid/private image URLs
|
|
457
|
+
}
|
|
458
|
+
i++;
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
const heading = /^(#{1,4})\s+(.+)$/.exec(trimmed);
|
|
462
|
+
if (heading) {
|
|
463
|
+
const level = Math.min(3, heading[1].length) as 1 | 2 | 3;
|
|
464
|
+
const text = stripInlineMarkdown(heading[2].trim());
|
|
465
|
+
if (level === 1 && blocks.length === 0 && (!inferredTitle || inferredTitle === text)) {
|
|
466
|
+
inferredTitle = text;
|
|
467
|
+
} else {
|
|
468
|
+
push({ kind: "heading", text, level });
|
|
469
|
+
}
|
|
470
|
+
i++;
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
if (/^>\s*/.test(trimmed)) {
|
|
474
|
+
const quoteLines: string[] = [];
|
|
475
|
+
while (i < lines.length && /^>\s*/.test(lines[i].trim())) {
|
|
476
|
+
quoteLines.push(lines[i].trim().replace(/^>\s*/, ""));
|
|
477
|
+
i++;
|
|
478
|
+
}
|
|
479
|
+
push({ kind: "quote", text: normalizeInline(stripInlineMarkdown(quoteLines.join(" "))) });
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
if (/^([-*+]|\d+\.)\s+/.test(trimmed)) {
|
|
483
|
+
push({
|
|
484
|
+
kind: "list",
|
|
485
|
+
text: normalizeInline(stripInlineMarkdown(trimmed.replace(/^([-*+]|\d+\.)\s+/, ""))),
|
|
486
|
+
});
|
|
487
|
+
i++;
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
const paraLines: string[] = [];
|
|
491
|
+
while (
|
|
492
|
+
i < lines.length &&
|
|
493
|
+
lines[i].trim() &&
|
|
494
|
+
!isMarkdownTableAt(lines, i) &&
|
|
495
|
+
!/^(```|\$\$|#{1,4}\s+|>\s*|([-*+]|\d+\.)\s+|!\[[^\]]*\]\(https?:\/\/)/.test(lines[i].trim())
|
|
496
|
+
) {
|
|
497
|
+
paraLines.push(lines[i].trim());
|
|
498
|
+
i++;
|
|
499
|
+
}
|
|
500
|
+
push({ kind: "paragraph", text: normalizeInline(stripInlineMarkdown(paraLines.join(" "))) });
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
if (!blocks.length) {
|
|
504
|
+
blocks.push({ id: "b1", kind: "paragraph", text: cleaned.slice(0, MAX_BLOCK_CHARS) });
|
|
505
|
+
}
|
|
506
|
+
const title = (inferredTitle || blocks[0].text.slice(0, 42)).slice(0, 120);
|
|
507
|
+
const firstProse = blocks.find((b) => b.kind !== "heading") ?? blocks[0];
|
|
508
|
+
return {
|
|
509
|
+
title,
|
|
510
|
+
...(siteName ? { siteName } : {}),
|
|
511
|
+
excerpt: firstProse.text.slice(0, 160),
|
|
512
|
+
blocks,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Resolves a quote inside a block's text to exact character offsets [startOffset, endOffset].
|
|
518
|
+
* Falls back to collapsed-whitespace matching if AI slightly normalized spaces.
|
|
519
|
+
*/
|
|
520
|
+
export function resolveQuoteAnchor(
|
|
521
|
+
blockText: string,
|
|
522
|
+
quote: string,
|
|
523
|
+
): { startOffset: number; endOffset: number; exactQuote: string } | undefined {
|
|
524
|
+
const trimmed = quote.trim();
|
|
525
|
+
if (!trimmed || !blockText) return undefined;
|
|
526
|
+
const direct = blockText.indexOf(trimmed);
|
|
527
|
+
if (direct >= 0) {
|
|
528
|
+
return {
|
|
529
|
+
startOffset: direct,
|
|
530
|
+
endOffset: direct + trimmed.length,
|
|
531
|
+
exactQuote: blockText.slice(direct, direct + trimmed.length),
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
// Collapsed whitespace fallback
|
|
535
|
+
const tokens = trimmed.split(/\s+/).map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
536
|
+
if (tokens.length > 1) {
|
|
537
|
+
const regex = new RegExp(tokens.join("\\s+"));
|
|
538
|
+
const match = regex.exec(blockText);
|
|
539
|
+
if (match && match[0]) {
|
|
540
|
+
return {
|
|
541
|
+
startOffset: match.index,
|
|
542
|
+
endOffset: match.index + match[0].length,
|
|
543
|
+
exactQuote: match[0],
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
return undefined;
|
|
548
|
+
}
|
package/server/index.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
|
+
import { createApp } from "./app.js";
|
|
3
|
+
import { loadConfig } from "./config.js";
|
|
4
|
+
import { ReaderStore } from "./store.js";
|
|
5
|
+
|
|
6
|
+
const config = loadConfig();
|
|
7
|
+
const dbPath = resolve(config.dataDir, "bell-reader.sqlite");
|
|
8
|
+
const store = new ReaderStore(dbPath);
|
|
9
|
+
|
|
10
|
+
const app = createApp({
|
|
11
|
+
store,
|
|
12
|
+
allowedOrigins: config.allowedOrigins,
|
|
13
|
+
suiteLinks: config.suiteLinks,
|
|
14
|
+
suiteFetch: config.suiteFetch,
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const server = app.listen(config.port, config.host, () => {
|
|
18
|
+
console.log(`BELL Web Reader listening on http://${config.host}:${config.port}`);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const shutdown = () => {
|
|
22
|
+
server.close(() => {
|
|
23
|
+
store.close();
|
|
24
|
+
process.exit(0);
|
|
25
|
+
});
|
|
26
|
+
};
|
|
27
|
+
process.on("SIGINT", shutdown);
|
|
28
|
+
process.on("SIGTERM", shutdown);
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { NextFunction, Request, Response } from "express";
|
|
2
|
+
|
|
3
|
+
function localHostsForPort(port: number | undefined): Set<string> {
|
|
4
|
+
const hosts = new Set(["127.0.0.1", "localhost", "[::1]"]);
|
|
5
|
+
if (port) {
|
|
6
|
+
hosts.add(`127.0.0.1:${port}`);
|
|
7
|
+
hosts.add(`localhost:${port}`);
|
|
8
|
+
hosts.add(`[::1]:${port}`);
|
|
9
|
+
}
|
|
10
|
+
return hosts;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function securityMiddleware(allowedOrigins: readonly string[]) {
|
|
14
|
+
const allowedOriginSet = new Set(allowedOrigins);
|
|
15
|
+
const allowedHostSet = new Set(
|
|
16
|
+
allowedOrigins.map((origin) => new URL(origin).host.toLowerCase()),
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
return (req: Request, res: Response, next: NextFunction): void => {
|
|
20
|
+
res.setHeader(
|
|
21
|
+
"Content-Security-Policy",
|
|
22
|
+
"default-src 'self'; img-src 'self' https: data:; style-src 'self' 'unsafe-inline'; font-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'none'",
|
|
23
|
+
);
|
|
24
|
+
res.setHeader("X-Frame-Options", "DENY");
|
|
25
|
+
res.setHeader("Referrer-Policy", "no-referrer");
|
|
26
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
27
|
+
|
|
28
|
+
const localPort = req.socket.localPort;
|
|
29
|
+
const localHosts = localHostsForPort(localPort);
|
|
30
|
+
const hostHeader = (req.headers.host ?? "").toLowerCase();
|
|
31
|
+
if (!hostHeader || (!localHosts.has(hostHeader) && !allowedHostSet.has(hostHeader))) {
|
|
32
|
+
res.status(403).json({ error: "不允許的 Host 來源。" });
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const isApi = req.path === "/api" || req.path.startsWith("/api/");
|
|
37
|
+
const secFetchSite = req.headers["sec-fetch-site"];
|
|
38
|
+
const isTopDocNav =
|
|
39
|
+
!isApi &&
|
|
40
|
+
(req.method === "GET" || req.method === "HEAD") &&
|
|
41
|
+
req.headers["sec-fetch-mode"] === "navigate" &&
|
|
42
|
+
req.headers["sec-fetch-dest"] === "document";
|
|
43
|
+
|
|
44
|
+
if (secFetchSite === "cross-site" && !isTopDocNav) {
|
|
45
|
+
res.status(403).json({ error: "拒絕跨站請求。" });
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const origin = req.headers.origin;
|
|
50
|
+
if (origin) {
|
|
51
|
+
let validOrigin = allowedOriginSet.has(origin);
|
|
52
|
+
if (!validOrigin) {
|
|
53
|
+
try {
|
|
54
|
+
const parsed = new URL(origin);
|
|
55
|
+
validOrigin =
|
|
56
|
+
(parsed.protocol === "http:" || parsed.protocol === "https:") &&
|
|
57
|
+
localHosts.has(parsed.host.toLowerCase());
|
|
58
|
+
} catch {
|
|
59
|
+
validOrigin = false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (!validOrigin) {
|
|
63
|
+
res.status(403).json({ error: "不允許的 Origin 來源。" });
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (req.method !== "GET" && req.method !== "HEAD" && req.method !== "OPTIONS") {
|
|
69
|
+
const contentType = req.headers["content-type"] ?? "";
|
|
70
|
+
if (!/^application\/json\b/i.test(contentType)) {
|
|
71
|
+
res.status(415).json({ error: "寫入請求必須使用 application/json。" });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
next();
|
|
77
|
+
};
|
|
78
|
+
}
|