dsh-thesis 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/DESIGN.md +390 -0
  2. package/LICENSE +21 -0
  3. package/NOTICE.md +10 -0
  4. package/README.md +253 -0
  5. package/cordis.patch.yml +17 -0
  6. package/docs/host-api.md +640 -0
  7. package/lib/codewalk/commands.d.ts +23 -0
  8. package/lib/codewalk/commands.js +168 -0
  9. package/lib/codewalk/engine.d.ts +108 -0
  10. package/lib/codewalk/engine.js +449 -0
  11. package/lib/codewalk/errors.d.ts +31 -0
  12. package/lib/codewalk/errors.js +20 -0
  13. package/lib/codewalk/fs-types.d.ts +34 -0
  14. package/lib/codewalk/fs-types.js +10 -0
  15. package/lib/codewalk/gate.d.ts +30 -0
  16. package/lib/codewalk/gate.js +60 -0
  17. package/lib/codewalk/host-types.d.ts +198 -0
  18. package/lib/codewalk/host-types.js +11 -0
  19. package/lib/codewalk/prestep.d.ts +17 -0
  20. package/lib/codewalk/prestep.js +62 -0
  21. package/lib/codewalk/register.d.ts +23 -0
  22. package/lib/codewalk/register.js +31 -0
  23. package/lib/codewalk/skill.d.ts +15 -0
  24. package/lib/codewalk/skill.js +57 -0
  25. package/lib/codewalk/tools.d.ts +18 -0
  26. package/lib/codewalk/tools.js +314 -0
  27. package/lib/codewalk/types.d.ts +151 -0
  28. package/lib/codewalk/types.js +9 -0
  29. package/lib/commands.d.ts +33 -0
  30. package/lib/commands.js +60 -0
  31. package/lib/config.d.ts +125 -0
  32. package/lib/config.js +139 -0
  33. package/lib/dedup/commands.d.ts +21 -0
  34. package/lib/dedup/commands.js +93 -0
  35. package/lib/dedup/index.d.ts +75 -0
  36. package/lib/dedup/index.js +331 -0
  37. package/lib/dedup/normalize.d.ts +125 -0
  38. package/lib/dedup/normalize.js +329 -0
  39. package/lib/dedup/report.d.ts +114 -0
  40. package/lib/dedup/report.js +309 -0
  41. package/lib/dedup/rewrite.d.ts +71 -0
  42. package/lib/dedup/rewrite.js +228 -0
  43. package/lib/dedup/similarity.d.ts +169 -0
  44. package/lib/dedup/similarity.js +323 -0
  45. package/lib/gates/dangerous.d.ts +22 -0
  46. package/lib/gates/dangerous.js +47 -0
  47. package/lib/gates/detect.d.ts +54 -0
  48. package/lib/gates/detect.js +106 -0
  49. package/lib/gates/messages.d.ts +39 -0
  50. package/lib/gates/messages.js +74 -0
  51. package/lib/gates/prompt.d.ts +21 -0
  52. package/lib/gates/prompt.js +56 -0
  53. package/lib/gates/writing-intent.d.ts +49 -0
  54. package/lib/gates/writing-intent.js +68 -0
  55. package/lib/index.d.ts +36 -0
  56. package/lib/index.js +76 -0
  57. package/lib/ingest/code.d.ts +71 -0
  58. package/lib/ingest/code.js +356 -0
  59. package/lib/ingest/commands.d.ts +23 -0
  60. package/lib/ingest/commands.js +74 -0
  61. package/lib/ingest/docx.d.ts +29 -0
  62. package/lib/ingest/docx.js +92 -0
  63. package/lib/ingest/index.d.ts +179 -0
  64. package/lib/ingest/index.js +640 -0
  65. package/lib/ingest/ooxml.d.ts +58 -0
  66. package/lib/ingest/ooxml.js +393 -0
  67. package/lib/ingest/pdf.d.ts +86 -0
  68. package/lib/ingest/pdf.js +1016 -0
  69. package/lib/ingest/pptx.d.ts +32 -0
  70. package/lib/ingest/pptx.js +116 -0
  71. package/lib/ingest/text.d.ts +25 -0
  72. package/lib/ingest/text.js +180 -0
  73. package/lib/ingest/types.d.ts +48 -0
  74. package/lib/ingest/types.js +47 -0
  75. package/lib/ingest/xlsx.d.ts +72 -0
  76. package/lib/ingest/xlsx.js +411 -0
  77. package/lib/ingest/zip.d.ts +76 -0
  78. package/lib/ingest/zip.js +393 -0
  79. package/lib/intake/definitions.d.ts +65 -0
  80. package/lib/intake/definitions.js +42 -0
  81. package/lib/intake/gate-rules.d.ts +57 -0
  82. package/lib/intake/gate-rules.js +118 -0
  83. package/lib/intake/gate.d.ts +24 -0
  84. package/lib/intake/gate.js +86 -0
  85. package/lib/intake/index.d.ts +76 -0
  86. package/lib/intake/index.js +468 -0
  87. package/lib/intake/questions.d.ts +169 -0
  88. package/lib/intake/questions.js +720 -0
  89. package/lib/intake/spec.d.ts +66 -0
  90. package/lib/intake/spec.js +221 -0
  91. package/lib/intake/state.d.ts +192 -0
  92. package/lib/intake/state.js +420 -0
  93. package/lib/memory/recall.d.ts +69 -0
  94. package/lib/memory/recall.js +177 -0
  95. package/lib/memory/register.d.ts +37 -0
  96. package/lib/memory/register.js +52 -0
  97. package/lib/memory/service.d.ts +39 -0
  98. package/lib/memory/service.js +39 -0
  99. package/lib/memory/store.d.ts +45 -0
  100. package/lib/memory/store.js +123 -0
  101. package/lib/memory/tools.d.ts +19 -0
  102. package/lib/memory/tools.js +195 -0
  103. package/lib/memory/types.d.ts +55 -0
  104. package/lib/memory/types.js +6 -0
  105. package/lib/paper/commands.d.ts +13 -0
  106. package/lib/paper/commands.js +84 -0
  107. package/lib/paper/lib/aicheck.d.ts +38 -0
  108. package/lib/paper/lib/aicheck.js +241 -0
  109. package/lib/paper/lib/bibtex.d.ts +21 -0
  110. package/lib/paper/lib/bibtex.js +114 -0
  111. package/lib/paper/lib/check.d.ts +32 -0
  112. package/lib/paper/lib/check.js +215 -0
  113. package/lib/paper/lib/defense.d.ts +48 -0
  114. package/lib/paper/lib/defense.js +220 -0
  115. package/lib/paper/lib/docx.d.ts +35 -0
  116. package/lib/paper/lib/docx.js +213 -0
  117. package/lib/paper/lib/layout.d.ts +41 -0
  118. package/lib/paper/lib/layout.js +572 -0
  119. package/lib/paper/lib/ledger.d.ts +112 -0
  120. package/lib/paper/lib/ledger.js +349 -0
  121. package/lib/paper/lib/lit-api.d.ts +59 -0
  122. package/lib/paper/lib/lit-api.js +296 -0
  123. package/lib/paper/lib/lit-cache.d.ts +32 -0
  124. package/lib/paper/lib/lit-cache.js +91 -0
  125. package/lib/paper/lib/project.d.ts +29 -0
  126. package/lib/paper/lib/project.js +93 -0
  127. package/lib/paper/lib/review.d.ts +37 -0
  128. package/lib/paper/lib/review.js +194 -0
  129. package/lib/paper/lib/zip.d.ts +14 -0
  130. package/lib/paper/lib/zip.js +124 -0
  131. package/lib/paper/register.d.ts +20 -0
  132. package/lib/paper/register.js +183 -0
  133. package/lib/paper/tools/aicheck.d.ts +6 -0
  134. package/lib/paper/tools/aicheck.js +38 -0
  135. package/lib/paper/tools/build.d.ts +27 -0
  136. package/lib/paper/tools/build.js +166 -0
  137. package/lib/paper/tools/check.d.ts +7 -0
  138. package/lib/paper/tools/check.js +79 -0
  139. package/lib/paper/tools/decision.d.ts +13 -0
  140. package/lib/paper/tools/decision.js +53 -0
  141. package/lib/paper/tools/defense.d.ts +11 -0
  142. package/lib/paper/tools/defense.js +74 -0
  143. package/lib/paper/tools/init.d.ts +23 -0
  144. package/lib/paper/tools/init.js +99 -0
  145. package/lib/paper/tools/lit.d.ts +27 -0
  146. package/lib/paper/tools/lit.js +236 -0
  147. package/lib/paper/tools/progress.d.ts +18 -0
  148. package/lib/paper/tools/progress.js +77 -0
  149. package/lib/paper/tools/review.d.ts +11 -0
  150. package/lib/paper/tools/review.js +41 -0
  151. package/lib/ppt/actions.d.ts +97 -0
  152. package/lib/ppt/actions.js +263 -0
  153. package/lib/ppt/check.d.ts +64 -0
  154. package/lib/ppt/check.js +185 -0
  155. package/lib/ppt/commands.d.ts +28 -0
  156. package/lib/ppt/commands.js +71 -0
  157. package/lib/ppt/convert.d.ts +146 -0
  158. package/lib/ppt/convert.js +423 -0
  159. package/lib/ppt/index.d.ts +45 -0
  160. package/lib/ppt/index.js +57 -0
  161. package/lib/ppt/marp.d.ts +74 -0
  162. package/lib/ppt/marp.js +165 -0
  163. package/lib/ppt/outline.d.ts +206 -0
  164. package/lib/ppt/outline.js +695 -0
  165. package/lib/shared/define-tool.d.ts +77 -0
  166. package/lib/shared/define-tool.js +235 -0
  167. package/lib/shared/fs-errors.d.ts +50 -0
  168. package/lib/shared/fs-errors.js +90 -0
  169. package/lib/shared/index.d.ts +31 -0
  170. package/lib/shared/index.js +32 -0
  171. package/lib/shared/spawn.d.ts +38 -0
  172. package/lib/shared/spawn.js +50 -0
  173. package/lib/shared/text.d.ts +14 -0
  174. package/lib/shared/text.js +26 -0
  175. package/package.json +82 -0
  176. package/skills/code-walkthrough/SKILL.md +94 -0
  177. package/skills/code-walkthrough/references/examples.md +129 -0
  178. package/skills/paper-dedup/SKILL.md +58 -0
  179. package/skills/paper-intake/SKILL.md +64 -0
  180. package/skills/thesis-citation/SKILL.md +67 -0
  181. package/skills/thesis-defense/SKILL.md +87 -0
  182. package/skills/thesis-eng-design/SKILL.md +97 -0
  183. package/skills/thesis-inquiry/SKILL.md +55 -0
  184. package/skills/thesis-inquiry/references/intent-spec-template.md +39 -0
  185. package/skills/thesis-literature/SKILL.md +56 -0
  186. package/skills/thesis-opener/SKILL.md +77 -0
  187. package/skills/thesis-pipeline/SKILL.md +121 -0
  188. package/skills/thesis-writing/SKILL.md +86 -0
@@ -0,0 +1,32 @@
1
+ /**
2
+ * `.pptx` 摄取:按 `ppt/slides/slideN.xml` 的**数字顺序**逐页抽文本。
3
+ *
4
+ * 顺序问题:ZIP 中央目录里的条目顺序不等于放映顺序,而且 `ppt/_rels/presentation.xml.rels`
5
+ * 只给出 rId→文件,真正的顺序在 `ppt/presentation.xml` 的 `<p:sldIdLst>` 里。
6
+ * 这里以"文件名里的数字"为准(PowerPoint 自身就是按数字命名并保持顺序的),
7
+ * 数字相同再按字典序,保证确定性。
8
+ *
9
+ * 每页渲染为:
10
+ * ```
11
+ * ## 第 3 页
12
+ * 标题文本
13
+ * 正文第一行
14
+ * 正文第二行
15
+ * ```
16
+ * 标题与正文之间只是换行(`a:p` 之间本就换行),不做"标题/正文"语义猜测。
17
+ *
18
+ * @module dsh-thesis/ingest
19
+ */
20
+ import { type ExtractOptions, type ExtractResult } from './types.ts';
21
+ /** 从 `ppt/slides/slide12.xml` 里取页码 12;取不到返回 Number.MAX_SAFE_INTEGER。 */
22
+ export declare function slideNumber(path: string): number;
23
+ /** 按页码排序幻灯片路径(同页码按字典序)。 */
24
+ export declare function sortSlides(paths: readonly string[]): string[];
25
+ /** 纯抽取:逐页渲染。 */
26
+ export declare function readPptx(bytes: Uint8Array): {
27
+ text: string;
28
+ slides: number;
29
+ skipped: number;
30
+ };
31
+ /** 门面用入口:pptx 字节 → 抽取结果。 */
32
+ export declare function extractPptx(bytes: Uint8Array, options: ExtractOptions): ExtractResult;
@@ -0,0 +1,116 @@
1
+ /**
2
+ * `.pptx` 摄取:按 `ppt/slides/slideN.xml` 的**数字顺序**逐页抽文本。
3
+ *
4
+ * 顺序问题:ZIP 中央目录里的条目顺序不等于放映顺序,而且 `ppt/_rels/presentation.xml.rels`
5
+ * 只给出 rId→文件,真正的顺序在 `ppt/presentation.xml` 的 `<p:sldIdLst>` 里。
6
+ * 这里以"文件名里的数字"为准(PowerPoint 自身就是按数字命名并保持顺序的),
7
+ * 数字相同再按字典序,保证确定性。
8
+ *
9
+ * 每页渲染为:
10
+ * ```
11
+ * ## 第 3 页
12
+ * 标题文本
13
+ * 正文第一行
14
+ * 正文第二行
15
+ * ```
16
+ * 标题与正文之间只是换行(`a:p` 之间本就换行),不做"标题/正文"语义猜测。
17
+ *
18
+ * @module dsh-thesis/ingest
19
+ */
20
+ import { extractPptxParagraphs, extractRawElement } from "./ooxml.js";
21
+ import { failureResult, successResult } from "./types.js";
22
+ import { ZipError, decodeBytes, openZip } from "./zip.js";
23
+ /** 单页最多字符数(防一张疯狂的幻灯片吃掉整个预算)。 */
24
+ const SLIDE_CHAR_LIMIT = 50_000;
25
+ /** 最多渲染的页数。 */
26
+ const SLIDE_LIMIT = 500;
27
+ /** 从 `ppt/slides/slide12.xml` 里取页码 12;取不到返回 Number.MAX_SAFE_INTEGER。 */
28
+ export function slideNumber(path) {
29
+ const match = /slide(\d+)\.xml$/i.exec(path);
30
+ if (match === null)
31
+ return Number.MAX_SAFE_INTEGER;
32
+ const value = Number.parseInt(match[1], 10);
33
+ return Number.isFinite(value) ? value : Number.MAX_SAFE_INTEGER;
34
+ }
35
+ /** 按页码排序幻灯片路径(同页码按字典序)。 */
36
+ export function sortSlides(paths) {
37
+ return [...paths].sort((a, b) => {
38
+ const na = slideNumber(a);
39
+ const nb = slideNumber(b);
40
+ if (na !== nb)
41
+ return na - nb;
42
+ return a.localeCompare(b);
43
+ });
44
+ }
45
+ /** 读取 `docProps/core.xml` 的 `dc:title`(演示文稿"标题"属性)。 */
46
+ function readCoreTitle(bytes) {
47
+ try {
48
+ const core = decodeBytes(openZip(bytes).require('docProps/core.xml'));
49
+ const raw = extractRawElement(core, ['dc:title', 'title']);
50
+ return raw === undefined || raw.trim() === '' ? undefined : raw;
51
+ }
52
+ catch {
53
+ return undefined;
54
+ }
55
+ }
56
+ /** 纯抽取:逐页渲染。 */
57
+ export function readPptx(bytes) {
58
+ const zip = openZip(bytes);
59
+ if (!zip.has('ppt/presentation.xml') && zip.names('ppt/slides/', '.xml').length === 0) {
60
+ throw new ZipError('missing-presentation', '这个 ZIP 里既没有 ppt/presentation.xml 也没有 ppt/slides/:不是有效的 .pptx(可能是 .ppt 改了扩展名)。请用 PowerPoint/WPS 另存为 .pptx。');
61
+ }
62
+ const slidePaths = sortSlides(zip.names('ppt/slides/', '.xml'));
63
+ if (slidePaths.length === 0) {
64
+ throw new ZipError('no-slides', '演示文稿里没有任何幻灯片:无法抽取内容。');
65
+ }
66
+ const blocks = [];
67
+ let rendered = 0;
68
+ let skipped = 0;
69
+ for (const path of slidePaths.slice(0, SLIDE_LIMIT)) {
70
+ const paragraphs = extractPptxParagraphs(decodeBytes(zip.require(path)));
71
+ const body = paragraphs.join('\n');
72
+ const page = slideNumber(path);
73
+ const label = page === Number.MAX_SAFE_INTEGER ? path.replace(/^ppt\/slides\//, '').replace(/\.xml$/, '') : String(page);
74
+ if (body.trim() === '') {
75
+ skipped += 1;
76
+ blocks.push(`## 第 ${label} 页\n(本页没有文本,可能只有图片)`);
77
+ continue;
78
+ }
79
+ const limited = body.length > SLIDE_CHAR_LIMIT ? `${body.slice(0, SLIDE_CHAR_LIMIT)}\n…(本页内容过长已截断)` : body;
80
+ blocks.push(`## 第 ${label} 页\n${limited}`);
81
+ rendered += 1;
82
+ }
83
+ const notes = [];
84
+ if (slidePaths.length > SLIDE_LIMIT)
85
+ notes.push(`共 ${slidePaths.length} 页,只渲染了前 ${SLIDE_LIMIT} 页。`);
86
+ if (skipped > 0)
87
+ notes.push(`其中 ${skipped} 页没有文本(图片页)。`);
88
+ return {
89
+ text: `${blocks.join('\n\n')}${notes.length > 0 ? `\n\n(${notes.join(' ')})` : ''}\n`,
90
+ slides: slidePaths.length,
91
+ skipped,
92
+ };
93
+ }
94
+ /** 门面用入口:pptx 字节 → 抽取结果。 */
95
+ export function extractPptx(bytes, options) {
96
+ let read;
97
+ try {
98
+ read = readPptx(bytes);
99
+ }
100
+ catch (error) {
101
+ const reason = error instanceof ZipError
102
+ ? error.message
103
+ : `读取 .pptx 失败:${error instanceof Error ? error.message : String(error)}`;
104
+ return failureResult(reason);
105
+ }
106
+ if (read.text.trim() === '') {
107
+ return failureResult('这个 .pptx 的所有页面都没有文本(可能全图):没有可摄取的文字。');
108
+ }
109
+ return successResult({
110
+ text: read.text,
111
+ maxChars: options.maxChars,
112
+ title: readCoreTitle(bytes),
113
+ note: `共 ${read.slides} 页。备注页(讲稿)不在其中:如需讲稿请另行提供。`,
114
+ lowConfidence: read.skipped > read.slides / 2,
115
+ });
116
+ }
@@ -0,0 +1,25 @@
1
+ /**
2
+ * 纯文本类材料摄取:`.md/.markdown/.txt/.csv/.tsv/.json/.yaml/.yml/.bib/.ris/.html`。
3
+ *
4
+ * 处理内容:
5
+ * - 编码判定:UTF-8 BOM / UTF-16 LE·BE BOM 先剥 BOM;再按"UTF-8 解码是否出现
6
+ * 替换字符"判断,必要时回退 GBK(中文 Windows 的记事本/CSV 常见);
7
+ * - 换行归一:CRLF/CR → LF;
8
+ * - HTML:去掉 `<script>/<style>`、块级标签转换行、去掉其余标签、解码常见实体,
9
+ * 得到可读纯文本(不引第三方解析器)。
10
+ *
11
+ * @module dsh-thesis/ingest
12
+ */
13
+ import { type ExtractOptions, type ExtractResult } from './types.ts';
14
+ /** `node:fs` 读进来的字节 → 文本(BOM 与编码归一)。 */
15
+ export declare function decodeTextBytes(bytes: Uint8Array): string;
16
+ /** CRLF / CR → LF。 */
17
+ export declare function normalizeNewlines(text: string): string;
18
+ /** 解 HTML 实体(数字实体 + 常见命名实体,其余原样保留)。 */
19
+ export declare function decodeHtmlEntities(input: string): string;
20
+ /** HTML → 纯文本(去标签、块级转换行、解码实体)。 */
21
+ export declare function htmlToText(html: string): string;
22
+ /** 是否像二进制(含 NUL 或大量控制字符)——防止把 .dat 当文本读。 */
23
+ export declare function looksBinary(bytes: Uint8Array): boolean;
24
+ /** 门面用入口:按扩展名给的信息做纯文本抽取。 */
25
+ export declare function extractText(bytes: Uint8Array, filename: string, options: ExtractOptions): ExtractResult;
@@ -0,0 +1,180 @@
1
+ /**
2
+ * 纯文本类材料摄取:`.md/.markdown/.txt/.csv/.tsv/.json/.yaml/.yml/.bib/.ris/.html`。
3
+ *
4
+ * 处理内容:
5
+ * - 编码判定:UTF-8 BOM / UTF-16 LE·BE BOM 先剥 BOM;再按"UTF-8 解码是否出现
6
+ * 替换字符"判断,必要时回退 GBK(中文 Windows 的记事本/CSV 常见);
7
+ * - 换行归一:CRLF/CR → LF;
8
+ * - HTML:去掉 `<script>/<style>`、块级标签转换行、去掉其余标签、解码常见实体,
9
+ * 得到可读纯文本(不引第三方解析器)。
10
+ *
11
+ * @module dsh-thesis/ingest
12
+ */
13
+ import { decodeXmlEntities } from "./ooxml.js";
14
+ import { failureResult, successResult } from "./types.js";
15
+ /** `node:fs` 读进来的字节 → 文本(BOM 与编码归一)。 */
16
+ export function decodeTextBytes(bytes) {
17
+ // UTF-16 BOM(Windows 记事本"Unicode"另存为)
18
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
19
+ return decodeUtf16(bytes.subarray(2), true);
20
+ }
21
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
22
+ return decodeUtf16(bytes.subarray(2), false);
23
+ }
24
+ // UTF-8 BOM
25
+ const body = bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
26
+ ? bytes.subarray(3)
27
+ : bytes;
28
+ const utf8 = decodeWith(new TextDecoder('utf-8'), body);
29
+ if (utf8 !== undefined && !utf8.includes('\ufffd'))
30
+ return normalizeNewlines(utf8);
31
+ // 回退 GBK(环境不支持时保留 UTF-8 结果)
32
+ try {
33
+ const gbk = decodeWith(new TextDecoder('gbk'), body);
34
+ if (gbk !== undefined && (utf8 === undefined || countReplacement(gbk) < countReplacement(utf8))) {
35
+ return normalizeNewlines(gbk);
36
+ }
37
+ }
38
+ catch {
39
+ // 没有 ICU:忽略
40
+ }
41
+ return normalizeNewlines(utf8 ?? '');
42
+ }
43
+ function decodeWith(decoder, bytes) {
44
+ try {
45
+ return decoder.decode(bytes);
46
+ }
47
+ catch {
48
+ return undefined;
49
+ }
50
+ }
51
+ function decodeUtf16(bytes, littleEndian) {
52
+ let out = '';
53
+ for (let i = 0; i + 1 < bytes.length; i += 2) {
54
+ const code = littleEndian ? bytes[i] | (bytes[i + 1] << 8) : (bytes[i] << 8) | bytes[i + 1];
55
+ out += String.fromCharCode(code);
56
+ }
57
+ return normalizeNewlines(out);
58
+ }
59
+ function countReplacement(text) {
60
+ let count = 0;
61
+ for (const ch of text)
62
+ if (ch === '\ufffd')
63
+ count += 1;
64
+ return count;
65
+ }
66
+ /** CRLF / CR → LF。 */
67
+ export function normalizeNewlines(text) {
68
+ return text.replace(/\r\n?/g, '\n');
69
+ }
70
+ // ---------------------------------------------------------------------------
71
+ // HTML
72
+ // ---------------------------------------------------------------------------
73
+ /** HTML 命名实体(够覆盖中文网页/文档导出的常见字符)。 */
74
+ const HTML_ENTITIES = {
75
+ nbsp: '\u00a0',
76
+ ensp: ' ',
77
+ emsp: ' ',
78
+ thinsp: ' ',
79
+ shy: '',
80
+ mdash: '—',
81
+ ndash: '–',
82
+ hellip: '…',
83
+ middot: '·',
84
+ bull: '•',
85
+ copy: '©',
86
+ reg: '®',
87
+ trade: '™',
88
+ laquo: '«',
89
+ raquo: '»',
90
+ ldquo: '“',
91
+ rdquo: '”',
92
+ lsquo: '‘',
93
+ rsquo: '’',
94
+ times: '×',
95
+ divide: '÷',
96
+ deg: '°',
97
+ plusmn: '±',
98
+ ne: '≠',
99
+ le: '≤',
100
+ ge: '≥',
101
+ rarr: '→',
102
+ larr: '←',
103
+ sect: '§',
104
+ para: '¶',
105
+ dagger: '†',
106
+ euro: '€',
107
+ pound: '£',
108
+ yen: '¥',
109
+ alpha: 'α',
110
+ beta: 'β',
111
+ gamma: 'γ',
112
+ delta: 'δ',
113
+ mu: 'μ',
114
+ pi: 'π',
115
+ sigma: 'σ',
116
+ omega: 'ω',
117
+ };
118
+ /** 解 HTML 实体(数字实体 + 常见命名实体,其余原样保留)。 */
119
+ export function decodeHtmlEntities(input) {
120
+ const named = /\&([a-zA-Z][a-zA-Z0-9]{1,31});/g;
121
+ return decodeXmlEntities(input).replace(named, (match, name) => HTML_ENTITIES[name.toLowerCase()] ?? match);
122
+ }
123
+ /** HTML → 纯文本(去标签、块级转换行、解码实体)。 */
124
+ export function htmlToText(html) {
125
+ let text = html;
126
+ // 注释
127
+ text = text.replace(/<!--[\s\S]*?-->/g, '');
128
+ // 脚本/样式整块丢弃
129
+ text = text.replace(/<(script|style|noscript|template)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, '');
130
+ // 表格单元格 → 制表符;换行标签 → 换行
131
+ text = text.replace(/<\/t[dh]\s*>/gi, '\t');
132
+ text = text.replace(/<br\s*\/?>/gi, '\n');
133
+ text = text.replace(/<\/(p|div|li|tr|h[1-6]|section|article|blockquote|pre|table|ul|ol|dl|dd|dt|figcaption)\s*>/gi, '\n');
134
+ text = text.replace(/<(hr|li)\b[^>]*>/gi, '\n');
135
+ // 其余标签直接去掉
136
+ text = text.replace(/<[^>]*>/g, '');
137
+ text = decodeHtmlEntities(text);
138
+ // 行内空白整理:去每行首尾空格、压掉 3 个以上空行
139
+ return text
140
+ .split('\n')
141
+ .map(line => line.replace(/[ \t]+$/, '').replace(/^[ \t]+/, ''))
142
+ .join('\n')
143
+ .replace(/\n{3,}/g, '\n\n')
144
+ .trim();
145
+ }
146
+ // ---------------------------------------------------------------------------
147
+ // 入口
148
+ // ---------------------------------------------------------------------------
149
+ /** 是否像二进制(含 NUL 或大量控制字符)——防止把 .dat 当文本读。 */
150
+ export function looksBinary(bytes) {
151
+ const sample = bytes.subarray(0, Math.min(bytes.length, 4096));
152
+ let control = 0;
153
+ for (const byte of sample) {
154
+ if (byte === 0)
155
+ return true;
156
+ if (byte < 9 || (byte > 13 && byte < 32))
157
+ control += 1;
158
+ }
159
+ return sample.length > 0 && control / sample.length > 0.05;
160
+ }
161
+ /** 门面用入口:按扩展名给的信息做纯文本抽取。 */
162
+ export function extractText(bytes, filename, options) {
163
+ const lower = filename.toLowerCase();
164
+ const isHtml = lower.endsWith('.html') || lower.endsWith('.htm');
165
+ if (looksBinary(bytes) && !isHtml) {
166
+ return failureResult('这个文件看起来是二进制(不是文本):请确认扩展名是否正确,或提供可读的文本版本。');
167
+ }
168
+ const decoded = decodeTextBytes(bytes);
169
+ const text = isHtml ? htmlToText(decoded) : decoded.replace(/^\ufeff/, '');
170
+ if (text.trim() === '') {
171
+ return failureResult('文件是空的(或去掉 HTML 标签后没有文字)。');
172
+ }
173
+ const extension = lower.slice(lower.lastIndexOf('.'));
174
+ const kind = isHtml ? 'HTML(已去标签)' : `${extension === lower ? '纯文本' : extension} 文本`;
175
+ return successResult({
176
+ text,
177
+ maxChars: options.maxChars,
178
+ note: `按${kind}读取。`,
179
+ });
180
+ }
@@ -0,0 +1,48 @@
1
+ /**
2
+ * 摄取模块的内部类型:各格式抽取器共用的结果/选项形状。
3
+ *
4
+ * 与门面 `index.ts` 的公开结果类型分开,是为了让 `docx.ts`/`pdf.ts` 等
5
+ * 只依赖一个小接口,不反过来依赖门面(避免循环 import)。
6
+ *
7
+ * @module dsh-thesis/ingest
8
+ */
9
+ /** 抽取选项:目前只有字符上限(超限即截断并标记)。 */
10
+ export interface ExtractOptions {
11
+ /** 文本上限(字符);超出部分截断,由调用方在结果里打 `truncated`。 */
12
+ readonly maxChars: number;
13
+ }
14
+ /** 单个文件抽取出的一段文本。 */
15
+ export interface ExtractResult {
16
+ /** 是否抽取成功;false 时 text 为空,note 是给人看的原因。 */
17
+ readonly ok: boolean;
18
+ /** 抽取到的文本(已按 maxChars 截断;ok=false 时为空串)。 */
19
+ readonly text: string;
20
+ /** 是否发生了截断(内容超过 maxChars)。 */
21
+ readonly truncated: boolean;
22
+ /** 文档标题(docx 的 dc:title、pptx 的 core.xml 标题等),可能没有。 */
23
+ readonly title?: string;
24
+ /** 额外说明:格式特性提示、降级原因、无法抽取的原因。 */
25
+ readonly note?: string;
26
+ /** 低置信标记:文本量明显偏少(如扫描版 PDF)。 */
27
+ readonly lowConfidence?: boolean;
28
+ }
29
+ /**
30
+ * 按 maxChars 截断文本,并保证「正文 + 截断提示」不超过 maxChars。
31
+ * 返回截断后的文本与是否发生截断。
32
+ */
33
+ export declare function truncateText(text: string, maxChars: number): {
34
+ text: string;
35
+ truncated: boolean;
36
+ };
37
+ /** 组装成功结果(统一做截断)。 */
38
+ export declare function successResult(options: {
39
+ text: string;
40
+ maxChars: number;
41
+ title?: string | undefined;
42
+ note?: string | undefined;
43
+ lowConfidence?: boolean | undefined;
44
+ }): ExtractResult;
45
+ /** 组装失败结果(ok=false,text 恒为空)。 */
46
+ export declare function failureResult(reason: string, options?: {
47
+ title?: string | undefined;
48
+ }): ExtractResult;
@@ -0,0 +1,47 @@
1
+ /**
2
+ * 摄取模块的内部类型:各格式抽取器共用的结果/选项形状。
3
+ *
4
+ * 与门面 `index.ts` 的公开结果类型分开,是为了让 `docx.ts`/`pdf.ts` 等
5
+ * 只依赖一个小接口,不反过来依赖门面(避免循环 import)。
6
+ *
7
+ * @module dsh-thesis/ingest
8
+ */
9
+ /**
10
+ * 按 maxChars 截断文本,并保证「正文 + 截断提示」不超过 maxChars。
11
+ * 返回截断后的文本与是否发生截断。
12
+ */
13
+ export function truncateText(text, maxChars) {
14
+ const limit = Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : 0;
15
+ if (text.length <= limit)
16
+ return { text, truncated: false };
17
+ const marker = '\n\n…(内容过长已截断)';
18
+ const room = Math.max(0, limit - marker.length);
19
+ return { text: `${text.slice(0, room)}${marker}`, truncated: true };
20
+ }
21
+ /** 组装成功结果(统一做截断)。 */
22
+ export function successResult(options) {
23
+ const { text, truncated } = truncateText(options.text, options.maxChars);
24
+ const notes = [];
25
+ if (options.note !== undefined && options.note !== '')
26
+ notes.push(options.note);
27
+ if (truncated)
28
+ notes.push(`内容超过 ${options.maxChars} 字符上限,已截断。`);
29
+ return {
30
+ ok: true,
31
+ text,
32
+ truncated,
33
+ ...(options.title !== undefined && options.title !== '' ? { title: options.title } : {}),
34
+ ...(notes.length > 0 ? { note: notes.join(' ') } : {}),
35
+ ...(options.lowConfidence === true ? { lowConfidence: true } : {}),
36
+ };
37
+ }
38
+ /** 组装失败结果(ok=false,text 恒为空)。 */
39
+ export function failureResult(reason, options) {
40
+ return {
41
+ ok: false,
42
+ text: '',
43
+ truncated: false,
44
+ ...(options?.title !== undefined && options.title !== '' ? { title: options.title } : {}),
45
+ note: reason,
46
+ };
47
+ }
@@ -0,0 +1,72 @@
1
+ /**
2
+ * `.xlsx` 摄取:每个工作表渲染成一段「表名 + 表头 + 行」的 TSV 文本。
3
+ *
4
+ * 需要处理的 OOXML 细节(都按"够用即止"实现,不做完整 SpreadsheetML):
5
+ * - `xl/workbook.xml` 给出 `<sheet name r:id>`;`xl/_rels/workbook.xml.rels`
6
+ * 把 r:id 映射到 `worksheets/sheetN.xml`(两个文件都可能缺 rels,故有回退猜测);
7
+ * - `xl/sharedStrings.xml`:共享字符串表(`<si>` 可含多个 `<r><t>` 富文本片段);
8
+ * - 内联字符串 `t="inlineStr"`:值直接写在 `<is><t>` 里;
9
+ * - 空单元格补空列:按 `<c r="C5">` 的列字母定位,缺列就用空串占位,保证列对齐;
10
+ * - 日期:Excel 把日期存成序列号(如 45000)。本模块**原样保留数字**,
11
+ * 但在表头行标注该列是「日期序列(Excel 序列号)」,避免把 45000 当数据。
12
+ *
13
+ * @module dsh-thesis/ingest
14
+ */
15
+ import { type ExtractOptions, type ExtractResult } from './types.ts';
16
+ /** 取一个标签串里某个属性的值(值可用单/双引号,也可无引号)。 */
17
+ export declare function attr(tag: string, name: string): string | undefined;
18
+ /** 取标签名 + 是否自闭合 + 属性串(供工作表遍历用)。 */
19
+ interface TagScan {
20
+ readonly name: string;
21
+ readonly selfClosing: boolean;
22
+ /** 是否闭标签(`</row>`)。 */
23
+ readonly closing: boolean;
24
+ readonly attrs: string;
25
+ /** 标签在源串中的起止下标(用于切出元素 body)。 */
26
+ readonly start: number;
27
+ readonly end: number;
28
+ }
29
+ /** 按出现顺序扫描标签(不构建 DOM,只给位置与属性)。 */
30
+ export declare function scanTags(xml: string): TagScan[];
31
+ /** 从 `xl/workbook.xml` 抽工作表顺序与名字。 */
32
+ export declare function parseWorkbookSheets(workbookXml: string): Array<{
33
+ name: string;
34
+ relId?: string;
35
+ }>;
36
+ /** 从 `xl/_rels/workbook.xml.rels` 抽 rId → 目标路径 的映射。 */
37
+ export declare function parseWorkbookRels(relsXml: string): Map<string, string>;
38
+ /** 解析 `xl/sharedStrings.xml`:索引 → 文本。 */
39
+ export declare function parseSharedStrings(xml: string): string[];
40
+ /** 自定义格式串是否像日期/时间(去掉引号、转义、颜色/条件段后看 ymdhs 字符)。 */
41
+ export declare function isDateFormatCode(code: string): boolean;
42
+ /**
43
+ * 解析 `xl/styles.xml`:返回「cellXfs 下标 → 是否日期格式」。
44
+ * 有些工具不写 cellXfs(此时全按非日期处理,数值原样保留)。
45
+ */
46
+ export declare function parseStyleDateFlags(stylesXml: string | undefined): boolean[];
47
+ /** `C12` → 列下标 2(0 基);非法引用返回 0。 */
48
+ export declare function columnIndex(ref: string): number;
49
+ interface CellValue {
50
+ readonly kind: 'text' | 'number' | 'date';
51
+ readonly text: string;
52
+ }
53
+ /** 解析一行里的单元格。 */
54
+ export declare function parseRow(rowXml: string, shared: readonly string[], dateFlags: readonly boolean[]): CellValue[];
55
+ interface SheetRender {
56
+ readonly text: string;
57
+ readonly rows: number;
58
+ readonly truncated: boolean;
59
+ readonly truncatedRows: boolean;
60
+ }
61
+ /** 渲染一个工作表:表名 + 表头 + TSV 行。 */
62
+ export declare function renderSheet(name: string, sheetXml: string, shared: readonly string[], dateFlags: readonly boolean[]): SheetRender;
63
+ /** 读取一个 .xlsx:返回按工作表顺序渲染的 TSV 文本。 */
64
+ export declare function readXlsx(bytes: Uint8Array): {
65
+ text: string;
66
+ sheets: number;
67
+ };
68
+ /** 门面用入口:xlsx 字节 → 抽取结果。 */
69
+ export declare function extractXlsx(bytes: Uint8Array, options: ExtractOptions): ExtractResult;
70
+ /** 工作表是否有任何非空数据行(供"空工作簿"判定)。 */
71
+ export declare function hasAnyData(text: string): boolean;
72
+ export {};