deep-read-summarize 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/CHANGELOG.md +86 -0
- package/LICENSE +21 -0
- package/README.md +222 -0
- package/SECURITY.md +27 -0
- package/cordis.patch.yml +22 -0
- package/docs/RELEASE-v0.1.0.md +61 -0
- package/docs/RELEASE.md +43 -0
- package/index.js +32 -0
- package/lib/cache.js +62 -0
- package/package.json +66 -0
- package/parsers/book.js +33 -0
- package/parsers/index.js +78 -0
- package/parsers/paper.js +31 -0
- package/parsers/video.js +32 -0
- package/parsers/web.js +30 -0
- package/schemas/index.js +110 -0
- package/scripts/lint.js +23 -0
- package/scripts/security-check.js +32 -0
- package/skills/deep-read-summarize/SKILL.md +68 -0
- package/tests/cache.test.js +69 -0
- package/tests/fixtures/fable.txt +28 -0
- package/tests/index.test.js +80 -0
- package/tests/quick_validate.js +52 -0
- package/tests/run-tests.js +79 -0
- package/workflow.js +43 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// tests/quick_validate.js — fast sanity check before release/tag
|
|
2
|
+
// 注意:不用 child_process(沙箱环境禁止 spawn),纯 fs 检查
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
|
|
5
|
+
console.log('== deep-read-summarize quick validation ==');
|
|
6
|
+
let failed = false;
|
|
7
|
+
|
|
8
|
+
// 1. All JS files exist
|
|
9
|
+
const files = [
|
|
10
|
+
'workflow.js', 'parsers/index.js', 'parsers/book.js', 'parsers/paper.js',
|
|
11
|
+
'parsers/video.js', 'parsers/web.js', 'schemas/index.js'
|
|
12
|
+
];
|
|
13
|
+
for (const f of files) {
|
|
14
|
+
if (!fs.existsSync(f)) { console.error('MISSING:', f); failed = true; }
|
|
15
|
+
}
|
|
16
|
+
if (!failed) console.log(' OK: all ' + files.length + ' JS files present');
|
|
17
|
+
|
|
18
|
+
// 2. README has compatibility + copyright sections (bilingual)
|
|
19
|
+
const readme = fs.readFileSync('README.md', 'utf8');
|
|
20
|
+
const hasCompat = readme.includes('兼容性') || readme.includes('Compatibility');
|
|
21
|
+
const hasCopyright = readme.includes('版权') || readme.includes('copyright');
|
|
22
|
+
if (!hasCompat || !hasCopyright) {
|
|
23
|
+
console.error('README must mention compatibility and copyright policy');
|
|
24
|
+
failed = true;
|
|
25
|
+
}
|
|
26
|
+
if (!failed) console.log(' OK: README has compatibility + copyright sections');
|
|
27
|
+
|
|
28
|
+
// 3. LICENSE is MIT
|
|
29
|
+
const lic = fs.readFileSync('LICENSE', 'utf8');
|
|
30
|
+
if (!lic.includes('MIT License')) { console.error('LICENSE must be MIT'); failed = true; }
|
|
31
|
+
if (!failed) console.log(' OK: LICENSE is MIT');
|
|
32
|
+
|
|
33
|
+
// 4. No secrets in source
|
|
34
|
+
let secretFound = false;
|
|
35
|
+
const all = files.map(f => fs.readFileSync(f, 'utf8')).join('\n');
|
|
36
|
+
const secretPatterns = [
|
|
37
|
+
/ghp_[A-Za-z0-9]{20,}/, /gho_[A-Za-z0-9]{20,}/, /sk-[A-Za-z0-9]{20,}/,
|
|
38
|
+
/AKIA[0-9A-Z]{16}/, /AIza[0-9A-Za-z_-]{30,}/, /-----BEGIN .*PRIVATE KEY-----/
|
|
39
|
+
];
|
|
40
|
+
for (const pat of secretPatterns) {
|
|
41
|
+
if (pat.test(all)) { console.error('SECRET PATTERN DETECTED'); secretFound = true; failed = true; }
|
|
42
|
+
}
|
|
43
|
+
if (!secretFound) console.log(' OK: no secrets in source');
|
|
44
|
+
|
|
45
|
+
// 5. SECURITY.md and templates exist
|
|
46
|
+
const required = ['SECURITY.md', '.github/ISSUE_TEMPLATE/bug_report.md', '.github/ISSUE_TEMPLATE/feature_request.md', '.github/PULL_REQUEST_TEMPLATE.md', 'docs/RELEASE.md'];
|
|
47
|
+
let missing = required.filter(f => !fs.existsSync(f));
|
|
48
|
+
if (missing.length) { console.error('MISSING:', missing.join(', ')); failed = true; }
|
|
49
|
+
if (!failed) console.log(' OK: SECURITY.md + issue/PR templates present');
|
|
50
|
+
|
|
51
|
+
if (failed) { console.error('== validation FAILED =='); process.exit(1); }
|
|
52
|
+
console.log('== validation OK — ready to tag ==');
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// tests/run-tests.js — fixture-driven tests
|
|
2
|
+
const assert = require('assert');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
let passed = 0, failed = 0;
|
|
7
|
+
function test(name, fn) {
|
|
8
|
+
try { fn(); passed++; console.log(' ✅', name); }
|
|
9
|
+
catch (e) { failed++; console.error(' ❌', name, '—', e.message); }
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// ---------- Test: parser registry ----------
|
|
13
|
+
const parsers = require('../parsers');
|
|
14
|
+
test('registry exposes 4 built-in parsers', () => {
|
|
15
|
+
const list = parsers.list();
|
|
16
|
+
const names = list.map(p => p.name).sort();
|
|
17
|
+
assert.deepStrictEqual(names, ['book', 'paper', 'video', 'web']);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test('resolve returns matching parser', () => {
|
|
21
|
+
assert.strictEqual(parsers.resolve('paper').name, 'paper');
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('resolve falls back to web for unknown type', () => {
|
|
25
|
+
assert.strictEqual(parsers.resolve('unknown').name, 'web');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('every parser builds a non-empty prompt', () => {
|
|
29
|
+
for (const type of ['book', 'paper', 'video', 'web']) {
|
|
30
|
+
const p = parsers.resolve(type);
|
|
31
|
+
const prompt = p.buildPrompt('https://example.com/x', { tempDir: '/tmp', tempFile: '/tmp/in.txt', maxChunks: 4 });
|
|
32
|
+
assert.ok(typeof prompt === 'string' && prompt.length > 50, p.name + ' prompt too short');
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// ---------- Test: schemas ----------
|
|
37
|
+
const schemas = require('../schemas');
|
|
38
|
+
test('fetchResultSchema requires saved', () => {
|
|
39
|
+
assert.ok(schemas.fetchResultSchema.required.includes('saved'));
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('qualityChecklistSchema requires 5 gates', () => {
|
|
43
|
+
assert.strictEqual(schemas.qualityChecklistSchema.required.length, 5);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// ---------- Test: workflow script parses ----------
|
|
47
|
+
const wf = require('../workflow');
|
|
48
|
+
test('workflow meta is valid', () => {
|
|
49
|
+
assert.strictEqual(wf.meta.name, 'deep-read-summarize');
|
|
50
|
+
assert.ok(wf.meta.phases.length >= 3);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('workflow script parses as async body', () => {
|
|
54
|
+
const fn = new Function('args','phase','agent','parallel','log', 'return (async () => { ' + wf.script + ' })()');
|
|
55
|
+
assert.ok(typeof fn === 'function');
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// ---------- Test: workflow exports parsers ----------
|
|
59
|
+
test('workflow.js exports parser registry', () => {
|
|
60
|
+
assert.ok(wf.parsers && typeof wf.parsers.resolve === 'function');
|
|
61
|
+
assert.strictEqual(wf.parsers.resolve('video').name, 'video');
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
// ---------- Test: fixture is self-authored (no copyright markers) ----------
|
|
65
|
+
test('fixture fable.txt exists and is public-domain self-authored', () => {
|
|
66
|
+
const fx = fs.readFileSync(path.join(__dirname, 'fixtures/fable.txt'), 'utf8');
|
|
67
|
+
assert.ok(fx.includes('released into the public domain'));
|
|
68
|
+
assert.ok(!fx.includes('Copyright ©'));
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// ---------- Run cache tests ----------
|
|
72
|
+
console.log('\n=== cache tests ===');
|
|
73
|
+
const cacheResult = require('./cache.test.js');
|
|
74
|
+
passed += cacheResult.passed;
|
|
75
|
+
failed += cacheResult.failed;
|
|
76
|
+
|
|
77
|
+
console.log('');
|
|
78
|
+
console.log('TOTAL: ' + passed + ' passed, ' + failed + ' failed');
|
|
79
|
+
process.exit(failed ? 1 : 0);
|
package/workflow.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// deep-read-summarize — Deep reading & summarization workflow for DSH
|
|
3
|
+
// Version: 3.0.0
|
|
4
|
+
// ============================================================================
|
|
5
|
+
// 【用法】meta → workflow meta;script → workflow script;
|
|
6
|
+
// args: { "input": "<url|path>", "type": "auto", "options": { "minWords": 2500, "fastMode": false, "maxChunks": 6, "requireCitations": true, "includeTimestamps": false, "cache": true, "outputDir": "./output", "tempDir": "./.tmp" } }
|
|
7
|
+
// ============================================================================
|
|
8
|
+
|
|
9
|
+
const meta = {
|
|
10
|
+
"name": "deep-read-summarize",
|
|
11
|
+
"description": "Deep reading & summarization of books/papers/videos/web (v3.0)",
|
|
12
|
+
"whenToUse": "User provides a book, paper, video link or web page and wants a deep-read summary saved to Obsidian",
|
|
13
|
+
"phases": [
|
|
14
|
+
{
|
|
15
|
+
"title": "解析器选择",
|
|
16
|
+
"detail": "按类型选择解析器插件"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"title": "获取与分块",
|
|
20
|
+
"detail": "Parser fetches content, writes temp file, builds chunk plan"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"title": "并行精读",
|
|
24
|
+
"detail": "N subagents deep-read chunks in parallel (Map)"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"title": "汇总成稿",
|
|
28
|
+
"detail": "Merge into final note (Reduce) with embedded self-check"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"title": "质量校验",
|
|
32
|
+
"detail": "Optional QC loop with retries"
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
// ===== script(复制到 workflow 工具的 script 参数)=====
|
|
38
|
+
const script = "// ========== deep-read-summarize v3.0 ==========\n// 3 波次:解析器获取分块 → 并行精读 → 成稿+质量校验\n// 特性:解析器插件化 / 失败纪律(fatal vs 降级) / 幂等 / 成本控制\n\n// ---------- 0. 配置校验(fatal 级错误:必须终止) ----------\nconst input = (args && args.input || \"\").trim();\nif (!input) throw new Error(\"[FATAL] args.input 缺失:请输入内容链接或文件路径\");\nconst forcedType = ((args && args.type) || \"auto\").toLowerCase();\nif ([\"auto\",\"book\",\"paper\",\"video\",\"web\"].indexOf(forcedType) === -1)\n throw new Error(\"[FATAL] args.type 非法:必须为 auto|book|paper|video|web\");\nconst options = (args && args.options) || {};\nconst minWords = options.minWords || 2500;\nconst fastMode = !!options.fastMode;\nconst maxChunks = Math.max(1, Math.min(options.maxChunks || 6, 12));\nconst requireCitations = options.requireCitations !== false;\nconst includeTimestamps = !!options.includeTimestamps;\nconst maxRetries = options.maxRetries || 1;\nconst cacheEnabled = options.cache !== false;\nconst outputDir = options.outputDir || \"./output\";\nconst tempDir = options.tempDir || \"./.tmp\";\nconst tempFile = tempDir + \"/input.txt\";\n\n// ---------- 0.5 幂等检查:输入指纹缓存 ----------\nlet cacheHit = false;\nif (cacheEnabled && Array.isArray(args._processedKeys)) {\n const fpKey = input.trim().slice(0, 300);\n cacheHit = args._processedKeys.indexOf(fpKey) !== -1;\n if (cacheHit) log(\"幂等命中:该输入已处理过,跳过\");\n}\nif (cacheHit) {\n return { ok: true, cached: true, kind: forcedType, title: (args && args.title) || \"cached\", filePath: \"\", chunksCount: 0, fastMode: fastMode, qualityPassed: true, qualityIssues: [], failedChunks: 0, note: \"\" };\n}\n\n// ---------- 1. 解析器选择(由 args._parsers 注入) ----------\nphase(\"解析器选择\");\nconst parsers = args._parsers || {};\nconst kindNames = { book: \"书籍\", paper: \"学术论文\", video: \"视频\", web: \"网页/文章\" };\nconst kindFocus = {\n book: \"按章节主题展开核心论点、论证、关键例子、章节间关系,收集关键概念与可引用原文(含页码/章节)\",\n paper: \"四维精读:1)研究问题与方法 2)结论与贡献 3)局限与批判 4)与同类工作比较;收集术语与可引用原文(含章节/段落)\",\n video: \"按主题脉络组织,提取核心论点、证据、例子、可操作建议,收集关键概念与主讲人原话\" + (includeTimestamps ? \"(含时间戳定位)\" : \"(不标时间戳)\"),\n web: \"提取核心论点、结构拆解、关键概念、可引用原文\"\n};\n\n// ---------- 2. 波次1:解析器获取 + 分块 ----------\nphase(\"获取与分块\");\nconst fetchPrompt = (parsers[forcedType !== \"auto\" ? forcedType : \"web\"] || parsers.web || { buildPrompt: function() { return [\"未知解析器\"]; } })\n .buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks });\nconst fetchSchema = {\n type: \"object\",\n properties: {\n kind: { type: \"string\" }, saved: { type: \"boolean\" }, message: { type: \"string\" }, totalLines: { type: \"number\" },\n metadata: { type: \"object\", properties: { title: { type: \"string\" }, author: { type: \"string\" }, year: { type: \"string\" }, language: { type: \"string\" } }, required: [], additionalProperties: true },\n chunkPlan: {\n type: \"array\",\n items: { type: \"object\", properties: { id: { type: \"number\" }, topic: { type: \"string\" }, startLine: { type: \"number\" }, endLine: { type: \"number\" } }, required: [\"id\",\"topic\",\"startLine\",\"endLine\"], additionalProperties: false }\n }\n },\n required: [\"saved\"], additionalProperties: false\n};\n\nconst fetched = await agent(fetchPrompt, { schema: fetchSchema, label: \"获取+分块\", phase: \"获取与分块\" });\n\n// 失败纪律:内容获取失败 = 可降级错误(返回失败结果,不抛 fatal)\nif (!fetched || !fetched.saved) {\n return { ok: false, stage: \"fetch\", fatal: false, message: (fetched && fetched.message) || \"获取失败(可降级:检查输入或手动提供文本)\" };\n}\nlet kind = (fetched.kind && forcedType === \"auto\") ? fetched.kind : forcedType;\nconst chunkPlan = (fetched && fetched.chunkPlan) || [];\nif (chunkPlan.length === 0) return { ok: false, stage: \"fetch\", fatal: false, message: \"分块计划为空\" };\nconst metadata = (fetched && fetched.metadata) || {};\nlog(\"类型:\" + kind + \" 行数:\" + fetched.totalLines + \" 块数:\" + chunkPlan.length);\n\n// ---------- 3. 波次2:并行精读(Map) ----------\nphase(\"并行精读\");\nconst chunkPrompts = chunkPlan.map(function (c) {\n return [\n \"请深度精读以下内容块,输出结构化精读笔记。\",\n \"内容类型:\" + (kindNames[kind] || kind) + \" | 本块主题:\" + c.topic,\n \"精读侧重:\" + (kindFocus[kind] || kindFocus.web),\n \"用 read 工具读取 \" + tempFile + \" 第 \" + c.startLine + \" 到 \" + c.endLine + \" 行(offset=\" + c.startLine + \", limit=\" + (c.endLine - c.startLine + 1) + \"),然后输出:\",\n \"## 本块主题:\" + c.topic,\n \"### 要点(详细展开,保留数据/例子/论证)\",\n \"### 关键概念(术语+解释)\",\n \"### 关键引用(最多2句原文+位置)\" + (requireCitations ? \",必须标注页码/章节/段落位置\" : \"\"),\n \"### 批判观察(如有)\"\n ].join(\"\\n\");\n});\n\nconst deepReads = await parallel(chunkPrompts.map(function (p) {\n return function () { return agent(p, { label: \"精读块\", phase: \"并行精读\" }); };\n}));\nconst validReads = deepReads.filter(function (r) { return r && r.trim && r.trim().length > 0; });\n\n// 失败纪律:某块精读失败 = 可降级(跳过该块,标记缺口继续)\nconst failedChunks = chunkPlan.length - validReads.length;\nif (validReads.length === 0) return { ok: false, stage: \"read\", fatal: false, message: \"所有分块精读均失败\" };\nif (failedChunks > 0) log(\"警告:\" + failedChunks + \" 块精读失败,标记缺口继续成稿\");\nlog(\"精读完成:\" + validReads.length + \"/\" + chunkPlan.length);\n\n// ---------- 4. 波次3:成稿 + 内嵌质量自检(Reduce) ----------\nphase(\"汇总成稿\");\nconst today = new Date().toISOString().slice(0, 10);\nconst finalTitle = metadata.title || \"未命名内容\";\nconst safeTitle = finalTitle.replace(/[\\\\/:*?\"<>|]/g, \" \").trim();\nconst mdPath = outputDir.replace(/\\\\$/, \"\") + \"/\" + safeTitle + \".md\";\n\nconst extraSections = fastMode ? [] : [\n \"\", \"## 5. 批判性分析(论证强处/弱处/可质疑之处)\", \"\",\n \"## 6. 个人思考与应用(与已有知识联系、如何应用、行动要点)\", \"\",\n \"## 7. 延伸阅读(相关主题/作者其他作品/参考文献)\", \"\"\n];\n\nconst draftPrompt = [\n \"将以下多块精读结果整合为完整 Obsidian 精读笔记,只输出 Markdown(含 YAML frontmatter)。\",\n \"模板:\", \"---\",\n 'title: \"《' + finalTitle + '》深度精读笔记\"',\n \"aliases: [\" + JSON.stringify(finalTitle) + \"]\",\n \"author: \" + (metadata.author || \"\"),\n \"year: \" + (metadata.year || \"\"),\n \"type: \" + kind,\n \"url: \" + (metadata.url || input),\n \"tags: [精读, \" + (kindNames[kind] || kind) + \"]\",\n \"created: \" + today,\n \"status: 已完成\",\n \"---\",\n \"# 《\" + finalTitle + \"》深度精读笔记\", \"\",\n \"> 一句话核心概括。\", \"\",\n \"## 1. 概述(约500字:主题/背景/核心问题/总体结论)\", \"\",\n \"## 2. 结构拆解(按主题组织,含详细要点;关键结论标注原文出处)\", \"\",\n \"## 3. 关键概念与术语表(表格)\", \"\",\n \"## 4. 关键引用(仅关键结论\" + (requireCitations ? \",必须标注页码/章节/段落\" : \"\") + \")\",\n].concat(extraSections, [\n \"---\",\n \"*笔记整理于 \" + today + \"*\", \"\",\n \"硬性要求:总篇幅≥\" + minWords + \"字;语言与原文一致;术语统一;引用必须真实;视频\" + (includeTimestamps ? \"标注时间戳\" : \"不标注时间戳\") + \"。\",\n \"输出前自检:1)覆盖所有块 2)frontmatter完整 3)各小节齐全 4)引用真实 5)篇幅达标 6)语言一致。\",\n \"\",\n \"各块精读结果:\",\n validReads.map(function (r, i) { return \"\\\\n===== 块 \" + (i + 1) + \" =====\\\\n\" + r; }).join(\"\\\\n\")\n]).join(\"\\n\");\n\nlet draft = await agent(draftPrompt, { label: \"汇总成稿\", phase: \"汇总成稿\" });\nif (!draft || !draft.trim) return { ok: false, stage: \"draft\", fatal: false, message: \"成稿失败\" };\n\n// ---------- 5. 质量校验(轻量重试) ----------\nphase(\"质量校验\");\nlet verified = draft;\nlet passed = false;\nlet issues = [];\nfor (let round = 0; round <= maxRetries; round++) {\n const checkPrompt = [\n \"请对总结笔记做质量校验,逐项检查并输出 JSON:\",\n \"1.coverage 2.citationsReal 3.terminologyConsistent 4.formatComplete 5.lengthMet(≥\" + minWords + \"字) 6.languageConsistent\",\n \"输出:{ \\\"pass\\\": true/false, \\\"issues\\\": [\\\"问题\\\"] },只输出 JSON。\",\n \"笔记:\", verified\n ].join(\"\\n\");\n const checkSchema = { type: \"object\", properties: { pass: { type: \"boolean\" }, issues: { type: \"array\", items: { type: \"string\" } } }, required: [\"pass\",\"issues\"], additionalProperties: false };\n const check = await agent(checkPrompt, { schema: checkSchema, label: \"质量校验\", phase: \"质量校验\" });\n passed = !!(check && check.pass);\n issues = (check && check.issues) || [];\n log(\"校验:\" + (passed ? \"通过\" : \"发现问题 \" + issues.length + \" 条\"));\n if (passed) break;\n if (round < maxRetries) {\n const fixPrompt = [\n \"修正以下笔记以解决校验问题,输出完整修正版:\",\n issues.map(function (s) { return \"- \" + s; }).join(\"\\n\"),\n \"当前笔记:\", verified\n ].join(\"\\n\");\n verified = await agent(fixPrompt, { label: \"修正笔记\", phase: \"质量校验\" });\n }\n}\n\n// ---------- 结果 ----------\nreturn {\n ok: true, kind: kind, title: finalTitle,\n filePath: mdPath, chunksCount: chunkPlan.length, fastMode: fastMode,\n qualityPassed: passed, qualityIssues: issues,\n failedChunks: failedChunks,\n note: verified\n};";
|
|
39
|
+
|
|
40
|
+
// ===== parsers 注册表(运行时注入,实现插件化)=====
|
|
41
|
+
const parsers = require("./parsers");
|
|
42
|
+
|
|
43
|
+
module.exports = { meta, script, parsers };
|