bookmark-research 0.5.0-beta.9 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.agents/plugins/marketplace.json +1 -1
  2. package/.codex-plugin/plugin.json +6 -1
  3. package/README.md +78 -56
  4. package/README.zh.md +74 -52
  5. package/bin/bookmark-research.js +9 -1
  6. package/config/providers.json +41 -0
  7. package/config/settings.example.json +36 -0
  8. package/docs/bookmark-research-architecture.md +8 -119
  9. package/docs/canvas-tree-reading-acceptance.md +98 -0
  10. package/docs/configuration.en.md +70 -0
  11. package/docs/configuration.md +70 -0
  12. package/docs/details.en.md +35 -32
  13. package/docs/details.md +12 -13
  14. package/docs/distribution.md +4 -7
  15. package/docs/harness-compatibility.md +10 -299
  16. package/docs/installation.en.md +21 -47
  17. package/docs/installation.md +24 -56
  18. package/docs/instructions.en.md +3 -3
  19. package/docs/instructions.md +3 -3
  20. package/docs/prompt-reference.en.md +3 -3
  21. package/docs/prompt-reference.md +3 -3
  22. package/docs/user-guide.en.md +5 -4
  23. package/docs/user-guide.md +7 -6
  24. package/hosts/claude/commands/research.md +7 -0
  25. package/hosts/codex/delegate.md +3 -1
  26. package/hosts/dsh/plugin.js +5 -0
  27. package/hosts/dsh/workflow-call.py +8 -4
  28. package/hosts/shared/research-call.py +15 -3
  29. package/hosts/shared/research-flow.js +3 -1
  30. package/install.sh +8 -8
  31. package/package.json +3 -8
  32. package/scripts/bench_preflight.py +189 -0
  33. package/scripts/bench_sync.py +191 -0
  34. package/scripts/bench_tools_list.py +129 -0
  35. package/scripts/build_zip.py +1 -10
  36. package/scripts/evaluate_ladder.py +149 -0
  37. package/scripts/export_bundle.py +6 -68
  38. package/scripts/host_assets.py +31 -31
  39. package/scripts/host_clients.py +32 -61
  40. package/scripts/host_install.py +24 -12
  41. package/scripts/install.py +233 -77
  42. package/scripts/launcher.py +28 -5
  43. package/scripts/npm_source.py +41 -0
  44. package/scripts/verify_fixture.py +1 -1
  45. package/scripts/verify_live.py +2 -2
  46. package/scripts/verify_wiki_retrieval.py +158 -0
  47. package/skills/bookmark-research/SKILL.md +31 -31
  48. package/skills/bookmark-research/references/cli.md +20 -8
  49. package/skills/bookmark-research/references/deep-research.md +12 -4
  50. package/skills/bookmark-research/references/host-workflows.md +4 -23
  51. package/skills/bookmark-research/references/package-semantics.md +67 -10
  52. package/skills/bookmark-research/references/research-methods.md +1 -1
  53. package/skills/bookmark-research/references/research-services.md +25 -47
  54. package/skills/bookmark-research/references/research-workflow.md +8 -4
  55. package/skills/bookmark-research/references/settings-and-archive.md +29 -11
  56. package/skills/bookmark-research/references/wiki.md +92 -0
  57. package/skills/bookmark-research/references/zh/cli.md +20 -8
  58. package/skills/bookmark-research/references/zh/deep-research.md +12 -4
  59. package/skills/bookmark-research/references/zh/host-workflows.md +4 -23
  60. package/skills/bookmark-research/references/zh/package-semantics.md +67 -10
  61. package/skills/bookmark-research/references/zh/research-methods.md +1 -1
  62. package/skills/bookmark-research/references/zh/research-services.md +27 -47
  63. package/skills/bookmark-research/references/zh/research-workflow.md +8 -4
  64. package/skills/bookmark-research/references/zh/settings-and-archive.md +28 -10
  65. package/skills/bookmark-research/references/zh/skill-guide.md +26 -33
  66. package/skills/bookmark-research/references/zh/wiki.md +94 -0
  67. package/src/archive.py +560 -25
  68. package/src/bookmark_dependencies.py +56 -0
  69. package/src/bookmark_index.py +1237 -58
  70. package/src/browser_client.py +342 -0
  71. package/src/cli.py +399 -33
  72. package/src/context_output.py +53 -0
  73. package/src/credentials.py +2 -1
  74. package/src/deepseek.py +137 -0
  75. package/src/discovery.py +118 -0
  76. package/src/dsh_client.py +206 -0
  77. package/src/fetch_cache.py +60 -0
  78. package/src/grok_search.py +272 -0
  79. package/src/jina.py +1 -1
  80. package/src/jobs.py +349 -0
  81. package/src/junk.py +33 -0
  82. package/src/locate.py +862 -0
  83. package/src/mcp_dispatch.py +99 -0
  84. package/src/mcp_server.py +899 -93
  85. package/src/netscape.py +124 -0
  86. package/src/onboarding.py +339 -114
  87. package/src/package_paths.py +137 -0
  88. package/src/perplexity.py +78 -0
  89. package/src/raw_library.py +255 -48
  90. package/src/readiness.py +43 -9
  91. package/src/ref_codec.py +70 -0
  92. package/src/release.py +1 -1
  93. package/src/research.py +449 -41
  94. package/src/research_plan.py +171 -0
  95. package/src/research_services.py +48 -21
  96. package/src/routing.py +50 -16
  97. package/src/search_results.py +84 -3
  98. package/src/settings.py +135 -18
  99. package/src/source_inputs.py +156 -23
  100. package/src/source_manager.py +79 -15
  101. package/src/source_versions.py +115 -0
  102. package/src/source_watcher.py +4 -2
  103. package/src/storage.py +60 -0
  104. package/src/tui.py +5 -2
  105. package/src/url_classify.py +45 -0
  106. package/src/web_search.py +662 -70
  107. package/src/wiki.py +788 -47
  108. package/src/wiki_search.py +251 -0
  109. package/src/wiki_work.py +229 -0
  110. package/docs/cli-launcher-plan.md +0 -58
  111. package/docs/diff-review-2026-10-09.md +0 -36
  112. package/docs/harness-sources.json +0 -342
  113. package/docs/host-validation.json +0 -340
  114. package/docs/install-design-0.2.0.md +0 -57
  115. package/docs/knowledge-storage-plan.md +0 -43
  116. package/docs/mcp-aggregation-0.2.0.md +0 -92
  117. package/docs/multi-host-installation-plan.md +0 -163
  118. package/docs/provider-smoke-0.2.0.json +0 -235
  119. package/docs/release-notes-0.5.0-beta.1.md +0 -33
  120. package/docs/release-notes-0.5.0-beta.2.md +0 -23
  121. package/docs/release-notes-0.5.0-beta.3.md +0 -28
  122. package/docs/release-notes-0.5.0-beta.4.md +0 -39
  123. package/docs/release-notes-0.5.0-beta.5.md +0 -31
  124. package/docs/release-notes-0.5.0-beta.6.md +0 -30
  125. package/docs/release-notes-0.5.0-beta.7.md +0 -57
  126. package/docs/release-notes-0.5.0-beta.8.md +0 -34
  127. package/docs/release-notes-0.5.0-beta.9.md +0 -35
  128. package/docs/research-0.2.0.md +0 -140
  129. package/docs/research-sources-0.2.0.json +0 -9360
  130. package/docs/validation-0.2.0.md +0 -126
  131. package/docs/validation-0.4.0.md +0 -154
  132. package/docs/wiki-quality.md +0 -107
  133. package/docs/wiki-quality.zh.md +0 -107
  134. package/hosts/pi/register-workflow.py +0 -72
  135. package/hosts/pi/runtime.js +0 -18
  136. package/scripts/verify_quality.py +0 -87
  137. package/skills/bookmark-research/references/wiki-and-evaluation.md +0 -69
  138. package/skills/bookmark-research/references/zh/wiki-and-evaluation.md +0 -69
  139. package/src/evaluation.py +0 -337
@@ -1,126 +0,0 @@
1
- # 0.2.0 验证记录与测试包
2
-
3
- 核验日期:2026-09-10。这里区分本地行为测试、真实数据包验证、服务连通性和客户端加载;某一层成功不替代其他层的验证。
4
-
5
- ## 需求与证据
6
-
7
- | 需求 | 实现与可核对证据 |
8
- | --- | --- |
9
- | 保持独立插件、版本 0.2.0 | `.codex-plugin/plugin.json`;导出器从同一 manifest 读取版本;原项目只作资料参考 |
10
- | 聚合 MCP | `src/provider_adapters.py`、`web_search.py`、`remote_mcp.py`;真实工具 schema fixture 与四个聚合相关测试模块 |
11
- | 三层检索路由 | `skills/bookmark-research/SKILL.md` 与 `references/deep-research.md`;官方出处与设计取舍见 [调研报告](research-0.2.0.md) |
12
- | 可继续的深度研究 | `src/research.py` 与 `tests/test_research.py`:规划、预算、检索、引用、冲突、修订、恢复与报告 |
13
- | 保留书签语境 | `bookmark_refs` → `context.json` → source 的书签实例关联;同 URL 在不同栏目保持独立身份 |
14
- | 有依据的结论 | 来源身份审阅、原句匹配、哈希校验、被拒绝来源拦截、撤回结论后重开问题 |
15
- | 学习 Qwen 安装结构 | [安装设计](install-design-0.2.0.md):原生 CLI、install/update/verify、来源选择、固定版本与配置分离 |
16
- | 可下载和验证的包 | `scripts/build_zip.py` 生成普通 ZIP 与 `--include-tests` 验证包;确定性文件清单与 SHA-256 |
17
- | 详细查看调研 URL | [v0.2.0 历史来源清单](https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/v0.2.0/docs/research-sources-0.2.0.json):逐项读取、失败、私人链接处理和来源质量状态;新版 ZIP 不附带实例清单 |
18
-
19
- ## 离线测试
20
-
21
- v0.2.0 发布快照测试:Python 3.9 / Codex CLI 0.153.4 环境中 **183 项全部通过,0 跳过**(6.010 秒)。Plugin validator、Skill quick validator、Python 编译检查与 `git diff --check` 通过。最终合成包验证完成两轮研究,输入 5 文件哈希未变。
22
-
23
- 发布后的 `main` 补充一条命令安装和首次使用引导;2026-09-10 的完整测试为 **193 项全部通过,0 跳过**(15.876 秒)。新增检查覆盖本地 Git 仓库的管道安装、临时目录清理后继续使用、固定 ref、已有 personal 安装识别,以及实际配置读取、默认配置无需文件、已有设置保留、配置损坏提示和 stdout JSON 与 stderr 引导分离。
24
-
25
- 在源码根目录或解压后的测试包根目录运行:
26
-
27
- ```sh
28
- python3 -B -m unittest discover -s tests -v
29
- python3 scripts/verify_fixture.py --output /tmp/bookmark-research-validation
30
- ```
31
-
32
- 测试包包含虚构域名、合成书签和录制的公开工具 schema,不包含个人书签库、API Key 或预建索引。默认不向网页服务发请求;系统有 Codex CLI 时,安装测试使用临时独立 profile 实际执行原生命令。没有 Codex 时相关客户端测试会 skip,不能把 skip 当作已验证安装。
33
-
34
- `verify_fixture.py` 使用固定离线响应,验证如下可观察行为:
35
-
36
- - 独立读取 JSON 计数并与 SQLite 对照;永久副本不增加总书签数。
37
- - 同 URL 的不同实例、栏目、组、子文件夹范围均保留。
38
- - 导入前后源文件 SHA-256 相同;第二次同步未变更。
39
- - 完成两轮研究,保留缺口、实际引用、失败页面与研究来源映射。
40
- - 关闭并重开运行时后,相同操作编号复用结果,不再次执行检索。
41
- - 交付独立的 `context.json`、`report.md`、`sources.json` 与原响应/正文快照。
42
-
43
- 测试返回 `network_calls:0`;研究计数中的 4 search calls / 3 fetch calls 是合成适配器调用,不能当作真实服务质量或费用测试。
44
-
45
- 单元测试还覆盖预算被并发调用消耗、请求中断后保留预留、没有正文/引用原句不存在/正文被修改、来源被拒绝、结论撤回、冲突解决依据失效、未回答问题无法标记完成,以及 CLI 和 MCP 的输入边界。
46
-
47
- 独立审查发现并修复了回执已写但状态未提交时的来源恢复、超过 2 MiB 的研究状态返回、MCP 错误标记、撤回结论被脚注渲染隐藏等问题。回归用例包括 45 条 claim/540 段长引用的完整分页与完成响应;审查者另用实际 Markdown 渲染器确认历史结论和跳转链接可见。
48
-
49
- 另一位代理按 Skill 自主探索合成画布,没有读取预设验证流程或答案。它找到 Investigation 组的 4 个书签及后续连线,发现 scenario 缺少 update 页后诚实交付 incomplete。补齐合成资料后,评估又暴露 incomplete 报告缺少恢复入口;现通过 `research_record kind=resume` 在同一任务继续。原报告、状态和来源快照保留,补读后完成报告,fetch 预算从 2/4 增至 3/4,原 unknown_outcome 不退款。初次失败和修复后结果保存在本机独立验收目录,没有把首次运行改写成全程成功。
50
-
51
- 恢复另经独立故障注入复核:连续两次恢复的 6 份快照逐字节一致,原报告的 9 个相对链接可解析;快照写入或状态保存失败时原产物保持完整,再次恢复不重复网络调用、不增加预算。
52
-
53
- ## 用户提供的数据包
54
-
55
- 实际在提供的示例画布上运行了:
56
-
57
- ```sh
58
- python3 scripts/verify_fixture.py --output /tmp/bookmark-research-0.2.0-fixture-check \
59
- --package '/absolute/path/to/书签画布-20260906'
60
- ```
61
-
62
- | 指标 | 实际结果 |
63
- | --- | --- |
64
- | 书签实例 | 638 |
65
- | 不同原始 URL | 557 |
66
- | 文件夹 | 122 |
67
- | 栏目/画布节点/连线 | 5/7/3 |
68
- | 已检查源文件 | 7,含包内指南 |
69
- | 源文件变化 | 无;逐文件 SHA-256 一致 |
70
- | 第二次同步 | `changed_files:0` |
71
-
72
- 计数使用原 JSON 的独立遍历与导入后的 SQLite 交叉核对。这里没有联网抓取这 557 个 URL。完整个人路径与逐文件哈希保存在本地验证输出,公开测试包只收录合成数据。
73
-
74
- 另一个 AI 升级调研包的 223 个书签实例/207 个去重 URL,其研究覆盖与无法读取项单独列在 [v0.2.0 历史来源清单](https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/v0.2.0/docs/research-sources-0.2.0.json),不把这些实例数混作公司或独立证据数。
75
-
76
- ## 真实服务验证
77
-
78
- 运行命令需要显式打开网络验证:
79
-
80
- ```sh
81
- python3 tests/live_provider_smoke.py --run-live --output /tmp/bookmark-provider-smoke.json
82
- ```
83
-
84
- 一次最多三次搜索与三次提取,测试进程清除服务密钥并使用临时设置。观察结果见 [provider-smoke-0.2.0.json](provider-smoke-0.2.0.json):三家工具目录与提取成功;Parallel、Tavily 搜索成功;首次 Exa 搜索连接失败,另一次明确发起的 Exa 搜索成功。原始集合保留 `passed:false`,成功复验单列,未把自动重试或持续可用性当成已验证。
85
-
86
- 这些检查未执行供应商托管 Deep Research、OAuth 或付费研究任务。Exa Agent、Parallel Task MCP、Tavily Research 的独立接口与认证依据见 [MCP 接入说明](mcp-aggregation-0.2.0.md)。本插件第三层是宿主模型使用可恢复研究工具执行的流程。
87
-
88
- ## 真实研究流程
89
-
90
- 使用工作区 0.2.0 的 `ResearchSessions`,由宿主模型实际完成“核验三层搜索与原生深度研究服务的边界”。任务 `r-e7090e9833194564` 保存在本机独立研究目录;网络步骤在不同进程中继续,读取已存来源、审核正文、记录原句后综合报告。
91
-
92
- | 观察项 | 实际结果 |
93
- | --- | --- |
94
- | 问题/搜索轮次 | 2/2 |
95
- | 已预留检索调用 | 4 search calls、3 fetch calls |
96
- | 实际来源快照 | 4,来自 3 个不同官方 URL |
97
- | 主张与修订 | 10 条,其中 1 条拆分后撤回;历史原句保留 |
98
- | 冲突 | 1 项,同 URL 摘录写 basic/fast;结论明确保留模式未知 |
99
- | 最终状态 | completed,报告、来源清单、操作回执与原文均存在 |
100
-
101
- 这次不是脚本预置回答:第二轮根据第一轮定义补读接口,发现不同快照的模式差异后额外读取并记录冲突。Exa 和 Parallel 的真实 search/fetch 调用均成功。完成标准是回答两道接口边界问题;不能据此声称已经运行供应商原生研究任务、核实服务器默认模式或通过通用研究质量基准。
102
-
103
- 实际摘录的 Search MCP 模式证据:Parallel 快照 SHA-256 `fbff308d1ca7bf7cdaa1fffc001dcb27cf2faa63a52423f2d6a1aa202d4bf925` 写 `Anonymous free-tier searches run in [fast mode] by default`(原 Markdown 带链接);Exa 快照 SHA-256 `cc9ab5f5e83ca5e7186ce1701f8567e2bae9626c2737990a3fa56c13ec0f49e2` 写 `Search runs in basic mode`(原 Markdown 带代码标记)。保存的逐字原句和限制见该任务 `report.md`。
104
-
105
- 调研来源清单另经复核:207 个原 URL 指纹、223 个实例映射、191 份正文/响应文件哈希和 16 处短引用全部匹配;原调研包 9 个文件哈希未变。覆盖不足和身份错配仍按原记录保留。
106
-
107
- ## 安装、更新与包完整性
108
-
109
- 安装测试使用 Codex CLI 0.153.4 和 Python 3.9:中文、空格、引号与字面 `$()` 路径;重复安装;同版本内容更新;版本更新;已存在索引和用户设置保留;来源冲突;安装缓存中的 doctor 与 MCP 初始化/工具发现。
110
-
111
- 普通发行 ZIP 与测试包使用同一版本源码构建,并作为 [v0.2.0 Release](https://github.com/Browser-bookmark-hub/Bookmark-Research/releases/tag/v0.2.0) 附件分发。Release 的 `SHA256SUMS` 校验整个 ZIP,各包的 `MANIFEST.sha256` 校验内部文件。复现构建:
112
-
113
- ```sh
114
- python3 scripts/build_zip.py --output dist/bookmark-research-0.2.0.zip
115
- python3 scripts/build_zip.py --include-tests --output dist/bookmark-research-test-pack-0.2.0.zip
116
- ```
117
-
118
- 输出路径应不存在;需要重新构建时用新路径或先自行移动旧产物。解压后核对 `MANIFEST.sha256`,再运行验证:
119
-
120
- ```sh
121
- shasum -a 256 -c MANIFEST.sha256
122
- python3 src/cli.py doctor
123
- python3 scripts/install.py verify
124
- ```
125
-
126
- 最后一条要求已在该客户端安装插件。测试包还可运行 `verify_fixture.py`;普通插件包不含测试资料。导出测试覆盖 Codex、Claude Code、Pi、DSH 与 Agent Plugins 五种目录结构,以及从任意工作目录启动。Claude Code、Pi、DSH 的客户端内实际加载仍未验证,不能仅凭导出成功宣称原生客户端已工作。
@@ -1,154 +0,0 @@
1
- # Bookmark Research 0.4.0 validation
2
-
3
- Version 0.4.0 combines whole-package research, host workflows, optional professional research, Wiki/evaluation, source lifecycle management and English/Chinese instructions. Earlier unpublished 0.3.0 work is included in this version; original validation measurements retain their observed versions and limits. A passing code test, a recognized plugin manifest and a successful model-driven workflow are separate observations.
4
-
5
- Manual export folders can disappear, and a later export can have a different filename or location. Stable source identity is separate from input paths, snapshots are recoverable, and explicitly persistent directories can be monitored. Directory, ZIP and single-card input use the existing Bookmark Canvas export protocol.
6
-
7
- ## Implemented behavior
8
-
9
- - Directory, wrapped/unwrapped ZIP, and permanent/temporary section JSON input. Unknown fields, exact raw protocol bytes, bookmark instances and relationships are retained. Ambiguous multi-package archives and loose cards mixed into a package fail visibly instead of reducing the imported scope.
10
- - `snapshot` and `live` source modes, persistent path aliases, immutable content versions and `source_history`. The current exported protocol has no reliable global canvas ID; moved exports of the same canvas reuse an explicit `source_id`, and distinct canvases are not merged by titles or URLs.
11
- - Separate `partial` and `complete` deletion rules. Single cards are partial inputs. Full mirrors remove absent files while preserving stable item identity through renames. Empty directories, Git write locks and invalid/inconsistent files retain the last valid index.
12
- - A process-scoped Python polling monitor, separate SQLite connections, consistent query reads, debounced updates, a longer deletion grace period, visible pending/error states and reconnect/query checks. Monitoring stops when the MCP process stops; standalone CLI users can run `watch`.
13
- - Full managed snapshots after partial imports, including recorded relationships needed to recover an older layout after a card-only rename. Recovery validates hashes. Legacy JSON recovered only from SQLite is explicitly labeled as lacking original whitespace.
14
- - Research status compares frozen scope with current indexed input. Wiki reads/lint flag changed bookmark input for review. No automatic webpage fetch, LLM research, evidence rewrite or Wiki publication is performed.
15
- - Installation discovery probes use an isolated empty data directory, so verifying an installation cannot start monitoring the user's sources. First-use instructions describe the supported input forms and continuous-directory mode.
16
-
17
- ## Task-driven user simulations (2026-09-14)
18
-
19
- Three delegated agents first followed the Skill on real tasks: a single GitHub bookmark check, an Exa/Parallel topic comparison within the supplied Canvas, and a resumable comparison of three ordinary bookmarks. The baseline was frozen at `baf1611`. Each case used isolated data, recorded CLI calls and actual retrieved text. For the revised Skill, the agents rotated to different cases without reading those cases' baseline answers. A fresh fourth agent could not be created because of the host thread limit. These are observed usability cases, not a controlled quality or speed benchmark.
20
-
21
- | User need | Baseline friction | Revised behavior observed |
22
- | --- | --- | --- |
23
- | Check one known URL | Main Skill plus four references and CLI help before reading; repeated provider text filled the response. | Main Skill alone was sufficient. Two fetches covered the requested facts: the default extract stopped before authentication details, so the agent requested a larger extract. No import, research session or Wiki. |
24
- | Compare a Canvas topic | One invalid CLI spelling; ten CLI calls, two fetch operations; seven related bookmarks retained. | Ten valid CLI calls, three fetch operations; all seven original URLs attempted, six had usable text. Official-document follow-up answered the topic while retaining the Smithery navigation-only gap. Fourteen substantive table claims passed independent semantic review. No whole-package investigation or Wiki. |
25
- | Research three ordinary bookmarks | Native inventory was empty; the agent maintained a separate manual list and created an unrequested Wiki. | `research_start.urls` froze all three originals without a Canvas import. Original-source review was 3/3; 17 substantive claims passed independent review of the draft and evidence. The saved report remained `incomplete` with 3/4 questions answered and Reader scanned-PDF OCR evidence still missing. A resume input was supplied; no Wiki was created. |
26
-
27
- The main English Skill decreased from **2,163 to 1,254 whitespace-delimited words**. References load by task; professional research remains optional support after the host's first pass. Wiki creation requires a request to retain knowledge there. Ordinary URL lists preserve duplicate positions and unsupported/local entries, and use the same frozen-scope completion checks as Canvas input. During the deep case, an inventory review linked to a supplementary-only claim was atomically rejected, then corrected to cite the original source.
28
-
29
- Default `fetch_web` / `fetch-web` now returns one selected text per URL, retaining every attempt's status and archive path. Full provider envelopes remain available through archives or `raw:true` / `--raw`. Reformatting the *same* three saved live responses with identical JSON serialization produced:
30
-
31
- | Response | Original JSON characters | Compact characters | Reduction |
32
- | --- | ---: | ---: | ---: |
33
- | Single GitHub README | 97,643 | 29,313 | 70.0% |
34
- | Canvas first fetch | 233,623 | 61,814 | 73.5% |
35
- | Canvas follow-up fetch | 387,546 | 102,871 | 73.5% |
36
-
37
- All eight selected texts matched their archived text and SHA-256 hashes. This measures response length, not latency, token use or research accuracy. Live provider availability and extracted contents differed between runs, and follow-up calls sometimes increased.
38
-
39
- Independent baseline review caught unsupported wording introduced by the final summary despite passing quotation/hash checks: treating page output as proof of PDF completeness, omitting same-page counterevidence, and presenting one OCR option as required. The revised instructions require review of the final synthesized answer as well as claims, with corrections before completion. Semantic reliability still depends on the readers and reviewers; the runtime enforces record consistency and scope, not factual truth.
40
-
41
- The local Codex source was backed up, re-exported and reinstalled at base version 0.4.0 with an installation-only cachebuster. All 61 source/cache files matched. A fresh stdio process exposed **35 tools**, accepted `research_start.urls` and `fetch_web.raw`, retained two distinct URLs/three instances with duplicate positions, and rejected unread input marked `completed`. The offline check saved an `incomplete` report without creating an index or making network calls. A new conversation loads the updated Skill and catalog.
42
-
43
- The full supplied Canvas was also rechecked offline: 207 distinct URLs, 223 instances, all three inventory pages, and all nine original files unchanged. This does not establish review of all 207 webpages. No professional research job, PDF conversion benchmark or new four-host end-to-end acceptance was performed. Raw case artifacts remain outside the repository and distribution.
44
-
45
- ## Whole-package scope
46
-
47
- The supplied package was read offline through the same index and research inventory APIs. Independent counts and the frozen inventory agree: **223 bookmark instances, 207 distinct original URL strings, 22 folders, 4 sections, 5 canvas nodes and 4 group memberships**. This package has no directed edges; the synthetic fixture separately exercises directed edges and copy cards.
48
-
49
- All three inventory pages were consumed and compared with the complete indexed inventory. Every original instance remains mapped to its URL and context. All nine files in the original directory had identical SHA-256 hashes before and after validation. Personal source files, paths and inventory contents are not included in the distribution.
50
-
51
- For 207 URLs, the default fetch plan reports a 26-call lower bound at eight URLs per call and a 38-call configured budget. Explicit user budgets remain unchanged. A budget below the first-pass capacity reports the shortfall without removing sources. This verifies data handling and capacity accounting; it does not mean all 207 webpages were fetched or reviewed during this offline check.
52
-
53
- Both professional-service payloads were also prepared offline from this actual frozen input. Each retains all 207 inventory IDs: 206 HTTP(S) URLs are supplied as retrievable sources and one `file://` input remains explicitly local-only. Nothing was submitted. A regression checks 207 URLs/223 instances, an explicit 150-ID subproblem and oversized private context without reducing source scope or uploading notes.
54
-
55
- ## Automated verification
56
-
57
- The latest integration tree passed **376 tests** in 39.178 seconds. New regressions cover ordinary URL inventories, duplicate instances, invalid/mixed inputs, pagination, partial-completion rejection, compact-response fidelity and MCP/CLI entry points. Plugin/Skill validators, local document-path checks and the installed-runtime check passed. The earlier consolidated checkpoint passed 319 tests in 34.511 seconds, including 63 JavaScript host workflow contract scenarios. The plugin manifest, local MCP server identity and remote MCP client identity all use 0.4.0; exporters derive their package versions from the manifest.
58
-
59
- The source-lifecycle checkpoint passed **313 tests** on Python 3.9.6 / SQLite 3.43.2 with FTS5. This includes 32 source-lifecycle cases and 20 installer cases. The source-lifecycle module was also run independently after the final consistency adjustment, and the installer module was run independently to verify it does not depend on test discovery order.
60
-
61
- ```sh
62
- python3 -B -m unittest discover -s tests -v
63
- python3 -B -m unittest discover -s tests -p 'test_source_manager.py'
64
- python3 -B -m unittest discover -s tests -p 'test_install.py'
65
- ```
66
-
67
- Cases cover persistent reopen, changed paths, duplicate source ambiguity, single-card merge, raw-byte recovery, renamed layout bindings, corrupt snapshots, ZIP traversal/symlinks/duplicates/multiple roots/size bounds, full-mirror deletion, partial retention, temporary directory loss, Git locks, changes during import, idle background updates, MCP reconnect, frozen research scope, Wiki review flags, and CLI/MCP behavior. The host workflow, research service and coverage regressions remain included.
68
-
69
- Skill/plugin validation and `git diff --check` passed. These checks establish implementation behavior and packaging validity, not semantic research quality.
70
-
71
- The tests cover full inventories, explicit subsets, repeated URLs, original context, concurrent evidence writes, legacy state, pagination, rejected evidence, partial review, external reports, unknown service outcomes and completion gates. Reviewing only part of the input cannot complete a whole-package investigation. Accounted items, accepted text, substantive review and answered questions have separate denominators.
72
-
73
- MCP/CLI integration tests establish a complete research session, cross interface boundaries when reading and importing, reject partial completion, and publish/search/lint a Wiki page from accepted evidence. They also check routing, settings and evaluation parity. The evaluation fixture is synthetic; its scores make no provider-quality claim. The earlier host/research checkpoint passed 280 tests, including 19 service lifecycle tests and 19 installer tests; its host wrapper exercised 36 JavaScript scenarios. These are checkpoint measurements, not the final combined test count.
74
-
75
- ```sh
76
- python3 -B -m unittest discover -s tests -v
77
- python3 -B scripts/verify_fixture.py --output /absolute/external/new-directory
78
- python3 -B scripts/verify_quality.py
79
- python3 -B -m unittest discover -s tests -p 'test_host_workflows.py'
80
- ```
81
-
82
- Append `--package /absolute/path/to/package` to the fixture verifier for read-only validation of a real package. Use a new output directory outside source packages and plugin caches. The fixture deliberately retains blocked sources and produces an incomplete report, proving that complete accounting is not complete research.
83
-
84
- ## English and Chinese instructions
85
-
86
- The execution Skill and all 11 method references are maintained in English, with 12 complete Chinese reading copies. There is one discoverable Skill. Starter prompts and onboarding are bilingual; Codex delegation and shared Claude/Pi/DSH workflows carry the requested output language. The [instruction index](instructions.en.md) and [prompt reference](prompt-reference.en.md) provide both reading languages.
87
-
88
- The bilingual update passed 56 relevant Python tests, including 63 JavaScript workflow contract scenarios. Tests cover language propagation, unchanged scope and status fields, all five export formats, one Skill entry point, and installer/bootstrap behavior. Local links and ZIP integrity manifests were also checked. Installer `--lang` selects onboarding language only; it does not persist a research-language preference.
89
-
90
- An independent agent followed the English Skill with an offline synthetic package: 12 successful CLI commands retained all seven bookmark instances and six unique URLs across four section cards. A copy anchor did not duplicate its underlying bookmarks. The brief, gaps and answer were in English; all five input files were unchanged. With zero retrieval budgets, no fetched pages or claims, the saved research correctly remained incomplete. This is an instruction-use check, not a live webpage or professional-service evaluation.
91
-
92
- The bilingual local update was reinstalled and enabled at base version 0.4.0. All shipped files matched the staged export apart from the installation cachebuster. Its actual stdio runtime exposed 35 MCP tools, with SQLite FTS5 available; isolated discovery created no database. Technical diagnostics, original evidence and historical validation records retain their original language.
93
-
94
- ## Real input and installed runtime
95
-
96
- Read-only verification against real user exports produced:
97
-
98
- | Input | Bookmark instances | Distinct original URLs | Protocol files |
99
- | --- | ---: | ---: | ---: |
100
- | Research directory export | 223 | 207 | 5 |
101
- | Sample ZIP export | 638 | 557 | 6 |
102
-
103
- Changing the research export's directory and reimporting with the same source ID produced zero changed files and zero inserted/updated items. Importing one of its cards preserved all 223 instances. Removing the disposable export copies still allowed queries. Restoring the managed snapshot into a new database reproduced the same counts and input fingerprint. A modification to a separate live copy was indexed without a query. Hashes confirmed that the original directory files and ZIP were unchanged.
104
-
105
- The existing personal Codex plugin was updated to 0.4.0 after preserving its previous source directory and taking a consistent SQLite backup. The installed shared runtime matched the repository bytes. Its actual stdio entry exposed **35 tools**, accepted the new source parameters, imported the complete 223-instance package, queried its saved snapshot and listed its history. Runtime/FTS5 checks passed, with no extra output on the MCP stream.
106
-
107
- The two previously registered manual exports were migrated to snapshot mode. Their source IDs, item primary keys, item revisions, counts and indexed input fingerprints remained unchanged; neither migration inserted, updated or deleted any bookmark item. A new Codex thread loads the updated Skill and tool catalog.
108
-
109
- ## Host and provider boundaries
110
-
111
- The [host validation evidence](host-validation.json) retains the actual client versions, tool counts and failures observed at each earlier checkpoint.
112
-
113
- | Route | Established evidence | Separate acceptance still required |
114
- | --- | --- | --- |
115
- | Codex | Native delegation recipe; preparation script reads all 207 IDs through real stdio MCP; delegated Skill acceptance on a fresh synthetic package produced a correctly incomplete report and two local-evidence Wiki pages | Live webpage research with multiple reader groups and independent child verification |
116
- | Claude Code | 2.1.247 strict plugin validation; live launch loaded the workflow command and connected all 34 MCP tools | Existing configured model endpoint returned 429 / Service Unavailable before any child ran; workflow execution and recovery remain unverified |
117
- | Pi | Project registrar, stdio bridge and isolated extension scenarios | Pi and the two required extensions are not installed in the validation environment |
118
- | DSH | Native call-object generator and isolated workflow scenarios | A configured DSH profile, engine, service and tool are not installed in the validation environment |
119
- | OpenAI / Parallel research APIs | Thin clients and lifecycle/contract tests, separate credentials, secondary report import | No research API credentials were configured; no paid task was submitted |
120
-
121
- At the earlier prerelease checkpoint, recorded as 0.3.0 in the original evidence, the personal Codex installation was updated with Codex CLI 0.153.4 after a byte-verified backup. The installed cache exposed all 34 tools over its actual stdio launch command; the CLI FTS5 check and read-only routing/service-description calls passed. All 36 non-manifest export files, including the three Codex host assets, matched the installed bytes. The installation-only cachebuster is separate from the repository version. A new Codex thread is required to load the updated Skill and tools.
122
-
123
- Workflow scenarios cover full pagination, independent verifier failures, gap-only follow-up, partial/fatal errors and false completion. The host remains responsible for checking authoritative saved state and artifacts after a workflow returns. These scenarios do not simulate a provider-quality benchmark or prove unavailable clients were run.
124
-
125
- An independently delegated Codex agent followed the revised Skill with only the original synthetic canvas input, without reading expected answers or changing code. It preserved seven instances and six URLs, read five local JSON/canvas files, saved six metadata claims, wrote two linked Wiki pages and checked saved state, coverage and hashes. All six URLs use the reserved `.test` domain, so they remained explicitly blocked: accounting was 6/6, original-page text and substantive review were both 0/6, and the investigation remained incomplete. No network or professional research API was called. An additional verifier could not start because the host thread limit was reached; the agent disclosed its own second review rather than claiming independent child verification. The acceptance exposed a report-label issue for local documents, now corrected to show their evidence and content kinds without changing coverage.
126
-
127
- The live Claude attempt used an isolated two-URL/three-instance synthetic package with pre-imported text and zero retrieval budget. The first isolated launch did not load user authentication; the second used the existing user model configuration with hooks disabled for that invocation. Both started zero subagents and reported zero usage cost. No global client settings changed. Tool discovery succeeded, but the research run did not; the successful manifest test is not presented as live execution acceptance.
128
-
129
- Actual search observations in this implementation session: Exa returned usable search results; Tavily reported `monthly_cap_reached_bonus_eligible` in a structured error envelope despite MCP `isError:false`; Parallel had an SSL EOF transport failure. Error handling now classifies these as provider failures instead of empty successful search. No bonus signup or payment was attempted, and these observations are not permanent claims about service availability.
130
-
131
- ## Wiki and quality claims
132
-
133
- Wiki verification checks accepted, active source claims, immutable revisions, optimistic updates, provenance, changed hashes, crosslinks, Chinese body search and stale-evidence exclusion. Lint reports semantic support as unscored. A semantic review declaration is recorded, not mechanically proven.
134
-
135
- The quality verifier uses six explicitly synthetic outputs to exercise answer F1, citation annotations, report rubric scores, four coverage metrics, cost/latency and repeat variance. Unknown labels and measurements remain null. It does not establish quality improvements over ordinary host research or professional services. Real comparisons require actual matched tasks, recorded conditions and independent judgments; see [Wiki and evaluation](wiki-quality.md).
136
-
137
- ## Limits and distribution
138
-
139
- The monitor uses periodic file checks, not native OS file events or a system daemon. Defaults are a 1-second poll, 2-second debounce and 5-second deletion grace; these are scheduling settings, not guaranteed update latency. It does not perform Git authentication or synchronization. Stable-file checks and Git locks cannot make an external exporter atomic. A complete mirror must have valid, agreeing references before its update can be committed.
140
-
141
- Snapshots are retained without automatic garbage collection. Local index freshness does not establish that remote webpages are current. Professional API access and full model-driven runs in all four hosts have not been newly validated by this version; the host and provider boundaries above retain those observations.
142
-
143
- ```sh
144
- python3 scripts/build_zip.py --output dist/bookmark-research-0.4.0.zip
145
- python3 scripts/build_zip.py --include-tests --output dist/bookmark-research-test-pack-0.4.0.zip
146
- ```
147
-
148
- Each distribution has an internal `MANIFEST.sha256`. User exports, source snapshots, databases, research artifacts, credentials and local validation directories are excluded. GitHub publication is separate from the completed local installation and validation.
149
-
150
- Each ZIP has a `MANIFEST.sha256`. Exports carry the shared runtime, Skill and the host-specific adapter; they exclude source packages, user settings, databases and credentials. Git installation remains independent of GitHub Releases. Host loading and optional service authentication follow their real interfaces, not the archive format.
151
-
152
- The historical 0.2.0 source-audit JSON contains original bookmark instance mappings, so it is excluded from both 0.4.0 ZIPs. Historical method documents link to that release's repository snapshot instead. The production package contains no synthetic test fixtures; the explicitly requested developer test pack adds only curated tests and synthetic fixtures.
153
-
154
- Installer checks require the Codex delegation recipe, preparation script and shared stdio bridge. Local installation and updates compare these assets with the selected source; standalone verification also rejects missing or symlinked host assets. Native Codex tests verify that changed assets reach the updated cache. These are installation integrity checks, not model-driven workflow acceptance.
@@ -1,107 +0,0 @@
1
- # Reviewed Wiki and annotated research evaluation
2
-
3
- **English** · [中文](wiki-quality.zh.md)
4
-
5
- The host writes and reviews knowledge pages. The plugin stores their evidence and revisions, checks their integrity, and searches the organized prose. Evaluation consumes explicitly annotated outputs; it does not call a model or infer semantic support from matching words.
6
-
7
- ## Wiki API
8
-
9
- `WikiStore(directory=None, settings=None, research_sessions=None)` uses an explicit directory first, then `settings.wiki.directory` (default: the data directory's `wiki/`). Plugin directories and canvas package directories are rejected. Research evidence remains in its research session, outside the original input package.
10
-
11
- ```python
12
- from wiki import WikiStore
13
-
14
- wiki = WikiStore(research_sessions=sessions)
15
- page = {
16
- "title": "Reusable sessions",
17
- "kind": "topic", # or entity
18
- "sections": [{
19
- "heading": "Supported behavior",
20
- "text": "The host's reviewed synthesis, including relevant limits.",
21
- "claims": [{"research_id": research_id, "claim_id": "c1"}],
22
- }],
23
- "links": [],
24
- "review": {
25
- "method": "model", # or human
26
- "reviewer": "The actual model/version or human reviewer",
27
- "note": "Explain the scope and support checks actually performed.",
28
- },
29
- }
30
- written = wiki.write("reusable-sessions", page, "Organize reviewed findings", expected_revision=0)
31
- current = wiki.get("reusable-sessions")
32
- wiki.write("reusable-sessions", revised_page, "Explain the new limitation", expected_revision=current["revision"])
33
- ```
34
-
35
- Every section requires explicit research claim references. A reference is accepted only when its claim is active, every cited source is `accepted`, the quote occurs in its saved body, and the body hash matches. The Wiki snapshot retains the research ID, claim ID, source ID, original-inventory IDs, bookmark instance references, exact quotes, reviews, retrieval metadata and artifact hashes. Up to 24 distinct claims may support one page; oversized topics must be split.
36
-
37
- The review declaration addresses whether the prose is supported, uses the correct entities and scope, preserves uncertainty, and acknowledges contrary evidence. The program records this declaration and its rubric. It cannot verify that the named human or model performed the judgment, and a successful write is not an independent semantic endorsement.
38
-
39
- An imported external report stays secondary evidence. Its provenance and quoted report text are retained; its bibliography is not expanded into pages supposedly read or original sources supposedly covered.
40
-
41
- ### Revisions, links and inspection
42
-
43
- Each update writes an immutable JSON revision and a rendered Markdown revision under `revisions/<page_id>/`, then atomically updates `index.json`. The index is the publication point. A failed publication can leave unreferenced files, while the preceding revision stays readable. `expected_revision=0` creates a page; an update must name the revision the caller read. After an interrupted write, inspect the current revision before retrying.
44
-
45
- Cross-links have the form `{"page_id": "existing-page", "relation": "The host's explanation"}`. The target must already exist. Rendered links point to the target revision present when the page was written, preserving the historical meaning. A later target update produces a lint warning; the host can explicitly revise the relationship. Prior page revisions remain available with `get(page_id, revision=N)`.
46
-
47
- | Method | Result |
48
- | --- | --- |
49
- | `write(page_id, page, change_note, expected_revision=0)` | Revision metadata and absolute artifact paths |
50
- | `get(page_id, revision=None)` | Authored page, evidence snapshot, update record and current provenance validation |
51
- | `list(offset=0, limit=20)` | Paginated current page metadata |
52
- | `lint(page_id=None)` | Source review, active-claim, citation, hash, link and history issues |
53
- | `search(query, offset=0, limit=10)` | Matching Wiki prose, revision dates and validation; stale pages excluded, `needs_review` warnings retained |
54
-
55
- Lint detects later rejection, withdrawal, source/claim changes and altered or missing artifacts. Historical text is preserved and identified as stale. Lint explicitly returns `semantic_support: "not_scored"`.
56
-
57
- Search checks titles, section headings and authored section text. It uses case-folded literal terms, including Chinese substrings, and simple occurrence ordering. Saving a provider extract alone does not make that extract searchable as organized Wiki knowledge. This is not embedding retrieval or semantic reranking; neither is selected without actual question-answering evaluation.
58
-
59
- New records use research-task-relative evidence paths. `get` resolves verified references at current research locations, including legacy records containing absolute paths, without rewriting immutable revisions. Identity, claim, quote and artifact hashes remain mandatory. Move the Wiki and cited research state/inventory/evidence together; preserve their relative layout for Markdown links and configure local research paths at the destination. Wiki Markdown alone is not a complete portable knowledge base.
60
-
61
- ## Quality evaluation input
62
-
63
- `evaluation.evaluate(suite, runs, judgments=None)` is a pure computation over supplied data. The complete runnable input is [research-quality.json](https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/main/tests/fixtures/research-quality.json), also included at `tests/fixtures/research-quality.json` in the source checkout and verification pack.
64
-
65
- - A **suite** identifies tasks, their original source/question IDs, gold answers and a versioned rubric. `gold_answers: null` means no answer-set benchmark is available. The rubric defines answer identity matching, citation support and report dimensions on a 0–4 scale.
66
- - A **run** records its task, system, mode, repeat, final report, atomic answers/claims/citations, coverage IDs and measurements. It also records model, tools, budget, input snapshot/version, time window and harness version. Citations are unique claim/source pairs.
67
- - A **judgment** identifies the rubric version and the actual human or model evaluator. It supplies predicted-answer → gold-ID matches, supported/unsupported/uncertain citation labels with reasons, and report rubric scores with reasons. Fixture judgments must identify themselves as synthetic.
68
-
69
- Answer matching is an annotation task. Evaluators must read the actual prediction, establish identity and scope, and use `gold_id: null` for an incorrect entity. The scorer does not guess aliases or use regex matches as a semantic judge. Multiple predictions matched to one gold entity count once for set precision/recall; unmatched predictions count as false positives. Duplicate matches are also reported.
70
-
71
- Each citation judgment concerns the complete claim and the actual source context, including identity, dates and scope. `supported` means the text supports the claim. `unsupported` and `uncertain` both remain in the full accuracy denominator. Exact-quote matching alone is not a support judgment.
72
-
73
- ### Metrics and missing information
74
-
75
- | Output | Meaning and limit |
76
- | --- | --- |
77
- | Answer precision, recall, F1 | Uses gold data and complete answer-match annotations; otherwise null |
78
- | Semantic citation accuracy | Supported / all citations when all are judged; partial-label statistics are shown separately |
79
- | Semantic claim support | Claims with a supported citation / all claims, once all citations are judged |
80
- | Citation presence | Claims with any citation / all claims; this is a structural measurement |
81
- | Report quality | Mean supplied dimension scores normalized from 0–4, only when every rubric dimension is scored |
82
- | Accounted/text/reviewed coverage | Separate rates against the original source set; failed sources remain in the denominator |
83
- | Question coverage | Recorded answered question IDs / original question IDs; this does not judge answer truth |
84
- | Cost and latency | Supplied USD/seconds with observed, reported or synthetic measurement basis; unknown values stay null |
85
- | Repeat variation | Count, mean, sample variance, sample standard deviation and range under the same declared conditions |
86
-
87
- Coverage must satisfy `reviewed ⊆ text ⊆ accounted ⊆ original scope`. These reported IDs still need the research evidence audit: an input label alone does not prove that a source was read. External tool and model usage unavailable to the plugin must remain unknown.
88
-
89
- Groups compare the same task. Differences in model, tools, budget, input version, time window, harness, evaluator or measurement basis are reported. Differing conditions form separate aggregates. Unknown budgets prevent a controlled-comparison declaration. A single observation has no variance estimate; missing costs are not averaged as zero. Failed and incomplete outputs remain in the supplied run set.
90
-
91
- The output pins suite, run and judgment hashes, retains evaluator/rubric identity and includes null-score diagnostics. It does not produce a provider ranking, a causal claim, a RACE/FACT score or a public leaderboard score. Real claims about improved research quality require real task runs and independently defensible annotations.
92
-
93
- ## Reproduce the offline demonstration
94
-
95
- These commands need a source checkout or verification pack. Runtime exports contain the guides, but not the test fixtures or verification scripts.
96
-
97
- ```sh
98
- python3 -B scripts/verify_quality.py
99
- python3 -B -m unittest discover -s tests -p 'test_wiki.py'
100
- python3 -B -m unittest discover -s tests -p 'test_evaluation.py'
101
- ```
102
-
103
- The verification script writes `inputs.json`, `evaluation.json` and a readable `evaluation.md` under a new external data-directory run. `--output /absolute/external/directory` selects another artifact directory. `--fixture /path/input.json` accepts a wrapper containing `suite`, `runs`, `judgments` and optional known fixture expectations.
104
-
105
- The included example contains four invented source records, two gold entities and six hand-authored outputs labeled as direct, workflow and professional modes. One source is unavailable, some outputs miss or invent an answer, and one citation has uncertain scope. Costs and durations are synthetic values; one cost is deliberately unknown. These cases exercise denominators, semantic-label completeness, repeat variance and missing measurements. **They do not measure those three execution modes or any real host/model/provider.**
106
-
107
- Wiki tests additionally verify immutable history, failed index publication, cross-links, Chinese prose search, source rejection/withdrawal, secondary-report provenance and source/package boundaries. Live host research, real professional-service comparisons and independent semantic judging remain separate validation work.
@@ -1,107 +0,0 @@
1
- # 经审阅的 Wiki 与有标注的研究评测
2
-
3
- [English](wiki-quality.md) · **中文**
4
-
5
- 宿主负责撰写、审阅知识页;插件保存证据和修订,核对完整性并检索整理后的正文。评测使用明确标注的输出,不调用模型,也不从文字匹配推断语义支持。
6
-
7
- ## Wiki API
8
-
9
- `WikiStore(directory=None, settings=None, research_sessions=None)` 优先使用显式目录,其次 `settings.wiki.directory`,默认数据目录中的 `wiki/`。插件目录和画布包目录会被拒绝。研究证据仍保存在原研究档案中,位于输入包之外。
10
-
11
- ```python
12
- from wiki import WikiStore
13
-
14
- wiki = WikiStore(research_sessions=sessions)
15
- page = {
16
- "title": "可复用会话",
17
- "kind": "topic", # 也可为 entity
18
- "sections": [{
19
- "heading": "已确认行为",
20
- "text": "宿主已审阅的综合说明,保留相关限制。",
21
- "claims": [{"research_id": research_id, "claim_id": "c1"}],
22
- }],
23
- "links": [],
24
- "review": {
25
- "method": "model", # 也可为 human
26
- "reviewer": "真实模型及版本或人工审阅者",
27
- "note": "说明实际检查的范围和证据支持。",
28
- },
29
- }
30
- written = wiki.write("reusable-sessions", page, "整理已审阅结论", expected_revision=0)
31
- current = wiki.get("reusable-sessions")
32
- wiki.write("reusable-sessions", revised_page, "解释新增限制", expected_revision=current["revision"])
33
- ```
34
-
35
- 每节必须明确引用研究 claim。只有 claim 仍有效、所有来源均为 accepted、引文存在于已存正文且正文哈希一致时才接受。Wiki 快照保留 research ID、claim ID、source ID、原清单 ID、书签实例引用、原样引文、审阅、检索元数据和产物哈希。每页最多 24 个不同 claim,过大主题需拆分。
36
-
37
- 审阅声明说明正文是否受支持、实体与范围是否正确、是否保留不确定性和反证。程序记录声明及 rubric,不能证明署名的人或模型实际做了判断。写入成功不是独立语义认可。
38
-
39
- 导入的外部报告始终是二手证据,保留其 provenance 与报告原句;参考文献不会被扩充成声称已读的原始网页或覆盖来源。
40
-
41
- ### 修订、链接与检查
42
-
43
- 每次更新在 `revisions/<page_id>/` 写入不可变 JSON 和渲染 Markdown,再原子更新 `index.json`。索引是发布点。发布失败可能留下未引用文件,旧版仍可读取。expected_revision=0 用于创建;更新必须指定调用者实际读过的版本。写入中断后先检查当前 revision,再判断是否重试。
44
-
45
- 交叉链接形如 `{"page_id":"existing-page","relation":"宿主对关系的解释"}`,目标页必须已存在。渲染链接指向写入时的目标版本,保留历史含义。目标后续变化会产生 lint 提示,宿主可明确修订关系。旧页版本可通过 `get(page_id, revision=N)` 读取。
46
-
47
- | 方法 | 返回 |
48
- | --- | --- |
49
- | `write(page_id, page, change_note, expected_revision=0)` | 修订元数据与绝对产物路径 |
50
- | `get(page_id, revision=None)` | 正文、证据快照、更新记录与当前 provenance 核验 |
51
- | `list(offset=0, limit=20)` | 当前页元数据的分页 |
52
- | `lint(page_id=None)` | 来源审阅、有效 claim、引文、哈希、链接和历史问题 |
53
- | `search(query, offset=0, limit=10)` | 匹配 Wiki 正文、修订日期及核验状态;排除已失效页,保留 `needs_review` 警告 |
54
-
55
- Lint 检测后续来源拒绝、结论撤回、来源/claim 变化和缺失/改动的产物。历史正文保留并标记过时,明确返回 `semantic_support:"not_scored"`。
56
-
57
- 检索匹配标题、章节标题和整理后的正文,采用忽略大小写的字面词匹配(包含中文子串)及简单出现次数排序。仅保存 provider 摘录不会使它成为可检索的 Wiki 知识。没有 embedding 检索或语义重排,也不会未经实际问答评测就选择它们。
58
-
59
- 新记录使用相对于研究任务的证据路径。`get` 将已验证引用解析到当前研究位置,兼容旧绝对路径记录,不改写不可变修订;身份、结论、引文与产物哈希仍须一致。迁移时同时保留 Wiki 和引用的研究 state/inventory/证据,保持相对布局以维持 Markdown 链接,并配置目标机器的本地研究路径。仅有 Wiki Markdown 不构成完整可迁移的知识库。
60
-
61
- ## 质量评测输入
62
-
63
- `evaluation.evaluate(suite, runs, judgments=None)` 只计算传入数据。完整可运行输入为 [research-quality.json](https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/main/tests/fixtures/research-quality.json),源码和验证包也包含 `tests/fixtures/research-quality.json`。
64
-
65
- - **suite** 定义任务、原始来源/问题 ID、金标准和版本化 rubric。gold_answers:null 表示没有答案集合基准。rubric 定义答案身份匹配、引用支持和 0–4 分的报告维度。
66
- - **run** 记录 task、system、mode、repeat、最终报告、原子答案/claims/citations、覆盖 ID 和测量,并保留模型、工具、预算、输入快照/版本、时间窗口及宿主版本。引文按唯一 claim/source 对计算。
67
- - **judgment** 标识 rubric 版本与真实人工/模型评审者,给出预测答案到 gold ID 的匹配、supported/unsupported/uncertain 引用标签及理由、报告维度分数及理由。合成样例的评审必须自报 synthetic。
68
-
69
- 答案匹配是标注任务。评审者必须阅读真实预测,核对身份与范围;错误实体用 gold_id:null。评分器不猜别名,也不把正则匹配当语义裁判。多个预测匹配同一 gold 实体时,集合 precision/recall 只计算一次;未匹配预测算假阳性,重复匹配另行报告。
70
-
71
- 每条引用判断针对完整命题及来源上下文,包括身份、日期和范围。supported 表示文本支持命题;unsupported 与 uncertain 都留在完整准确率分母中。原句匹配不能代替支持性判断。
72
-
73
- ### 指标与缺失信息
74
-
75
- | 输出 | 含义与边界 |
76
- | --- | --- |
77
- | 答案 precision、recall、F1 | 需要金标准与完整答案匹配,否则 null |
78
- | 语义引用准确率 | 完整标注时为 supported/全部引文;局部标注另行显示 |
79
- | 结论语义支持 | 全部引文已标注后,有 supported 引文的 claim/全部 claim |
80
- | 引用存在率 | 有任意引文的 claim/全部 claim,仅是结构指标 |
81
- | 报告质量 | 全部 rubric 维度已评分时,将提供的 0–4 分均值归一化 |
82
- | accounted/text/reviewed 覆盖 | 各自相对原始来源集合计算,失败来源仍留在分母 |
83
- | 问题覆盖 | 已记录回答的问题 ID/原始问题 ID,不判断答案真假 |
84
- | 成本与时延 | 提供的美元/秒,注明 observed、reported 或 synthetic;未知保持 null |
85
- | 重复运行波动 | 相同声明条件下的数量、均值、样本方差、样本标准差和极差 |
86
-
87
- 覆盖必须满足 reviewed ⊆ text ⊆ accounted ⊆ original scope。报告的 ID 仍需证据审计,输入标签本身不证明实际阅读。插件无法取得的外部工具和模型用量保持未知。
88
-
89
- 比较按同一任务分组。模型、工具、预算、输入版本、时间窗口、宿主、评审者或测量依据不同会被披露,并分开聚合。未知预算不能声称受控比较。只有一次观察时没有方差估计,缺失费用不会当 0 求均值。失败和未完成输出仍保留在运行集合中。
90
-
91
- 输出固定 suite、run 和 judgment 哈希,保留评审者/rubric 身份及 null 分数原因。它不会生成 provider 排名、因果结论、RACE/FACT 分数或公共榜单分数。声称研究质量提高需要真实任务运行与可辩护的独立标注。
92
-
93
- ## 重现离线演示
94
-
95
- 以下命令需要源码或验证包;运行时导出带指南,但不含测试资料和验证脚本。
96
-
97
- ```sh
98
- python3 -B scripts/verify_quality.py
99
- python3 -B -m unittest discover -s tests -p 'test_wiki.py'
100
- python3 -B -m unittest discover -s tests -p 'test_evaluation.py'
101
- ```
102
-
103
- 验证脚本在数据目录的新运行目录下写入 inputs.json、evaluation.json 和可读的 evaluation.md。`--output /absolute/external/directory` 可另选目录;`--fixture /path/input.json` 接受含 suite、runs、judgments 和可选已知样例期望的包装对象。
104
-
105
- 示例包含四条虚构来源、两个 gold 实体和六份手写输出,标成 direct、workflow、professional。一个来源不可用,部分输出遗漏或虚构答案,一个引文范围不确定。费用和时长是合成值,其中一个费用故意未知。这些情况验证分母、语义标签完整性、重复方差及缺失测量,**不测量这三条路线或任何真实宿主/模型/服务**。
106
-
107
- Wiki 测试另覆盖不可变历史、索引发布失败、交叉链接、中文正文检索、来源拒绝/结论撤回、二手报告 provenance 及来源/包边界。真实宿主研究、专业服务比较和独立语义评审仍是另外的验证工作。
@@ -1,72 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Register the saved workflow in one project, without changing Pi settings."""
3
-
4
- import argparse
5
- import json
6
- from pathlib import Path
7
- import sys
8
- import tempfile
9
-
10
-
11
- def register(project):
12
- project = Path(project).expanduser().absolute()
13
- if not project.is_dir() or project.is_symlink():
14
- raise ValueError("Project must be an existing, nonsymlink directory")
15
- project = project.resolve()
16
- root = Path(__file__).resolve().parents[2]
17
- source = root / "workflows/bookmark-research"
18
- if source.is_dir():
19
- manifest = json.loads((source / "workflow.json").read_text(encoding="utf-8"))
20
- script = (source / "script.js").read_bytes()
21
- else:
22
- sys.path.insert(0, str(root / "scripts"))
23
- from host_assets import build_workflow
24
- definition, script = build_workflow(root, "pi")
25
- manifest = {**definition["meta"], "parameters": definition["parameters"]}
26
- script = script.encode("utf-8")
27
- manifest["parameters"]["bridge_path"] = {
28
- "type": "string", "default": str(root / "hosts/shared/research-call.py"),
29
- "description": "Shared research tool bridge in this stable bundle location",
30
- }
31
- target = project / ".pi/subagent-workflows/bookmark-research"
32
- current = project
33
- for part in target.relative_to(project).parts:
34
- current /= part
35
- if current.is_symlink():
36
- raise ValueError("Refusing a symlink registration path: " + str(current))
37
- payload = {"workflow.json": (json.dumps(manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
38
- "script.js": script}
39
- if target.exists():
40
- same = target.is_dir() and {p.name for p in target.iterdir()} == set(payload) and all(
41
- (target / name).is_file() and not (target / name).is_symlink() and
42
- (target / name).read_bytes() == content for name, content in payload.items())
43
- if not same:
44
- raise ValueError("Existing workflow differs; preserve it and choose a clean project registration location")
45
- return {"path": str(target), "changed": False, "extensions_installed": False}
46
- target.parent.mkdir(parents=True, exist_ok=True)
47
- with tempfile.TemporaryDirectory(prefix=".bookmark-workflow-", dir=target.parent) as temporary:
48
- stage = Path(temporary) / "bookmark-research"
49
- stage.mkdir()
50
- for name, content in payload.items():
51
- (stage / name).write_bytes(content)
52
- stage.rename(target)
53
- return {"path": str(target), "changed": True, "extensions_installed": False}
54
-
55
-
56
- def main():
57
- for stream in (sys.stdout, sys.stderr): # Windows pipes default to the ANSI code page
58
- if hasattr(stream, "reconfigure"):
59
- stream.reconfigure(encoding="utf-8")
60
- parser = argparse.ArgumentParser(description=__doc__)
61
- parser.add_argument("--project", required=True, help="Existing project whose .pi/subagent-workflows directory to use")
62
- options = parser.parse_args()
63
- try:
64
- print(json.dumps(register(options.project), ensure_ascii=False, indent=2))
65
- except (OSError, ValueError) as error:
66
- print(json.dumps({"error": str(error)}, ensure_ascii=False), file=sys.stderr)
67
- return 1
68
- return 0
69
-
70
-
71
- if __name__ == "__main__":
72
- raise SystemExit(main())
@@ -1,18 +0,0 @@
1
- // pi-subagents-workflows injects args/cwd; pi-subagents owns runs and lifecycle.
2
- const host = {
3
- name: "pi",
4
- all(jobs) {
5
- return runs.all(jobs.map(job => ({
6
- key: job.key, agent: "delegate", task: job.prompt, cwd,
7
- context: "fresh", outputSchema: job.schema,
8
- }))).then(values => jobs.map((job, index) => {
9
- const run = values[index];
10
- const ok = Boolean(run && !run.error && run.ok !== false && run.structuredOutput);
11
- return {
12
- key: job.key, ok, data: ok ? run.structuredOutput : null,
13
- error: ok ? null : String(run?.error || "Missing structuredOutput or failed child"),
14
- child_run_id: run?.runId || null,
15
- };
16
- }));
17
- },
18
- };