foxclaw 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (495) hide show
  1. package/README.md +54 -29
  2. package/bin/foxclaw.cjs +51 -0
  3. package/index.js +11 -10
  4. package/package.json +46 -38
  5. package/python/.dockerignore +35 -0
  6. package/python/.env.example +165 -0
  7. package/python/.gitattributes +46 -0
  8. package/python/.github/workflows/ci.yml +122 -0
  9. package/python/.github/workflows/release.yml +183 -0
  10. package/python/Dockerfile +64 -0
  11. package/python/LICENSE +21 -0
  12. package/python/MANIFEST.in +14 -0
  13. package/python/README.md +192 -0
  14. package/python/README_CN.md +324 -0
  15. package/python/bench/bm25_audit.py +199 -0
  16. package/python/bench/bm25_regression.py +178 -0
  17. package/python/bench/bm25_stress.py +159 -0
  18. package/python/bench/cd_audit.py +238 -0
  19. package/python/bench/chaos.py +142 -0
  20. package/python/bench/concurrency.py +103 -0
  21. package/python/bench/e2e_real_models.py +95 -0
  22. package/python/bench/llm_retry_test.py +248 -0
  23. package/python/bench/load_test.py +196 -0
  24. package/python/bench/vector_ivf_upgrade.py +134 -0
  25. package/python/docs/CHANNELS.md +195 -0
  26. package/python/docs/GOVERNANCE.md +164 -0
  27. package/python/docs/RELEASE.md +109 -0
  28. package/python/docs/RUN.md +75 -0
  29. package/python/docs/RUN_CN.md +121 -0
  30. package/python/docs//351/205/215/347/275/256/346/214/207/345/215/227.md +152 -0
  31. package/python/evals/__init__.py +0 -0
  32. package/python/evals/governance/README.md +61 -0
  33. package/python/evals/governance/__init__.py +0 -0
  34. package/python/evals/governance/cases.py +152 -0
  35. package/python/evals/governance/compare_embeddings.py +94 -0
  36. package/python/evals/governance/convergence.svg +11 -0
  37. package/python/evals/governance/report.json +371 -0
  38. package/python/evals/governance/run_evals.py +400 -0
  39. package/python/examples/governance_middleware_demo.py +77 -0
  40. package/python/examples/langgraph_governance_demo.py +108 -0
  41. package/python/examples/sample_itinerary.csv +6 -0
  42. package/python/examples/sample_knowledge.csv +9 -0
  43. package/python/examples/sample_transactions.csv +14 -0
  44. package/python/foxclaw/__init__.py +526 -0
  45. package/python/foxclaw/agents/__init__.py +12 -0
  46. package/python/foxclaw/agents/agents.py +35 -0
  47. package/python/foxclaw/agents/base.py +865 -0
  48. package/python/foxclaw/agents/impl/__init__.py +1 -0
  49. package/python/foxclaw/agents/impl/_helpers.py +78 -0
  50. package/python/foxclaw/agents/impl/finance.py +248 -0
  51. package/python/foxclaw/agents/impl/office.py +367 -0
  52. package/python/foxclaw/agents/impl/supervisor.py +659 -0
  53. package/python/foxclaw/agents/mixins/_context.py +151 -0
  54. package/python/foxclaw/agents/mixins/augment.py +149 -0
  55. package/python/foxclaw/agents/mixins/llm.py +66 -0
  56. package/python/foxclaw/agents/mixins/memory.py +35 -0
  57. package/python/foxclaw/agents/mixins/preference.py +81 -0
  58. package/python/foxclaw/agents/mixins/prompt.py +77 -0
  59. package/python/foxclaw/agents/mixins/skillbox.py +58 -0
  60. package/python/foxclaw/agents/mixins/tool_exec.py +443 -0
  61. package/python/foxclaw/api/__init__.py +283 -0
  62. package/python/foxclaw/api/app.py +123 -0
  63. package/python/foxclaw/api/middleware.py +317 -0
  64. package/python/foxclaw/api/routes/ahs.py +417 -0
  65. package/python/foxclaw/api/routes/attachments.py +117 -0
  66. package/python/foxclaw/api/routes/browser.py +39 -0
  67. package/python/foxclaw/api/routes/domain.py +314 -0
  68. package/python/foxclaw/api/routes/knowledge.py +163 -0
  69. package/python/foxclaw/api/routes/llm_admin.py +461 -0
  70. package/python/foxclaw/api/routes/memory.py +89 -0
  71. package/python/foxclaw/api/routes/orchestration.py +41 -0
  72. package/python/foxclaw/api/routes/recovery.py +43 -0
  73. package/python/foxclaw/api/routes/review.py +275 -0
  74. package/python/foxclaw/api/routes/search.py +37 -0
  75. package/python/foxclaw/api/routes/system.py +327 -0
  76. package/python/foxclaw/api/routes/task.py +281 -0
  77. package/python/foxclaw/api/routes/ws.py +68 -0
  78. package/python/foxclaw/api/spine.py +55 -0
  79. package/python/foxclaw/browser/__init__.py +13 -0
  80. package/python/foxclaw/browser/action/__init__.py +1 -0
  81. package/python/foxclaw/browser/action/action_executor.py +54 -0
  82. package/python/foxclaw/browser/action/action_types.py +45 -0
  83. package/python/foxclaw/browser/action/action_verifier.py +68 -0
  84. package/python/foxclaw/browser/action/url_policy.py +147 -0
  85. package/python/foxclaw/browser/adapter/__init__.py +1 -0
  86. package/python/foxclaw/browser/adapter/base_adapter.py +23 -0
  87. package/python/foxclaw/browser/adapter/web_adapter.py +46 -0
  88. package/python/foxclaw/browser/driver/__init__.py +1 -0
  89. package/python/foxclaw/browser/driver/base_driver.py +61 -0
  90. package/python/foxclaw/browser/driver/playwright_driver.py +166 -0
  91. package/python/foxclaw/browser/perception/__init__.py +1 -0
  92. package/python/foxclaw/browser/perception/dom_extractor.py +43 -0
  93. package/python/foxclaw/browser/perception/element_indexer.py +57 -0
  94. package/python/foxclaw/browser/perception/semantic_reducer.py +33 -0
  95. package/python/foxclaw/browser/perception/snapshot.py +29 -0
  96. package/python/foxclaw/browser/service.py +148 -0
  97. package/python/foxclaw/browser/session/__init__.py +1 -0
  98. package/python/foxclaw/browser/session/session_manager.py +57 -0
  99. package/python/foxclaw/capability/__init__.py +12 -0
  100. package/python/foxclaw/capability/embedding_service.py +67 -0
  101. package/python/foxclaw/channels/__init__.py +140 -0
  102. package/python/foxclaw/channels/discord.py +184 -0
  103. package/python/foxclaw/channels/dispatcher.py +204 -0
  104. package/python/foxclaw/channels/feishu.py +225 -0
  105. package/python/foxclaw/channels/openclaw_bridge.py +156 -0
  106. package/python/foxclaw/channels/signing.py +187 -0
  107. package/python/foxclaw/channels/telegram.py +121 -0
  108. package/python/foxclaw/channels/weixin.py +225 -0
  109. package/python/foxclaw/cli.py +154 -0
  110. package/python/foxclaw/cognitive/__init__.py +4 -0
  111. package/python/foxclaw/cognitive/reasoner.py +82 -0
  112. package/python/foxclaw/cognitive/state.py +147 -0
  113. package/python/foxclaw/core/__init__.py +9 -0
  114. package/python/foxclaw/core/audit_chain.py +133 -0
  115. package/python/foxclaw/core/audit_sink.py +100 -0
  116. package/python/foxclaw/core/compaction.py +283 -0
  117. package/python/foxclaw/core/config.py +65 -0
  118. package/python/foxclaw/core/confirm_nonce.py +157 -0
  119. package/python/foxclaw/core/dispatch.py +27 -0
  120. package/python/foxclaw/core/envfile.py +48 -0
  121. package/python/foxclaw/core/failure_policy.py +151 -0
  122. package/python/foxclaw/core/gateway.py +85 -0
  123. package/python/foxclaw/core/governance_audit.py +175 -0
  124. package/python/foxclaw/core/idempotency.py +289 -0
  125. package/python/foxclaw/core/injection_guard.py +221 -0
  126. package/python/foxclaw/core/memory_adapter.py +516 -0
  127. package/python/foxclaw/core/metrics.py +89 -0
  128. package/python/foxclaw/core/model_router.py +1097 -0
  129. package/python/foxclaw/core/paths.py +103 -0
  130. package/python/foxclaw/core/recovery.py +362 -0
  131. package/python/foxclaw/core/scheduler.py +144 -0
  132. package/python/foxclaw/core/secrets.py +134 -0
  133. package/python/foxclaw/core/sessions.py +157 -0
  134. package/python/foxclaw/core/settings.py +430 -0
  135. package/python/foxclaw/core/shared_write.py +198 -0
  136. package/python/foxclaw/core/steering_queue.py +153 -0
  137. package/python/foxclaw/core/system_bus.py +84 -0
  138. package/python/foxclaw/core/thought_chain.py +318 -0
  139. package/python/foxclaw/core/token_meter.py +235 -0
  140. package/python/foxclaw/core/tools.py +172 -0
  141. package/python/foxclaw/core/tracing.py +390 -0
  142. package/python/foxclaw/core/truth_pipeline.py +114 -0
  143. package/python/foxclaw/core/vec_recall_adapter.py +503 -0
  144. package/python/foxclaw/core/verify.py +181 -0
  145. package/python/foxclaw/core/web_search.py +242 -0
  146. package/python/foxclaw/finance/__init__.py +15 -0
  147. package/python/foxclaw/finance/evidence.py +153 -0
  148. package/python/foxclaw/finance/fx.py +100 -0
  149. package/python/foxclaw/finance/graph.py +188 -0
  150. package/python/foxclaw/finance/models.py +150 -0
  151. package/python/foxclaw/finance/nlparse.py +575 -0
  152. package/python/foxclaw/finance/policy.py +329 -0
  153. package/python/foxclaw/finance/reasoner.py +129 -0
  154. package/python/foxclaw/finance/reasoning.py +340 -0
  155. package/python/foxclaw/finance/reconcile.py +180 -0
  156. package/python/foxclaw/finance/runtime.py +527 -0
  157. package/python/foxclaw/governance/__init__.py +24 -0
  158. package/python/foxclaw/governance/middleware.py +184 -0
  159. package/python/foxclaw/harness/__init__.py +17 -0
  160. package/python/foxclaw/harness/_base.py +72 -0
  161. package/python/foxclaw/harness/_policy.py +359 -0
  162. package/python/foxclaw/harness/drift.py +89 -0
  163. package/python/foxclaw/harness/event_bus.py +31 -0
  164. package/python/foxclaw/harness/harness.py +446 -0
  165. package/python/foxclaw/harness/loop_detection.py +152 -0
  166. package/python/foxclaw/itinerary/__init__.py +10 -0
  167. package/python/foxclaw/itinerary/amap_client.py +117 -0
  168. package/python/foxclaw/itinerary/models.py +80 -0
  169. package/python/foxclaw/itinerary/reasoner.py +101 -0
  170. package/python/foxclaw/itinerary/reasoning.py +174 -0
  171. package/python/foxclaw/itinerary/runtime.py +148 -0
  172. package/python/foxclaw/kernel/__init__.py +13 -0
  173. package/python/foxclaw/kernel/capabilities.py +73 -0
  174. package/python/foxclaw/kernel/policy_gate.py +114 -0
  175. package/python/foxclaw/kernel/process_manager.py +205 -0
  176. package/python/foxclaw/kernel/scheduler.py +131 -0
  177. package/python/foxclaw/kernel/sentinel.py +453 -0
  178. package/python/foxclaw/kernel/sentinel_client.py +246 -0
  179. package/python/foxclaw/kernel/spine.py +97 -0
  180. package/python/foxclaw/knowledge/__init__.py +376 -0
  181. package/python/foxclaw/knowledge/truth_crystallizer.py +92 -0
  182. package/python/foxclaw/learning/__init__.py +12 -0
  183. package/python/foxclaw/learning/knowledge_graph.py +149 -0
  184. package/python/foxclaw/learning/models.py +93 -0
  185. package/python/foxclaw/learning/reasoner.py +70 -0
  186. package/python/foxclaw/learning/runtime.py +363 -0
  187. package/python/foxclaw/learning/spaced_repetition.py +96 -0
  188. package/python/foxclaw/mcp_server.py +189 -0
  189. package/python/foxclaw/orchestration/__init__.py +184 -0
  190. package/python/foxclaw/pipeline/__init__.py +23 -0
  191. package/python/foxclaw/pipeline/cognitive_pipeline.py +481 -0
  192. package/python/foxclaw/pipeline/supervisor_stages.py +230 -0
  193. package/python/foxclaw/planning/__init__.py +17 -0
  194. package/python/foxclaw/planning/llm_planner.py +232 -0
  195. package/python/foxclaw/planning/planner.py +99 -0
  196. package/python/foxclaw/search/__init__.py +24 -0
  197. package/python/foxclaw/search/agent_port.py +194 -0
  198. package/python/foxclaw/search/evidence.py +113 -0
  199. package/python/foxclaw/search/external_search.py +50 -0
  200. package/python/foxclaw/search/hkc_read_port.py +60 -0
  201. package/python/foxclaw/search/internal_search.py +73 -0
  202. package/python/foxclaw/search/models.py +129 -0
  203. package/python/foxclaw/search/planner.py +44 -0
  204. package/python/foxclaw/search/retriever.py +39 -0
  205. package/python/foxclaw/search/runtime.py +134 -0
  206. package/python/foxclaw/search/strategy.py +44 -0
  207. package/python/foxclaw/skills/__init__.py +613 -0
  208. package/python/foxclaw/skills/hooks_spec.py +73 -0
  209. package/python/foxclaw/skills/lifecycle.py +278 -0
  210. package/python/foxclaw/skills/markdown_skill.py +174 -0
  211. package/python/foxclaw/skills/markdown_store.py +39 -0
  212. package/python/foxclaw/skills/mcp_client.py +190 -0
  213. package/python/foxclaw/skills/mcp_store.py +43 -0
  214. package/python/foxclaw/skills/permission.py +180 -0
  215. package/python/foxclaw/skills/pipeline.py +386 -0
  216. package/python/foxclaw/skills/policy.py +80 -0
  217. package/python/foxclaw/skills/review.py +72 -0
  218. package/python/foxclaw/skills/store.py +57 -0
  219. package/python/frontend/assets/core_bg.jpg +0 -0
  220. package/python/frontend/deck/auth.js +32 -0
  221. package/python/frontend/deck/chat_model_picker.js +537 -0
  222. package/python/frontend/deck/domain.js +296 -0
  223. package/python/frontend/deck/main.js +1950 -0
  224. package/python/frontend/deck/markdown.js +48 -0
  225. package/python/frontend/deck/settings.js +324 -0
  226. package/python/frontend/deck/state.js +6 -0
  227. package/python/frontend/deck/util.js +6 -0
  228. package/python/frontend/deck/visuals.js +212 -0
  229. package/python/frontend/deck.css +855 -0
  230. package/python/frontend/deck.html +686 -0
  231. package/python/frontend/deck_v2.html +341 -0
  232. package/python/frontend/planet/index.html +266 -0
  233. package/python/frontend/planet/planet.css +579 -0
  234. package/python/frontend/planet/planet.js +1637 -0
  235. package/python/frontend/planet/vendor/three.core.min.js +6 -0
  236. package/python/frontend/planet/vendor/three.module.min.js +6 -0
  237. package/python/frontend/v3/assets/index-BD7VPbMV.js +33 -0
  238. package/python/frontend/v3/assets/index-CAvnYWGH.css +1 -0
  239. package/python/frontend/v3/index.html +15 -0
  240. package/python/frontend/widget_runtime.js +295 -0
  241. package/python/frontend/widgets.js +696 -0
  242. package/python/frontend_src/index.html +14 -0
  243. package/python/frontend_src/package-lock.json +1376 -0
  244. package/python/frontend_src/package.json +19 -0
  245. package/python/frontend_src/src/App.vue +62 -0
  246. package/python/frontend_src/src/api.js +89 -0
  247. package/python/frontend_src/src/components/ChatDock.vue +377 -0
  248. package/python/frontend_src/src/components/ComposerMenu.vue +75 -0
  249. package/python/frontend_src/src/components/DocViewer.vue +171 -0
  250. package/python/frontend_src/src/components/Inspector.vue +241 -0
  251. package/python/frontend_src/src/components/LeftRail.vue +394 -0
  252. package/python/frontend_src/src/components/MenuPopover.vue +130 -0
  253. package/python/frontend_src/src/components/MessageStream.vue +401 -0
  254. package/python/frontend_src/src/components/ModelPicker.vue +355 -0
  255. package/python/frontend_src/src/components/QuickIngest.vue +118 -0
  256. package/python/frontend_src/src/components/Stage.vue +226 -0
  257. package/python/frontend_src/src/components/ToolMenu.vue +144 -0
  258. package/python/frontend_src/src/components/TopBar.vue +90 -0
  259. package/python/frontend_src/src/components/panels/ActivityPanel.vue +84 -0
  260. package/python/frontend_src/src/components/panels/AgentsPanel.vue +86 -0
  261. package/python/frontend_src/src/components/panels/KnowledgePanel.vue +441 -0
  262. package/python/frontend_src/src/components/panels/MemoryPanel.vue +118 -0
  263. package/python/frontend_src/src/components/panels/PanelBox.vue +51 -0
  264. package/python/frontend_src/src/components/panels/ReviewPanel.vue +308 -0
  265. package/python/frontend_src/src/components/panels/SettingsPanel.vue +254 -0
  266. package/python/frontend_src/src/components/panels/TasksPanel.vue +74 -0
  267. package/python/frontend_src/src/components/panels/ToolsPanel.vue +640 -0
  268. package/python/frontend_src/src/components/panels/UsagePanel.vue +105 -0
  269. package/python/frontend_src/src/lib/fmt.js +48 -0
  270. package/python/frontend_src/src/lib/md.js +204 -0
  271. package/python/frontend_src/src/main.js +7 -0
  272. package/python/frontend_src/src/stores/app.js +106 -0
  273. package/python/frontend_src/src/stores/chat.js +484 -0
  274. package/python/frontend_src/src/stores/model.js +104 -0
  275. package/python/frontend_src/src/stores/panels.js +330 -0
  276. package/python/frontend_src/src/styles/base.css +138 -0
  277. package/python/frontend_src/src/styles/panel.css +117 -0
  278. package/python/frontend_src/src/styles/tokens.css +78 -0
  279. package/python/frontend_src/vite.config.js +46 -0
  280. package/python/pyproject.toml +265 -0
  281. package/python/skills/README.md +123 -0
  282. package/python/skills/execution-contract/SKILL.md +30 -0
  283. package/python/skills/general-assistant/SKILL.md +34 -0
  284. package/python/skills/planner/SKILL.md +33 -0
  285. package/python/skills/schedule-itinerary/SKILL.md +35 -0
  286. package/python/skills/skill-composition/SKILL.md +31 -0
  287. package/python/skills/skill-guard/SKILL.md +67 -0
  288. package/python/skills/skill-memory/SKILL.md +50 -0
  289. package/python/skills/skill-router/SKILL.md +35 -0
  290. package/python/tests/conftest.py +76 -0
  291. package/python/tests/run_all.py +263 -0
  292. package/python/tests/test_agent_lifecycle.py +360 -0
  293. package/python/tests/test_ahs_p2.py +414 -0
  294. package/python/tests/test_anthropic_tools.py +206 -0
  295. package/python/tests/test_api.py +87 -0
  296. package/python/tests/test_api_auth.py +83 -0
  297. package/python/tests/test_api_host_guard.py +69 -0
  298. package/python/tests/test_attachments.py +167 -0
  299. package/python/tests/test_audit_chain.py +121 -0
  300. package/python/tests/test_audit_sink.py +136 -0
  301. package/python/tests/test_browser.py +153 -0
  302. package/python/tests/test_browser_request_guard.py +184 -0
  303. package/python/tests/test_capabilities.py +158 -0
  304. package/python/tests/test_channel_signing.py +192 -0
  305. package/python/tests/test_channel_trust.py +109 -0
  306. package/python/tests/test_channels.py +184 -0
  307. package/python/tests/test_cli.py +46 -0
  308. package/python/tests/test_compaction_toolpair.py +119 -0
  309. package/python/tests/test_config_layering.py +65 -0
  310. package/python/tests/test_confirm_nonce.py +197 -0
  311. package/python/tests/test_context_compaction_wiring.py +130 -0
  312. package/python/tests/test_dispatch.py +95 -0
  313. package/python/tests/test_domain_evals.py +403 -0
  314. package/python/tests/test_domain_import.py +78 -0
  315. package/python/tests/test_domain_reasoners.py +148 -0
  316. package/python/tests/test_e2e.py +178 -0
  317. package/python/tests/test_emb_cache.py +107 -0
  318. package/python/tests/test_embedding_singleton.py +89 -0
  319. package/python/tests/test_envfile.py +83 -0
  320. package/python/tests/test_evals_smoke.py +114 -0
  321. package/python/tests/test_event_spine.py +182 -0
  322. package/python/tests/test_evidence_semantic.py +101 -0
  323. package/python/tests/test_evidence_sse.py +97 -0
  324. package/python/tests/test_file_ingest.py +146 -0
  325. package/python/tests/test_finance.py +153 -0
  326. package/python/tests/test_finance_ledger.py +252 -0
  327. package/python/tests/test_finance_persist.py +178 -0
  328. package/python/tests/test_gateway.py +144 -0
  329. package/python/tests/test_governance.py +468 -0
  330. package/python/tests/test_governance_audit.py +359 -0
  331. package/python/tests/test_governance_middleware.py +143 -0
  332. package/python/tests/test_hkc.py +286 -0
  333. package/python/tests/test_hkc_index_build_manager.py +339 -0
  334. package/python/tests/test_hkc_manifest_reuse.py +317 -0
  335. package/python/tests/test_hkc_search_event_sync.py +533 -0
  336. package/python/tests/test_import_path_guard.py +161 -0
  337. package/python/tests/test_injection_guard.py +211 -0
  338. package/python/tests/test_integration.py +141 -0
  339. package/python/tests/test_itinerary.py +103 -0
  340. package/python/tests/test_kee_semantic.py +89 -0
  341. package/python/tests/test_kernel.py +222 -0
  342. package/python/tests/test_learning.py +110 -0
  343. package/python/tests/test_llm_admin.py +160 -0
  344. package/python/tests/test_llm_planner.py +366 -0
  345. package/python/tests/test_markdown_skills.py +73 -0
  346. package/python/tests/test_markdown_skills_openclaw.py +71 -0
  347. package/python/tests/test_mcp_connectors.py +109 -0
  348. package/python/tests/test_mcp_guardian_discovery.py +91 -0
  349. package/python/tests/test_mcp_no_memory.py +43 -0
  350. package/python/tests/test_mcp_policy_gate.py +77 -0
  351. package/python/tests/test_mcp_server.py +95 -0
  352. package/python/tests/test_memory_recall.py +95 -0
  353. package/python/tests/test_metrics.py +66 -0
  354. package/python/tests/test_multi_chain.py +102 -0
  355. package/python/tests/test_multimodal.py +135 -0
  356. package/python/tests/test_negotiation.py +146 -0
  357. package/python/tests/test_orchestration.py +119 -0
  358. package/python/tests/test_orchestration_decoupled.py +109 -0
  359. package/python/tests/test_p3_p5_hardening.py +338 -0
  360. package/python/tests/test_parallel_dispatch.py +215 -0
  361. package/python/tests/test_pipeline.py +533 -0
  362. package/python/tests/test_plan_visible.py +86 -0
  363. package/python/tests/test_planner.py +95 -0
  364. package/python/tests/test_policy_gate.py +109 -0
  365. package/python/tests/test_reasoner.py +185 -0
  366. package/python/tests/test_release_check.py +263 -0
  367. package/python/tests/test_resume_checkpoint.py +156 -0
  368. package/python/tests/test_routing.py +95 -0
  369. package/python/tests/test_scheduler.py +141 -0
  370. package/python/tests/test_search.py +204 -0
  371. package/python/tests/test_secrets.py +159 -0
  372. package/python/tests/test_semantic.py +78 -0
  373. package/python/tests/test_sessions.py +91 -0
  374. package/python/tests/test_settings.py +281 -0
  375. package/python/tests/test_shadow_learning.py +330 -0
  376. package/python/tests/test_shared_write.py +207 -0
  377. package/python/tests/test_skill_taint_gate.py +154 -0
  378. package/python/tests/test_skills.py +133 -0
  379. package/python/tests/test_stats_and_events.py +165 -0
  380. package/python/tests/test_streaming.py +60 -0
  381. package/python/tests/test_supervisor_pipeline.py +177 -0
  382. package/python/tests/test_supervisor_triage.py +108 -0
  383. package/python/tests/test_taint.py +117 -0
  384. package/python/tests/test_task_stream.py +85 -0
  385. package/python/tests/test_thinking_sse.py +67 -0
  386. package/python/tests/test_thinking_stream.py +76 -0
  387. package/python/tests/test_tool_center.py +129 -0
  388. package/python/tests/test_tool_parallel.py +113 -0
  389. package/python/tests/test_tool_params.py +103 -0
  390. package/python/tests/test_tool_use.py +242 -0
  391. package/python/tests/test_toplevel_concurrency.py +117 -0
  392. package/python/tests/test_tracing.py +290 -0
  393. package/python/tests/test_truth_crystallizer.py +127 -0
  394. package/python/tests/test_truth_flow.py +93 -0
  395. package/python/tests/test_url_policy.py +53 -0
  396. package/python/tests/test_verify_evidence.py +225 -0
  397. package/python/tests/test_version_sync.py +71 -0
  398. package/python/tests/test_web_trigger.py +99 -0
  399. package/python/tools/check_faiss_type.py +31 -0
  400. package/python/tools/check_log.ps1 +46 -0
  401. package/python/tools/ivf_status.py +72 -0
  402. package/python/tools/mypy_baseline.txt +5 -0
  403. package/python/tools/mypy_ratchet.py +97 -0
  404. package/python/tools/pack.py +178 -0
  405. package/python/tools/release_check.py +349 -0
  406. package/python/tools/restart_foxclaw.ps1 +74 -0
  407. package/python/tools/shared_write_audit.py +204 -0
  408. package/python/tools/size_baseline.txt +28 -0
  409. package/python/tools/size_check.py +102 -0
  410. package/python/tools/verify_env.bat +2 -0
  411. package/python/tools/verify_env.py +25 -0
  412. package/python/vendor/HMR_LICENSE +37 -0
  413. package/python/vendor/HMR_README.md +65 -0
  414. package/python/vendor/hkc/LICENSE +37 -0
  415. package/python/vendor/hkc/hkc_ace/__init__.py +0 -0
  416. package/python/vendor/hkc/hkc_ace/ace.py +457 -0
  417. package/python/vendor/hkc/hkc_ace/skill_taxonomy.json +465 -0
  418. package/python/vendor/hkc/hkc_api/__init__.py +0 -0
  419. package/python/vendor/hkc/hkc_api/container.py +416 -0
  420. package/python/vendor/hkc/hkc_api/ingest_jobs.py +103 -0
  421. package/python/vendor/hkc/hkc_api/main.py +157 -0
  422. package/python/vendor/hkc/hkc_api/routes/__init__.py +0 -0
  423. package/python/vendor/hkc/hkc_api/routes/ability.py +151 -0
  424. package/python/vendor/hkc/hkc_api/routes/config.py +51 -0
  425. package/python/vendor/hkc/hkc_api/routes/knowledge.py +630 -0
  426. package/python/vendor/hkc/hkc_api/routes/search.py +88 -0
  427. package/python/vendor/hkc/hkc_api/schemas.py +153 -0
  428. package/python/vendor/hkc/hkc_api/synthesis.py +179 -0
  429. package/python/vendor/hkc/hkc_core/__init__.py +0 -0
  430. package/python/vendor/hkc/hkc_core/graph/__init__.py +0 -0
  431. package/python/vendor/hkc/hkc_core/graph/base.py +188 -0
  432. package/python/vendor/hkc/hkc_core/graph/sqlite_store.py +789 -0
  433. package/python/vendor/hkc/hkc_core/models/__init__.py +0 -0
  434. package/python/vendor/hkc/hkc_core/models/enums.py +65 -0
  435. package/python/vendor/hkc/hkc_core/models/ku.py +307 -0
  436. package/python/vendor/hkc/hkc_core/utils/__init__.py +0 -0
  437. package/python/vendor/hkc/hkc_core/utils/id_gen.py +113 -0
  438. package/python/vendor/hkc/hkc_crystallizer/__init__.py +44 -0
  439. package/python/vendor/hkc/hkc_crystallizer/adapters.py +94 -0
  440. package/python/vendor/hkc/hkc_crystallizer/candidate.py +145 -0
  441. package/python/vendor/hkc/hkc_crystallizer/crystallizer.py +179 -0
  442. package/python/vendor/hkc/hkc_crystallizer/ingress.py +90 -0
  443. package/python/vendor/hkc/hkc_kde/__init__.py +0 -0
  444. package/python/vendor/hkc/hkc_kde/chunker.py +197 -0
  445. package/python/vendor/hkc/hkc_kde/extractor.py +623 -0
  446. package/python/vendor/hkc/hkc_kde/kde.py +298 -0
  447. package/python/vendor/hkc/hkc_kde/loaders/__init__.py +0 -0
  448. package/python/vendor/hkc/hkc_kde/loaders/ebook_loader.py +153 -0
  449. package/python/vendor/hkc/hkc_kde/loaders/office_loader.py +194 -0
  450. package/python/vendor/hkc/hkc_kde/loaders/pdf_loader.py +162 -0
  451. package/python/vendor/hkc/hkc_kde/loaders/text_loader.py +157 -0
  452. package/python/vendor/hkc/hkc_kde/loaders/web_loader.py +91 -0
  453. package/python/vendor/hkc/hkc_kde/models.py +74 -0
  454. package/python/vendor/hkc/hkc_kde/packager.py +329 -0
  455. package/python/vendor/hkc/hkc_kde/relation_disc.py +151 -0
  456. package/python/vendor/hkc/hkc_kee/__init__.py +0 -0
  457. package/python/vendor/hkc/hkc_kee/dedup.py +227 -0
  458. package/python/vendor/hkc/hkc_kee/fingerprint.py +106 -0
  459. package/python/vendor/hkc/hkc_kee/kee.py +541 -0
  460. package/python/vendor/hkc/hkc_kep/__init__.py +0 -0
  461. package/python/vendor/hkc/hkc_kep/event_bus.py +205 -0
  462. package/python/vendor/hkc/hkc_sdk/__init__.py +63 -0
  463. package/python/vendor/hkc/hkc_sdk/base.py +151 -0
  464. package/python/vendor/hkc/hkc_sdk/direct_client.py +153 -0
  465. package/python/vendor/hkc/hkc_sdk/exceptions.py +39 -0
  466. package/python/vendor/hkc/hkc_sdk/http_client.py +166 -0
  467. package/python/vendor/hkc/hkc_sdk/models.py +171 -0
  468. package/python/vendor/hkc/hkc_search/__init__.py +0 -0
  469. package/python/vendor/hkc/hkc_search/bm25.py +325 -0
  470. package/python/vendor/hkc/hkc_search/embedding_backends.py +374 -0
  471. package/python/vendor/hkc/hkc_search/graph_search.py +132 -0
  472. package/python/vendor/hkc/hkc_search/hybrid.py +331 -0
  473. package/python/vendor/hkc/hkc_search/index_build_manager.py +378 -0
  474. package/python/vendor/hkc/hkc_search/vector_search.py +484 -0
  475. package/python/vendor/hmr/__init__.py +0 -0
  476. package/python/vendor/hmr/core/__init__.py +0 -0
  477. package/python/vendor/hmr/core/hmr.py +684 -0
  478. package/python/vendor/hmr/core/models.py +310 -0
  479. package/python/vendor/hmr/engines/__init__.py +0 -0
  480. package/python/vendor/hmr/engines/evolution.py +710 -0
  481. package/python/vendor/hmr/engines/jit_compiler.py +472 -0
  482. package/python/vendor/hmr/engines/lifecycle.py +264 -0
  483. package/python/vendor/hmr/engines/policy.py +625 -0
  484. package/python/vendor/hmr/engines/recall.py +260 -0
  485. package/python/vendor/hmr/engines/runtime_state.py +158 -0
  486. package/python/vendor/hmr/engines/scheduler.py +323 -0
  487. package/python/vendor/hmr/engines/semantic.py +150 -0
  488. package/python/vendor/hmr/engines/temporal.py +261 -0
  489. package/python/vendor/hmr/engines/thought_chain.py +635 -0
  490. package/python/vendor/hmr/graph/__init__.py +0 -0
  491. package/python/vendor/hmr/graph/cwg.py +338 -0
  492. package/python/vendor/hmr/graph/memory_graph.py +530 -0
  493. package/python/vendor/hmr/storage/__init__.py +0 -0
  494. package/python/vendor/hmr/storage/memory_fs.py +300 -0
  495. package/python/vendor/hmr/storage/vector_store.py +520 -0
@@ -0,0 +1,298 @@
1
+ """
2
+ hkc-kde / kde.py
3
+ Knowledge Digest Engine 主引擎。
4
+
5
+ 完整 Pipeline:
6
+ 文件 / URL
7
+ → Stage 1: Loader(解析原始格式)
8
+ → Stage 2: Chunker(分块)+ Extractor(LLM 提取)
9
+ → Stage 3: RelationshipDiscovery(关系发现)
10
+ → Stage 4: Packager(写入 GraphStore,发布事件)
11
+ """
12
+ # TODO: Replace with proper package install before v1 release
13
+
14
+ import hashlib
15
+ import logging
16
+ import os
17
+ from pathlib import Path
18
+ from typing import Optional
19
+
20
+ from hkc_core.graph.base import GraphStore
21
+ from hkc_core.utils.id_gen import IDGenerator
22
+ from hkc_kep.event_bus import EventBus
23
+
24
+ from .models import ParsedDocument
25
+ from .chunker import Chunker, MAX_CHARS, OVERLAP_CHARS
26
+
27
+ # 单文档最多处理的 chunk 数(防止误传超大文件导致数小时的 LLM 调用与不可控账单)。
28
+ #
29
+ # 为什么是 800:按实际步长(MAX_CHARS - OVERLAP_CHARS = 1568 字符/段)折算,
30
+ # 800 段约覆盖 125 万字符,足以吞下一本普通电子书/一份长文档;按 batch=4、
31
+ # 并发 8 的实测口径约 1.3 小时、几元成本,属于"能跑完、看得见进度、账单可控"的
32
+ # 折中。而上传上限 20MB(约 700 万字符 ≈ 4400 段 ≈ 7 小时)远超这个量级,
33
+ # 若把上限调到 0(不限制),一次误传就会在后台连续跑大半天并持续产生费用,
34
+ # 中途既看不到头也无法回退,因此默认不给"全量跑完"的权限。
35
+ #
36
+ # 可经环境变量 HKC_MAX_CHUNKS_PER_DOC 覆盖:设 0 或负数 = 不限制(完整摄入整本书)。
37
+ # 真需要整本摄入时再显式放开,并自行承担耗时与费用。
38
+ # 注意:超出上限只处理前 N 段,而整文件内容哈希已入库,同一文件重传会被去重拦截,
39
+ # 想补齐剩余部分必须先把文件拆分成多份再分别上传。
40
+ DEFAULT_MAX_CHUNKS_PER_DOC = 800
41
+
42
+
43
+ class DuplicateDocumentError(Exception):
44
+ """同一内容已摄入(内容 SHA-256 命中),拒绝重复摄入。"""
45
+ pass
46
+
47
+
48
+ def content_hash_of(data: bytes) -> str:
49
+ """计算内容 SHA-256 指纹,作为文档精确去重身份。"""
50
+ return hashlib.sha256(data).hexdigest()
51
+
52
+
53
+ def _max_chunks_per_doc() -> int:
54
+ """运行时读取单文档 chunk 上限(可经 HKC_MAX_CHUNKS_PER_DOC 覆盖)。<=0 表示不限制。"""
55
+ raw = os.environ.get("HKC_MAX_CHUNKS_PER_DOC", "").strip()
56
+ if not raw:
57
+ return DEFAULT_MAX_CHUNKS_PER_DOC
58
+ try:
59
+ return int(raw)
60
+ except ValueError:
61
+ return DEFAULT_MAX_CHUNKS_PER_DOC
62
+ from .extractor import Extractor
63
+ from .packager import KUPackager
64
+ from .loaders.text_loader import MarkdownLoader, TxtLoader
65
+ from .loaders.web_loader import WebLoader
66
+
67
+ logger = logging.getLogger(__name__)
68
+
69
+
70
+ class KnowledgeDigestEngine:
71
+
72
+ def __init__(
73
+ self,
74
+ graph_store: GraphStore,
75
+ event_bus: EventBus,
76
+ id_gen: IDGenerator,
77
+ api_key: Optional[str] = None,
78
+ model: Optional[str] = None,
79
+ provider: str = "anthropic",
80
+ base_url: Optional[str] = None,
81
+ ):
82
+ self.store = graph_store
83
+ self.bus = event_bus
84
+ self.id_gen = id_gen
85
+ self.chunker = Chunker()
86
+ self.extractor = Extractor(api_key=api_key, model=model, provider=provider, base_url=base_url)
87
+ self.packager = KUPackager(graph_store, event_bus, id_gen)
88
+
89
+ # ── 公开接口 ─────────────────────────────────────────────
90
+
91
+ def ingest_file(
92
+ self,
93
+ path: str,
94
+ source_title: str = "",
95
+ source_year: int = 0,
96
+ domain: str = "",
97
+ source: str = "",
98
+ collection: str = "",
99
+ progress_cb=None,
100
+ ) -> list:
101
+ """
102
+ 摄入本地文件。
103
+ 支持:.pdf / .md / .markdown / .txt / .epub / .mobi / .azw3 / .docx / .xlsx
104
+ 返回写入的 KU 列表。
105
+
106
+ source: 有意义的来源标识(如原始上传文件名)。留空时回退到文件路径。
107
+ 上传场景下 path 是临时盘路径,必须传 source 覆盖,否则
108
+ Evidence 的来源会变成临时路径(可追溯链 KU←Evidence 失真)。
109
+ progress_cb: 可选 (processed,total)->None,逐 chunk 上报抽取进度(异步摄入用)。
110
+ """
111
+ path = Path(path)
112
+ suffix = path.suffix.lower()
113
+
114
+ # 方向一:入口精确哈希查重。同一文件(内容 SHA-256 相同)重复上传直接拒绝,
115
+ # 不让它流到知识级去重(否则会重复建 Evidence 再靠 KEE 合并,触发大量关系重定向)。
116
+ content_hash = content_hash_of(path.read_bytes())
117
+ if self.find_evidence_by_content_hash(content_hash):
118
+ raise DuplicateDocumentError(
119
+ f"文件内容已摄入过(指纹 {content_hash[:12]}),请勿重复上传"
120
+ )
121
+
122
+ if suffix == ".pdf":
123
+ doc = self._load_pdf(str(path))
124
+ elif suffix in (".md", ".markdown"):
125
+ doc = MarkdownLoader().load(str(path))
126
+ elif suffix == ".txt":
127
+ doc = TxtLoader().load(str(path))
128
+ elif suffix in (".epub", ".mobi", ".azw3", ".azw"):
129
+ from .loaders.ebook_loader import EbookLoader
130
+ doc = EbookLoader().load(str(path))
131
+ elif suffix == ".docx":
132
+ from .loaders.office_loader import DocxLoader
133
+ doc = DocxLoader().load(str(path))
134
+ elif suffix == ".xlsx":
135
+ from .loaders.office_loader import XlsxLoader
136
+ doc = XlsxLoader().load(str(path))
137
+ else:
138
+ raise ValueError(
139
+ f"不支持的文件格式: {suffix},"
140
+ "支持: .pdf .md .txt .epub .mobi .azw3 .docx .xlsx")
141
+
142
+ if source:
143
+ doc.source = source # 用调用方提供的来源标识覆盖临时盘路径
144
+
145
+ return self._run_pipeline(
146
+ doc,
147
+ source_title = source_title or path.stem,
148
+ source_year = source_year,
149
+ domain = domain,
150
+ collection = collection,
151
+ content_hash = content_hash,
152
+ progress_cb = progress_cb,
153
+ )
154
+
155
+ def ingest_url(
156
+ self,
157
+ url: str,
158
+ source_title: str = "",
159
+ source_year: int = 0,
160
+ domain: str = "",
161
+ ) -> list:
162
+ """摄入网页 URL。"""
163
+ doc = WebLoader().load(url)
164
+ return self._run_pipeline(
165
+ doc,
166
+ source_title = source_title or url,
167
+ source_year = source_year,
168
+ domain = domain,
169
+ )
170
+
171
+ def ingest_text(
172
+ self,
173
+ text: str,
174
+ source: str = "inline",
175
+ source_title: str = "",
176
+ source_year: int = 0,
177
+ domain: str = "",
178
+ ) -> list:
179
+ """
180
+ 直接摄入文本字符串。
181
+ 适用于:已有文本内容、测试、管道传入。
182
+ """
183
+ doc = MarkdownLoader().load_string(text, source=source)
184
+ return self._run_pipeline(
185
+ doc,
186
+ source_title = source_title or source,
187
+ source_year = source_year,
188
+ domain = domain,
189
+ )
190
+
191
+ # ── 内部流程 ─────────────────────────────────────────────
192
+
193
+ def _run_pipeline(
194
+ self,
195
+ doc: ParsedDocument,
196
+ source_title: str,
197
+ source_year: int,
198
+ domain: str,
199
+ collection: str = "",
200
+ content_hash: str = "",
201
+ progress_cb=None,
202
+ ) -> list:
203
+ logger.info(
204
+ "KDE pipeline 开始: doc_id=%s title=%s sections=%d",
205
+ doc.doc_id, doc.title,
206
+ sum(len(ch.sections) for ch in doc.chapters),
207
+ )
208
+
209
+ # Stage 2a: Chunk
210
+ chunks = self.chunker.chunk(doc)
211
+ logger.info("Chunker: %d chunks", len(chunks))
212
+
213
+ if not chunks:
214
+ logger.warning("文档无有效内容: %s", doc.source)
215
+ return []
216
+
217
+ # 保护:超大文档(如整本电子书)会切出成百上千 chunk,每个都调一次 LLM,
218
+ # 极慢且烧费用。超过上限时截断,只处理前 N 个,并警告。
219
+ # 上限可经 HKC_MAX_CHUNKS_PER_DOC 覆盖;设 0 = 不限制(完整摄入)。
220
+ _cap = _max_chunks_per_doc()
221
+ if _cap > 0 and len(chunks) > _cap:
222
+ logger.warning(
223
+ "文档过大:切出 %d 个 chunk,超过单文档上限 %d。仅处理前 %d 个"
224
+ "(约 %d 字符)。如需完整摄入,请调大 HKC_MAX_CHUNKS_PER_DOC"
225
+ "(设 0 = 不限制,但耗时与费用会同步放大);"
226
+ "注意同一文件重传会被内容去重拦截,补齐剩余部分需先拆分文件。",
227
+ len(chunks), _cap, _cap, _cap * (MAX_CHARS - OVERLAP_CHARS),
228
+ )
229
+ chunks = chunks[:_cap]
230
+
231
+ # Stage 2b: LLM 提取(有进度回调则带上;无则按旧签名调用,兼容 MockExtractor)
232
+ if progress_cb is not None:
233
+ result = self.extractor.extract(chunks, progress_cb=progress_cb)
234
+ else:
235
+ result = self.extractor.extract(chunks)
236
+ logger.info(
237
+ "Extractor: facts=%d claims=%d entities=%d concepts=%d",
238
+ len(result.all_facts()), len(result.all_claims()),
239
+ len(result.all_entities()), len(result.all_concepts()),
240
+ )
241
+
242
+ # 把 domain 注入 Claim(如果 LLM 没有填写 domain)
243
+ if domain:
244
+ for item in result.all_claims():
245
+ if not item.get("domain"):
246
+ item["domain"] = domain
247
+
248
+ # Stage 3 + 4: 关系发现 + 打包写入
249
+ new_kus = self.packager.package(
250
+ result,
251
+ source_title = source_title,
252
+ source_year = source_year,
253
+ source_id = doc.source, # 精确来源标识(如 memory:agent:finance:mem_001)
254
+ collection = collection,
255
+ content_hash = content_hash,
256
+ )
257
+
258
+ logger.info("KDE pipeline 完成: 写入 %d KU", len(new_kus))
259
+ return new_kus
260
+
261
+ def find_evidence_by_content_hash(self, content_hash: str):
262
+ """按内容 SHA-256 指纹查找已摄入的文件 Evidence。命中返回 KU,否则 None。
263
+
264
+ 哈希存在 Evidence.extra["content_hash"]。单机少用户(几十本书)场景下
265
+ 全量遍历 Evidence 即可,简单可靠,无需引入额外索引存储。
266
+ """
267
+ if not content_hash:
268
+ return None
269
+ from hkc_core.models.enums import KUType
270
+ for ku in self.store.query_by_type(
271
+ KUType.EVIDENCE, status="active", limit=10_000_000
272
+ ):
273
+ if (getattr(ku, "extra", None) or {}).get("content_hash") == content_hash:
274
+ return ku
275
+ return None
276
+
277
+ def is_duplicate_content(self, data: bytes) -> bool:
278
+ """供 API 层在提交/创建 job 前提前拦截重复上传(避免起后台任务)。"""
279
+ return self.find_evidence_by_content_hash(content_hash_of(data)) is not None
280
+
281
+ def _load_pdf(self, path: str) -> ParsedDocument:
282
+ try:
283
+ from .loaders.pdf_loader import PDFLoader
284
+ return PDFLoader().load(path)
285
+ except ImportError:
286
+ # PyMuPDF 未安装,降级为纯文本提取
287
+ logger.warning("PyMuPDF 未安装,PDF 降级为文本提取")
288
+ import subprocess, shlex
289
+ safe_path = shlex.quote(path)
290
+ result = subprocess.run(
291
+ ["python3", "-c",
292
+ f"import fitz; d=fitz.open({safe_path}); "
293
+ "print(chr(10).join(p.get_text() for p in d))"],
294
+ capture_output=True, text=True
295
+ )
296
+ if result.returncode == 0:
297
+ return MarkdownLoader().load_string(result.stdout, source=path)
298
+ raise
File without changes
@@ -0,0 +1,153 @@
1
+ """
2
+ hkc-kde / loaders / ebook_loader.py
3
+ 电子书解析器:支持 .epub / .mobi / .azw3
4
+
5
+ - epub:标准格式(zip + xhtml),用 ebooklib + BeautifulSoup 提取文本
6
+ - mobi / azw3:Amazon 格式,用 mobi 库解包为 html 后提取
7
+ (azw3 即 KF8,mobi 库可处理)
8
+
9
+ 所有格式最终转成纯文本,按 ParsedDocument 返回,后续 pipeline 与其它格式一致。
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import os
14
+ import uuid
15
+ import logging
16
+ import shutil
17
+ from pathlib import Path
18
+
19
+ from ..models import ParsedDocument, Chapter, Section
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ # 软依赖检测
24
+ try:
25
+ import ebooklib
26
+ from ebooklib import epub
27
+ HAS_EBOOKLIB = True
28
+ except ImportError:
29
+ HAS_EBOOKLIB = False
30
+
31
+ try:
32
+ from bs4 import BeautifulSoup
33
+ HAS_BS4 = True
34
+ except ImportError:
35
+ HAS_BS4 = False
36
+
37
+ try:
38
+ import mobi
39
+ HAS_MOBI = True
40
+ except ImportError:
41
+ HAS_MOBI = False
42
+
43
+
44
+ _MIN_SECTION_CHARS = 12 # 太短的片段(导航/版权页碎片)忽略;阈值偏低以兼容中文短段落
45
+
46
+
47
+ def _html_to_text(html: str) -> str:
48
+ """从 HTML 提取纯文本。"""
49
+ if HAS_BS4:
50
+ soup = BeautifulSoup(html, "html.parser")
51
+ # 去掉脚本/样式
52
+ for tag in soup(["script", "style"]):
53
+ tag.decompose()
54
+ return soup.get_text("\n", strip=True)
55
+ # 没有 bs4 时的兜底:粗暴去标签
56
+ import re
57
+ text = re.sub(r"<[^>]+>", " ", html)
58
+ return re.sub(r"\s+", " ", text).strip()
59
+
60
+
61
+ class EbookLoader:
62
+ """epub / mobi / azw3 加载器。"""
63
+
64
+ def load(self, path: str) -> ParsedDocument:
65
+ path = Path(path)
66
+ if not path.exists():
67
+ raise FileNotFoundError(f"文件不存在: {path}")
68
+
69
+ suffix = path.suffix.lower()
70
+ if suffix == ".epub":
71
+ title, blocks = self._load_epub(path)
72
+ elif suffix in (".mobi", ".azw3", ".azw"):
73
+ title, blocks = self._load_mobi(path)
74
+ else:
75
+ raise ValueError(f"EbookLoader 不支持的格式: {suffix}")
76
+
77
+ # blocks: list[str] 纯文本段落 → 组装成单一章节的多个 Section
78
+ sections = []
79
+ for i, blk in enumerate(blocks):
80
+ blk = blk.strip()
81
+ if len(blk) < _MIN_SECTION_CHARS:
82
+ continue
83
+ sections.append(Section(section_id=f"SEC_{i:04d}", content=blk))
84
+
85
+ chapter = Chapter(chapter_id="CH_0001", title=title or path.stem)
86
+ chapter.sections = sections
87
+
88
+ return ParsedDocument(
89
+ doc_id = f"DOC_{uuid.uuid4().hex[:8].upper()}",
90
+ title = title or path.stem,
91
+ source = str(path),
92
+ doc_type = "EBOOK",
93
+ chapters = [chapter],
94
+ metadata = {
95
+ "format": suffix.lstrip("."),
96
+ "file_size": path.stat().st_size,
97
+ "section_count": len(sections),
98
+ },
99
+ )
100
+
101
+ def _load_epub(self, path: Path):
102
+ if not HAS_EBOOKLIB:
103
+ raise ImportError("解析 epub 需要 ebooklib,请运行: pip install ebooklib beautifulsoup4")
104
+ book = epub.read_epub(str(path))
105
+ # 标题
106
+ title = ""
107
+ try:
108
+ md = book.get_metadata("DC", "title")
109
+ if md:
110
+ title = md[0][0]
111
+ except Exception:
112
+ pass
113
+ # 按文档顺序提取每个 html 文档的文本
114
+ blocks = []
115
+ for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
116
+ try:
117
+ html = item.get_content().decode("utf-8", errors="ignore")
118
+ text = _html_to_text(html)
119
+ if text:
120
+ blocks.append(text)
121
+ except Exception as e:
122
+ logger.warning("epub 文档解析失败: %s", e)
123
+ return title, blocks
124
+
125
+ def _load_mobi(self, path: Path):
126
+ if not HAS_MOBI:
127
+ raise ImportError("解析 mobi/azw3 需要 mobi 库,请运行: pip install mobi")
128
+ tmp_dir = None
129
+ try:
130
+ # mobi.extract 返回 (tmp_dir, 解包出的主文件路径)
131
+ tmp_dir, out_path = mobi.extract(str(path))
132
+ text = ""
133
+ op = Path(out_path)
134
+ if op.suffix.lower() in (".html", ".htm", ".xhtml"):
135
+ html = op.read_text(encoding="utf-8", errors="ignore")
136
+ text = _html_to_text(html)
137
+ elif op.suffix.lower() in (".txt", ""):
138
+ text = op.read_text(encoding="utf-8", errors="ignore")
139
+ else:
140
+ # 兜底:扫描解包目录里的 (x)html。
141
+ # KF8/azw3(MOBI8)会被解包成 epub 结构,正文是 .xhtml(out_path 为 .epub),
142
+ # 仅扫 *.html 会漏掉全部正文 → 必须同时覆盖 .xhtml/.htm。
143
+ # 排序保证 part0000/part0001… 的段落顺序稳定。
144
+ for f in sorted(Path(tmp_dir).rglob("*")):
145
+ if f.suffix.lower() in (".html", ".htm", ".xhtml"):
146
+ text += _html_to_text(f.read_text(encoding="utf-8", errors="ignore")) + "\n\n"
147
+ # 按段落切分(双换行)
148
+ blocks = [b for b in text.split("\n\n")] if "\n\n" in text else [text]
149
+ title = path.stem
150
+ return title, blocks
151
+ finally:
152
+ if tmp_dir and os.path.isdir(tmp_dir):
153
+ shutil.rmtree(tmp_dir, ignore_errors=True)
@@ -0,0 +1,194 @@
1
+ """
2
+ hkc-kde / loaders / office_loader.py
3
+ Office 文档解析器:支持 .docx(Word)/ .xlsx(Excel)。
4
+
5
+ - docx:python-docx 读取。段落按「标题样式(Heading/Title)」切章节,正文段落归入当前章节,
6
+ 表格逐行转成 "单元格 | 单元格" 文本一并纳入(很多知识就藏在 Word 表格里)。
7
+ - xlsx:openpyxl 只读+data_only 读取(取缓存值而非公式)。每个工作表 → 一个 Chapter,
8
+ 行按批合并成 Section(避免上千行切出上千个碎 Section 拖垮 chunker/LLM)。
9
+
10
+ 所有格式最终转成纯文本,按 ParsedDocument 返回,后续 pipeline 与其它格式一致。
11
+ 软依赖:未装 python-docx / openpyxl 时抛清晰的可操作错误(提示装 knowledge-files),
12
+ 不静默产出空文档(否则用户以为摄入成功却啥都没进库)。
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import uuid
17
+ import logging
18
+ from pathlib import Path
19
+
20
+ from ..models import ParsedDocument, Chapter, Section
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ # 软依赖检测(缺失时在 load 抛可操作错误,而非 import 期崩溃整个 KDE)
25
+ try:
26
+ import docx as _docx # python-docx
27
+ HAS_DOCX = True
28
+ except ImportError:
29
+ HAS_DOCX = False
30
+
31
+ try:
32
+ import openpyxl as _openpyxl
33
+ HAS_OPENPYXL = True
34
+ except ImportError:
35
+ HAS_OPENPYXL = False
36
+
37
+
38
+ _MIN_SECTION_CHARS = 2 # docx 段落可能很短(标题项/列表项),阈值给低些保内容
39
+ _XLSX_ROWS_PER_SECTION = 20 # 每 20 行合并为一个 Section
40
+ _XLSX_MAX_ROWS_PER_SHEET = 5000 # 单表行上限,超出截断(防超大表切出海量 chunk 烧 LLM)
41
+ _XLSX_MAX_CELL_CHARS = 500 # 单元格文本上限,防个别巨长单元格
42
+
43
+
44
+ def _flush_chapter(chapters, title, paras, counter):
45
+ """把累积段落收敛成一个 Chapter;空则跳过。返回新的 counter。"""
46
+ secs = [
47
+ Section(f"SEC_{counter:04d}_{i:04d}", p)
48
+ for i, p in enumerate(paras)
49
+ if len(p) >= _MIN_SECTION_CHARS
50
+ ]
51
+ if secs:
52
+ chapters.append(Chapter(
53
+ chapter_id=f"CH_{counter:04d}",
54
+ title=title,
55
+ sections=secs,
56
+ ))
57
+ return counter + 1
58
+ return counter
59
+
60
+
61
+ class DocxLoader:
62
+ """Word .docx → ParsedDocument。按标题样式切章节,含表格文本。"""
63
+
64
+ def load(self, path: str) -> ParsedDocument:
65
+ if not HAS_DOCX:
66
+ raise RuntimeError(
67
+ "解析 .docx 需要 python-docx,未安装。请安装可选依赖:"
68
+ "pip install -e \".[knowledge-files]\" 或 pip install python-docx"
69
+ )
70
+ path = Path(path)
71
+ document = _docx.Document(str(path))
72
+
73
+ chapters: list[Chapter] = []
74
+ current_title = ""
75
+ current_paras: list[str] = []
76
+ counter = 1
77
+
78
+ def _add_para(para):
79
+ nonlocal current_title, current_paras, counter
80
+ text = (para.text or "").strip()
81
+ if not text:
82
+ return
83
+ style = (getattr(para.style, "name", "") or "")
84
+ if style.startswith("Heading") or style == "Title":
85
+ # 遇到新标题:先收敛上一章,再切换标题
86
+ counter = _flush_chapter(chapters, current_title, current_paras, counter)
87
+ current_title = text
88
+ current_paras = []
89
+ else:
90
+ current_paras.append(text)
91
+
92
+ def _add_table(table):
93
+ # 逐行转 "cell | cell | cell",追加到【当前】章节(表格归它实际出现的那一节)
94
+ for row in table.rows:
95
+ cells = [(c.text or "").strip() for c in row.cells]
96
+ line = " | ".join(c for c in cells if c)
97
+ if line:
98
+ current_paras.append(line)
99
+
100
+ # 按文档 body 子元素的真实顺序遍历,使表格归入它出现位置的章节(而非全塞末章,
101
+ # 否则 Evidence 溯源标题失真)。用 tag 后缀判断以耐 python-docx 版本差异;
102
+ # 任何异常回退到「先所有段落、再所有表格」的稳妥旧路径。
103
+ try:
104
+ from docx.text.paragraph import Paragraph as _Para
105
+ from docx.table import Table as _Table
106
+ body = document.element.body
107
+ for child in body.iterchildren():
108
+ tag = getattr(child, "tag", "")
109
+ if tag.endswith("}p"):
110
+ _add_para(_Para(child, document))
111
+ elif tag.endswith("}tbl"):
112
+ _add_table(_Table(child, document))
113
+ except Exception:
114
+ chapters.clear(); current_title = ""; current_paras = []; counter = 1
115
+ for para in document.paragraphs:
116
+ _add_para(para)
117
+ for table in document.tables:
118
+ _add_table(table)
119
+
120
+ counter = _flush_chapter(chapters, current_title, current_paras, counter)
121
+
122
+ if not chapters:
123
+ # 空文档也返回一个占位章节,让上层 "文档无有效内容" 分支统一处理
124
+ chapters = [Chapter("CH_0001", "", [])]
125
+
126
+ return ParsedDocument(
127
+ doc_id=f"DOC_{uuid.uuid4().hex[:8].upper()}",
128
+ title=path.stem,
129
+ source=str(path),
130
+ doc_type="DOCX",
131
+ chapters=chapters,
132
+ )
133
+
134
+
135
+ class XlsxLoader:
136
+ """Excel .xlsx → ParsedDocument。每工作表一章,行按批合并为 Section。"""
137
+
138
+ def load(self, path: str) -> ParsedDocument:
139
+ if not HAS_OPENPYXL:
140
+ raise RuntimeError(
141
+ "解析 .xlsx 需要 openpyxl,未安装。请安装可选依赖:"
142
+ "pip install -e \".[knowledge-files]\" 或 pip install openpyxl"
143
+ )
144
+ path = Path(path)
145
+ # read_only + data_only:大表内存友好,且取缓存值而非公式串。
146
+ wb = _openpyxl.load_workbook(str(path), read_only=True, data_only=True)
147
+ chapters: list[Chapter] = []
148
+ counter = 1
149
+ try:
150
+ for ws in wb.worksheets:
151
+ rows_text: list[str] = []
152
+ for ri, row in enumerate(ws.iter_rows(values_only=True)):
153
+ if ri >= _XLSX_MAX_ROWS_PER_SHEET:
154
+ logger.warning(
155
+ "工作表 %s 超过 %d 行,已截断", ws.title, _XLSX_MAX_ROWS_PER_SHEET)
156
+ break
157
+ cells = []
158
+ for v in row:
159
+ if v is None:
160
+ continue
161
+ s = str(v).strip()
162
+ if s:
163
+ cells.append(s[:_XLSX_MAX_CELL_CHARS])
164
+ if cells:
165
+ rows_text.append(" | ".join(cells))
166
+
167
+ if not rows_text:
168
+ continue
169
+
170
+ # 按批合并成 Section
171
+ secs = []
172
+ for bi, start in enumerate(range(0, len(rows_text), _XLSX_ROWS_PER_SECTION)):
173
+ batch = rows_text[start:start + _XLSX_ROWS_PER_SECTION]
174
+ secs.append(Section(f"SEC_{counter:04d}_{bi:04d}", "\n".join(batch)))
175
+ chapters.append(Chapter(
176
+ chapter_id=f"CH_{counter:04d}",
177
+ title=ws.title or f"Sheet{counter}",
178
+ sections=secs,
179
+ ))
180
+ counter += 1
181
+ finally:
182
+ # read_only 模式必须显式关,否则底层 zip 文件句柄泄漏(Windows 上还会锁文件)
183
+ wb.close()
184
+
185
+ if not chapters:
186
+ chapters = [Chapter("CH_0001", "", [])]
187
+
188
+ return ParsedDocument(
189
+ doc_id=f"DOC_{uuid.uuid4().hex[:8].upper()}",
190
+ title=path.stem,
191
+ source=str(path),
192
+ doc_type="XLSX",
193
+ chapters=chapters,
194
+ )