@octopi-agent/engine 0.70.7 → 0.90.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -201
- package/NOTICE +7 -7
- package/README.md +64 -26
- package/dist/config-schema/agent.d.ts +89 -1
- package/dist/config-schema/agent.js +81 -5
- package/dist/config-schema/context.d.ts +24 -5
- package/dist/config-schema/context.js +28 -1
- package/dist/config-schema/engine.d.ts +205 -13
- package/dist/config-schema/engine.js +31 -2
- package/dist/config-schema/governance.d.ts +95 -0
- package/dist/config-schema/governance.js +93 -1
- package/dist/config-schema/models.d.ts +3 -3
- package/dist/config-schema/observability.d.ts +6 -0
- package/dist/config-schema/observability.js +16 -0
- package/dist/config-schema/run.d.ts +10 -0
- package/dist/config-schema/run.js +18 -0
- package/dist/config-schema/shared.d.ts +13 -0
- package/dist/config-schema/shared.js +22 -0
- package/dist/config-schema/substrate.d.ts +2 -2
- package/dist/config-schema/subsystems.d.ts +1 -1
- package/dist/config-schema/subsystems.js +2 -1
- package/dist/config.d.ts +201 -8
- package/dist/config.js +105 -3
- package/dist/harness/activation/compiler.js +3 -0
- package/dist/harness/activation/runtime.d.ts +7 -0
- package/dist/harness/activation/runtime.js +60 -2
- package/dist/harness/activation/sources/agent-signal.d.ts +11 -0
- package/dist/harness/activation/sources/agent-signal.js +33 -0
- package/dist/harness/activation/types.d.ts +6 -1
- package/dist/harness/agent/builder.d.ts +50 -5
- package/dist/harness/agent/builder.js +189 -59
- package/dist/harness/agent/config-bridge.d.ts +33 -25
- package/dist/harness/agent/config-bridge.js +84 -291
- package/dist/harness/agent/index.d.ts +3 -1
- package/dist/harness/agent/index.js +2 -1
- package/dist/harness/agent/tool-purpose.d.ts +46 -0
- package/dist/harness/agent/tool-purpose.js +127 -0
- package/dist/harness/agent/wire-agent-builder.d.ts +133 -0
- package/dist/harness/agent/wire-agent-builder.js +370 -0
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +13 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.js +16 -1
- package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +8 -1
- package/dist/harness/collaboration/autonomous-subsystem/runtime.js +17 -9
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.d.ts +12 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.js +19 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/session-lifecycle-bridge.d.ts +6 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/session-lifecycle-bridge.js +24 -7
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.d.ts +5 -2
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +11 -3
- package/dist/harness/collaboration/autonomous-subsystem/think/index.d.ts +1 -1
- package/dist/harness/collaboration/autonomous-subsystem/think/index.js +1 -1
- package/dist/harness/collaboration/autonomous-subsystem/think/llm-port.d.ts +7 -4
- package/dist/harness/collaboration/autonomous-subsystem/think/llm-port.js +8 -9
- package/dist/harness/collaboration/autonomous-subsystem/think/model-resolver.d.ts +36 -10
- package/dist/harness/collaboration/autonomous-subsystem/think/model-resolver.js +60 -7
- package/dist/harness/collaboration/delegation/child-run.d.ts +144 -0
- package/dist/harness/collaboration/delegation/child-run.js +241 -0
- package/dist/harness/collaboration/delegation/index.d.ts +26 -0
- package/dist/harness/collaboration/delegation/index.js +25 -0
- package/dist/harness/collaboration/delegation/notifier.d.ts +46 -0
- package/dist/harness/collaboration/delegation/notifier.js +84 -0
- package/dist/harness/collaboration/delegation/registry-types.d.ts +200 -0
- package/dist/harness/collaboration/delegation/registry-types.js +61 -0
- package/dist/harness/collaboration/delegation/registry.d.ts +110 -0
- package/dist/harness/collaboration/delegation/registry.js +220 -0
- package/dist/harness/collaboration/delegation/service.d.ts +249 -0
- package/dist/harness/collaboration/delegation/service.js +499 -0
- package/dist/harness/collaboration/delegation/store.d.ts +155 -0
- package/dist/harness/collaboration/delegation/store.js +271 -0
- package/dist/harness/collaboration/delegation/transport.d.ts +73 -0
- package/dist/harness/collaboration/delegation/transport.js +61 -0
- package/dist/harness/collaboration/delegation/types.d.ts +231 -0
- package/dist/harness/collaboration/delegation/types.js +70 -0
- package/dist/harness/collaboration/delegation/wiring.d.ts +111 -0
- package/dist/harness/collaboration/delegation/wiring.js +145 -0
- package/dist/harness/collaboration/orchestration/index.d.ts +11 -13
- package/dist/harness/collaboration/orchestration/index.js +11 -11
- package/dist/harness/context/assembler.d.ts +2 -2
- package/dist/harness/context/assembler.js +12 -7
- package/dist/harness/context/capabilities/compact/engine.js +80 -38
- package/dist/harness/context/capabilities/compact/types.d.ts +12 -1
- package/dist/harness/context/capabilities/summary/registry.js +10 -0
- package/dist/harness/context/constitution/default-agents.md +230 -228
- package/dist/harness/context/default-context-engine.d.ts +101 -23
- package/dist/harness/context/default-context-engine.js +483 -172
- package/dist/harness/context/hybrid-compressor.d.ts +3 -3
- package/dist/harness/context/hybrid-compressor.js +18 -23
- package/dist/harness/context/index.d.ts +5 -1
- package/dist/harness/context/index.js +2 -0
- package/dist/harness/context/layer-snapshot.d.ts +4 -0
- package/dist/harness/context/layer-snapshot.js +30 -18
- package/dist/harness/context/layer-types.d.ts +85 -1
- package/dist/harness/context/layer-types.js +110 -4
- package/dist/harness/context/layers.d.ts +5 -2
- package/dist/harness/context/layers.js +11 -5
- package/dist/harness/context/llm-summarizer.d.ts +15 -1
- package/dist/harness/context/llm-summarizer.js +107 -56
- package/dist/harness/context/message-selector.js +6 -2
- package/dist/harness/context/root-instructions.d.ts +72 -0
- package/dist/harness/context/root-instructions.js +227 -0
- package/dist/harness/context/runtime-datetime.d.ts +34 -7
- package/dist/harness/context/runtime-datetime.js +66 -10
- package/dist/harness/context/smart-router.d.ts +1 -1
- package/dist/harness/context/smart-router.js +1 -1
- package/dist/harness/context/system-prompt-assembler.d.ts +30 -0
- package/dist/harness/context/system-prompt-assembler.js +73 -6
- package/dist/harness/context/token-constants.d.ts +2 -2
- package/dist/harness/context/token-constants.js +2 -2
- package/dist/harness/context/token-estimate-fns.d.ts +1 -3
- package/dist/harness/context/token-estimate-fns.js +2 -4
- package/dist/harness/context/token-estimator.d.ts +1 -1
- package/dist/harness/context/token-estimator.js +14 -6
- package/dist/harness/context/tool-result-aging.d.ts +140 -0
- package/dist/harness/context/tool-result-aging.js +331 -0
- package/dist/harness/context/tool-result-body.d.ts +10 -0
- package/dist/harness/context/tool-result-body.js +27 -0
- package/dist/harness/context/turn-envelope.d.ts +123 -0
- package/dist/harness/context/turn-envelope.js +188 -0
- package/dist/harness/context/types.d.ts +35 -47
- package/dist/harness/extension/execution-environment/effect-policy.d.ts +258 -0
- package/dist/harness/extension/execution-environment/effect-policy.js +358 -0
- package/dist/harness/extension/execution-environment/index.d.ts +18 -2
- package/dist/harness/extension/execution-environment/index.js +11 -2
- package/dist/harness/extension/execution-environment/isolation.js +5 -1
- package/dist/harness/extension/execution-environment/memory-undo.d.ts +101 -0
- package/dist/harness/extension/execution-environment/memory-undo.js +192 -0
- package/dist/harness/extension/execution-environment/process-isolation.d.ts +84 -0
- package/dist/harness/extension/execution-environment/process-isolation.js +315 -0
- package/dist/harness/extension/execution-environment/resource-limits.d.ts +36 -0
- package/dist/harness/extension/execution-environment/resource-limits.js +84 -0
- package/dist/harness/extension/execution-environment/sandbox.d.ts +125 -4
- package/dist/harness/extension/execution-environment/sandbox.js +521 -51
- package/dist/harness/extension/execution-environment/shell-journal.d.ts +100 -0
- package/dist/harness/extension/execution-environment/shell-journal.js +299 -0
- package/dist/harness/extension/execution-environment/tool-capabilities.d.ts +45 -0
- package/dist/harness/extension/execution-environment/tool-capabilities.js +126 -0
- package/dist/harness/extension/execution-environment/types.d.ts +67 -39
- package/dist/harness/extension/execution-environment/types.js +2 -2
- package/dist/harness/extension/execution-environment/undo-log.d.ts +203 -0
- package/dist/harness/extension/execution-environment/undo-log.js +627 -0
- package/dist/harness/extension/execution-environment/undo-merge.d.ts +40 -0
- package/dist/harness/extension/execution-environment/undo-merge.js +167 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.d.ts +1 -1
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.d.ts +9 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/bridge.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/api.d.ts +1 -6
- package/dist/harness/extension/plugin-ecosystem/plugins/api.js +3 -8
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.d.ts +0 -3
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.js +0 -3
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.d.ts +2 -10
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.js +2 -9
- package/dist/harness/extension/plugin-ecosystem/plugins/index.d.ts +1 -3
- package/dist/harness/extension/plugin-ecosystem/plugins/index.js +1 -3
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.d.ts +2 -4
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.js +4 -21
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.d.ts +3 -9
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.js +4 -8
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.d.ts +1 -4
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.js +1 -3
- package/dist/harness/extension/plugin-ecosystem/skills/manager.d.ts +1 -1
- package/dist/harness/extension/plugin-ecosystem/skills/manager.js +1 -1
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.d.ts +51 -0
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +153 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +18 -3
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +277 -81
- package/dist/harness/extension/plugin-ecosystem/tools/delegation.d.ts +31 -0
- package/dist/harness/extension/plugin-ecosystem/tools/delegation.js +291 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +10 -2
- package/dist/harness/extension/plugin-ecosystem/tools/env-info.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/eol.d.ts +35 -0
- package/dist/harness/extension/plugin-ecosystem/tools/eol.js +43 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-edit.js +40 -9
- package/dist/harness/extension/plugin-ecosystem/tools/file-ops.d.ts +33 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-ops.js +376 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.d.ts +12 -2
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +241 -70
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.d.ts +31 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.js +104 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.d.ts +111 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.js +197 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.d.ts +27 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.js +463 -0
- package/dist/harness/extension/plugin-ecosystem/tools/http.d.ts +10 -6
- package/dist/harness/extension/plugin-ecosystem/tools/http.js +41 -47
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +6 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +4 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +4 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +25 -1
- package/dist/harness/extension/plugin-ecosystem/tools/package-tools.d.ts +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/package-tools.js +391 -0
- package/dist/harness/extension/plugin-ecosystem/tools/platform.d.ts +51 -2
- package/dist/harness/extension/plugin-ecosystem/tools/platform.js +151 -12
- package/dist/harness/extension/plugin-ecosystem/tools/session-history.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.d.ts +10 -1
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.js +35 -19
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +8 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +5 -0
- package/dist/harness/extension/plugin-ecosystem/tools/web-search.js +1 -0
- package/dist/harness/governance/credentials/index.d.ts +1 -1
- package/dist/harness/governance/credentials/store.d.ts +16 -4
- package/dist/harness/governance/credentials/store.js +33 -6
- package/dist/harness/governance/credentials/types.d.ts +11 -0
- package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +105 -0
- package/dist/harness/governance/human-in-the-loop/decision-cache.js +184 -0
- package/dist/harness/governance/human-in-the-loop/index.d.ts +5 -5
- package/dist/harness/governance/human-in-the-loop/index.js +3 -3
- package/dist/harness/governance/human-in-the-loop/types.d.ts +7 -62
- package/dist/harness/governance/human-in-the-loop/types.js +6 -2
- package/dist/harness/governance/security/capability-enforcer.d.ts +9 -1
- package/dist/harness/governance/security/capability-enforcer.js +31 -8
- package/dist/harness/governance/security/degradation.js +5 -7
- package/dist/harness/governance/security/risk-evaluator.d.ts +1 -1
- package/dist/harness/governance/security/risk-evaluator.js +4 -82
- package/dist/harness/governance/security/temp-path.d.ts +41 -0
- package/dist/harness/governance/security/temp-path.js +117 -0
- package/dist/harness/governance/session-acl/index.d.ts +1 -0
- package/dist/harness/governance/session-acl/index.js +1 -0
- package/dist/harness/governance/session-acl/rights.d.ts +12 -0
- package/dist/harness/governance/session-acl/rights.js +89 -7
- package/dist/harness/governance/session-acl/seed-roles.js +9 -1
- package/dist/harness/governance/session-acl/service.js +21 -29
- package/dist/harness/governance/session-acl/tool-rights.d.ts +81 -0
- package/dist/harness/governance/session-acl/tool-rights.js +238 -0
- package/dist/harness/governance/session-acl/types.d.ts +22 -0
- package/dist/harness/index.d.ts +16 -12
- package/dist/harness/index.js +10 -6
- package/dist/harness/knowledge/file-identity.d.ts +8 -0
- package/dist/harness/knowledge/file-identity.js +26 -1
- package/dist/harness/knowledge/file-index-store.js +11 -11
- package/dist/harness/knowledge/grounding.d.ts +2 -1
- package/dist/harness/knowledge/grounding.js +8 -5
- package/dist/harness/knowledge/index-store.d.ts +31 -4
- package/dist/harness/knowledge/index-store.js +121 -39
- package/dist/harness/knowledge/index.d.ts +2 -2
- package/dist/harness/knowledge/index.js +1 -1
- package/dist/harness/knowledge/ingest.d.ts +115 -115
- package/dist/harness/knowledge/ingest.js +154 -151
- package/dist/harness/knowledge/job-control-state.js +4 -4
- package/dist/harness/knowledge/job-control.js +11 -11
- package/dist/harness/knowledge/knowledge-serve-child.js +1 -1
- package/dist/harness/knowledge/membership-store.js +11 -11
- package/dist/harness/knowledge/network-guard.d.ts +61 -4
- package/dist/harness/knowledge/network-guard.js +390 -113
- package/dist/harness/knowledge/purge.js +4 -1
- package/dist/harness/knowledge/query-service.js +3 -3
- package/dist/harness/knowledge/writer-service.d.ts +2 -0
- package/dist/harness/knowledge/writer-service.js +14 -13
- package/dist/harness/memory/backfill-coverage.js +3 -1
- package/dist/harness/memory/backfill-trigger.d.ts +2 -2
- package/dist/harness/memory/backfill-trigger.js +4 -4
- package/dist/harness/memory/index.d.ts +1 -0
- package/dist/harness/memory/index.js +1 -0
- package/dist/harness/memory/sqlite/agent-db.js +2 -0
- package/dist/harness/memory/sqlite/memory-store.d.ts +1 -0
- package/dist/harness/memory/sqlite/memory-store.js +14 -4
- package/dist/harness/memory/sqlite/wisdom-store.js +9 -9
- package/dist/harness/memory/store.d.ts +1 -0
- package/dist/harness/memory/store.js +3 -0
- package/dist/harness/memory/time-format.d.ts +28 -0
- package/dist/harness/memory/time-format.js +52 -0
- package/dist/harness/memory/types.d.ts +12 -0
- package/dist/harness/observability/observer/hub.js +53 -2
- package/dist/harness/observability/observer/index.d.ts +2 -2
- package/dist/harness/observability/observer/index.js +1 -1
- package/dist/harness/observability/observer/types.d.ts +80 -0
- package/dist/harness/observability/observer/types.js +118 -9
- package/dist/harness/run/agent/agent.d.ts +18 -0
- package/dist/harness/run/agent/agent.js +26 -0
- package/dist/harness/run/concurrency/tool-loop-detection-core.d.ts +1 -1
- package/dist/harness/run/concurrency/tool-loop-detection-core.js +1 -1
- package/dist/harness/run/reliability/harness-events.d.ts +4 -0
- package/dist/harness/run/reliability/run-agent.js +110 -18
- package/dist/harness/run/reliability/run-metrics-collector.d.ts +28 -2
- package/dist/harness/run/reliability/run-metrics-collector.js +92 -2
- package/dist/harness/run/run-guard/default-run-guard.d.ts +0 -5
- package/dist/harness/run/run-guard/default-run-guard.js +104 -44
- package/dist/harness/run/run-scope.d.ts +52 -0
- package/dist/harness/run/runner.d.ts +117 -16
- package/dist/harness/run/runner.js +668 -157
- package/dist/harness/session/attachments/llm-intent.js +17 -17
- package/dist/harness/session/history/port.js +16 -5
- package/dist/harness/session/in-memory-store.d.ts +6 -0
- package/dist/harness/session/in-memory-store.js +55 -3
- package/dist/harness/session/schedules/index.d.ts +10 -0
- package/dist/harness/session/schedules/index.js +10 -0
- package/dist/harness/session/schedules/render.d.ts +13 -0
- package/dist/harness/session/schedules/render.js +63 -0
- package/dist/harness/session/schedules/service.d.ts +179 -0
- package/dist/harness/session/schedules/service.js +695 -0
- package/dist/harness/session/schedules/store.d.ts +119 -0
- package/dist/harness/session/schedules/store.js +323 -0
- package/dist/harness/session/schedules/tools.d.ts +39 -0
- package/dist/harness/session/schedules/tools.js +442 -0
- package/dist/harness/session/schedules/types.d.ts +104 -0
- package/dist/harness/session/schedules/types.js +97 -0
- package/dist/harness/session/tasks/tools.js +22 -0
- package/dist/harness/session/title.js +25 -9
- package/dist/harness/session/types.d.ts +35 -0
- package/dist/harness/shared/events/agent-event-map.d.ts +39 -0
- package/dist/harness/shared/events/agent-event-map.js +2 -0
- package/dist/harness/shared/types/agent-definition.d.ts +23 -0
- package/dist/index.d.ts +25 -9
- package/dist/index.js +11 -2
- package/dist/integration/observability/run-telemetry.d.ts +8 -0
- package/dist/integration/observability/run-telemetry.js +31 -8
- package/dist/integration/observability/trace-collector.js +26 -4
- package/dist/integration/observability/trace-events.d.ts +1 -0
- package/dist/integration/observability/trace-events.js +2 -0
- package/dist/integration/observability/trace-logger.js +27 -17
- package/dist/integration/providers/anthropic.d.ts +10 -0
- package/dist/integration/providers/anthropic.js +44 -1
- package/dist/integration/providers/openai.d.ts +10 -0
- package/dist/integration/providers/openai.js +58 -2
- package/dist/integration/providers/wire-dump-diff.d.ts +191 -0
- package/dist/integration/providers/wire-dump-diff.js +351 -0
- package/dist/integration/providers/wire-dump.d.ts +178 -0
- package/dist/integration/providers/wire-dump.js +396 -0
- package/dist/integration/storage/jsonl.d.ts +29 -0
- package/dist/integration/storage/jsonl.js +243 -41
- package/dist/integration/storage/session-index.js +63 -8
- package/dist/paths.d.ts +19 -0
- package/dist/paths.js +38 -1
- package/dist/subsystems/cognition-steward/conceptualize/SUBSYSTEM.md +43 -43
- package/dist/subsystems/cognition-steward/conceptualize/config.yaml +50 -50
- package/dist/subsystems/cognition-steward/consolidate/SUBSYSTEM.md +6 -6
- package/dist/subsystems/cognition-steward/consolidate/config.yaml +54 -54
- package/dist/subsystems/memory-steward/backfill/handler.js +9 -0
- package/dist/subsystems/safety-guard/config.yaml +3 -1
- package/dist/subsystems/wisdom-steward/evaluate/config.yaml +50 -50
- package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +42 -42
- package/dist/subsystems/wisdom-steward/formation/config.yaml +54 -54
- package/dist/subsystems/wisdom-steward/govern/config.yaml +55 -55
- package/package.json +115 -115
- package/dist/harness/collaboration/multi-agent/agent-registry-types.d.ts +0 -144
- package/dist/harness/collaboration/multi-agent/agent-registry-types.js +0 -25
- package/dist/harness/collaboration/multi-agent/index.d.ts +0 -17
- package/dist/harness/collaboration/multi-agent/index.js +0 -15
- package/dist/harness/collaboration/multi-agent/process.d.ts +0 -167
- package/dist/harness/collaboration/multi-agent/process.js +0 -256
- package/dist/harness/collaboration/multi-agent/registry.d.ts +0 -36
- package/dist/harness/collaboration/multi-agent/registry.js +0 -137
- package/dist/harness/collaboration/multi-agent/swarm.d.ts +0 -135
- package/dist/harness/collaboration/multi-agent/swarm.js +0 -317
- package/dist/harness/collaboration/multi-agent/types.d.ts +0 -66
- package/dist/harness/collaboration/multi-agent/types.js +0 -15
- package/dist/harness/collaboration/orchestration/async-task-store.d.ts +0 -107
- package/dist/harness/collaboration/orchestration/async-task-store.js +0 -12
- package/dist/harness/collaboration/orchestration/async-task.d.ts +0 -156
- package/dist/harness/collaboration/orchestration/async-task.js +0 -359
- package/dist/harness/collaboration/orchestration/planner/hybrid-planner.d.ts +0 -47
- package/dist/harness/collaboration/orchestration/planner/hybrid-planner.js +0 -61
- package/dist/harness/collaboration/orchestration/planner/index.d.ts +0 -9
- package/dist/harness/collaboration/orchestration/planner/index.js +0 -6
- package/dist/harness/collaboration/orchestration/planner/llm-planner.d.ts +0 -54
- package/dist/harness/collaboration/orchestration/planner/llm-planner.js +0 -167
- package/dist/harness/collaboration/orchestration/planner/rule-planner.d.ts +0 -85
- package/dist/harness/collaboration/orchestration/planner/rule-planner.js +0 -175
- package/dist/harness/collaboration/orchestration/quality/classifier.d.ts +0 -21
- package/dist/harness/collaboration/orchestration/quality/classifier.js +0 -235
- package/dist/harness/collaboration/orchestration/quality/gate.d.ts +0 -29
- package/dist/harness/collaboration/orchestration/quality/gate.js +0 -409
- package/dist/harness/collaboration/orchestration/quality/index.d.ts +0 -9
- package/dist/harness/collaboration/orchestration/quality/index.js +0 -8
- package/dist/harness/collaboration/orchestration/quality/types.d.ts +0 -159
- package/dist/harness/collaboration/orchestration/quality/types.js +0 -33
- package/dist/harness/collaboration/orchestration/scheduler/index.d.ts +0 -5
- package/dist/harness/collaboration/orchestration/scheduler/index.js +0 -4
- package/dist/harness/collaboration/orchestration/scheduler/task-scheduler.d.ts +0 -143
- package/dist/harness/collaboration/orchestration/scheduler/task-scheduler.js +0 -297
- package/dist/harness/collaboration/orchestration/strategy/classifier.d.ts +0 -12
- package/dist/harness/collaboration/orchestration/strategy/classifier.js +0 -81
- package/dist/harness/collaboration/orchestration/strategy/index.d.ts +0 -3
- package/dist/harness/collaboration/orchestration/strategy/index.js +0 -2
- package/dist/harness/collaboration/orchestration/strategy/router.d.ts +0 -28
- package/dist/harness/collaboration/orchestration/strategy/router.js +0 -105
- package/dist/harness/collaboration/orchestration/strategy/types.d.ts +0 -49
- package/dist/harness/collaboration/orchestration/strategy/types.js +0 -4
- package/dist/harness/collaboration/orchestration/workflow/engine.d.ts +0 -86
- package/dist/harness/collaboration/orchestration/workflow/engine.js +0 -298
- package/dist/harness/collaboration/orchestration/workflow/index.d.ts +0 -9
- package/dist/harness/collaboration/orchestration/workflow/index.js +0 -7
- package/dist/harness/collaboration/orchestration/workflow/types.d.ts +0 -143
- package/dist/harness/collaboration/orchestration/workflow/types.js +0 -7
- package/dist/harness/governance/human-in-the-loop/approval-manager.d.ts +0 -32
- package/dist/harness/governance/human-in-the-loop/approval-manager.js +0 -93
- /package/dist/harness/collaboration/{multi-agent → delegation}/message-channel-types.d.ts +0 -0
- /package/dist/harness/collaboration/{multi-agent → delegation}/message-channel-types.js +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
|
|
5
5
|
*/
|
|
6
6
|
import { randomUUID, createHash } from 'node:crypto';
|
|
7
7
|
import { open, stat } from 'node:fs/promises';
|
|
@@ -65,7 +65,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
65
65
|
debounceMs;
|
|
66
66
|
maxFileBytes;
|
|
67
67
|
fileLimits;
|
|
68
|
-
/**
|
|
68
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
69
69
|
parseTimeoutMs;
|
|
70
70
|
embedding;
|
|
71
71
|
embedBatch;
|
|
@@ -85,12 +85,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
85
85
|
embedRunner;
|
|
86
86
|
pollTimer = null;
|
|
87
87
|
reconcileTimer = null;
|
|
88
|
-
/**
|
|
88
|
+
/** 源级中止:DB 权威 + 本进程 AbortSignal(跨重启) */
|
|
89
89
|
jobControl;
|
|
90
90
|
jobQueue;
|
|
91
|
-
/**
|
|
91
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
92
92
|
onSourceSettled;
|
|
93
|
-
/**
|
|
93
|
+
/** 上次 parse 缺口扫描时间 */
|
|
94
94
|
lastParseGapScanAt = new Map();
|
|
95
95
|
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
96
96
|
gapScanUrgent = new Set();
|
|
@@ -100,9 +100,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
100
100
|
runningEmbed = 0;
|
|
101
101
|
draining = false;
|
|
102
102
|
kickAgain = false;
|
|
103
|
-
/** source
|
|
103
|
+
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
104
104
|
sourceLocks = new Set();
|
|
105
|
-
/** walk 发现的文件数(Phase A coverage
|
|
105
|
+
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
106
106
|
discoveredFiles = new Map();
|
|
107
107
|
watchers = new Map();
|
|
108
108
|
debounceTimers = new Map();
|
|
@@ -129,7 +129,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
129
129
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
130
130
|
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
131
131
|
this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
|
|
132
|
-
// watch
|
|
132
|
+
// watch 批量落盘时缩短静默窗,新增文件更快入队
|
|
133
133
|
this.debounceMs = options.debounceMs ?? 800;
|
|
134
134
|
this.fileLimits = resolveKnowledgeFileLimits({
|
|
135
135
|
...options.fileLimits,
|
|
@@ -141,7 +141,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
141
141
|
this.embedding = options.embeddingProvider ?? null;
|
|
142
142
|
this.embedBatch = options.embedBatch ?? 32;
|
|
143
143
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
144
|
-
// embed
|
|
144
|
+
// embed 与 parse 分槽;缺省 4 路并行打 embedding API
|
|
145
145
|
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
146
146
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
147
147
|
this.credentials = options.credentials ?? null;
|
|
@@ -161,7 +161,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
161
161
|
emitProgress: (evt) => this.emitProgress(evt),
|
|
162
162
|
recordSecretHit: (input) => this.recordEmbedSecretHit(input),
|
|
163
163
|
});
|
|
164
|
-
// embedding
|
|
164
|
+
// embedding 外发审计(合规可查;非权威计量)
|
|
165
165
|
this.sources.database.raw.exec(`
|
|
166
166
|
CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
|
|
167
167
|
id TEXT PRIMARY KEY,
|
|
@@ -181,17 +181,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
181
181
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
182
182
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
183
183
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
184
|
-
//
|
|
184
|
+
// 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
|
|
185
185
|
this.reclaimOrphanRunningJobs();
|
|
186
|
-
//
|
|
186
|
+
// 历史脏行:目录等非文件曾被标成 no_adapter,启动即清
|
|
187
187
|
this.cleanupNonFileIndexRows();
|
|
188
188
|
}
|
|
189
189
|
lastCleanupAt = 0;
|
|
190
190
|
/**
|
|
191
|
-
*
|
|
191
|
+
* 清掉「非普通文件」误入 knowledge_files 的行(目录曾被标 no_adapter)。
|
|
192
192
|
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
193
|
+
* 仅处理 skipped/error 且无 chunk 的行:有 chunk 的仍走 pruneMissing/磁盘对齐。
|
|
194
|
+
* 默认 60s 节流,避免每轮 reconcile 扫全表 stat。
|
|
195
195
|
*
|
|
196
196
|
* @param sourceId - 限定源;省略则全部源
|
|
197
197
|
* @param opts.force - 忽略节流
|
|
@@ -223,15 +223,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
223
223
|
}
|
|
224
224
|
}
|
|
225
225
|
catch {
|
|
226
|
-
//
|
|
226
|
+
// 路径不存在:留给 pruneMissing(磁盘对齐),避免误删有效但暂时不可见的文件
|
|
227
227
|
}
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
230
|
return removed;
|
|
231
231
|
}
|
|
232
232
|
/**
|
|
233
|
-
* 无跨进程 Lease
|
|
234
|
-
*
|
|
233
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
234
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
235
235
|
*/
|
|
236
236
|
reclaimOrphanRunningJobs() {
|
|
237
237
|
this.jobQueue.reclaimOrphanRunning();
|
|
@@ -240,10 +240,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
240
240
|
return this.index;
|
|
241
241
|
}
|
|
242
242
|
/**
|
|
243
|
-
* 过滤:文档扩展名交给 worker
|
|
243
|
+
* 过滤:文档扩展名交给 worker 抽取,不被 BINARY 短路。
|
|
244
244
|
*
|
|
245
|
-
* @param p -
|
|
246
|
-
* @returns
|
|
245
|
+
* @param p - 文件路径
|
|
246
|
+
* @returns 是否跳过
|
|
247
247
|
*/
|
|
248
248
|
shouldSkipFile(p) {
|
|
249
249
|
if (isDocumentPath(p))
|
|
@@ -277,11 +277,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
277
277
|
}
|
|
278
278
|
}
|
|
279
279
|
/**
|
|
280
|
-
*
|
|
281
|
-
*
|
|
280
|
+
* 路径归属:`parse_file` / 本地读文件只允许读源 root 内的路径。
|
|
281
|
+
* 并做 **realpath** 比较,防符号链接逃出 root。
|
|
282
282
|
*
|
|
283
|
-
* @param source -
|
|
284
|
-
* @param path -
|
|
283
|
+
* @param source - 知识源
|
|
284
|
+
* @param path - 请求路径(本地绝对路径,或外源逻辑键)
|
|
285
285
|
*/
|
|
286
286
|
sourceOwnsPath(source, path) {
|
|
287
287
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
@@ -290,13 +290,13 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
290
290
|
const root = resolve(source.location);
|
|
291
291
|
const target = resolve(path);
|
|
292
292
|
const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
|
|
293
|
-
//
|
|
293
|
+
// 词法比较(路径不存在时也拒绝明显越界)
|
|
294
294
|
if (!eq(root, target)) {
|
|
295
295
|
const relLex = relative(root, target);
|
|
296
296
|
if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
|
|
297
297
|
return false;
|
|
298
298
|
}
|
|
299
|
-
// realpath
|
|
299
|
+
// realpath 比较(symlink stat 失败回词法结果)
|
|
300
300
|
try {
|
|
301
301
|
const rootReal = realpathSync(root);
|
|
302
302
|
const targetReal = realpathSync(target);
|
|
@@ -310,10 +310,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
310
310
|
}
|
|
311
311
|
}
|
|
312
312
|
/**
|
|
313
|
-
* 全量/增量索引某源(Phase A
|
|
313
|
+
* 全量/增量索引某源(Phase A)
|
|
314
314
|
*
|
|
315
|
-
* @param opts.full - 显式重建(supersede
|
|
316
|
-
* @param opts.incremental -
|
|
315
|
+
* @param opts.full - 显式重建(supersede 未完成任务);poll/增量默认不打断在跑
|
|
316
|
+
* @param opts.incremental - 增量同步(poll):有 active 任务则跳过本轮
|
|
317
317
|
*/
|
|
318
318
|
async ingestSource(sourceId, opts) {
|
|
319
319
|
// fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
|
|
@@ -353,7 +353,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
353
353
|
return;
|
|
354
354
|
}
|
|
355
355
|
if (this.sourceLocks.has(sourceId)) {
|
|
356
|
-
//
|
|
356
|
+
// 已有同源任务在跑:入队 walk 即可(去重)
|
|
357
357
|
await this.enqueue(sourceId, 'walk_source', null, 1);
|
|
358
358
|
return;
|
|
359
359
|
}
|
|
@@ -362,20 +362,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
362
362
|
// 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
|
|
363
363
|
this.sources.update(sourceId, { status: 'discovering' });
|
|
364
364
|
this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
|
|
365
|
-
// full 不再 clearSource
|
|
365
|
+
// full 不再 clearSource:靠本轮 keepPaths 差量 prune,discover 失败不丢库
|
|
366
366
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
367
367
|
await this.ingestRemoteSource(source);
|
|
368
368
|
return;
|
|
369
369
|
}
|
|
370
|
-
// watch 策略索引同时挂上 fs
|
|
370
|
+
// watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失)
|
|
371
371
|
this.startWatch(sourceId);
|
|
372
372
|
const localDiscovery = await this.localFetcher.discover(source);
|
|
373
373
|
const files = localDiscovery.docs;
|
|
374
374
|
this.discoveredFiles.set(sourceId, files.length);
|
|
375
|
-
//
|
|
375
|
+
// 中止后不 prune、不入队(半截 keep 会误删)
|
|
376
376
|
if (this.isAborted(sourceId))
|
|
377
377
|
return;
|
|
378
|
-
//
|
|
378
|
+
// 差量 prune:仅 walk 完整时执行;出错/截断时 keep 不完整会误删
|
|
379
379
|
if (localDiscovery.complete) {
|
|
380
380
|
const localKeep = new Set(files.map((f) => f.path));
|
|
381
381
|
// walk 成功且目录为空时允许清空 Membership(§6.2)
|
|
@@ -456,7 +456,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
456
456
|
return;
|
|
457
457
|
}
|
|
458
458
|
const refs = discovered.docs;
|
|
459
|
-
// 条件 GET
|
|
459
|
+
// 条件 GET:带上已有 etag/Last-Modified,304 则整文档跳过
|
|
460
460
|
const enriched = refs.map((ref) => {
|
|
461
461
|
const existing = this.index.getFile(source.id, ref.path);
|
|
462
462
|
return {
|
|
@@ -476,7 +476,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
476
476
|
let okCount = 0;
|
|
477
477
|
let unchanged = 0;
|
|
478
478
|
for (const ref of enriched) {
|
|
479
|
-
//
|
|
479
|
+
// 中止后立刻停:不再外发、不再写索引
|
|
480
480
|
if (this.isAborted(source.id)) {
|
|
481
481
|
this.emitProgress({
|
|
482
482
|
type: 'source',
|
|
@@ -489,7 +489,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
489
489
|
try {
|
|
490
490
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
491
491
|
if (!doc) {
|
|
492
|
-
// 304 /
|
|
492
|
+
// 304 / 未变:保持原索引
|
|
493
493
|
unchanged += 1;
|
|
494
494
|
okCount += 1;
|
|
495
495
|
continue;
|
|
@@ -509,8 +509,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
509
509
|
});
|
|
510
510
|
}
|
|
511
511
|
}
|
|
512
|
-
//
|
|
513
|
-
//
|
|
512
|
+
// 差量 prune:仅当 discover **完整** 时才可删;截断/walk 失败时 keep 不完整,prune 会误删
|
|
513
|
+
// 中止后不做 prune:半截 keep 会误删已入库文档
|
|
514
514
|
if (!this.isAborted(source.id) && discovered.complete) {
|
|
515
515
|
const keep = new Set(enriched.map((r) => r.path));
|
|
516
516
|
const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
|
|
@@ -536,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
536
536
|
});
|
|
537
537
|
}
|
|
538
538
|
/**
|
|
539
|
-
*
|
|
539
|
+
* 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
|
|
540
540
|
*
|
|
541
|
-
* @returns
|
|
542
|
-
* @throws authRef
|
|
541
|
+
* @returns 凭证或 null(仅当源未配置 authRef)
|
|
542
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
|
|
543
543
|
*/
|
|
544
544
|
async resolveCred(source) {
|
|
545
545
|
if (!source.authRef)
|
|
@@ -547,23 +547,26 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
547
547
|
if (!this.credentials) {
|
|
548
548
|
throw new Error(`source ${source.id} has authRef "${source.authRef}" but CredentialStore is not configured`);
|
|
549
549
|
}
|
|
550
|
-
const resolved = await this.credentials.resolve(source.authRef
|
|
550
|
+
const resolved = await this.credentials.resolve(source.authRef, {
|
|
551
|
+
kind: 'knowledge_source',
|
|
552
|
+
id: source.id,
|
|
553
|
+
});
|
|
551
554
|
if (!resolved) {
|
|
552
555
|
throw new Error(`credential "${source.authRef}" unavailable (missing secret, expired, or not found)`);
|
|
553
556
|
}
|
|
554
557
|
return resolved;
|
|
555
558
|
}
|
|
556
559
|
/**
|
|
557
|
-
*
|
|
560
|
+
* 将 VirtualDocument 写入索引(外源 / 规范化后文本)
|
|
558
561
|
*
|
|
559
|
-
* @returns
|
|
562
|
+
* @returns 是否成功写入
|
|
560
563
|
*/
|
|
561
564
|
async indexVirtualDoc(sourceId, doc, authRef) {
|
|
562
565
|
if (!doc.content?.trim()) {
|
|
563
566
|
this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
|
|
564
567
|
return false;
|
|
565
568
|
}
|
|
566
|
-
//
|
|
569
|
+
// 逻辑键可能无扩展名(connector id / URL path):按 MIME → 扩展名 → 内容形态回退
|
|
567
570
|
let adapter = this.adapters.match(doc.path, doc.contentType);
|
|
568
571
|
if (!adapter) {
|
|
569
572
|
if (looksLikeHtml(doc.content, doc.contentType))
|
|
@@ -612,7 +615,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
612
615
|
}
|
|
613
616
|
}
|
|
614
617
|
/**
|
|
615
|
-
*
|
|
618
|
+
* 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
|
|
616
619
|
*/
|
|
617
620
|
async ingestFileNow(sourceId, filePath) {
|
|
618
621
|
const ok = await this.parseOne(sourceId, filePath);
|
|
@@ -620,17 +623,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
620
623
|
return ok;
|
|
621
624
|
}
|
|
622
625
|
/**
|
|
623
|
-
*
|
|
626
|
+
* 重做指定文件:强制重新 解析 → 分块 → 向量
|
|
624
627
|
*
|
|
625
|
-
* -
|
|
626
|
-
* - 失效 contentHash
|
|
627
|
-
* -
|
|
628
|
-
* -
|
|
628
|
+
* - 路径必须归属本源(防越权读盘);外源逻辑键走 fetch_doc 重取
|
|
629
|
+
* - 失效 contentHash(忽略 isFresh),旧 chunks 保留到 upsert 成功
|
|
630
|
+
* - **源处于中止态时自动 resume**:显式重做 = 意图要跑活,不允许静默丢弃
|
|
631
|
+
* - 目录等非文件:直接清脏行,不假装入队
|
|
629
632
|
* - 完成后走既有 embed 链路
|
|
630
633
|
*
|
|
631
|
-
* @param sourceId -
|
|
632
|
-
* @param paths -
|
|
633
|
-
* @returns queued=新入队;alreadyActive
|
|
634
|
+
* @param sourceId - 源 id
|
|
635
|
+
* @param paths - 文件路径列表
|
|
636
|
+
* @returns queued=新入队;alreadyActive=已在队列/执行;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属本源
|
|
634
637
|
*/
|
|
635
638
|
async reprocessFiles(sourceId, paths) {
|
|
636
639
|
if (!paths.length) {
|
|
@@ -656,7 +659,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
656
659
|
continue;
|
|
657
660
|
}
|
|
658
661
|
if (isRemote) {
|
|
659
|
-
// 外源:path
|
|
662
|
+
// 外源:path 是逻辑键,重做 = 重新 fetch + 索引(不读本地盘)
|
|
660
663
|
this.index.invalidateFileForReparse(sourceId, p);
|
|
661
664
|
if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
|
|
662
665
|
queued += 1;
|
|
@@ -666,7 +669,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
666
669
|
}
|
|
667
670
|
continue;
|
|
668
671
|
}
|
|
669
|
-
//
|
|
672
|
+
// 目录/设备节点:清脏行即可,没有「解析」可跑
|
|
670
673
|
try {
|
|
671
674
|
const st = statSync(p);
|
|
672
675
|
if (!st.isFile()) {
|
|
@@ -676,7 +679,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
676
679
|
}
|
|
677
680
|
}
|
|
678
681
|
catch {
|
|
679
|
-
//
|
|
682
|
+
// 路径不存在:仍按文件流程,让 parse 自清理
|
|
680
683
|
}
|
|
681
684
|
// 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
|
|
682
685
|
this.index.invalidateFileForReparse(sourceId, p);
|
|
@@ -690,15 +693,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
690
693
|
this.kick();
|
|
691
694
|
return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
|
|
692
695
|
}
|
|
693
|
-
/**
|
|
696
|
+
/** 路径上是否已有 queued/running 的 parse/fetch 任务 */
|
|
694
697
|
hasActiveParseJob(sourceId, path) {
|
|
695
698
|
return this.jobQueue.hasActivePathJob(sourceId, path);
|
|
696
699
|
}
|
|
697
700
|
/**
|
|
698
|
-
*
|
|
701
|
+
* 路径级任务/文件状态(UI 跟踪重做完成)
|
|
699
702
|
*
|
|
700
|
-
* @param sourceId -
|
|
701
|
-
* @param paths -
|
|
703
|
+
* @param sourceId - 源 id
|
|
704
|
+
* @param paths - 路径列表
|
|
702
705
|
*/
|
|
703
706
|
jobStateForPaths(sourceId, paths) {
|
|
704
707
|
return paths.map((p) => {
|
|
@@ -725,10 +728,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
725
728
|
});
|
|
726
729
|
}
|
|
727
730
|
/**
|
|
728
|
-
*
|
|
731
|
+
* 按筛选批量重做(当前列表的 status/ext/q 条件)
|
|
729
732
|
*
|
|
730
|
-
* @param sourceId -
|
|
731
|
-
* @param opts -
|
|
733
|
+
* @param sourceId - 源 id
|
|
734
|
+
* @param opts - 与文件列表筛选一致
|
|
732
735
|
* @returns 入队条数
|
|
733
736
|
*/
|
|
734
737
|
async reprocessByFilter(sourceId, opts) {
|
|
@@ -747,7 +750,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
747
750
|
clearTimeout(prev);
|
|
748
751
|
this.debounceTimers.set(key, setTimeout(() => {
|
|
749
752
|
this.debounceTimers.delete(key);
|
|
750
|
-
//
|
|
753
|
+
// 统一先 stat:目录误报 change 时不得 parse 成 no_adapter
|
|
751
754
|
void stat(filePath)
|
|
752
755
|
.then(async (st) => {
|
|
753
756
|
if (st.isFile()) {
|
|
@@ -773,7 +776,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
773
776
|
}, this.debounceMs));
|
|
774
777
|
}
|
|
775
778
|
/**
|
|
776
|
-
*
|
|
779
|
+
* 启动目录 watch(幂等)
|
|
777
780
|
*/
|
|
778
781
|
startWatch(sourceId) {
|
|
779
782
|
const source = this.sources.get(sourceId);
|
|
@@ -817,7 +820,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
817
820
|
}
|
|
818
821
|
}
|
|
819
822
|
/**
|
|
820
|
-
*
|
|
823
|
+
* 等待队列排空(测试/管理)
|
|
821
824
|
*/
|
|
822
825
|
async idle(timeoutMs = 30_000) {
|
|
823
826
|
const start = Date.now();
|
|
@@ -843,10 +846,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
843
846
|
this.stopWatch(id);
|
|
844
847
|
}
|
|
845
848
|
/**
|
|
846
|
-
*
|
|
849
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
847
850
|
*
|
|
848
|
-
* @param opts.sourceId -
|
|
849
|
-
* @returns
|
|
851
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
852
|
+
* @returns 取消统计
|
|
850
853
|
*/
|
|
851
854
|
abortJobs(opts) {
|
|
852
855
|
const sid = opts?.sourceId?.trim();
|
|
@@ -860,11 +863,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
860
863
|
let cancelledQueued = 0;
|
|
861
864
|
let abortedRunning = 0;
|
|
862
865
|
for (const id of unique) {
|
|
863
|
-
// 1) DB
|
|
866
|
+
// 1) DB 落中止态(跨重启)+ 打断本进程信号
|
|
864
867
|
const newly = this.jobControl.markAborted(id);
|
|
865
868
|
if (newly)
|
|
866
869
|
abortedRunning += 1;
|
|
867
|
-
// 2) 清空 queued
|
|
870
|
+
// 2) 清空 queued → cancelled(共享 File 的 parse 不杀:仍被领用则保留)
|
|
868
871
|
const res = this.sources.database.raw
|
|
869
872
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
870
873
|
WHERE source_id = ? AND status = 'queued'
|
|
@@ -879,7 +882,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
879
882
|
.run(Date.now(), id, id);
|
|
880
883
|
cancelledQueued += Number(res.changes ?? 0);
|
|
881
884
|
}
|
|
882
|
-
// 3)
|
|
885
|
+
// 3) 看门狗不要再把 aborted running 收回 queued(限定本次涉及的源)
|
|
883
886
|
if (sid) {
|
|
884
887
|
this.sources.database.raw
|
|
885
888
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
@@ -907,15 +910,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
907
910
|
return { cancelledQueued, abortedRunning, runningJobs };
|
|
908
911
|
}
|
|
909
912
|
/**
|
|
910
|
-
*
|
|
913
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
911
914
|
*
|
|
912
|
-
* -
|
|
913
|
-
* -
|
|
915
|
+
* - 清除该源中止态
|
|
916
|
+
* - 把 cancelled 任务放回 queued
|
|
914
917
|
* - 若仍缺向量则排队 embed
|
|
915
|
-
* -
|
|
918
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
916
919
|
*
|
|
917
|
-
* @param opts.sourceId -
|
|
918
|
-
* @returns
|
|
920
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
921
|
+
* @returns 恢复统计
|
|
919
922
|
*/
|
|
920
923
|
resumeJobs(opts) {
|
|
921
924
|
const sid = opts?.sourceId?.trim();
|
|
@@ -930,15 +933,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
930
933
|
let restoredCancelled = 0;
|
|
931
934
|
let embedQueued = 0;
|
|
932
935
|
for (const id of ids) {
|
|
933
|
-
// 1)
|
|
936
|
+
// 1) 清除中止态(DB + 新纪元信号)
|
|
934
937
|
this.beginAbortEpoch(id);
|
|
935
|
-
// 2) cancelled
|
|
938
|
+
// 2) cancelled → queued(用户中止时被杀的活)
|
|
936
939
|
const res = this.sources.database.raw
|
|
937
940
|
.prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
|
|
938
941
|
WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
|
|
939
942
|
.run(Date.now(), id);
|
|
940
943
|
restoredCancelled += Number(res.changes ?? 0);
|
|
941
|
-
// 3)
|
|
944
|
+
// 3) 仍缺向量则确保 embed 在队
|
|
942
945
|
if (this.embedding) {
|
|
943
946
|
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
944
947
|
const active = this.countActiveJobs(id);
|
|
@@ -957,23 +960,23 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
957
960
|
});
|
|
958
961
|
return { restoredCancelled, embedQueued };
|
|
959
962
|
}
|
|
960
|
-
/**
|
|
963
|
+
/** 开启新的未中止纪元(resume / supersede 后用;清 DB 中止态) */
|
|
961
964
|
beginAbortEpoch(sourceId) {
|
|
962
965
|
this.jobControl.beginEpoch(sourceId);
|
|
963
966
|
}
|
|
964
967
|
/**
|
|
965
|
-
* 重建前作废本源未完成任务(supersede
|
|
968
|
+
* 重建前作废本源未完成任务(supersede)
|
|
966
969
|
*
|
|
967
|
-
* - queued
|
|
968
|
-
* - running 打断(worker
|
|
969
|
-
* - 随后 beginAbortEpoch
|
|
970
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
971
|
+
* - running 打断(worker/循环退出)
|
|
972
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
970
973
|
*/
|
|
971
974
|
supersedeSourceWork(sourceId) {
|
|
972
975
|
this.sources.database.raw
|
|
973
976
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
974
977
|
WHERE source_id = ? AND status = 'queued'`)
|
|
975
978
|
.run(Date.now(), sourceId);
|
|
976
|
-
// 打断在跑 parse/embed
|
|
979
|
+
// 打断在跑 parse/embed;再开新纪元(清中止态),避免 walk 自己被 abort 卡死
|
|
977
980
|
this.jobControl.markAborted(sourceId);
|
|
978
981
|
this.beginAbortEpoch(sourceId);
|
|
979
982
|
this.emitProgress({
|
|
@@ -1057,12 +1060,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1057
1060
|
this.gapScanTimer.unref?.();
|
|
1058
1061
|
}
|
|
1059
1062
|
/**
|
|
1060
|
-
*
|
|
1063
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
1061
1064
|
*
|
|
1062
|
-
*
|
|
1063
|
-
* embed
|
|
1065
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
1066
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
1064
1067
|
*
|
|
1065
|
-
* @param sourceId -
|
|
1068
|
+
* @param sourceId - 知识源 id
|
|
1066
1069
|
* @returns 删除的文件数
|
|
1067
1070
|
*/
|
|
1068
1071
|
async pruneMissingOnDisk(sourceId) {
|
|
@@ -1074,12 +1077,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1074
1077
|
try {
|
|
1075
1078
|
const found = await this.localFetcher.discover(source);
|
|
1076
1079
|
if (!found.complete) {
|
|
1077
|
-
// walk
|
|
1080
|
+
// walk 不完整:退回逐文件 stat,禁止用残缺 keep 清库
|
|
1078
1081
|
removed = await this.pruneByStatScan(sourceId);
|
|
1079
1082
|
}
|
|
1080
1083
|
else {
|
|
1081
1084
|
const keep = new Set(found.docs.map((f) => f.path));
|
|
1082
|
-
// walk
|
|
1085
|
+
// walk 完整:即使空目录也可清空(与逐文件 stat 语义一致)
|
|
1083
1086
|
removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
|
|
1084
1087
|
}
|
|
1085
1088
|
}
|
|
@@ -1101,7 +1104,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1101
1104
|
}
|
|
1102
1105
|
return removed;
|
|
1103
1106
|
}
|
|
1104
|
-
/**
|
|
1107
|
+
/** 逐文件 stat(file 源 / walk 失败回退);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
1105
1108
|
async pruneByStatScan(sourceId, concurrency = 4) {
|
|
1106
1109
|
const files = this.index.listFiles(sourceId);
|
|
1107
1110
|
let removed = 0;
|
|
@@ -1138,10 +1141,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1138
1141
|
return this.jobControl.isAborted(sourceId);
|
|
1139
1142
|
}
|
|
1140
1143
|
/**
|
|
1141
|
-
*
|
|
1144
|
+
* 中止/继续 控制面读数(互斥按钮用)
|
|
1142
1145
|
*
|
|
1143
|
-
* @param sourceId -
|
|
1144
|
-
* @returns 队列计数 +
|
|
1146
|
+
* @param sourceId - 源 id
|
|
1147
|
+
* @returns 队列计数 + 是否已中止 + 是否仍缺向量
|
|
1145
1148
|
*/
|
|
1146
1149
|
jobControlState(sourceId) {
|
|
1147
1150
|
return readJobControlState(this.sources.database, this.index, sourceId, {
|
|
@@ -1149,12 +1152,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1149
1152
|
});
|
|
1150
1153
|
}
|
|
1151
1154
|
/**
|
|
1152
|
-
*
|
|
1155
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
1153
1156
|
*
|
|
1154
|
-
*
|
|
1155
|
-
* 1. 回收超时 running
|
|
1156
|
-
* 2. 有缺向量且无 active embed
|
|
1157
|
-
* 3.
|
|
1157
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
1158
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
1159
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
1160
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
1158
1161
|
*/
|
|
1159
1162
|
startReconciler(intervalMs = 15_000) {
|
|
1160
1163
|
if (this.reconcileTimer || this.disposed)
|
|
@@ -1177,21 +1180,21 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1177
1180
|
this.reconcileTimer = null;
|
|
1178
1181
|
}
|
|
1179
1182
|
}
|
|
1180
|
-
/**
|
|
1183
|
+
/** 对账入口(也可手工调用) */
|
|
1181
1184
|
async reconcileJobs() {
|
|
1182
1185
|
if (this.disposed)
|
|
1183
1186
|
return;
|
|
1184
|
-
// stale
|
|
1187
|
+
// stale 阈值必须 **大于** 硬超时,否则合法的长 parse 会被收回后双跑
|
|
1185
1188
|
this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
|
|
1186
1189
|
// path: 与 win:/unix: 身份分裂的零认领残留
|
|
1187
1190
|
this.index.purgeOrphanPathIdentityFiles();
|
|
1188
|
-
//
|
|
1191
|
+
// 目录脏行等历史误入项(轻量;非每轮全表)
|
|
1189
1192
|
this.cleanupNonFileIndexRows();
|
|
1190
|
-
//
|
|
1193
|
+
// 终态 job 定期清理,防表无限增长
|
|
1191
1194
|
this.cleanupTerminalJobs();
|
|
1192
|
-
// 存量 FTS
|
|
1195
|
+
// 存量 FTS 后台补齐(分批让出事件循环;构造期禁止同步 rebuild)
|
|
1193
1196
|
void this.index.ensureFtsBackfill();
|
|
1194
|
-
// 存量向量回填:小批量 +
|
|
1197
|
+
// 存量向量回填:小批量 + 让出事件循环,禁止 2000 条同步写(会堵死 abort)
|
|
1195
1198
|
await this.index.backfillVecFromEmbeddings(25);
|
|
1196
1199
|
for (const s of this.sources.list()) {
|
|
1197
1200
|
if (s.status === 'removed' || s.status === 'disabled')
|
|
@@ -1205,8 +1208,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1205
1208
|
(s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
|
|
1206
1209
|
await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
|
|
1207
1210
|
}
|
|
1208
|
-
// 稳定态收尾(auto-describe
|
|
1209
|
-
// 先看 active,再 gap
|
|
1211
|
+
// 稳定态收尾(auto-describe 等)——纯关键词部署同样要触发
|
|
1212
|
+
// 先看 active,再 gap;避免刚入队的 gap job 挡住 settled
|
|
1210
1213
|
if (active.total === 0) {
|
|
1211
1214
|
this.refreshCoverage(s.id, { force: true });
|
|
1212
1215
|
try {
|
|
@@ -1219,7 +1222,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1219
1222
|
else {
|
|
1220
1223
|
this.flushCoverageDirty(s.id);
|
|
1221
1224
|
}
|
|
1222
|
-
// watch 漏事件 / skip
|
|
1225
|
+
// watch 漏事件 / skip 可重试时补齐 parse(与是否配置 embedding 无关)
|
|
1223
1226
|
// urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
|
|
1224
1227
|
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
1225
1228
|
const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
|
|
@@ -1242,11 +1245,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1242
1245
|
}
|
|
1243
1246
|
this.kick();
|
|
1244
1247
|
}
|
|
1245
|
-
/**
|
|
1248
|
+
/** 终态 job 清理(done/failed/cancelled 超过保留期) */
|
|
1246
1249
|
cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
|
|
1247
1250
|
this.jobQueue.cleanupTerminal(retainMs);
|
|
1248
1251
|
}
|
|
1249
|
-
/** running 超时(无 heartbeat)→ 收回 queued
|
|
1252
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
1250
1253
|
reclaimStaleRunningJobs(staleMs) {
|
|
1251
1254
|
this.jobQueue.reclaimStaleRunning(staleMs);
|
|
1252
1255
|
}
|
|
@@ -1261,7 +1264,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1261
1264
|
return Boolean(row?.ok);
|
|
1262
1265
|
}
|
|
1263
1266
|
/**
|
|
1264
|
-
*
|
|
1267
|
+
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
1265
1268
|
*/
|
|
1266
1269
|
startPolling() {
|
|
1267
1270
|
if (this.pollTimer || this.disposed)
|
|
@@ -1276,7 +1279,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1276
1279
|
});
|
|
1277
1280
|
});
|
|
1278
1281
|
}, this.pollTickMs);
|
|
1279
|
-
//
|
|
1282
|
+
// 不阻止进程退出
|
|
1280
1283
|
this.pollTimer.unref?.();
|
|
1281
1284
|
}
|
|
1282
1285
|
stopPolling() {
|
|
@@ -1286,9 +1289,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1286
1289
|
}
|
|
1287
1290
|
}
|
|
1288
1291
|
/**
|
|
1289
|
-
*
|
|
1292
|
+
* 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
|
|
1290
1293
|
*
|
|
1291
|
-
* @returns
|
|
1294
|
+
* @returns 本轮实际触发的 sourceId 列表
|
|
1292
1295
|
*/
|
|
1293
1296
|
async pollDueSources() {
|
|
1294
1297
|
if (this.disposed)
|
|
@@ -1318,7 +1321,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1318
1321
|
status: 'poll_due',
|
|
1319
1322
|
detail: source.sync.intervalMs?.toString(),
|
|
1320
1323
|
});
|
|
1321
|
-
//
|
|
1324
|
+
// 异步入队,不阻塞 tick;增量:有 active 则跳过,禁止 supersede 打断
|
|
1322
1325
|
void this.ingestSource(source.id, { incremental: true }).catch((err) => {
|
|
1323
1326
|
this.emitProgress({
|
|
1324
1327
|
type: 'error',
|
|
@@ -1330,7 +1333,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1330
1333
|
}
|
|
1331
1334
|
return triggered;
|
|
1332
1335
|
}
|
|
1333
|
-
//
|
|
1336
|
+
// ── 内部 ──
|
|
1334
1337
|
hasQueuedJobs() {
|
|
1335
1338
|
return this.jobQueue.hasQueuedJobs();
|
|
1336
1339
|
}
|
|
@@ -1401,7 +1404,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1401
1404
|
if (this.disposed)
|
|
1402
1405
|
return;
|
|
1403
1406
|
if (this.draining) {
|
|
1404
|
-
//
|
|
1407
|
+
// 等待中的 drain 会再 kick;标记一次避免丢信号
|
|
1405
1408
|
this.kickAgain = true;
|
|
1406
1409
|
return;
|
|
1407
1410
|
}
|
|
@@ -1437,7 +1440,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1437
1440
|
}
|
|
1438
1441
|
const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
|
|
1439
1442
|
if (!job) {
|
|
1440
|
-
//
|
|
1443
|
+
// 队列清空:合并刷新被 parse/drop 标脏的 coverage,避免每文件一次 COUNT
|
|
1441
1444
|
this.flushCoverageDirty();
|
|
1442
1445
|
return;
|
|
1443
1446
|
}
|
|
@@ -1500,7 +1503,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1500
1503
|
return;
|
|
1501
1504
|
}
|
|
1502
1505
|
try {
|
|
1503
|
-
// 归属闸门:任何会碰本地盘 /
|
|
1506
|
+
// 归属闸门:任何会碰本地盘 / 外源逻辑键的 path 任务都必须属于本源
|
|
1504
1507
|
if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
|
|
1505
1508
|
const owner = this.sources.get(job.source_id);
|
|
1506
1509
|
if (owner && !this.sourceOwnsPath(owner, job.path)) {
|
|
@@ -1508,7 +1511,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1508
1511
|
}
|
|
1509
1512
|
}
|
|
1510
1513
|
if (job.kind === 'parse_file' && job.path) {
|
|
1511
|
-
// 单文件解析硬超时:大 xlsx/pdf
|
|
1514
|
+
// 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环;超时随体积放大
|
|
1512
1515
|
let parseTimeoutMs = this.parseTimeoutMs;
|
|
1513
1516
|
try {
|
|
1514
1517
|
const st = await stat(job.path);
|
|
@@ -1516,7 +1519,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1516
1519
|
parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
|
|
1517
1520
|
}
|
|
1518
1521
|
catch {
|
|
1519
|
-
//
|
|
1522
|
+
// 路径已删时交给 parseOne 自清理
|
|
1520
1523
|
}
|
|
1521
1524
|
this.heartbeatJob(job.id);
|
|
1522
1525
|
await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
@@ -1558,7 +1561,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1558
1561
|
if (!this.embedding) {
|
|
1559
1562
|
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1560
1563
|
if (stillMissing) {
|
|
1561
|
-
//
|
|
1564
|
+
// 禁止空跑标 done:迁移后缺向量却显示 queue=0
|
|
1562
1565
|
throw new Error('embedding_provider_missing');
|
|
1563
1566
|
}
|
|
1564
1567
|
return;
|
|
@@ -1568,7 +1571,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1568
1571
|
throw new Error('aborted');
|
|
1569
1572
|
}
|
|
1570
1573
|
}
|
|
1571
|
-
//
|
|
1574
|
+
// 统一收尾闸门:任何 kind 中止后都不得标 done
|
|
1572
1575
|
if (this.isAborted(job.source_id)) {
|
|
1573
1576
|
throw new Error('aborted');
|
|
1574
1577
|
}
|
|
@@ -1581,7 +1584,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1581
1584
|
path: job.path ?? undefined,
|
|
1582
1585
|
status: 'done',
|
|
1583
1586
|
});
|
|
1584
|
-
//
|
|
1587
|
+
// 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
|
|
1585
1588
|
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1586
1589
|
this.ensureEmbedJob(job.source_id);
|
|
1587
1590
|
this.refreshCoverage(job.source_id, { force: true });
|
|
@@ -1590,7 +1593,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1590
1593
|
catch (err) {
|
|
1591
1594
|
const message = err instanceof Error ? err.message : String(err);
|
|
1592
1595
|
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1593
|
-
//
|
|
1596
|
+
// 归属拒绝不是「文件解析失败」,不得写入 knowledge_files
|
|
1594
1597
|
const ownership = message.includes('not owned by source');
|
|
1595
1598
|
if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
|
|
1596
1599
|
this.index.markFileError(job.source_id, job.path, message);
|
|
@@ -1607,8 +1610,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1607
1610
|
}
|
|
1608
1611
|
}
|
|
1609
1612
|
/**
|
|
1610
|
-
*
|
|
1611
|
-
*
|
|
1613
|
+
* 带截止时间的可取消执行:超时会 abort signal,工作侧必须在副作用前检查。
|
|
1614
|
+
* 禁止再用 Promise.race 包不可取消的 Promise——那会在超时后留下僵尸写入。
|
|
1612
1615
|
*/
|
|
1613
1616
|
async withTimeout(work, ms, label, parent) {
|
|
1614
1617
|
const controller = new AbortController();
|
|
@@ -1633,7 +1636,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1633
1636
|
}
|
|
1634
1637
|
}
|
|
1635
1638
|
/**
|
|
1636
|
-
*
|
|
1639
|
+
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
1637
1640
|
*/
|
|
1638
1641
|
async fetchDocJob(sourceId, path, signal) {
|
|
1639
1642
|
const source = this.sources.get(sourceId);
|
|
@@ -1660,7 +1663,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1660
1663
|
}
|
|
1661
1664
|
async parseOne(sourceId, filePath, signal) {
|
|
1662
1665
|
const source = this.sources.get(sourceId);
|
|
1663
|
-
//
|
|
1666
|
+
// 防御深度:即使队列被污染,也不读源 root 外的本地文件
|
|
1664
1667
|
if (source && !this.sourceOwnsPath(source, filePath)) {
|
|
1665
1668
|
throw new Error(`path not owned by source: ${filePath}`);
|
|
1666
1669
|
}
|
|
@@ -1670,7 +1673,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1670
1673
|
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
1671
1674
|
return false;
|
|
1672
1675
|
}
|
|
1673
|
-
//
|
|
1676
|
+
// 先确认是普通文件:目录/设备节点不得进 knowledge_files(历史上曾误标 no_adapter)
|
|
1674
1677
|
let st;
|
|
1675
1678
|
try {
|
|
1676
1679
|
st = await stat(filePath);
|
|
@@ -1685,7 +1688,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1685
1688
|
}
|
|
1686
1689
|
const adapter = this.adapters.match(filePath);
|
|
1687
1690
|
if (!adapter) {
|
|
1688
|
-
// worker
|
|
1691
|
+
// 文档优先 worker 抽取;注意进程内 port 仅作 worker 不可用时的回退
|
|
1689
1692
|
if (isDocumentPath(filePath)) {
|
|
1690
1693
|
return this.parseDocumentFile(sourceId, filePath, signal);
|
|
1691
1694
|
}
|
|
@@ -1746,12 +1749,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1746
1749
|
}
|
|
1747
1750
|
}
|
|
1748
1751
|
/**
|
|
1749
|
-
* PDF/Office:DocumentPort
|
|
1750
|
-
* 新鲜度以原件字节 hash
|
|
1752
|
+
* PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
|
|
1753
|
+
* 新鲜度以原件字节 hash 为准。
|
|
1751
1754
|
*
|
|
1752
|
-
* @param sourceId -
|
|
1753
|
-
* @param filePath -
|
|
1754
|
-
* @returns
|
|
1755
|
+
* @param sourceId - 知识源 id
|
|
1756
|
+
* @param filePath - 本地文件路径
|
|
1757
|
+
* @returns 是否成功入索引
|
|
1755
1758
|
*/
|
|
1756
1759
|
async parseDocumentFile(sourceId, filePath, signal) {
|
|
1757
1760
|
if (signal?.aborted)
|
|
@@ -1801,12 +1804,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1801
1804
|
}
|
|
1802
1805
|
catch (err) {
|
|
1803
1806
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1804
|
-
//
|
|
1807
|
+
// 中止/超时不是「这个文件解析失败」,必须上抛让 job 标 failed,禁止静默 done
|
|
1805
1808
|
if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
|
|
1806
1809
|
throw err;
|
|
1807
1810
|
}
|
|
1808
1811
|
if (isDocumentExtractError(err)) {
|
|
1809
|
-
//
|
|
1812
|
+
// 结构化跳过(密码/老格式/超大等),不挡整库
|
|
1810
1813
|
const identKey = await this.identityKeyFor(filePath);
|
|
1811
1814
|
this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
|
|
1812
1815
|
return false;
|
|
@@ -1871,8 +1874,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1871
1874
|
/**
|
|
1872
1875
|
* Phase B:嵌入待处理 chunk
|
|
1873
1876
|
*
|
|
1874
|
-
*
|
|
1875
|
-
*
|
|
1877
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
1878
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
1876
1879
|
*/
|
|
1877
1880
|
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1878
1881
|
await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
|
|
@@ -1883,7 +1886,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1883
1886
|
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
1884
1887
|
.run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
|
|
1885
1888
|
}
|
|
1886
|
-
/**
|
|
1889
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
1887
1890
|
ensureEmbedJob(sourceId) {
|
|
1888
1891
|
if (!this.embedding)
|
|
1889
1892
|
return;
|
|
@@ -1898,7 +1901,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1898
1901
|
heartbeatJob(jobId) {
|
|
1899
1902
|
this.jobQueue.heartbeat(jobId);
|
|
1900
1903
|
}
|
|
1901
|
-
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile
|
|
1904
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷新 */
|
|
1902
1905
|
coverageDirty = new Set();
|
|
1903
1906
|
markCoverageDirty(sourceId) {
|
|
1904
1907
|
this.coverageDirty.add(sourceId);
|
|
@@ -1926,7 +1929,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1926
1929
|
if (!source)
|
|
1927
1930
|
return;
|
|
1928
1931
|
const discovered = this.discoveredFiles.get(sourceId) ?? 0;
|
|
1929
|
-
// Phase A:已落库文件(indexed+skipped+error
|
|
1932
|
+
// Phase A:已落库文件(indexed+skipped+error)/ walk 发现数
|
|
1930
1933
|
const phaseA = discovered > 0
|
|
1931
1934
|
? Math.min(1, stats.files / discovered)
|
|
1932
1935
|
: stats.files > 0
|
|
@@ -1984,7 +1987,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1984
1987
|
this.emit('progress', out);
|
|
1985
1988
|
this.emit('knowledge.index.progress', out);
|
|
1986
1989
|
}
|
|
1987
|
-
/**
|
|
1990
|
+
/** 磁盘水位告警(仅告警,不丢任务) */
|
|
1988
1991
|
async warnDiskWatermark(sourceId) {
|
|
1989
1992
|
try {
|
|
1990
1993
|
const { statfs } = await import('node:fs/promises');
|