@octopi-agent/engine 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +7 -0
- package/README.md +26 -0
- package/dist/builtin-model-info.d.ts +30 -0
- package/dist/builtin-model-info.js +118 -0
- package/dist/config-schema/agent.d.ts +72 -0
- package/dist/config-schema/agent.js +38 -0
- package/dist/config-schema/context.d.ts +122 -0
- package/dist/config-schema/context.js +100 -0
- package/dist/config-schema/engine.d.ts +665 -0
- package/dist/config-schema/engine.js +55 -0
- package/dist/config-schema/governance.d.ts +81 -0
- package/dist/config-schema/governance.js +55 -0
- package/dist/config-schema/models.d.ts +159 -0
- package/dist/config-schema/models.js +92 -0
- package/dist/config-schema/observability.d.ts +57 -0
- package/dist/config-schema/observability.js +59 -0
- package/dist/config-schema/run.d.ts +102 -0
- package/dist/config-schema/run.js +99 -0
- package/dist/config-schema/shared.d.ts +54 -0
- package/dist/config-schema/shared.js +43 -0
- package/dist/config-schema/substrate.d.ts +156 -0
- package/dist/config-schema/substrate.js +191 -0
- package/dist/config-schema/subsystems.d.ts +11 -0
- package/dist/config-schema/subsystems.js +13 -0
- package/dist/config-schema/web-search.d.ts +32 -0
- package/dist/config-schema/web-search.js +38 -0
- package/dist/config.d.ts +840 -0
- package/dist/config.js +161 -0
- package/dist/harness/activation/coalesce.d.ts +38 -0
- package/dist/harness/activation/coalesce.js +80 -0
- package/dist/harness/activation/compiler.d.ts +15 -0
- package/dist/harness/activation/compiler.js +136 -0
- package/dist/harness/activation/dispatcher.d.ts +19 -0
- package/dist/harness/activation/dispatcher.js +31 -0
- package/dist/harness/activation/event-source-types.d.ts +63 -0
- package/dist/harness/activation/event-source-types.js +13 -0
- package/dist/harness/activation/index.d.ts +20 -0
- package/dist/harness/activation/index.js +14 -0
- package/dist/harness/activation/router.d.ts +16 -0
- package/dist/harness/activation/router.js +28 -0
- package/dist/harness/activation/runtime.d.ts +78 -0
- package/dist/harness/activation/runtime.js +310 -0
- package/dist/harness/activation/sources/agent-signal.d.ts +34 -0
- package/dist/harness/activation/sources/agent-signal.js +93 -0
- package/dist/harness/activation/sources/escalate-bridge.d.ts +43 -0
- package/dist/harness/activation/sources/escalate-bridge.js +85 -0
- package/dist/harness/activation/sources/index.d.ts +6 -0
- package/dist/harness/activation/sources/index.js +3 -0
- package/dist/harness/activation/sources/schedule.d.ts +43 -0
- package/dist/harness/activation/sources/schedule.js +102 -0
- package/dist/harness/activation/types.d.ts +166 -0
- package/dist/harness/activation/types.js +15 -0
- package/dist/harness/agent/builder.d.ts +456 -0
- package/dist/harness/agent/builder.js +1414 -0
- package/dist/harness/agent/config-bridge.d.ts +80 -0
- package/dist/harness/agent/config-bridge.js +424 -0
- package/dist/harness/agent/index.d.ts +15 -0
- package/dist/harness/agent/index.js +12 -0
- package/dist/harness/agent/persona.d.ts +52 -0
- package/dist/harness/agent/persona.js +166 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/index.d.ts +4 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/index.js +2 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/reader.d.ts +18 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/reader.js +83 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/writer.d.ts +36 -0
- package/dist/harness/collaboration/autonomous-subsystem/audit/writer.js +54 -0
- package/dist/harness/collaboration/autonomous-subsystem/boundary/index.d.ts +2 -0
- package/dist/harness/collaboration/autonomous-subsystem/boundary/index.js +1 -0
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +24 -0
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +142 -0
- package/dist/harness/collaboration/autonomous-subsystem/errors.d.ts +8 -0
- package/dist/harness/collaboration/autonomous-subsystem/errors.js +11 -0
- package/dist/harness/collaboration/autonomous-subsystem/index.d.ts +25 -0
- package/dist/harness/collaboration/autonomous-subsystem/index.js +28 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +34 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.js +368 -0
- package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +131 -0
- package/dist/harness/collaboration/autonomous-subsystem/runtime.js +531 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/engine.d.ts +116 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/engine.js +551 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/index.d.ts +5 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/index.js +3 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/input-builder.d.ts +13 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/input-builder.js +128 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.d.ts +46 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.js +57 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/session-lifecycle-bridge.d.ts +48 -0
- package/dist/harness/collaboration/autonomous-subsystem/sense/session-lifecycle-bridge.js +108 -0
- package/dist/harness/collaboration/autonomous-subsystem/session/index.d.ts +2 -0
- package/dist/harness/collaboration/autonomous-subsystem/session/index.js +1 -0
- package/dist/harness/collaboration/autonomous-subsystem/session/manager.d.ts +26 -0
- package/dist/harness/collaboration/autonomous-subsystem/session/manager.js +78 -0
- package/dist/harness/collaboration/autonomous-subsystem/signal/bus.d.ts +110 -0
- package/dist/harness/collaboration/autonomous-subsystem/signal/bus.js +190 -0
- package/dist/harness/collaboration/autonomous-subsystem/signal/index.d.ts +2 -0
- package/dist/harness/collaboration/autonomous-subsystem/signal/index.js +1 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.d.ts +107 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +405 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/index.d.ts +6 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/index.js +3 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/llm-port.d.ts +65 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/llm-port.js +116 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/model-resolver.d.ts +64 -0
- package/dist/harness/collaboration/autonomous-subsystem/think/model-resolver.js +89 -0
- package/dist/harness/collaboration/autonomous-subsystem/types.d.ts +559 -0
- package/dist/harness/collaboration/autonomous-subsystem/types.js +21 -0
- package/dist/harness/collaboration/multi-agent/agent-registry-types.d.ts +144 -0
- package/dist/harness/collaboration/multi-agent/agent-registry-types.js +25 -0
- package/dist/harness/collaboration/multi-agent/index.d.ts +17 -0
- package/dist/harness/collaboration/multi-agent/index.js +15 -0
- package/dist/harness/collaboration/multi-agent/message-channel-types.d.ts +69 -0
- package/dist/harness/collaboration/multi-agent/message-channel-types.js +16 -0
- package/dist/harness/collaboration/multi-agent/process.d.ts +167 -0
- package/dist/harness/collaboration/multi-agent/process.js +256 -0
- package/dist/harness/collaboration/multi-agent/registry.d.ts +36 -0
- package/dist/harness/collaboration/multi-agent/registry.js +137 -0
- package/dist/harness/collaboration/multi-agent/swarm.d.ts +135 -0
- package/dist/harness/collaboration/multi-agent/swarm.js +317 -0
- package/dist/harness/collaboration/multi-agent/types.d.ts +66 -0
- package/dist/harness/collaboration/multi-agent/types.js +15 -0
- package/dist/harness/collaboration/orchestration/async-task-store.d.ts +107 -0
- package/dist/harness/collaboration/orchestration/async-task-store.js +12 -0
- package/dist/harness/collaboration/orchestration/async-task.d.ts +156 -0
- package/dist/harness/collaboration/orchestration/async-task.js +359 -0
- package/dist/harness/collaboration/orchestration/cognitive-loop.d.ts +93 -0
- package/dist/harness/collaboration/orchestration/cognitive-loop.js +6 -0
- package/dist/harness/collaboration/orchestration/index.d.ts +19 -0
- package/dist/harness/collaboration/orchestration/index.js +16 -0
- package/dist/harness/collaboration/orchestration/planner/hybrid-planner.d.ts +47 -0
- package/dist/harness/collaboration/orchestration/planner/hybrid-planner.js +61 -0
- package/dist/harness/collaboration/orchestration/planner/index.d.ts +9 -0
- package/dist/harness/collaboration/orchestration/planner/index.js +6 -0
- package/dist/harness/collaboration/orchestration/planner/llm-planner.d.ts +54 -0
- package/dist/harness/collaboration/orchestration/planner/llm-planner.js +167 -0
- package/dist/harness/collaboration/orchestration/planner/rule-planner.d.ts +85 -0
- package/dist/harness/collaboration/orchestration/planner/rule-planner.js +175 -0
- package/dist/harness/collaboration/orchestration/quality/classifier.d.ts +21 -0
- package/dist/harness/collaboration/orchestration/quality/classifier.js +235 -0
- package/dist/harness/collaboration/orchestration/quality/gate.d.ts +29 -0
- package/dist/harness/collaboration/orchestration/quality/gate.js +409 -0
- package/dist/harness/collaboration/orchestration/quality/index.d.ts +9 -0
- package/dist/harness/collaboration/orchestration/quality/index.js +8 -0
- package/dist/harness/collaboration/orchestration/quality/types.d.ts +159 -0
- package/dist/harness/collaboration/orchestration/quality/types.js +33 -0
- package/dist/harness/collaboration/orchestration/reflector/index.d.ts +5 -0
- package/dist/harness/collaboration/orchestration/reflector/index.js +4 -0
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.d.ts +41 -0
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.js +142 -0
- package/dist/harness/collaboration/orchestration/scheduler/index.d.ts +5 -0
- package/dist/harness/collaboration/orchestration/scheduler/index.js +4 -0
- package/dist/harness/collaboration/orchestration/scheduler/task-scheduler.d.ts +143 -0
- package/dist/harness/collaboration/orchestration/scheduler/task-scheduler.js +297 -0
- package/dist/harness/collaboration/orchestration/strategy/classifier.d.ts +12 -0
- package/dist/harness/collaboration/orchestration/strategy/classifier.js +81 -0
- package/dist/harness/collaboration/orchestration/strategy/index.d.ts +3 -0
- package/dist/harness/collaboration/orchestration/strategy/index.js +2 -0
- package/dist/harness/collaboration/orchestration/strategy/router.d.ts +28 -0
- package/dist/harness/collaboration/orchestration/strategy/router.js +105 -0
- package/dist/harness/collaboration/orchestration/strategy/types.d.ts +49 -0
- package/dist/harness/collaboration/orchestration/strategy/types.js +4 -0
- package/dist/harness/collaboration/orchestration/workflow/engine.d.ts +86 -0
- package/dist/harness/collaboration/orchestration/workflow/engine.js +298 -0
- package/dist/harness/collaboration/orchestration/workflow/index.d.ts +9 -0
- package/dist/harness/collaboration/orchestration/workflow/index.js +7 -0
- package/dist/harness/collaboration/orchestration/workflow/types.d.ts +143 -0
- package/dist/harness/collaboration/orchestration/workflow/types.js +7 -0
- package/dist/harness/context/assembler.d.ts +74 -0
- package/dist/harness/context/assembler.js +315 -0
- package/dist/harness/context/budget-allocator.d.ts +41 -0
- package/dist/harness/context/budget-allocator.js +80 -0
- package/dist/harness/context/capabilities/compact/engine.d.ts +15 -0
- package/dist/harness/context/capabilities/compact/engine.js +320 -0
- package/dist/harness/context/capabilities/compact/index.d.ts +7 -0
- package/dist/harness/context/capabilities/compact/index.js +6 -0
- package/dist/harness/context/capabilities/compact/types.d.ts +78 -0
- package/dist/harness/context/capabilities/compact/types.js +6 -0
- package/dist/harness/context/capabilities/index.d.ts +10 -0
- package/dist/harness/context/capabilities/index.js +10 -0
- package/dist/harness/context/capabilities/summary/budget.d.ts +23 -0
- package/dist/harness/context/capabilities/summary/budget.js +26 -0
- package/dist/harness/context/capabilities/summary/executor.d.ts +37 -0
- package/dist/harness/context/capabilities/summary/executor.js +189 -0
- package/dist/harness/context/capabilities/summary/gate.d.ts +43 -0
- package/dist/harness/context/capabilities/summary/gate.js +69 -0
- package/dist/harness/context/capabilities/summary/index.d.ts +17 -0
- package/dist/harness/context/capabilities/summary/index.js +15 -0
- package/dist/harness/context/capabilities/summary/memory-cache.d.ts +16 -0
- package/dist/harness/context/capabilities/summary/memory-cache.js +42 -0
- package/dist/harness/context/capabilities/summary/oversized.d.ts +51 -0
- package/dist/harness/context/capabilities/summary/oversized.js +80 -0
- package/dist/harness/context/capabilities/summary/policies.d.ts +12 -0
- package/dist/harness/context/capabilities/summary/policies.js +139 -0
- package/dist/harness/context/capabilities/summary/port.d.ts +29 -0
- package/dist/harness/context/capabilities/summary/port.js +124 -0
- package/dist/harness/context/capabilities/summary/registry.d.ts +50 -0
- package/dist/harness/context/capabilities/summary/registry.js +182 -0
- package/dist/harness/context/capabilities/summary/resolver.d.ts +35 -0
- package/dist/harness/context/capabilities/summary/resolver.js +145 -0
- package/dist/harness/context/capabilities/summary/structured.d.ts +35 -0
- package/dist/harness/context/capabilities/summary/structured.js +163 -0
- package/dist/harness/context/capabilities/summary/tool-binding.d.ts +86 -0
- package/dist/harness/context/capabilities/summary/tool-binding.js +166 -0
- package/dist/harness/context/capabilities/summary/types.d.ts +171 -0
- package/dist/harness/context/capabilities/summary/types.js +6 -0
- package/dist/harness/context/compact-key.d.ts +13 -0
- package/dist/harness/context/compact-key.js +17 -0
- package/dist/harness/context/constitution/default-agents.md +227 -0
- package/dist/harness/context/constitution/load-constitution.d.ts +30 -0
- package/dist/harness/context/constitution/load-constitution.js +66 -0
- package/dist/harness/context/default-context-engine.d.ts +171 -0
- package/dist/harness/context/default-context-engine.js +836 -0
- package/dist/harness/context/hybrid-compressor.d.ts +41 -0
- package/dist/harness/context/hybrid-compressor.js +158 -0
- package/dist/harness/context/index.d.ts +26 -0
- package/dist/harness/context/index.js +17 -0
- package/dist/harness/context/layer-health.d.ts +58 -0
- package/dist/harness/context/layer-health.js +160 -0
- package/dist/harness/context/layer-snapshot.d.ts +69 -0
- package/dist/harness/context/layer-snapshot.js +133 -0
- package/dist/harness/context/layer-types.d.ts +184 -0
- package/dist/harness/context/layer-types.js +79 -0
- package/dist/harness/context/layers.d.ts +146 -0
- package/dist/harness/context/layers.js +371 -0
- package/dist/harness/context/llm-summarizer.d.ts +38 -0
- package/dist/harness/context/llm-summarizer.js +220 -0
- package/dist/harness/context/message-selector.d.ts +44 -0
- package/dist/harness/context/message-selector.js +180 -0
- package/dist/harness/context/runtime-datetime.d.ts +49 -0
- package/dist/harness/context/runtime-datetime.js +74 -0
- package/dist/harness/context/smart-router.d.ts +75 -0
- package/dist/harness/context/smart-router.js +118 -0
- package/dist/harness/context/summarize.d.ts +39 -0
- package/dist/harness/context/summarize.js +50 -0
- package/dist/harness/context/system-prompt-assembler.d.ts +56 -0
- package/dist/harness/context/system-prompt-assembler.js +122 -0
- package/dist/harness/context/token-constants.d.ts +24 -0
- package/dist/harness/context/token-constants.js +27 -0
- package/dist/harness/context/token-estimate-fns.d.ts +70 -0
- package/dist/harness/context/token-estimate-fns.js +117 -0
- package/dist/harness/context/token-estimator.d.ts +63 -0
- package/dist/harness/context/token-estimator.js +126 -0
- package/dist/harness/context/truncate-compressor.d.ts +28 -0
- package/dist/harness/context/truncate-compressor.js +100 -0
- package/dist/harness/context/types.d.ts +316 -0
- package/dist/harness/context/types.js +9 -0
- package/dist/harness/extension/execution-environment/index.d.ts +10 -0
- package/dist/harness/extension/execution-environment/index.js +8 -0
- package/dist/harness/extension/execution-environment/isolation.d.ts +56 -0
- package/dist/harness/extension/execution-environment/isolation.js +56 -0
- package/dist/harness/extension/execution-environment/sandbox.d.ts +12 -0
- package/dist/harness/extension/execution-environment/sandbox.js +87 -0
- package/dist/harness/extension/execution-environment/types.d.ts +157 -0
- package/dist/harness/extension/execution-environment/types.js +6 -0
- package/dist/harness/extension/execution-environment/workspace.d.ts +26 -0
- package/dist/harness/extension/execution-environment/workspace.js +171 -0
- package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +34 -0
- package/dist/harness/extension/plugin-ecosystem/channel-types.js +12 -0
- package/dist/harness/extension/plugin-ecosystem/commands/builtin.d.ts +34 -0
- package/dist/harness/extension/plugin-ecosystem/commands/builtin.js +185 -0
- package/dist/harness/extension/plugin-ecosystem/commands/index.d.ts +15 -0
- package/dist/harness/extension/plugin-ecosystem/commands/index.js +11 -0
- package/dist/harness/extension/plugin-ecosystem/commands/parse.d.ts +17 -0
- package/dist/harness/extension/plugin-ecosystem/commands/parse.js +41 -0
- package/dist/harness/extension/plugin-ecosystem/commands/plugin-bridge.d.ts +6 -0
- package/dist/harness/extension/plugin-ecosystem/commands/plugin-bridge.js +49 -0
- package/dist/harness/extension/plugin-ecosystem/commands/router.d.ts +71 -0
- package/dist/harness/extension/plugin-ecosystem/commands/router.js +302 -0
- package/dist/harness/extension/plugin-ecosystem/commands/skill-bridge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/commands/skill-bridge.js +59 -0
- package/dist/harness/extension/plugin-ecosystem/commands/types.d.ts +126 -0
- package/dist/harness/extension/plugin-ecosystem/commands/types.js +7 -0
- package/dist/harness/extension/plugin-ecosystem/commands/user-source.d.ts +16 -0
- package/dist/harness/extension/plugin-ecosystem/commands/user-source.js +102 -0
- package/dist/harness/extension/plugin-ecosystem/index.d.ts +13 -0
- package/dist/harness/extension/plugin-ecosystem/index.js +10 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/bridge.d.ts +42 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/bridge.js +88 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/discovery.d.ts +39 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/discovery.js +117 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/index.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/index.js +8 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/manager.d.ts +74 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/manager.js +146 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/types.d.ts +149 -0
- package/dist/harness/extension/plugin-ecosystem/mcp/types.js +12 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/api.d.ts +294 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/api.js +270 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.d.ts +100 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.js +158 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.d.ts +101 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.js +65 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/hooks.d.ts +7 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/hooks.js +7 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/index.d.ts +26 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/index.js +26 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/lifecycle.d.ts +86 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/lifecycle.js +139 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.d.ts +135 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.js +353 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.d.ts +504 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.js +368 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.d.ts +261 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.js +44 -0
- package/dist/harness/extension/plugin-ecosystem/skills/manager.d.ts +123 -0
- package/dist/harness/extension/plugin-ecosystem/skills/manager.js +210 -0
- package/dist/harness/extension/plugin-ecosystem/skills/types.d.ts +29 -0
- package/dist/harness/extension/plugin-ecosystem/skills/types.js +7 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +34 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +92 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +101 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +570 -0
- package/dist/harness/extension/plugin-ecosystem/tools/env-info.d.ts +8 -0
- package/dist/harness/extension/plugin-ecosystem/tools/env-info.js +49 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-edit.d.ts +14 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-edit.js +126 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.d.ts +23 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +659 -0
- package/dist/harness/extension/plugin-ecosystem/tools/http.d.ts +28 -0
- package/dist/harness/extension/plugin-ecosystem/tools/http.js +170 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +14 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +12 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +25 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +164 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.d.ts +19 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +197 -0
- package/dist/harness/extension/plugin-ecosystem/tools/platform.d.ts +82 -0
- package/dist/harness/extension/plugin-ecosystem/tools/platform.js +251 -0
- package/dist/harness/extension/plugin-ecosystem/tools/session-history.d.ts +21 -0
- package/dist/harness/extension/plugin-ecosystem/tools/session-history.js +165 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.d.ts +65 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.js +262 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +34 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +30 -0
- package/dist/harness/extension/plugin-ecosystem/tools/web-search-types.d.ts +66 -0
- package/dist/harness/extension/plugin-ecosystem/tools/web-search-types.js +12 -0
- package/dist/harness/extension/plugin-ecosystem/tools/web-search.d.ts +23 -0
- package/dist/harness/extension/plugin-ecosystem/tools/web-search.js +84 -0
- package/dist/harness/governance/accounting/index.d.ts +5 -0
- package/dist/harness/governance/accounting/index.js +3 -0
- package/dist/harness/governance/accounting/session-ledger.d.ts +67 -0
- package/dist/harness/governance/accounting/session-ledger.js +84 -0
- package/dist/harness/governance/accounting/usage-ledger.d.ts +107 -0
- package/dist/harness/governance/accounting/usage-ledger.js +137 -0
- package/dist/harness/governance/credentials/crypto.d.ts +34 -0
- package/dist/harness/governance/credentials/crypto.js +79 -0
- package/dist/harness/governance/credentials/db.d.ts +32 -0
- package/dist/harness/governance/credentials/db.js +87 -0
- package/dist/harness/governance/credentials/index.d.ts +9 -0
- package/dist/harness/governance/credentials/index.js +7 -0
- package/dist/harness/governance/credentials/store.d.ts +47 -0
- package/dist/harness/governance/credentials/store.js +228 -0
- package/dist/harness/governance/credentials/types.d.ts +67 -0
- package/dist/harness/governance/credentials/types.js +30 -0
- package/dist/harness/governance/human-in-the-loop/approval-manager.d.ts +32 -0
- package/dist/harness/governance/human-in-the-loop/approval-manager.js +93 -0
- package/dist/harness/governance/human-in-the-loop/index.d.ts +9 -0
- package/dist/harness/governance/human-in-the-loop/index.js +7 -0
- package/dist/harness/governance/human-in-the-loop/types.d.ts +66 -0
- package/dist/harness/governance/human-in-the-loop/types.js +6 -0
- package/dist/harness/governance/index.d.ts +10 -0
- package/dist/harness/governance/index.js +10 -0
- package/dist/harness/governance/security/capability-enforcer.d.ts +56 -0
- package/dist/harness/governance/security/capability-enforcer.js +132 -0
- package/dist/harness/governance/security/default-risk-policy.d.ts +66 -0
- package/dist/harness/governance/security/default-risk-policy.js +75 -0
- package/dist/harness/governance/security/default-security-guard.d.ts +105 -0
- package/dist/harness/governance/security/default-security-guard.js +578 -0
- package/dist/harness/governance/security/degradation.d.ts +14 -0
- package/dist/harness/governance/security/degradation.js +208 -0
- package/dist/harness/governance/security/index.d.ts +10 -0
- package/dist/harness/governance/security/index.js +10 -0
- package/dist/harness/governance/security/risk-evaluator.d.ts +68 -0
- package/dist/harness/governance/security/risk-evaluator.js +1043 -0
- package/dist/harness/governance/security/risk-types.d.ts +74 -0
- package/dist/harness/governance/security/risk-types.js +7 -0
- package/dist/harness/governance/security/shell-parser.d.ts +35 -0
- package/dist/harness/governance/security/shell-parser.js +525 -0
- package/dist/harness/governance/security/tool-call-risk-policy.d.ts +6 -0
- package/dist/harness/governance/security/tool-call-risk-policy.js +6 -0
- package/dist/harness/governance/session-acl/index.d.ts +11 -0
- package/dist/harness/governance/session-acl/index.js +7 -0
- package/dist/harness/governance/session-acl/rights.d.ts +78 -0
- package/dist/harness/governance/session-acl/rights.js +142 -0
- package/dist/harness/governance/session-acl/seed-roles.d.ts +16 -0
- package/dist/harness/governance/session-acl/seed-roles.js +85 -0
- package/dist/harness/governance/session-acl/service.d.ts +136 -0
- package/dist/harness/governance/session-acl/service.js +407 -0
- package/dist/harness/governance/session-acl/types.d.ts +102 -0
- package/dist/harness/governance/session-acl/types.js +5 -0
- package/dist/harness/index.d.ts +137 -0
- package/dist/harness/index.js +106 -0
- package/dist/harness/knowledge/adapters.d.ts +68 -0
- package/dist/harness/knowledge/adapters.js +421 -0
- package/dist/harness/knowledge/catalog-index.d.ts +7 -0
- package/dist/harness/knowledge/catalog-index.js +7 -0
- package/dist/harness/knowledge/catalog-types.d.ts +33 -0
- package/dist/harness/knowledge/catalog-types.js +10 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +13 -0
- package/dist/harness/knowledge/connector-fetcher.js +27 -0
- package/dist/harness/knowledge/connectors.d.ts +55 -0
- package/dist/harness/knowledge/connectors.js +181 -0
- package/dist/harness/knowledge/db.d.ts +26 -0
- package/dist/harness/knowledge/db.js +235 -0
- package/dist/harness/knowledge/describe.d.ts +40 -0
- package/dist/harness/knowledge/describe.js +52 -0
- package/dist/harness/knowledge/fetchers.d.ts +66 -0
- package/dist/harness/knowledge/fetchers.js +342 -0
- package/dist/harness/knowledge/grounding.d.ts +98 -0
- package/dist/harness/knowledge/grounding.js +208 -0
- package/dist/harness/knowledge/hit-log.d.ts +74 -0
- package/dist/harness/knowledge/hit-log.js +99 -0
- package/dist/harness/knowledge/html.d.ts +20 -0
- package/dist/harness/knowledge/html.js +103 -0
- package/dist/harness/knowledge/index-store.d.ts +150 -0
- package/dist/harness/knowledge/index-store.js +394 -0
- package/dist/harness/knowledge/index.d.ts +38 -0
- package/dist/harness/knowledge/index.js +24 -0
- package/dist/harness/knowledge/ingest.d.ts +162 -0
- package/dist/harness/knowledge/ingest.js +814 -0
- package/dist/harness/knowledge/network-guard.d.ts +57 -0
- package/dist/harness/knowledge/network-guard.js +242 -0
- package/dist/harness/knowledge/paths.d.ts +14 -0
- package/dist/harness/knowledge/paths.js +17 -0
- package/dist/harness/knowledge/purge.d.ts +34 -0
- package/dist/harness/knowledge/purge.js +51 -0
- package/dist/harness/knowledge/retriever.d.ts +83 -0
- package/dist/harness/knowledge/retriever.js +184 -0
- package/dist/harness/knowledge/secret-scan.d.ts +11 -0
- package/dist/harness/knowledge/secret-scan.js +32 -0
- package/dist/harness/knowledge/source-store.d.ts +107 -0
- package/dist/harness/knowledge/source-store.js +516 -0
- package/dist/harness/knowledge/types.d.ts +129 -0
- package/dist/harness/knowledge/types.js +14 -0
- package/dist/harness/memory/backfill-coverage.d.ts +80 -0
- package/dist/harness/memory/backfill-coverage.js +146 -0
- package/dist/harness/memory/backfill-trigger.d.ts +83 -0
- package/dist/harness/memory/backfill-trigger.js +247 -0
- package/dist/harness/memory/cognition-types.d.ts +4 -0
- package/dist/harness/memory/cognition-types.js +4 -0
- package/dist/harness/memory/cognition.d.ts +16 -0
- package/dist/harness/memory/cognition.js +107 -0
- package/dist/harness/memory/confidence.d.ts +63 -0
- package/dist/harness/memory/confidence.js +157 -0
- package/dist/harness/memory/decay-policy.d.ts +22 -0
- package/dist/harness/memory/decay-policy.js +28 -0
- package/dist/harness/memory/gates.d.ts +39 -0
- package/dist/harness/memory/gates.js +176 -0
- package/dist/harness/memory/health-probe.d.ts +36 -0
- package/dist/harness/memory/health-probe.js +78 -0
- package/dist/harness/memory/index.d.ts +32 -0
- package/dist/harness/memory/index.js +26 -0
- package/dist/harness/memory/similarity.d.ts +33 -0
- package/dist/harness/memory/similarity.js +69 -0
- package/dist/harness/memory/sqlite/agent-db.d.ts +73 -0
- package/dist/harness/memory/sqlite/agent-db.js +277 -0
- package/dist/harness/memory/sqlite/backfill-coverage.d.ts +13 -0
- package/dist/harness/memory/sqlite/backfill-coverage.js +57 -0
- package/dist/harness/memory/sqlite/cognition-store.d.ts +34 -0
- package/dist/harness/memory/sqlite/cognition-store.js +188 -0
- package/dist/harness/memory/sqlite/embedding-from-models.d.ts +47 -0
- package/dist/harness/memory/sqlite/embedding-from-models.js +90 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +93 -0
- package/dist/harness/memory/sqlite/embedding.js +268 -0
- package/dist/harness/memory/sqlite/index.d.ts +22 -0
- package/dist/harness/memory/sqlite/index.js +17 -0
- package/dist/harness/memory/sqlite/keyword-search.d.ts +37 -0
- package/dist/harness/memory/sqlite/keyword-search.js +113 -0
- package/dist/harness/memory/sqlite/knowledge-registry.d.ts +45 -0
- package/dist/harness/memory/sqlite/knowledge-registry.js +59 -0
- package/dist/harness/memory/sqlite/memory-store.d.ts +80 -0
- package/dist/harness/memory/sqlite/memory-store.js +548 -0
- package/dist/harness/memory/sqlite/sqlite-vec.d.ts +45 -0
- package/dist/harness/memory/sqlite/sqlite-vec.js +159 -0
- package/dist/harness/memory/sqlite/vector-search.d.ts +41 -0
- package/dist/harness/memory/sqlite/vector-search.js +72 -0
- package/dist/harness/memory/sqlite/wisdom-store.d.ts +23 -0
- package/dist/harness/memory/sqlite/wisdom-store.js +64 -0
- package/dist/harness/memory/steward-policy.d.ts +122 -0
- package/dist/harness/memory/steward-policy.js +369 -0
- package/dist/harness/memory/store.d.ts +33 -0
- package/dist/harness/memory/store.js +173 -0
- package/dist/harness/memory/types.d.ts +170 -0
- package/dist/harness/memory/types.js +7 -0
- package/dist/harness/memory/wisdom-types.d.ts +4 -0
- package/dist/harness/memory/wisdom-types.js +4 -0
- package/dist/harness/observability/diagnostics/index.d.ts +5 -0
- package/dist/harness/observability/diagnostics/index.js +4 -0
- package/dist/harness/observability/diagnostics/registry.d.ts +19 -0
- package/dist/harness/observability/diagnostics/registry.js +74 -0
- package/dist/harness/observability/diagnostics/types.d.ts +42 -0
- package/dist/harness/observability/diagnostics/types.js +6 -0
- package/dist/harness/observability/observer/hub.d.ts +107 -0
- package/dist/harness/observability/observer/hub.js +907 -0
- package/dist/harness/observability/observer/index.d.ts +4 -0
- package/dist/harness/observability/observer/index.js +3 -0
- package/dist/harness/observability/observer/run-snapshot.d.ts +47 -0
- package/dist/harness/observability/observer/run-snapshot.js +65 -0
- package/dist/harness/observability/observer/types.d.ts +370 -0
- package/dist/harness/observability/observer/types.js +379 -0
- package/dist/harness/observability/run-telemetry.d.ts +51 -0
- package/dist/harness/observability/run-telemetry.js +28 -0
- package/dist/harness/run/agent/agent.d.ts +142 -0
- package/dist/harness/run/agent/agent.js +192 -0
- package/dist/harness/run/agent/index.d.ts +8 -0
- package/dist/harness/run/agent/index.js +7 -0
- package/dist/harness/run/budget/budget.d.ts +94 -0
- package/dist/harness/run/budget/budget.js +124 -0
- package/dist/harness/run/budget/index.d.ts +7 -0
- package/dist/harness/run/budget/index.js +6 -0
- package/dist/harness/run/concurrency/index.d.ts +18 -0
- package/dist/harness/run/concurrency/index.js +14 -0
- package/dist/harness/run/concurrency/provider-pool.d.ts +128 -0
- package/dist/harness/run/concurrency/provider-pool.js +329 -0
- package/dist/harness/run/concurrency/rate-limiter.d.ts +109 -0
- package/dist/harness/run/concurrency/rate-limiter.js +233 -0
- package/dist/harness/run/concurrency/session-gate.d.ts +85 -0
- package/dist/harness/run/concurrency/session-gate.js +176 -0
- package/dist/harness/run/concurrency/session-lease.d.ts +79 -0
- package/dist/harness/run/concurrency/session-lease.js +100 -0
- package/dist/harness/run/concurrency/tool-loop-detection-core.d.ts +41 -0
- package/dist/harness/run/concurrency/tool-loop-detection-core.js +239 -0
- package/dist/harness/run/concurrency/tool-loop-detection.d.ts +8 -0
- package/dist/harness/run/concurrency/tool-loop-detection.js +7 -0
- package/dist/harness/run/concurrency/tool-validator.d.ts +96 -0
- package/dist/harness/run/concurrency/tool-validator.js +195 -0
- package/dist/harness/run/index.d.ts +14 -0
- package/dist/harness/run/index.js +14 -0
- package/dist/harness/run/model/index.d.ts +9 -0
- package/dist/harness/run/model/index.js +7 -0
- package/dist/harness/run/model/resolver.d.ts +111 -0
- package/dist/harness/run/model/resolver.js +192 -0
- package/dist/harness/run/model/run-scope.d.ts +43 -0
- package/dist/harness/run/model/run-scope.js +64 -0
- package/dist/harness/run/model/types.d.ts +47 -0
- package/dist/harness/run/model/types.js +9 -0
- package/dist/harness/run/reliability/circuit-breaker.d.ts +87 -0
- package/dist/harness/run/reliability/circuit-breaker.js +130 -0
- package/dist/harness/run/reliability/fallback-provider.d.ts +40 -0
- package/dist/harness/run/reliability/fallback-provider.js +118 -0
- package/dist/harness/run/reliability/harness-events.d.ts +109 -0
- package/dist/harness/run/reliability/harness-events.js +7 -0
- package/dist/harness/run/reliability/index.d.ts +12 -0
- package/dist/harness/run/reliability/index.js +7 -0
- package/dist/harness/run/reliability/model-binding.d.ts +22 -0
- package/dist/harness/run/reliability/model-binding.js +27 -0
- package/dist/harness/run/reliability/provider-wrapper.d.ts +26 -0
- package/dist/harness/run/reliability/provider-wrapper.js +61 -0
- package/dist/harness/run/reliability/run-agent.d.ts +86 -0
- package/dist/harness/run/reliability/run-agent.js +932 -0
- package/dist/harness/run/reliability/run-metrics-collector.d.ts +80 -0
- package/dist/harness/run/reliability/run-metrics-collector.js +194 -0
- package/dist/harness/run/reliability/run-model-context.d.ts +19 -0
- package/dist/harness/run/reliability/run-model-context.js +32 -0
- package/dist/harness/run/run-guard/default-run-guard.d.ts +75 -0
- package/dist/harness/run/run-guard/default-run-guard.js +347 -0
- package/dist/harness/run/run-guard/index.d.ts +9 -0
- package/dist/harness/run/run-guard/index.js +8 -0
- package/dist/harness/run/run-scope.d.ts +81 -0
- package/dist/harness/run/run-scope.js +58 -0
- package/dist/harness/run/runner.d.ts +362 -0
- package/dist/harness/run/runner.js +1290 -0
- package/dist/harness/session/attachments/extract.d.ts +35 -0
- package/dist/harness/session/attachments/extract.js +72 -0
- package/dist/harness/session/attachments/index.d.ts +11 -0
- package/dist/harness/session/attachments/index.js +11 -0
- package/dist/harness/session/attachments/inject.d.ts +71 -0
- package/dist/harness/session/attachments/inject.js +142 -0
- package/dist/harness/session/attachments/intent.d.ts +32 -0
- package/dist/harness/session/attachments/intent.js +69 -0
- package/dist/harness/session/attachments/llm-intent.d.ts +17 -0
- package/dist/harness/session/attachments/llm-intent.js +74 -0
- package/dist/harness/session/attachments/paths.d.ts +15 -0
- package/dist/harness/session/attachments/paths.js +18 -0
- package/dist/harness/session/attachments/sanitize.d.ts +23 -0
- package/dist/harness/session/attachments/sanitize.js +55 -0
- package/dist/harness/session/attachments/service.d.ts +105 -0
- package/dist/harness/session/attachments/service.js +320 -0
- package/dist/harness/session/attachments/types.d.ts +64 -0
- package/dist/harness/session/attachments/types.js +17 -0
- package/dist/harness/session/compact.d.ts +23 -0
- package/dist/harness/session/compact.js +53 -0
- package/dist/harness/session/history/index.d.ts +4 -0
- package/dist/harness/session/history/index.js +3 -0
- package/dist/harness/session/history/port.d.ts +51 -0
- package/dist/harness/session/history/port.js +480 -0
- package/dist/harness/session/history/score.d.ts +71 -0
- package/dist/harness/session/history/score.js +225 -0
- package/dist/harness/session/history/types.d.ts +157 -0
- package/dist/harness/session/history/types.js +29 -0
- package/dist/harness/session/in-memory-store.d.ts +21 -0
- package/dist/harness/session/in-memory-store.js +48 -0
- package/dist/harness/session/index.d.ts +12 -0
- package/dist/harness/session/index.js +12 -0
- package/dist/harness/session/state-machine.d.ts +18 -0
- package/dist/harness/session/state-machine.js +31 -0
- package/dist/harness/session/tasks/index.d.ts +11 -0
- package/dist/harness/session/tasks/index.js +9 -0
- package/dist/harness/session/tasks/render.d.ts +12 -0
- package/dist/harness/session/tasks/render.js +57 -0
- package/dist/harness/session/tasks/service.d.ts +69 -0
- package/dist/harness/session/tasks/service.js +327 -0
- package/dist/harness/session/tasks/tools.d.ts +17 -0
- package/dist/harness/session/tasks/tools.js +210 -0
- package/dist/harness/session/tasks/types.d.ts +29 -0
- package/dist/harness/session/tasks/types.js +11 -0
- package/dist/harness/session/types.d.ts +88 -0
- package/dist/harness/session/types.js +7 -0
- package/dist/harness/shared/events/agent-event-map.d.ts +264 -0
- package/dist/harness/shared/events/agent-event-map.js +51 -0
- package/dist/harness/shared/events/index.d.ts +6 -0
- package/dist/harness/shared/events/index.js +4 -0
- package/dist/harness/shared/events/scenario-events.d.ts +96 -0
- package/dist/harness/shared/events/scenario-events.js +7 -0
- package/dist/harness/shared/types/agent-definition.d.ts +51 -0
- package/dist/harness/shared/types/agent-definition.js +8 -0
- package/dist/harness/shared/types/hook-context.d.ts +10 -0
- package/dist/harness/shared/types/hook-context.js +6 -0
- package/dist/harness/shared/types/index.d.ts +7 -0
- package/dist/harness/shared/types/index.js +4 -0
- package/dist/harness/shared/types/queue-mode.d.ts +4 -0
- package/dist/harness/shared/types/queue-mode.js +1 -0
- package/dist/harness/shared/types/thinking-level.d.ts +4 -0
- package/dist/harness/shared/types/thinking-level.js +1 -0
- package/dist/index.d.ts +117 -0
- package/dist/index.js +80 -0
- package/dist/integration/agent-runtime/channel-message-source.d.ts +27 -0
- package/dist/integration/agent-runtime/channel-message-source.js +58 -0
- package/dist/integration/agent-runtime/file-watch-source.d.ts +31 -0
- package/dist/integration/agent-runtime/file-watch-source.js +63 -0
- package/dist/integration/agent-runtime/index.d.ts +6 -0
- package/dist/integration/agent-runtime/index.js +3 -0
- package/dist/integration/agent-runtime/webhook-source.d.ts +39 -0
- package/dist/integration/agent-runtime/webhook-source.js +112 -0
- package/dist/integration/index.d.ts +23 -0
- package/dist/integration/index.js +23 -0
- package/dist/integration/mcp/index.d.ts +6 -0
- package/dist/integration/mcp/index.js +6 -0
- package/dist/integration/mcp/sdk-client.d.ts +39 -0
- package/dist/integration/mcp/sdk-client.js +157 -0
- package/dist/integration/observability/exporters.d.ts +111 -0
- package/dist/integration/observability/exporters.js +167 -0
- package/dist/integration/observability/index.d.ts +21 -0
- package/dist/integration/observability/index.js +16 -0
- package/dist/integration/observability/log-observer.d.ts +12 -0
- package/dist/integration/observability/log-observer.js +59 -0
- package/dist/integration/observability/metrics.d.ts +116 -0
- package/dist/integration/observability/metrics.js +237 -0
- package/dist/integration/observability/noop-observer.d.ts +12 -0
- package/dist/integration/observability/noop-observer.js +22 -0
- package/dist/integration/observability/observer-bridge.d.ts +56 -0
- package/dist/integration/observability/observer-bridge.js +195 -0
- package/dist/integration/observability/run-telemetry.d.ts +28 -0
- package/dist/integration/observability/run-telemetry.js +83 -0
- package/dist/integration/observability/trace-collector.d.ts +124 -0
- package/dist/integration/observability/trace-collector.js +253 -0
- package/dist/integration/observability/trace-events.d.ts +69 -0
- package/dist/integration/observability/trace-events.js +83 -0
- package/dist/integration/observability/trace-logger.d.ts +110 -0
- package/dist/integration/observability/trace-logger.js +219 -0
- package/dist/integration/providers/anthropic.d.ts +92 -0
- package/dist/integration/providers/anthropic.js +432 -0
- package/dist/integration/providers/index.d.ts +4 -0
- package/dist/integration/providers/index.js +2 -0
- package/dist/integration/providers/openai.d.ts +120 -0
- package/dist/integration/providers/openai.js +429 -0
- package/dist/integration/providers/usage.d.ts +20 -0
- package/dist/integration/providers/usage.js +69 -0
- package/dist/integration/storage/archive-manager.d.ts +79 -0
- package/dist/integration/storage/archive-manager.js +223 -0
- package/dist/integration/storage/jsonl.d.ts +51 -0
- package/dist/integration/storage/jsonl.js +246 -0
- package/dist/integration/storage/memory.d.ts +7 -0
- package/dist/integration/storage/memory.js +7 -0
- package/dist/integration/storage/session-index.d.ts +57 -0
- package/dist/integration/storage/session-index.js +379 -0
- package/dist/integration/web-search/brave.d.ts +13 -0
- package/dist/integration/web-search/brave.js +64 -0
- package/dist/integration/web-search/duckduckgo.d.ts +12 -0
- package/dist/integration/web-search/duckduckgo.js +102 -0
- package/dist/integration/web-search/factory.d.ts +91 -0
- package/dist/integration/web-search/factory.js +166 -0
- package/dist/integration/web-search/http.d.ts +22 -0
- package/dist/integration/web-search/http.js +94 -0
- package/dist/integration/web-search/index.d.ts +16 -0
- package/dist/integration/web-search/index.js +9 -0
- package/dist/integration/web-search/mimo.d.ts +57 -0
- package/dist/integration/web-search/mimo.js +226 -0
- package/dist/integration/web-search/serper.d.ts +13 -0
- package/dist/integration/web-search/serper.js +59 -0
- package/dist/integration/web-search/tavily.d.ts +16 -0
- package/dist/integration/web-search/tavily.js +56 -0
- package/dist/paths.d.ts +14 -0
- package/dist/paths.js +22 -0
- package/dist/subsystems/memory-steward/backfill/SUBSYSTEM.md +30 -0
- package/dist/subsystems/memory-steward/backfill/config.yaml +51 -0
- package/dist/subsystems/memory-steward/backfill/handler.d.ts +18 -0
- package/dist/subsystems/memory-steward/backfill/handler.js +343 -0
- package/dist/subsystems/memory-steward/govern/SUBSYSTEM.md +12 -0
- package/dist/subsystems/memory-steward/govern/config.yaml +65 -0
- package/dist/subsystems/memory-steward/govern/handler.d.ts +19 -0
- package/dist/subsystems/memory-steward/govern/handler.js +79 -0
- package/dist/subsystems/safety-guard/SUBSYSTEM.md +60 -0
- package/dist/subsystems/safety-guard/config.yaml +40 -0
- package/package.json +110 -0
|
@@ -0,0 +1,814 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
|
|
3
|
+
*
|
|
4
|
+
* 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
|
|
5
|
+
*/
|
|
6
|
+
import { randomUUID } from 'node:crypto';
|
|
7
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
8
|
+
import { watch } from 'node:fs';
|
|
9
|
+
import { join, sep } from 'node:path';
|
|
10
|
+
import { EventEmitter } from 'node:events';
|
|
11
|
+
import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter } from './adapters.js';
|
|
12
|
+
import { looksLikeHtml } from './html.js';
|
|
13
|
+
import { hashContent, KnowledgeIndexStore } from './index-store.js';
|
|
14
|
+
import { LocalFsFetcher, UrlFetcher } from './fetchers.js';
|
|
15
|
+
import { ConnectorRegistry } from './connectors.js';
|
|
16
|
+
import { ConnectorFetcher } from './connector-fetcher.js';
|
|
17
|
+
export class KnowledgeIngest extends EventEmitter {
|
|
18
|
+
sources;
|
|
19
|
+
index;
|
|
20
|
+
adapters;
|
|
21
|
+
parseConcurrency;
|
|
22
|
+
maxQueueDepth;
|
|
23
|
+
debounceMs;
|
|
24
|
+
maxFileBytes;
|
|
25
|
+
embedding;
|
|
26
|
+
embedBatch;
|
|
27
|
+
embedMinIntervalMs;
|
|
28
|
+
embedConcurrency;
|
|
29
|
+
diskWatermarkAlert;
|
|
30
|
+
credentials;
|
|
31
|
+
localFetcher;
|
|
32
|
+
urlFetcher;
|
|
33
|
+
connectorFetcher;
|
|
34
|
+
pollTickMs;
|
|
35
|
+
pollMinIntervalMs;
|
|
36
|
+
maxPollPerTick;
|
|
37
|
+
lastEmbedAt = 0;
|
|
38
|
+
pollTimer = null;
|
|
39
|
+
running = 0;
|
|
40
|
+
runningEmbed = 0;
|
|
41
|
+
draining = false;
|
|
42
|
+
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
43
|
+
sourceLocks = new Set();
|
|
44
|
+
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
45
|
+
discoveredFiles = new Map();
|
|
46
|
+
watchers = new Map();
|
|
47
|
+
debounceTimers = new Map();
|
|
48
|
+
disposed = false;
|
|
49
|
+
constructor(options) {
|
|
50
|
+
super();
|
|
51
|
+
this.sources = options.sourceStore;
|
|
52
|
+
this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
|
|
53
|
+
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
54
|
+
this.parseConcurrency = options.parseConcurrency ?? 4;
|
|
55
|
+
this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
|
|
56
|
+
this.debounceMs = options.debounceMs ?? 2000;
|
|
57
|
+
this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
|
|
58
|
+
this.embedding = options.embeddingProvider ?? null;
|
|
59
|
+
this.embedBatch = options.embedBatch ?? 32;
|
|
60
|
+
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
61
|
+
this.embedConcurrency = options.embedConcurrency ?? options.parseConcurrency ?? 2;
|
|
62
|
+
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
63
|
+
this.credentials = options.credentials ?? null;
|
|
64
|
+
this.localFetcher =
|
|
65
|
+
options.fetchers?.local ?? new LocalFsFetcher((p) => this.adapters.shouldSkipPath(p));
|
|
66
|
+
this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
|
|
67
|
+
this.connectorFetcher =
|
|
68
|
+
options.fetchers?.connector ??
|
|
69
|
+
new ConnectorFetcher(options.connectors ?? new ConnectorRegistry());
|
|
70
|
+
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
71
|
+
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
72
|
+
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
73
|
+
}
|
|
74
|
+
get indexStore() {
|
|
75
|
+
return this.index;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* 全量/增量索引某源(Phase A)
|
|
79
|
+
*/
|
|
80
|
+
async ingestSource(sourceId, opts) {
|
|
81
|
+
const source = this.sources.get(sourceId);
|
|
82
|
+
if (!source || source.status === 'removed') {
|
|
83
|
+
throw new Error(`knowledge source not found: ${sourceId}`);
|
|
84
|
+
}
|
|
85
|
+
if (source.kind !== 'directory' &&
|
|
86
|
+
source.kind !== 'file' &&
|
|
87
|
+
source.kind !== 'workspace' &&
|
|
88
|
+
source.kind !== 'url' &&
|
|
89
|
+
source.kind !== 'connector') {
|
|
90
|
+
this.sources.update(sourceId, {
|
|
91
|
+
status: 'error',
|
|
92
|
+
errors: [
|
|
93
|
+
{
|
|
94
|
+
message: `kind ${source.kind} not supported`,
|
|
95
|
+
at: Date.now(),
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
});
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (this.sourceLocks.has(sourceId)) {
|
|
102
|
+
// 已有同源任务在跑:入队 walk 即可(去重)
|
|
103
|
+
this.enqueue(sourceId, 'walk_source', null, 1, source.location);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
this.sourceLocks.add(sourceId);
|
|
107
|
+
try {
|
|
108
|
+
this.sources.update(sourceId, { status: 'discovering', coverage: 0 });
|
|
109
|
+
this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
|
|
110
|
+
// full 不再 clearSource:靠本轮 keepPaths 差量 prune,discover 失败不丢库
|
|
111
|
+
if (source.kind === 'url' || source.kind === 'connector') {
|
|
112
|
+
await this.ingestRemoteSource(source);
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const files = await this.localFetcher.discover(source);
|
|
116
|
+
this.discoveredFiles.set(sourceId, files.length);
|
|
117
|
+
// 差量 prune:磁盘上已消失的 path
|
|
118
|
+
const localKeep = new Set(files.map((f) => f.path));
|
|
119
|
+
this.index.pruneMissing(sourceId, localKeep);
|
|
120
|
+
this.sources.update(sourceId, { status: 'partial', coverage: 0 });
|
|
121
|
+
this.emitProgress({
|
|
122
|
+
type: 'source',
|
|
123
|
+
sourceId,
|
|
124
|
+
status: 'partial',
|
|
125
|
+
detail: `${files.length} files`,
|
|
126
|
+
});
|
|
127
|
+
for (const file of files) {
|
|
128
|
+
this.enqueue(sourceId, 'parse_file', file.path, 2, file.path);
|
|
129
|
+
}
|
|
130
|
+
if (this.embedding) {
|
|
131
|
+
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
132
|
+
}
|
|
133
|
+
this.kick();
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
this.sourceLocks.delete(sourceId);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* URL / connector 源:discover + fetch + 索引
|
|
141
|
+
*/
|
|
142
|
+
async ingestRemoteSource(source) {
|
|
143
|
+
let cred;
|
|
144
|
+
try {
|
|
145
|
+
cred = await this.resolveCred(source);
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
149
|
+
// fail-closed:不降级匿名抓取,也不动已有索引
|
|
150
|
+
this.sources.update(source.id, {
|
|
151
|
+
status: 'error',
|
|
152
|
+
errors: [...(source.errors ?? []), { message: msg, at: Date.now() }].slice(-5),
|
|
153
|
+
});
|
|
154
|
+
this.emitProgress({ type: 'error', sourceId: source.id, status: 'credential_failed', detail: msg });
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
const fetcher = source.kind === 'connector' ? this.connectorFetcher : this.urlFetcher;
|
|
158
|
+
let refs;
|
|
159
|
+
try {
|
|
160
|
+
refs = await fetcher.discover(source, cred);
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
164
|
+
this.sources.update(source.id, {
|
|
165
|
+
status: 'error',
|
|
166
|
+
errors: [...(source.errors ?? []), { message: msg, at: Date.now() }].slice(-5),
|
|
167
|
+
});
|
|
168
|
+
this.emitProgress({ type: 'error', sourceId: source.id, status: 'discover_failed', detail: msg });
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
// 条件 GET:带上已有 etag/Last-Modified,304 则整文档跳过
|
|
172
|
+
const enriched = refs.map((ref) => {
|
|
173
|
+
const existing = this.index.getFile(source.id, ref.path);
|
|
174
|
+
return {
|
|
175
|
+
...ref,
|
|
176
|
+
etag: ref.etag ?? existing?.etag,
|
|
177
|
+
lastModified: ref.lastModified ?? existing?.lastModified,
|
|
178
|
+
};
|
|
179
|
+
});
|
|
180
|
+
this.discoveredFiles.set(source.id, enriched.length);
|
|
181
|
+
this.sources.update(source.id, { status: 'partial', coverage: 0 });
|
|
182
|
+
this.emitProgress({
|
|
183
|
+
type: 'source',
|
|
184
|
+
sourceId: source.id,
|
|
185
|
+
status: 'partial',
|
|
186
|
+
detail: `${enriched.length} docs`,
|
|
187
|
+
});
|
|
188
|
+
let okCount = 0;
|
|
189
|
+
let unchanged = 0;
|
|
190
|
+
for (const ref of enriched) {
|
|
191
|
+
try {
|
|
192
|
+
const doc = await fetcher.fetch(source, ref, cred);
|
|
193
|
+
if (!doc) {
|
|
194
|
+
// 304 / 未变:保持原索引
|
|
195
|
+
unchanged += 1;
|
|
196
|
+
okCount += 1;
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
if (this.indexVirtualDoc(source.id, doc)) {
|
|
200
|
+
okCount += 1;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
catch (err) {
|
|
204
|
+
this.index.markFileError(source.id, ref.path, err instanceof Error ? err.message : String(err));
|
|
205
|
+
this.emitProgress({
|
|
206
|
+
type: 'error',
|
|
207
|
+
sourceId: source.id,
|
|
208
|
+
path: ref.path,
|
|
209
|
+
status: 'fetch_failed',
|
|
210
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// 差量 prune:discover 成功后,源上已消失的文档才删
|
|
215
|
+
const keep = new Set(enriched.map((r) => r.path));
|
|
216
|
+
const pruned = this.index.pruneMissing(source.id, keep);
|
|
217
|
+
if (pruned > 0) {
|
|
218
|
+
this.emitProgress({
|
|
219
|
+
type: 'source',
|
|
220
|
+
sourceId: source.id,
|
|
221
|
+
status: 'pruned',
|
|
222
|
+
detail: `removed=${pruned}`,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
if (this.embedding) {
|
|
226
|
+
this.enqueue(source.id, 'embed_source', null, 3);
|
|
227
|
+
this.kick();
|
|
228
|
+
}
|
|
229
|
+
this.refreshCoverage(source.id);
|
|
230
|
+
this.emitProgress({
|
|
231
|
+
type: 'source',
|
|
232
|
+
sourceId: source.id,
|
|
233
|
+
status: 'ready',
|
|
234
|
+
detail: `url docs ok=${okCount}/${enriched.length} unchanged=${unchanged}`,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
|
|
239
|
+
*
|
|
240
|
+
* @returns 凭证或 null(仅当源未配置 authRef)
|
|
241
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
|
|
242
|
+
*/
|
|
243
|
+
async resolveCred(source) {
|
|
244
|
+
if (!source.authRef)
|
|
245
|
+
return null;
|
|
246
|
+
if (!this.credentials) {
|
|
247
|
+
throw new Error(`source ${source.id} has authRef "${source.authRef}" but CredentialStore is not configured`);
|
|
248
|
+
}
|
|
249
|
+
const resolved = await this.credentials.resolve(source.authRef);
|
|
250
|
+
if (!resolved) {
|
|
251
|
+
throw new Error(`credential "${source.authRef}" unavailable (missing secret, expired, or not found)`);
|
|
252
|
+
}
|
|
253
|
+
return resolved;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* 将 VirtualDocument 写入索引(外源 / 规范化后文本)
|
|
257
|
+
*
|
|
258
|
+
* @returns 是否成功写入
|
|
259
|
+
*/
|
|
260
|
+
indexVirtualDoc(sourceId, doc) {
|
|
261
|
+
if (!doc.content?.trim()) {
|
|
262
|
+
this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
// 逻辑键可能无扩展名(connector id / URL path):按 MIME → 扩展名 → 内容形态回退
|
|
266
|
+
let adapter = this.adapters.match(doc.path, doc.contentType);
|
|
267
|
+
if (!adapter) {
|
|
268
|
+
if (looksLikeHtml(doc.content, doc.contentType))
|
|
269
|
+
adapter = htmlAdapter;
|
|
270
|
+
else if (/^#{1,6}\s|\n\n/.test(doc.content))
|
|
271
|
+
adapter = markdownAdapter;
|
|
272
|
+
else
|
|
273
|
+
adapter = textAdapter;
|
|
274
|
+
}
|
|
275
|
+
const contentHash = hashContent(doc.content);
|
|
276
|
+
if (this.index.isFresh(sourceId, doc.path, contentHash)) {
|
|
277
|
+
return true;
|
|
278
|
+
}
|
|
279
|
+
try {
|
|
280
|
+
const chunks = adapter.chunk(doc.content, doc.path);
|
|
281
|
+
this.index.upsertFile({
|
|
282
|
+
sourceId,
|
|
283
|
+
path: doc.path,
|
|
284
|
+
contentHash,
|
|
285
|
+
size: doc.size,
|
|
286
|
+
mtime: Math.floor(Date.now() / 1000),
|
|
287
|
+
adapterId: adapter.id,
|
|
288
|
+
chunks,
|
|
289
|
+
externalUrl: doc.externalUrl,
|
|
290
|
+
etag: doc.etag,
|
|
291
|
+
lastModified: doc.lastModified,
|
|
292
|
+
});
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
catch (err) {
|
|
296
|
+
this.index.markFileError(sourceId, doc.path, err instanceof Error ? err.message : String(err));
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
|
|
302
|
+
*/
|
|
303
|
+
async ingestFileNow(sourceId, filePath) {
|
|
304
|
+
const ok = await this.parseOne(sourceId, filePath);
|
|
305
|
+
this.refreshCoverage(sourceId);
|
|
306
|
+
return ok;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* 处理 fs 变更(debounce 后入队)
|
|
310
|
+
*/
|
|
311
|
+
handleFsChange(sourceId, filePath, event) {
|
|
312
|
+
if (this.disposed)
|
|
313
|
+
return;
|
|
314
|
+
const key = `${sourceId}:${filePath}`;
|
|
315
|
+
const prev = this.debounceTimers.get(key);
|
|
316
|
+
if (prev)
|
|
317
|
+
clearTimeout(prev);
|
|
318
|
+
this.debounceTimers.set(key, setTimeout(() => {
|
|
319
|
+
this.debounceTimers.delete(key);
|
|
320
|
+
if (event === 'rename') {
|
|
321
|
+
// inotify:create/move 也报 rename — 按存在性分支,勿一律 drop
|
|
322
|
+
void stat(filePath)
|
|
323
|
+
.then((st) => {
|
|
324
|
+
if (st.isFile()) {
|
|
325
|
+
this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
|
|
329
|
+
}
|
|
330
|
+
this.kick();
|
|
331
|
+
})
|
|
332
|
+
.catch(() => {
|
|
333
|
+
this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
|
|
334
|
+
this.kick();
|
|
335
|
+
});
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
|
|
339
|
+
this.kick();
|
|
340
|
+
}, this.debounceMs));
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* 启动目录 watch(幂等)
|
|
344
|
+
*/
|
|
345
|
+
startWatch(sourceId) {
|
|
346
|
+
const source = this.sources.get(sourceId);
|
|
347
|
+
if (!source || source.sync.strategy !== 'watch' || source.sync.enabled === false)
|
|
348
|
+
return;
|
|
349
|
+
if (this.watchers.has(sourceId))
|
|
350
|
+
return;
|
|
351
|
+
if (source.kind !== 'directory' && source.kind !== 'workspace')
|
|
352
|
+
return;
|
|
353
|
+
try {
|
|
354
|
+
const watcher = watch(source.location, { recursive: true }, (event, filename) => {
|
|
355
|
+
if (!filename)
|
|
356
|
+
return;
|
|
357
|
+
const abs = join(source.location, filename.toString().split(sep).join(sep));
|
|
358
|
+
if (this.adapters.shouldSkipPath(abs))
|
|
359
|
+
return;
|
|
360
|
+
this.handleFsChange(sourceId, abs, event === 'rename' ? 'rename' : 'change');
|
|
361
|
+
});
|
|
362
|
+
this.watchers.set(sourceId, watcher);
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
this.emitProgress({
|
|
366
|
+
type: 'error',
|
|
367
|
+
sourceId,
|
|
368
|
+
status: 'watch_failed',
|
|
369
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
stopWatch(sourceId) {
|
|
374
|
+
const w = this.watchers.get(sourceId);
|
|
375
|
+
if (w) {
|
|
376
|
+
w.close();
|
|
377
|
+
this.watchers.delete(sourceId);
|
|
378
|
+
}
|
|
379
|
+
for (const [key, t] of this.debounceTimers) {
|
|
380
|
+
if (key.startsWith(`${sourceId}:`)) {
|
|
381
|
+
clearTimeout(t);
|
|
382
|
+
this.debounceTimers.delete(key);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* 等待队列排空(测试/管理)
|
|
388
|
+
*/
|
|
389
|
+
async idle(timeoutMs = 30_000) {
|
|
390
|
+
const start = Date.now();
|
|
391
|
+
while (this.running > 0 || this.hasQueuedJobs()) {
|
|
392
|
+
if (Date.now() - start > timeoutMs) {
|
|
393
|
+
throw new Error('knowledge ingest idle timeout');
|
|
394
|
+
}
|
|
395
|
+
this.kick();
|
|
396
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
dispose() {
|
|
400
|
+
this.disposed = true;
|
|
401
|
+
this.stopPolling();
|
|
402
|
+
for (const id of this.watchers.keys())
|
|
403
|
+
this.stopWatch(id);
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
407
|
+
*/
|
|
408
|
+
startPolling() {
|
|
409
|
+
if (this.pollTimer || this.disposed)
|
|
410
|
+
return;
|
|
411
|
+
this.pollTimer = setInterval(() => {
|
|
412
|
+
void this.pollDueSources().catch((err) => {
|
|
413
|
+
this.emitProgress({
|
|
414
|
+
type: 'error',
|
|
415
|
+
sourceId: '*',
|
|
416
|
+
status: 'poll_tick_failed',
|
|
417
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
418
|
+
});
|
|
419
|
+
});
|
|
420
|
+
}, this.pollTickMs);
|
|
421
|
+
// 不阻止进程退出
|
|
422
|
+
this.pollTimer.unref?.();
|
|
423
|
+
}
|
|
424
|
+
stopPolling() {
|
|
425
|
+
if (this.pollTimer) {
|
|
426
|
+
clearInterval(this.pollTimer);
|
|
427
|
+
this.pollTimer = null;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
|
|
432
|
+
*
|
|
433
|
+
* @returns 本轮实际触发的 sourceId 列表
|
|
434
|
+
*/
|
|
435
|
+
async pollDueSources() {
|
|
436
|
+
if (this.disposed)
|
|
437
|
+
return [];
|
|
438
|
+
const now = Date.now();
|
|
439
|
+
const due = this.sources
|
|
440
|
+
.list()
|
|
441
|
+
.filter((s) => {
|
|
442
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
443
|
+
return false;
|
|
444
|
+
if (s.sync.enabled === false)
|
|
445
|
+
return false;
|
|
446
|
+
if (s.sync.strategy !== 'poll')
|
|
447
|
+
return false;
|
|
448
|
+
const interval = s.sync.intervalMs ?? this.pollMinIntervalMs;
|
|
449
|
+
const last = s.lastPolledAt ?? 0;
|
|
450
|
+
return now - last >= Math.max(interval, this.pollMinIntervalMs);
|
|
451
|
+
})
|
|
452
|
+
.slice(0, this.maxPollPerTick);
|
|
453
|
+
const triggered = [];
|
|
454
|
+
for (const source of due) {
|
|
455
|
+
triggered.push(source.id);
|
|
456
|
+
this.sources.update(source.id, { lastPolledAt: now });
|
|
457
|
+
this.emitProgress({
|
|
458
|
+
type: 'source',
|
|
459
|
+
sourceId: source.id,
|
|
460
|
+
status: 'poll_due',
|
|
461
|
+
detail: source.sync.intervalMs?.toString(),
|
|
462
|
+
});
|
|
463
|
+
// 异步入队,不阻塞 tick
|
|
464
|
+
void this.ingestSource(source.id).catch((err) => {
|
|
465
|
+
this.emitProgress({
|
|
466
|
+
type: 'error',
|
|
467
|
+
sourceId: source.id,
|
|
468
|
+
status: 'poll_failed',
|
|
469
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
470
|
+
});
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
return triggered;
|
|
474
|
+
}
|
|
475
|
+
// ── 内部 ──
|
|
476
|
+
hasQueuedJobs() {
|
|
477
|
+
const row = this.sources.database.raw
|
|
478
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
479
|
+
.get();
|
|
480
|
+
return (row?.n ?? 0) > 0;
|
|
481
|
+
}
|
|
482
|
+
enqueue(sourceId, kind, path, priority, _detail) {
|
|
483
|
+
// 同 source+kind 去重(queued + running,避免并发重复 embed)
|
|
484
|
+
const activeStatuses = ['queued', 'running'];
|
|
485
|
+
if (path != null) {
|
|
486
|
+
const placeholders = activeStatuses.map(() => '?').join(',');
|
|
487
|
+
const dup = this.sources.database.raw
|
|
488
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
489
|
+
WHERE source_id = ? AND kind = ? AND path = ? AND status IN (${placeholders})`)
|
|
490
|
+
.get(sourceId, kind, path, ...activeStatuses);
|
|
491
|
+
if (dup?.id)
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
else {
|
|
495
|
+
const placeholders = activeStatuses.map(() => '?').join(',');
|
|
496
|
+
const dup = this.sources.database.raw
|
|
497
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
498
|
+
WHERE source_id = ? AND kind = ? AND path IS NULL AND status IN (${placeholders})`)
|
|
499
|
+
.get(sourceId, kind, ...activeStatuses);
|
|
500
|
+
if (dup?.id)
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
const now = Date.now();
|
|
504
|
+
const depth = this.sources.database.raw
|
|
505
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
506
|
+
.get();
|
|
507
|
+
if ((depth?.n ?? 0) >= this.maxQueueDepth) {
|
|
508
|
+
this.emitProgress({
|
|
509
|
+
type: 'error',
|
|
510
|
+
sourceId,
|
|
511
|
+
status: 'queue_backpressure',
|
|
512
|
+
detail: `queued=${depth.n} (delay only, jobs kept)`,
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
if (this.diskWatermarkAlert) {
|
|
516
|
+
void this.warnDiskWatermark(sourceId);
|
|
517
|
+
}
|
|
518
|
+
this.sources.database.raw
|
|
519
|
+
.prepare(`INSERT INTO knowledge_jobs (id, source_id, kind, path, priority, status, attempts, created_at, updated_at)
|
|
520
|
+
VALUES (?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
|
|
521
|
+
.run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
|
|
522
|
+
}
|
|
523
|
+
kick() {
|
|
524
|
+
if (this.disposed || this.draining)
|
|
525
|
+
return;
|
|
526
|
+
this.draining = true;
|
|
527
|
+
queueMicrotask(() => {
|
|
528
|
+
this.draining = false;
|
|
529
|
+
this.drain();
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
drain() {
|
|
533
|
+
while (this.running < this.parseConcurrency) {
|
|
534
|
+
const job = this.claimJob();
|
|
535
|
+
if (!job)
|
|
536
|
+
return;
|
|
537
|
+
const isEmbed = job.kind === 'embed_source';
|
|
538
|
+
if (isEmbed && this.runningEmbed >= this.embedConcurrency) {
|
|
539
|
+
this.sources.database.raw
|
|
540
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued' WHERE id = ? AND status = 'running'`)
|
|
541
|
+
.run(job.id);
|
|
542
|
+
// embed 满:本轮不再认领,避免空转把同一 job 抢来抢去
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
this.running += 1;
|
|
546
|
+
if (isEmbed)
|
|
547
|
+
this.runningEmbed += 1;
|
|
548
|
+
void this.runJob(job)
|
|
549
|
+
.catch((err) => {
|
|
550
|
+
this.emitProgress({
|
|
551
|
+
type: 'error',
|
|
552
|
+
sourceId: job.source_id,
|
|
553
|
+
path: job.path ?? undefined,
|
|
554
|
+
status: 'job_failed',
|
|
555
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
556
|
+
});
|
|
557
|
+
})
|
|
558
|
+
.finally(() => {
|
|
559
|
+
this.running -= 1;
|
|
560
|
+
if (isEmbed)
|
|
561
|
+
this.runningEmbed -= 1;
|
|
562
|
+
this.kick();
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
claimJob() {
|
|
567
|
+
const row = this.sources.database.raw
|
|
568
|
+
.prepare(`SELECT * FROM knowledge_jobs
|
|
569
|
+
WHERE status = 'queued'
|
|
570
|
+
ORDER BY priority ASC, created_at ASC
|
|
571
|
+
LIMIT 1`)
|
|
572
|
+
.get();
|
|
573
|
+
if (!row)
|
|
574
|
+
return null;
|
|
575
|
+
const res = this.sources.database.raw
|
|
576
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
|
|
577
|
+
WHERE id = ? AND status = 'queued'`)
|
|
578
|
+
.run(Date.now(), row.id);
|
|
579
|
+
if (Number(res.changes ?? 0) === 0)
|
|
580
|
+
return null;
|
|
581
|
+
return row;
|
|
582
|
+
}
|
|
583
|
+
async runJob(job) {
|
|
584
|
+
const now = Date.now();
|
|
585
|
+
try {
|
|
586
|
+
if (job.kind === 'parse_file' && job.path) {
|
|
587
|
+
await this.parseOne(job.source_id, job.path);
|
|
588
|
+
this.refreshCoverage(job.source_id);
|
|
589
|
+
if (this.embedding) {
|
|
590
|
+
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
591
|
+
this.kick();
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
else if (job.kind === 'fetch_doc' && job.path) {
|
|
595
|
+
await this.fetchDocJob(job.source_id, job.path);
|
|
596
|
+
this.refreshCoverage(job.source_id);
|
|
597
|
+
if (this.embedding) {
|
|
598
|
+
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
599
|
+
this.kick();
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
else if (job.kind === 'drop_file' && job.path) {
|
|
603
|
+
this.index.removeFile(job.source_id, job.path);
|
|
604
|
+
this.refreshCoverage(job.source_id);
|
|
605
|
+
}
|
|
606
|
+
else if (job.kind === 'walk_source') {
|
|
607
|
+
await this.ingestSource(job.source_id);
|
|
608
|
+
}
|
|
609
|
+
else if (job.kind === 'embed_source') {
|
|
610
|
+
await this.embedPending(job.source_id);
|
|
611
|
+
}
|
|
612
|
+
this.sources.database.raw
|
|
613
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
614
|
+
.run(now, job.id);
|
|
615
|
+
this.emitProgress({
|
|
616
|
+
type: 'job',
|
|
617
|
+
sourceId: job.source_id,
|
|
618
|
+
path: job.path ?? undefined,
|
|
619
|
+
status: 'done',
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
catch (err) {
|
|
623
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
624
|
+
this.sources.database.raw
|
|
625
|
+
.prepare(`UPDATE knowledge_jobs
|
|
626
|
+
SET status = 'failed', attempts = attempts + 1, last_error = ?, updated_at = ?
|
|
627
|
+
WHERE id = ?`)
|
|
628
|
+
.run(message, now, job.id);
|
|
629
|
+
throw err;
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
634
|
+
*/
|
|
635
|
+
async fetchDocJob(sourceId, path) {
|
|
636
|
+
const source = this.sources.get(sourceId);
|
|
637
|
+
if (!source || source.status === 'removed')
|
|
638
|
+
return;
|
|
639
|
+
const cred = await this.resolveCred(source);
|
|
640
|
+
const existing = this.index.getFile(sourceId, path);
|
|
641
|
+
const ref = {
|
|
642
|
+
path,
|
|
643
|
+
externalUrl: existing?.externalUrl ?? source.location,
|
|
644
|
+
etag: existing?.etag,
|
|
645
|
+
lastModified: existing?.lastModified,
|
|
646
|
+
};
|
|
647
|
+
const fetcher = source.kind === 'url'
|
|
648
|
+
? this.urlFetcher
|
|
649
|
+
: source.kind === 'connector'
|
|
650
|
+
? this.connectorFetcher
|
|
651
|
+
: this.localFetcher;
|
|
652
|
+
const doc = await fetcher.fetch(source, ref, cred);
|
|
653
|
+
if (doc)
|
|
654
|
+
this.indexVirtualDoc(sourceId, doc);
|
|
655
|
+
}
|
|
656
|
+
async parseOne(sourceId, filePath) {
|
|
657
|
+
if (this.adapters.shouldSkipPath(filePath)) {
|
|
658
|
+
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
659
|
+
return false;
|
|
660
|
+
}
|
|
661
|
+
const adapter = this.adapters.match(filePath);
|
|
662
|
+
if (!adapter) {
|
|
663
|
+
this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
|
|
664
|
+
return false;
|
|
665
|
+
}
|
|
666
|
+
let st;
|
|
667
|
+
try {
|
|
668
|
+
st = await stat(filePath);
|
|
669
|
+
}
|
|
670
|
+
catch {
|
|
671
|
+
this.index.removeFile(sourceId, filePath);
|
|
672
|
+
return false;
|
|
673
|
+
}
|
|
674
|
+
if (!st.isFile())
|
|
675
|
+
return false;
|
|
676
|
+
if (st.size > this.maxFileBytes) {
|
|
677
|
+
this.index.markFileSkipped(sourceId, filePath, 'oversize');
|
|
678
|
+
return false;
|
|
679
|
+
}
|
|
680
|
+
const content = await readFile(filePath, 'utf8');
|
|
681
|
+
const contentHash = hashContent(content);
|
|
682
|
+
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
683
|
+
return true;
|
|
684
|
+
}
|
|
685
|
+
try {
|
|
686
|
+
const chunks = adapter.chunk(content, filePath);
|
|
687
|
+
this.index.upsertFile({
|
|
688
|
+
sourceId,
|
|
689
|
+
path: filePath,
|
|
690
|
+
contentHash,
|
|
691
|
+
size: st.size,
|
|
692
|
+
mtime: Math.floor(st.mtimeMs),
|
|
693
|
+
adapterId: adapter.id,
|
|
694
|
+
chunks,
|
|
695
|
+
});
|
|
696
|
+
return true;
|
|
697
|
+
}
|
|
698
|
+
catch (err) {
|
|
699
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
|
|
700
|
+
return false;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
/**
|
|
704
|
+
* Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
|
|
705
|
+
*/
|
|
706
|
+
async embedPending(sourceId) {
|
|
707
|
+
if (!this.embedding)
|
|
708
|
+
return;
|
|
709
|
+
const pending = this.index.listChunksMissingEmbedding(sourceId, this.embedBatch);
|
|
710
|
+
if (pending.length === 0)
|
|
711
|
+
return;
|
|
712
|
+
if (this.embedMinIntervalMs > 0) {
|
|
713
|
+
const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
|
|
714
|
+
if (wait > 0)
|
|
715
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
716
|
+
}
|
|
717
|
+
try {
|
|
718
|
+
const vectors = await this.embedding.embedBatch(pending.map((p) => p.text));
|
|
719
|
+
for (let i = 0; i < pending.length; i++) {
|
|
720
|
+
const vec = vectors[i];
|
|
721
|
+
if (vec?.length)
|
|
722
|
+
this.index.setChunkEmbedding(pending[i].id, vec);
|
|
723
|
+
}
|
|
724
|
+
this.lastEmbedAt = Date.now();
|
|
725
|
+
this.emitProgress({
|
|
726
|
+
type: 'embed',
|
|
727
|
+
sourceId,
|
|
728
|
+
status: 'batch_done',
|
|
729
|
+
detail: `embedded=${pending.length}`,
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
catch (err) {
|
|
733
|
+
this.emitProgress({
|
|
734
|
+
type: 'error',
|
|
735
|
+
sourceId,
|
|
736
|
+
status: 'embed_failed',
|
|
737
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
738
|
+
});
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
// 仍有 pending 则继续排队
|
|
742
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
|
|
743
|
+
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
744
|
+
this.kick();
|
|
745
|
+
}
|
|
746
|
+
this.refreshCoverage(sourceId);
|
|
747
|
+
}
|
|
748
|
+
refreshCoverage(sourceId) {
|
|
749
|
+
const stats = this.index.sourceStats(sourceId);
|
|
750
|
+
const source = this.sources.get(sourceId);
|
|
751
|
+
if (!source)
|
|
752
|
+
return;
|
|
753
|
+
const discovered = this.discoveredFiles.get(sourceId) ?? 0;
|
|
754
|
+
// Phase A:已落库文件(indexed+skipped+error)/ walk 发现数
|
|
755
|
+
const phaseA = discovered > 0
|
|
756
|
+
? Math.min(1, stats.files / discovered)
|
|
757
|
+
: stats.files > 0
|
|
758
|
+
? 1
|
|
759
|
+
: 0;
|
|
760
|
+
const embCov = this.embedding ? this.index.embeddingCoverage(sourceId) : 1;
|
|
761
|
+
const blended = discovered === 0 && stats.files === 0 && stats.chunks === 0
|
|
762
|
+
? source.coverage ?? 0
|
|
763
|
+
: this.embedding
|
|
764
|
+
? phaseA * 0.5 + embCov * 0.5
|
|
765
|
+
: phaseA;
|
|
766
|
+
const stillQueued = this.sources.database.raw
|
|
767
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'queued'`)
|
|
768
|
+
.get(sourceId);
|
|
769
|
+
const stillRunning = this.sources.database.raw
|
|
770
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'running'`)
|
|
771
|
+
.get(sourceId);
|
|
772
|
+
const busy = (stillQueued?.n ?? 0) > 0 || (stillRunning?.n ?? 0) > 0;
|
|
773
|
+
const status = busy
|
|
774
|
+
? 'partial'
|
|
775
|
+
: stats.chunks > 0 || stats.files > 0
|
|
776
|
+
? 'ready'
|
|
777
|
+
: source.status;
|
|
778
|
+
this.sources.update(sourceId, {
|
|
779
|
+
status: status,
|
|
780
|
+
coverage: blended,
|
|
781
|
+
});
|
|
782
|
+
this.emitProgress({
|
|
783
|
+
type: 'source',
|
|
784
|
+
sourceId,
|
|
785
|
+
status,
|
|
786
|
+
detail: `files=${stats.files} chunks=${stats.chunks} emb=${embCov.toFixed(2)}`,
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
emitProgress(evt) {
|
|
790
|
+
this.emit('progress', evt);
|
|
791
|
+
this.emit('knowledge.index.progress', evt);
|
|
792
|
+
}
|
|
793
|
+
/** 磁盘水位告警(仅告警,不丢任务) */
|
|
794
|
+
async warnDiskWatermark(sourceId) {
|
|
795
|
+
try {
|
|
796
|
+
const { statfs } = await import('node:fs/promises');
|
|
797
|
+
const root = process.env.OCTOPI_HOME || process.cwd();
|
|
798
|
+
const st = await statfs(root);
|
|
799
|
+
const freeBytes = Number(st.bavail) * Number(st.bsize);
|
|
800
|
+
const freeMb = freeBytes / (1024 * 1024);
|
|
801
|
+
if (freeMb < 100) {
|
|
802
|
+
this.emitProgress({
|
|
803
|
+
type: 'error',
|
|
804
|
+
sourceId,
|
|
805
|
+
status: 'disk_watermark',
|
|
806
|
+
detail: `free≈${Math.round(freeMb)}MB (jobs kept)`,
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
catch {
|
|
811
|
+
// statfs 不可用时静默
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|