@monoes/monomindcli 2.16.12 → 2.16.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/core/coder.md +1 -1
- package/.claude/agents/engineering/engineering-devops-automator.md +1 -1
- package/.claude/agents/github/repo-architect.md +1 -1
- package/.claude/agents/testing/production-validator.md +1 -1
- package/.claude/agents/testing/testing-tool-evaluator.md +1 -1
- package/.claude/commands/github/pr-manager.md +1 -1
- package/.claude/helpers/pick-rank.cjs +34 -4
- package/.claude/skills/monolean-audit/SKILL.md +1 -1
- package/README.md +1 -1
- package/dist/src/commands/cleanup-data.d.ts +30 -0
- package/dist/src/commands/cleanup-data.d.ts.map +1 -0
- package/dist/src/commands/cleanup-data.js +138 -0
- package/dist/src/commands/cleanup-data.js.map +1 -0
- package/dist/src/commands/cleanup-scratch.d.ts +43 -0
- package/dist/src/commands/cleanup-scratch.d.ts.map +1 -0
- package/dist/src/commands/cleanup-scratch.js +125 -0
- package/dist/src/commands/cleanup-scratch.js.map +1 -0
- package/dist/src/commands/cleanup.d.ts +2 -39
- package/dist/src/commands/cleanup.d.ts.map +1 -1
- package/dist/src/commands/cleanup.js +5 -247
- package/dist/src/commands/cleanup.js.map +1 -1
- package/dist/src/commands/doctor-gitignore-checks.d.ts +10 -0
- package/dist/src/commands/doctor-gitignore-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-gitignore-checks.js +141 -0
- package/dist/src/commands/doctor-gitignore-checks.js.map +1 -0
- package/dist/src/commands/doctor-helpers-checks.d.ts +10 -0
- package/dist/src/commands/doctor-helpers-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-helpers-checks.js +292 -0
- package/dist/src/commands/doctor-helpers-checks.js.map +1 -0
- package/dist/src/commands/doctor-memory-checks.d.ts +47 -0
- package/dist/src/commands/doctor-memory-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-memory-checks.js +298 -0
- package/dist/src/commands/doctor-memory-checks.js.map +1 -0
- package/dist/src/commands/doctor-monograph-checks.d.ts +9 -0
- package/dist/src/commands/doctor-monograph-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-monograph-checks.js +331 -0
- package/dist/src/commands/doctor-monograph-checks.js.map +1 -0
- package/dist/src/commands/doctor-project-checks.d.ts +7 -66
- package/dist/src/commands/doctor-project-checks.d.ts.map +1 -1
- package/dist/src/commands/doctor-project-checks.js +10 -1331
- package/dist/src/commands/doctor-project-checks.js.map +1 -1
- package/dist/src/commands/doctor-routing-checks.d.ts +10 -0
- package/dist/src/commands/doctor-routing-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-routing-checks.js +194 -0
- package/dist/src/commands/doctor-routing-checks.js.map +1 -0
- package/dist/src/commands/doctor-worker-checks.d.ts +17 -0
- package/dist/src/commands/doctor-worker-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-worker-checks.js +122 -0
- package/dist/src/commands/doctor-worker-checks.js.map +1 -0
- package/dist/src/commands/hooks-routing-commands.js +1 -1
- package/dist/src/commands/hooks-routing-commands.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +6 -15
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/org-control.d.ts +117 -0
- package/dist/src/commands/org-control.d.ts.map +1 -0
- package/dist/src/commands/org-control.js +274 -0
- package/dist/src/commands/org-control.js.map +1 -0
- package/dist/src/commands/org-lifecycle.d.ts +7 -0
- package/dist/src/commands/org-lifecycle.d.ts.map +1 -0
- package/dist/src/commands/org-lifecycle.js +298 -0
- package/dist/src/commands/org-lifecycle.js.map +1 -0
- package/dist/src/commands/org-manage.d.ts +7 -0
- package/dist/src/commands/org-manage.d.ts.map +1 -0
- package/dist/src/commands/org-manage.js +357 -0
- package/dist/src/commands/org-manage.js.map +1 -0
- package/dist/src/commands/org-memory-command.d.ts +3 -0
- package/dist/src/commands/org-memory-command.d.ts.map +1 -0
- package/dist/src/commands/org-memory-command.js +178 -0
- package/dist/src/commands/org-memory-command.js.map +1 -0
- package/dist/src/commands/org-observe-approvals.d.ts +8 -0
- package/dist/src/commands/org-observe-approvals.d.ts.map +1 -0
- package/dist/src/commands/org-observe-approvals.js +211 -0
- package/dist/src/commands/org-observe-approvals.js.map +1 -0
- package/dist/src/commands/org-observe-checkpoint.d.ts +26 -0
- package/dist/src/commands/org-observe-checkpoint.d.ts.map +1 -0
- package/dist/src/commands/org-observe-checkpoint.js +180 -0
- package/dist/src/commands/org-observe-checkpoint.js.map +1 -0
- package/dist/src/commands/org-observe-config.d.ts +8 -0
- package/dist/src/commands/org-observe-config.d.ts.map +1 -0
- package/dist/src/commands/org-observe-config.js +160 -0
- package/dist/src/commands/org-observe-config.js.map +1 -0
- package/dist/src/commands/org-observe-gates.d.ts +4 -0
- package/dist/src/commands/org-observe-gates.d.ts.map +1 -0
- package/dist/src/commands/org-observe-gates.js +182 -0
- package/dist/src/commands/org-observe-gates.js.map +1 -0
- package/dist/src/commands/org-observe-inbox.d.ts +18 -0
- package/dist/src/commands/org-observe-inbox.d.ts.map +1 -0
- package/dist/src/commands/org-observe-inbox.js +154 -0
- package/dist/src/commands/org-observe-inbox.js.map +1 -0
- package/dist/src/commands/org-observe-logs.d.ts +31 -0
- package/dist/src/commands/org-observe-logs.d.ts.map +1 -0
- package/dist/src/commands/org-observe-logs.js +272 -0
- package/dist/src/commands/org-observe-logs.js.map +1 -0
- package/dist/src/commands/org-observe-questions.d.ts +8 -0
- package/dist/src/commands/org-observe-questions.d.ts.map +1 -0
- package/dist/src/commands/org-observe-questions.js +201 -0
- package/dist/src/commands/org-observe-questions.js.map +1 -0
- package/dist/src/commands/org-observe-report.d.ts +8 -0
- package/dist/src/commands/org-observe-report.d.ts.map +1 -0
- package/dist/src/commands/org-observe-report.js +366 -0
- package/dist/src/commands/org-observe-report.js.map +1 -0
- package/dist/src/commands/org-observe-shared.d.ts +20 -0
- package/dist/src/commands/org-observe-shared.d.ts.map +1 -0
- package/dist/src/commands/org-observe-shared.js +55 -0
- package/dist/src/commands/org-observe-shared.js.map +1 -0
- package/dist/src/commands/org-observe.d.ts +9 -101
- package/dist/src/commands/org-observe.d.ts.map +1 -1
- package/dist/src/commands/org-observe.js +12 -1675
- package/dist/src/commands/org-observe.js.map +1 -1
- package/dist/src/commands/org-poll.d.ts +57 -0
- package/dist/src/commands/org-poll.d.ts.map +1 -0
- package/dist/src/commands/org-poll.js +194 -0
- package/dist/src/commands/org-poll.js.map +1 -0
- package/dist/src/commands/org-run.d.ts +3 -0
- package/dist/src/commands/org-run.d.ts.map +1 -0
- package/dist/src/commands/org-run.js +449 -0
- package/dist/src/commands/org-run.js.map +1 -0
- package/dist/src/commands/org-serve.d.ts +19 -0
- package/dist/src/commands/org-serve.d.ts.map +1 -0
- package/dist/src/commands/org-serve.js +395 -0
- package/dist/src/commands/org-serve.js.map +1 -0
- package/dist/src/commands/org-subcommands-config.d.ts +8 -0
- package/dist/src/commands/org-subcommands-config.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-config.js +93 -0
- package/dist/src/commands/org-subcommands-config.js.map +1 -0
- package/dist/src/commands/org-subcommands-hil.d.ts +11 -0
- package/dist/src/commands/org-subcommands-hil.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-hil.js +216 -0
- package/dist/src/commands/org-subcommands-hil.js.map +1 -0
- package/dist/src/commands/org-subcommands-observe.d.ts +12 -0
- package/dist/src/commands/org-subcommands-observe.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-observe.js +249 -0
- package/dist/src/commands/org-subcommands-observe.js.map +1 -0
- package/dist/src/commands/org-subcommands-runtime.d.ts +12 -0
- package/dist/src/commands/org-subcommands-runtime.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-runtime.js +163 -0
- package/dist/src/commands/org-subcommands-runtime.js.map +1 -0
- package/dist/src/commands/org.d.ts +3 -151
- package/dist/src/commands/org.d.ts.map +1 -1
- package/dist/src/commands/org.js +43 -2735
- package/dist/src/commands/org.js.map +1 -1
- package/dist/src/index.js +2 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/init/never-commit.d.ts +3 -3
- package/dist/src/init/types.d.ts +1 -1
- package/dist/src/init/types.js +1 -1
- package/dist/src/knowledge/document-store.js +1 -1
- package/dist/src/knowledge/document-store.js.map +1 -1
- package/dist/src/mcp-tools/hooks-edit-command.d.ts +10 -0
- package/dist/src/mcp-tools/hooks-edit-command.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-edit-command.js +247 -0
- package/dist/src/mcp-tools/hooks-edit-command.js.map +1 -0
- package/dist/src/mcp-tools/hooks-learning.d.ts +9 -0
- package/dist/src/mcp-tools/hooks-learning.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-learning.js +408 -0
- package/dist/src/mcp-tools/hooks-learning.js.map +1 -0
- package/dist/src/mcp-tools/hooks-metrics-list.d.ts +8 -0
- package/dist/src/mcp-tools/hooks-metrics-list.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-metrics-list.js +205 -0
- package/dist/src/mcp-tools/hooks-metrics-list.js.map +1 -0
- package/dist/src/mcp-tools/hooks-route.d.ts +12 -0
- package/dist/src/mcp-tools/hooks-route.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-route.js +251 -0
- package/dist/src/mcp-tools/hooks-route.js.map +1 -0
- package/dist/src/mcp-tools/hooks-routing.d.ts +6 -27
- package/dist/src/mcp-tools/hooks-routing.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-routing.js +6 -1727
- package/dist/src/mcp-tools/hooks-routing.js.map +1 -1
- package/dist/src/mcp-tools/hooks-session.d.ts +9 -0
- package/dist/src/mcp-tools/hooks-session.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-session.js +243 -0
- package/dist/src/mcp-tools/hooks-session.js.map +1 -0
- package/dist/src/mcp-tools/hooks-task.d.ts +17 -0
- package/dist/src/mcp-tools/hooks-task.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-task.js +427 -0
- package/dist/src/mcp-tools/hooks-task.js.map +1 -0
- package/dist/src/mcp-tools/hooks-tools.d.ts +2 -1
- package/dist/src/mcp-tools/hooks-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-tools.js +2 -1
- package/dist/src/mcp-tools/hooks-tools.js.map +1 -1
- package/dist/src/mcp-tools/memory-tools.js +1 -1
- package/dist/src/mcp-tools/memory-tools.js.map +1 -1
- package/dist/src/memory/bm25-index.d.ts +2 -2
- package/dist/src/memory/bm25-index.js +2 -2
- package/dist/src/memory/embedding-operations.js +2 -2
- package/dist/src/memory/embedding-operations.js.map +1 -1
- package/dist/src/memory/entity-name-key.d.ts +1 -1
- package/dist/src/memory/entity-name-key.js +1 -1
- package/dist/src/memory/hnsw-operations.d.ts +1 -1
- package/dist/src/memory/hnsw-operations.js +1 -1
- package/dist/src/memory/memory-bridge-backend.d.ts +43 -0
- package/dist/src/memory/memory-bridge-backend.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-backend.js +363 -0
- package/dist/src/memory/memory-bridge-backend.js.map +1 -0
- package/dist/src/memory/memory-bridge-core.d.ts +42 -0
- package/dist/src/memory/memory-bridge-core.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-core.js +126 -0
- package/dist/src/memory/memory-bridge-core.js.map +1 -0
- package/dist/src/memory/memory-bridge-entries.d.ts +131 -0
- package/dist/src/memory/memory-bridge-entries.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-entries.js +227 -0
- package/dist/src/memory/memory-bridge-entries.js.map +1 -0
- package/dist/src/memory/memory-bridge-feedback.d.ts +60 -0
- package/dist/src/memory/memory-bridge-feedback.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-feedback.js +118 -0
- package/dist/src/memory/memory-bridge-feedback.js.map +1 -0
- package/dist/src/memory/memory-bridge-paths.d.ts +123 -0
- package/dist/src/memory/memory-bridge-paths.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-paths.js +335 -0
- package/dist/src/memory/memory-bridge-paths.js.map +1 -0
- package/dist/src/memory/memory-bridge-search.d.ts +48 -0
- package/dist/src/memory/memory-bridge-search.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-search.js +383 -0
- package/dist/src/memory/memory-bridge-search.js.map +1 -0
- package/dist/src/memory/memory-bridge-sessions.d.ts +38 -0
- package/dist/src/memory/memory-bridge-sessions.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-sessions.js +109 -0
- package/dist/src/memory/memory-bridge-sessions.js.map +1 -0
- package/dist/src/memory/memory-bridge-store.d.ts +58 -0
- package/dist/src/memory/memory-bridge-store.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-store.js +201 -0
- package/dist/src/memory/memory-bridge-store.js.map +1 -0
- package/dist/src/memory/memory-bridge.d.ts +10 -438
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +14 -1787
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/memory/memory-kg-claims.d.ts +52 -0
- package/dist/src/memory/memory-kg-claims.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-claims.js +99 -0
- package/dist/src/memory/memory-kg-claims.js.map +1 -0
- package/dist/src/memory/memory-kg-index.d.ts +86 -0
- package/dist/src/memory/memory-kg-index.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-index.js +256 -0
- package/dist/src/memory/memory-kg-index.js.map +1 -0
- package/dist/src/memory/memory-kg-ingest.d.ts +44 -0
- package/dist/src/memory/memory-kg-ingest.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-ingest.js +361 -0
- package/dist/src/memory/memory-kg-ingest.js.map +1 -0
- package/dist/src/memory/memory-kg-model.d.ts +168 -0
- package/dist/src/memory/memory-kg-model.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-model.js +158 -0
- package/dist/src/memory/memory-kg-model.js.map +1 -0
- package/dist/src/memory/memory-kg-names.d.ts +64 -0
- package/dist/src/memory/memory-kg-names.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-names.js +187 -0
- package/dist/src/memory/memory-kg-names.js.map +1 -0
- package/dist/src/memory/memory-kg-promote.d.ts +79 -0
- package/dist/src/memory/memory-kg-promote.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-promote.js +248 -0
- package/dist/src/memory/memory-kg-promote.js.map +1 -0
- package/dist/src/memory/memory-kg-rebuild.d.ts +77 -0
- package/dist/src/memory/memory-kg-rebuild.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rebuild.js +299 -0
- package/dist/src/memory/memory-kg-rebuild.js.map +1 -0
- package/dist/src/memory/memory-kg-rollback.d.ts +47 -0
- package/dist/src/memory/memory-kg-rollback.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rollback.js +193 -0
- package/dist/src/memory/memory-kg-rollback.js.map +1 -0
- package/dist/src/memory/memory-kg-rules.d.ts +64 -0
- package/dist/src/memory/memory-kg-rules.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rules.js +251 -0
- package/dist/src/memory/memory-kg-rules.js.map +1 -0
- package/dist/src/memory/memory-kg-scan.d.ts +118 -0
- package/dist/src/memory/memory-kg-scan.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-scan.js +209 -0
- package/dist/src/memory/memory-kg-scan.js.map +1 -0
- package/dist/src/memory/memory-kg-search.d.ts +80 -0
- package/dist/src/memory/memory-kg-search.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-search.js +239 -0
- package/dist/src/memory/memory-kg-search.js.map +1 -0
- package/dist/src/memory/memory-kg.d.ts +19 -547
- package/dist/src/memory/memory-kg.d.ts.map +1 -1
- package/dist/src/memory/memory-kg.js +10 -2389
- package/dist/src/memory/memory-kg.js.map +1 -1
- package/dist/src/orgrt/daemon-types.d.ts +196 -0
- package/dist/src/orgrt/daemon-types.d.ts.map +1 -0
- package/dist/src/orgrt/daemon-types.js +38 -0
- package/dist/src/orgrt/daemon-types.js.map +1 -0
- package/dist/src/orgrt/daemon.d.ts +70 -300
- package/dist/src/orgrt/daemon.d.ts.map +1 -1
- package/dist/src/orgrt/daemon.js +41 -2536
- package/dist/src/orgrt/daemon.js.map +1 -1
- package/dist/src/orgrt/idle-watchdog.d.ts +39 -0
- package/dist/src/orgrt/idle-watchdog.d.ts.map +1 -0
- package/dist/src/orgrt/idle-watchdog.js +289 -0
- package/dist/src/orgrt/idle-watchdog.js.map +1 -0
- package/dist/src/orgrt/org-reload.d.ts +7 -0
- package/dist/src/orgrt/org-reload.d.ts.map +1 -0
- package/dist/src/orgrt/org-reload.js +111 -0
- package/dist/src/orgrt/org-reload.js.map +1 -0
- package/dist/src/orgrt/org-start-steps.d.ts +21 -0
- package/dist/src/orgrt/org-start-steps.d.ts.map +1 -0
- package/dist/src/orgrt/org-start-steps.js +305 -0
- package/dist/src/orgrt/org-start-steps.js.map +1 -0
- package/dist/src/orgrt/org-start.d.ts +7 -0
- package/dist/src/orgrt/org-start.d.ts.map +1 -0
- package/dist/src/orgrt/org-start.js +435 -0
- package/dist/src/orgrt/org-start.js.map +1 -0
- package/dist/src/orgrt/org-state-file.d.ts +8 -0
- package/dist/src/orgrt/org-state-file.d.ts.map +1 -0
- package/dist/src/orgrt/org-state-file.js +116 -0
- package/dist/src/orgrt/org-state-file.js.map +1 -0
- package/dist/src/orgrt/org-stop.d.ts +6 -0
- package/dist/src/orgrt/org-stop.d.ts.map +1 -0
- package/dist/src/orgrt/org-stop.js +332 -0
- package/dist/src/orgrt/org-stop.js.map +1 -0
- package/dist/src/orgrt/org-tools.d.ts +11 -0
- package/dist/src/orgrt/org-tools.d.ts.map +1 -0
- package/dist/src/orgrt/org-tools.js +305 -0
- package/dist/src/orgrt/org-tools.js.map +1 -0
- package/dist/src/orgrt/role-incarnation.d.ts +18 -0
- package/dist/src/orgrt/role-incarnation.d.ts.map +1 -0
- package/dist/src/orgrt/role-incarnation.js +374 -0
- package/dist/src/orgrt/role-incarnation.js.map +1 -0
- package/dist/src/orgrt/role-respawn.d.ts +7 -0
- package/dist/src/orgrt/role-respawn.d.ts.map +1 -0
- package/dist/src/orgrt/role-respawn.js +318 -0
- package/dist/src/orgrt/role-respawn.js.map +1 -0
- package/dist/src/orgrt/role-session-opts.d.ts +389 -0
- package/dist/src/orgrt/role-session-opts.d.ts.map +1 -0
- package/dist/src/orgrt/role-session-opts.js +289 -0
- package/dist/src/orgrt/role-session-opts.js.map +1 -0
- package/dist/src/orgrt/runner-resolve.d.ts +43 -0
- package/dist/src/orgrt/runner-resolve.d.ts.map +1 -0
- package/dist/src/orgrt/runner-resolve.js +77 -0
- package/dist/src/orgrt/runner-resolve.js.map +1 -0
- package/dist/src/orgrt/session-gate.d.ts +34 -0
- package/dist/src/orgrt/session-gate.d.ts.map +1 -0
- package/dist/src/orgrt/session-gate.js +74 -0
- package/dist/src/orgrt/session-gate.js.map +1 -0
- package/dist/src/orgrt/session-prompt.d.ts +31 -0
- package/dist/src/orgrt/session-prompt.d.ts.map +1 -0
- package/dist/src/orgrt/session-prompt.js +115 -0
- package/dist/src/orgrt/session-prompt.js.map +1 -0
- package/dist/src/orgrt/session-run.d.ts +18 -0
- package/dist/src/orgrt/session-run.d.ts.map +1 -0
- package/dist/src/orgrt/session-run.js +429 -0
- package/dist/src/orgrt/session-run.js.map +1 -0
- package/dist/src/orgrt/session-stream.d.ts +21 -0
- package/dist/src/orgrt/session-stream.d.ts.map +1 -0
- package/dist/src/orgrt/session-stream.js +173 -0
- package/dist/src/orgrt/session-stream.js.map +1 -0
- package/dist/src/orgrt/session-types.d.ts +219 -0
- package/dist/src/orgrt/session-types.d.ts.map +1 -0
- package/dist/src/orgrt/session-types.js +2 -0
- package/dist/src/orgrt/session-types.js.map +1 -0
- package/dist/src/orgrt/session-usage.d.ts +29 -0
- package/dist/src/orgrt/session-usage.d.ts.map +1 -0
- package/dist/src/orgrt/session-usage.js +64 -0
- package/dist/src/orgrt/session-usage.js.map +1 -0
- package/dist/src/orgrt/session.d.ts +5 -281
- package/dist/src/orgrt/session.d.ts.map +1 -1
- package/dist/src/orgrt/session.js +8 -1117
- package/dist/src/orgrt/session.js.map +1 -1
- package/dist/src/ui/org-hil.mjs +1 -1
- package/dist/src/ui/routes-org-agents.mjs +318 -0
- package/dist/src/ui/routes-org-approvals.mjs +133 -0
- package/dist/src/ui/routes-org-config.mjs +459 -0
- package/dist/src/ui/routes-org-control.mjs +300 -0
- package/dist/src/ui/routes-org-files.mjs +172 -0
- package/dist/src/ui/routes-org-helpers.mjs +84 -0
- package/dist/src/ui/routes-org-knowledge.mjs +262 -0
- package/dist/src/ui/routes-org-lifecycle.mjs +224 -0
- package/dist/src/ui/routes-org-live.mjs +278 -0
- package/dist/src/ui/routes-org-mastermind.mjs +346 -0
- package/dist/src/ui/routes-org-planning.mjs +398 -0
- package/dist/src/ui/routes-org-runs.mjs +430 -0
- package/dist/src/ui/routes-org-status.mjs +342 -0
- package/dist/src/ui/routes-org-workflows.mjs +212 -0
- package/dist/src/ui/routes-org.mjs +26 -3847
- package/dist/src/ui/server.mjs +1 -1
- package/dist/src/utils/native-error.d.ts +1 -1
- package/dist/src/utils/native-error.js +1 -1
- package/dist/src/utils/paths.d.ts +2 -1
- package/dist/src/utils/paths.d.ts.map +1 -1
- package/dist/src/utils/paths.js +2 -1
- package/dist/src/utils/paths.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/org-skills/adversarial-reviewer/SKILL.md +1 -1
- package/org-skills/board-deck-builder/SKILL.md +1 -1
- package/org-skills/code-reviewer/SKILL.md +1 -1
- package/org-skills/debugging-wizard/SKILL.md +1 -1
- package/org-skills/scientific-thinking-literature-review/SKILL.md +1 -1
- package/org-skills/scientific-thinking-scholar-evaluation/SKILL.md +1 -1
- package/package.json +5 -5
|
@@ -88,2395 +88,16 @@
|
|
|
88
88
|
*
|
|
89
89
|
* @module v1/cli/memory/memory-kg
|
|
90
90
|
*/
|
|
91
|
-
import {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
export
|
|
95
|
-
export
|
|
96
|
-
export
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
export
|
|
101
|
-
/** Derived adjacency index namespace (K7). See `KgNamespaces.adj`. */
|
|
102
|
-
export const KG_ADJ_NS = 'kg:adj';
|
|
103
|
-
/** Derived origin-support index namespace (K7). See `KgNamespaces.originIdx`. */
|
|
104
|
-
export const KG_ORIGIN_IDX_NS = 'kg:origin-idx';
|
|
105
|
-
/** Derived-index status namespace (K7). See `KgNamespaces.indexStatus`. */
|
|
106
|
-
export const KG_INDEX_STATUS_NS = 'kg:index-status';
|
|
107
|
-
const MAX_NAME_LEN = 200;
|
|
108
|
-
const MAX_DESC_LEN = 2000;
|
|
109
|
-
/** Nodes/edges/rules accepted per call. Overflow is REPORTED, never sliced away
|
|
110
|
-
* in silence — see `KgIngestResult.nodesTruncated`. */
|
|
111
|
-
const MAX_NODES_PER_CALL = 500;
|
|
112
|
-
const MAX_EDGES_PER_CALL = 1000;
|
|
113
|
-
const MAX_RULES_PER_CALL = 50;
|
|
114
|
-
/** Per-origin description contributions retained on one element.
|
|
115
|
-
*
|
|
116
|
-
* A cap has to exist — the bridge caps a stored value, and an element asserted
|
|
117
|
-
* by ten thousand runs would otherwise stop being writable. What must NOT
|
|
118
|
-
* happen is the old `origin_refs.slice(-100)`, which dropped the oldest
|
|
119
|
-
* provenance and left the entry claiming complete history. Past this cap the
|
|
120
|
-
* oldest contributions are dropped AND the entry records `origins_dropped`
|
|
121
|
-
* with `provenance_complete: false`, so a reader can tell that a rollback of an
|
|
122
|
-
* old origin may find nothing to withdraw. */
|
|
123
|
-
const MAX_CLAIMS = 200;
|
|
124
|
-
// ── Identity ────────────────────────────────────────────────────────
|
|
125
|
-
/**
|
|
126
|
-
* Version of the identity scheme implemented by `entityId`/`edgeKey`/`ruleKey`.
|
|
127
|
-
*
|
|
128
|
-
* Bump when a derivation changes. Entries written under an older scheme are not
|
|
129
|
-
* orphaned: resolution probes the previous key shape and adopts the row in
|
|
130
|
-
* place (see `resolveEntity`), so a bump costs a second keyed lookup on the
|
|
131
|
-
* miss path rather than a migration.
|
|
132
|
-
*
|
|
133
|
-
* Version 1 was the name-only scheme (`n:<normalized-name>`, truncated to 200
|
|
134
|
-
* characters), under which `Person:Alex` and `Service:Alex` were one entity and
|
|
135
|
-
* two names first differing at character 250 were one entity.
|
|
136
|
-
*/
|
|
137
|
-
export const KG_ID_VERSION = 2;
|
|
138
|
-
/** Identity-grade name normalization: the same folding as `normalizeName` but
|
|
139
|
-
* WITHOUT its 200-character truncation, which silently merged long names that
|
|
140
|
-
* differ only past the cut. `normalizeName` keeps truncating because it feeds
|
|
141
|
-
* tags and display, where length matters and collisions do not. */
|
|
142
|
-
function canonicalName(name) {
|
|
143
|
-
return String(name).trim().toLowerCase().replace(/['’]/g, '').replace(/\s+/g, '_');
|
|
144
|
-
}
|
|
145
|
-
/** Types that assert nothing. A generic label must not fork an entity away from
|
|
146
|
-
* its typed self, so it maps to the empty discriminator and is resolved by
|
|
147
|
-
* name (see `resolveEntity`). */
|
|
148
|
-
const GENERIC_TYPES = new Set(['', 'entity', 'unknown']);
|
|
149
|
-
/** The identity-bearing part of a type: '' when the caller told us nothing. */
|
|
150
|
-
function typeBucket(type) {
|
|
151
|
-
const t = canonicalName(type ?? '');
|
|
152
|
-
return GENERIC_TYPES.has(t) ? '' : t;
|
|
153
|
-
}
|
|
154
|
-
/** Hash a tuple injectively: every component is length-prefixed, so no
|
|
155
|
-
* component's content can masquerade as a delimiter or bleed into its
|
|
156
|
-
* neighbour, and no two distinct tuples share an input string. Same discipline
|
|
157
|
-
* as monograph's `symbolId`/`fileId` (packages/@monomind/monograph/src/types.ts),
|
|
158
|
-
* and adopted here for the same reason — the previous scheme lost distinctions
|
|
159
|
-
* to truncation and to an unescaped separator. */
|
|
160
|
-
function hashTuple(components) {
|
|
161
|
-
const canonical = components.map((c) => `${Buffer.byteLength(c, 'utf8')}:${c}`).join('');
|
|
162
|
-
return createHash('sha256').update(canonical, 'utf8').digest('hex').slice(0, 32);
|
|
163
|
-
}
|
|
164
|
-
/** Mint an entity id from the full identity tuple.
|
|
165
|
-
*
|
|
166
|
-
* This MINTS; it does not RESOLVE. A caller that computes an id and writes to
|
|
167
|
-
* it bypasses the name index and re-forks the entities the index exists to
|
|
168
|
-
* keep together — every ingest path goes through `resolveEntity` instead. */
|
|
169
|
-
export function nodeKey(type, name) {
|
|
170
|
-
return `n:${hashTuple([String(KG_ID_VERSION), typeBucket(type), canonicalName(name)])}`;
|
|
171
|
-
}
|
|
172
|
-
/** The pre-`KG_ID_VERSION` key for a name: name-only, truncated at 200. Probed
|
|
173
|
-
* on a miss so existing graphs keep working without being re-keyed. */
|
|
174
|
-
function legacyNodeKey(name) {
|
|
175
|
-
return `n:${normalizeName(name)}`;
|
|
176
|
-
}
|
|
177
|
-
/** Rows per `bridgeListEntries` call while scanning a namespace.
|
|
178
|
-
*
|
|
179
|
-
* A single list call cannot return more than the backend's own 10,000-row
|
|
180
|
-
* ceiling (`MAX_QUERY_LIMIT` in sql-backend.ts), so the old `limit: MAX_LIST`
|
|
181
|
-
* scans were not "large enough to be safe" — they were exactly the point past
|
|
182
|
-
* which the graph goes silently invisible. Everything below pages instead.
|
|
183
|
-
*
|
|
184
|
-
* 1,000 balances the two costs: one page is at most ~1 MB even at the bridge's
|
|
185
|
-
* 16 KB value cap (typical KG entries are far smaller), and it takes a tenth
|
|
186
|
-
* of the round trips a 100-row page would. */
|
|
187
|
-
const SCAN_PAGE = 1_000;
|
|
188
|
-
/** Edge rows `kgSearch` will read before giving up and reporting `truncated`.
|
|
189
|
-
* Search is interactive and runs a full scan per query, so unlike rollback it
|
|
190
|
-
* keeps a ceiling — five times the old silent one, and now stated in the
|
|
191
|
-
* result rather than hidden. */
|
|
192
|
-
const SEARCH_EDGE_SCAN_MAX = 50_000;
|
|
193
|
-
/** Page through an entire namespace, handing each page to `onPage` so callers
|
|
194
|
-
* fold as they go instead of materializing the namespace.
|
|
195
|
-
*
|
|
196
|
-
* `onPage` returning exactly `false` stops the scan early (only `kgSearch`
|
|
197
|
-
* does that); any other return value continues. Returns false when the backend
|
|
198
|
-
* was unavailable — an unreadable namespace must never be mistaken for an
|
|
199
|
-
* empty one.
|
|
200
|
-
*
|
|
201
|
-
* Callers that MUTATE what they find must collect during the scan and mutate
|
|
202
|
-
* afterwards: deleting a row pulls every later row back one position, so under
|
|
203
|
-
* an advancing offset the row that slid into the gap is never read. (The
|
|
204
|
-
* bridge's upsert no longer reorders — it reuses the existing entry id and
|
|
205
|
-
* preserves `createdAt` — so rewrites alone are safe; deletes are not, and
|
|
206
|
-
* both callers here delete.) */
|
|
207
|
-
async function scanNamespace(namespace, dbPath, onPage) {
|
|
208
|
-
for (let offset = 0;; offset += SCAN_PAGE) {
|
|
209
|
-
const res = await bridgeListEntries({ namespace, limit: SCAN_PAGE, offset, dbPath });
|
|
210
|
-
if (!res)
|
|
211
|
-
return false;
|
|
212
|
-
if (res.entries.length && onPage(res.entries) === false)
|
|
213
|
-
return true;
|
|
214
|
-
if (res.entries.length < SCAN_PAGE)
|
|
215
|
-
return true;
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
/** Delete every entry in `namespace`. Used only to reset a DERIVED-index
|
|
219
|
-
* namespace (`kg:adj`/`kg:origin-idx`) before a fresh `kgRebuildIndex` —
|
|
220
|
-
* never a canonical one. Collect-then-delete, same reason `kgRollback`
|
|
221
|
-
* does: deleting mid-scan pulls later rows back under an advancing offset.
|
|
222
|
-
* Returns false (nothing deleted) on an unreadable or partially-deletable
|
|
223
|
-
* namespace, so the caller never proceeds as if a clear that didn't fully
|
|
224
|
-
* happen did. */
|
|
225
|
-
async function clearNamespace(namespace, dbPath) {
|
|
226
|
-
const doomed = [];
|
|
227
|
-
const covered = await scanNamespace(namespace, dbPath, (page) => {
|
|
228
|
-
for (const e of page)
|
|
229
|
-
doomed.push(e);
|
|
230
|
-
});
|
|
231
|
-
if (!covered)
|
|
232
|
-
return false;
|
|
233
|
-
let ok = true;
|
|
234
|
-
for (const e of doomed) {
|
|
235
|
-
const del = await bridgeDeleteEntry({ id: e.id, namespace, dbPath });
|
|
236
|
-
if (!del?.deleted)
|
|
237
|
-
ok = false;
|
|
238
|
-
}
|
|
239
|
-
return ok;
|
|
240
|
-
}
|
|
241
|
-
/** Cap on reported failure messages — a dead backend fails every write, and a
|
|
242
|
-
* 500-entry failure list is noise, not signal. `error` carries the true count. */
|
|
243
|
-
const MAX_FAILURES = 20;
|
|
244
|
-
/** `bridgeStoreEntry` never throws: it returns `null` when no backend is
|
|
245
|
-
* reachable and `{ success: false, error }` when the write itself failed.
|
|
246
|
-
* Both look like success to an `await` that ignores the result, which is how
|
|
247
|
-
* the graph came to claim knowledge it had not persisted. Every write in this
|
|
248
|
-
* module goes through here.
|
|
249
|
-
*
|
|
250
|
-
* @returns a failure message, or null when the write landed. */
|
|
251
|
-
function storeFailure(res, what) {
|
|
252
|
-
if (!res)
|
|
253
|
-
return `${what}: memory backend unavailable`;
|
|
254
|
-
if (!res.success)
|
|
255
|
-
return `${what}: ${res.error ?? 'store rejected'}`;
|
|
256
|
-
return null;
|
|
257
|
-
}
|
|
258
|
-
/** Accumulates write failures across a multi-write operation. The memory
|
|
259
|
-
* bridge exposes no transaction primitive, so ingest CANNOT be atomic: some
|
|
260
|
-
* writes land and some do not. Rather than hide that, callers get exact
|
|
261
|
-
* counters for what persisted plus the failure list for what did not. */
|
|
262
|
-
class FailureLog {
|
|
263
|
-
messages = [];
|
|
264
|
-
count = 0;
|
|
265
|
-
/** @returns true when the write failed (caller should not count it). */
|
|
266
|
-
add(res, what) {
|
|
267
|
-
const msg = storeFailure(res, what);
|
|
268
|
-
if (!msg)
|
|
269
|
-
return false;
|
|
270
|
-
this.count++;
|
|
271
|
-
if (this.messages.length < MAX_FAILURES)
|
|
272
|
-
this.messages.push(msg);
|
|
273
|
-
return true;
|
|
274
|
-
}
|
|
275
|
-
note(message) {
|
|
276
|
-
this.count++;
|
|
277
|
-
if (this.messages.length < MAX_FAILURES)
|
|
278
|
-
this.messages.push(message);
|
|
279
|
-
}
|
|
280
|
-
get failed() {
|
|
281
|
-
return this.count > 0;
|
|
282
|
-
}
|
|
283
|
-
/** Summary for the result's `error` field; undefined when everything landed. */
|
|
284
|
-
summary() {
|
|
285
|
-
if (!this.count)
|
|
286
|
-
return undefined;
|
|
287
|
-
return `${this.count} bridge operation(s) failed; graph state is partial`;
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
/** Attempts a compare-and-swap claim write, retrying while the bridge reports
|
|
291
|
-
* a version conflict (K5): a concurrent writer applied its own claim to the
|
|
292
|
-
* same row between our read and our write. `attempt()` must re-read,
|
|
293
|
-
* re-merge (via `applyClaim`), and re-attempt the write itself on every call
|
|
294
|
-
* — retrying with the SAME stale write would just lose the update again,
|
|
295
|
-
* the exact bug this exists to close.
|
|
296
|
-
*
|
|
297
|
-
* Exhausting `maxAttempts` under sustained contention is returned as-is (the
|
|
298
|
-
* last conflict response) rather than retried forever: the caller's
|
|
299
|
-
* `FailureLog` reports it as a real, visible failure — never a silent lost
|
|
300
|
-
* update — same as any other write the bridge refused. */
|
|
301
|
-
async function withCasRetry(attempt, maxAttempts = 3) {
|
|
302
|
-
let res = null;
|
|
303
|
-
for (let i = 0; i < maxAttempts; i++) {
|
|
304
|
-
res = await attempt();
|
|
305
|
-
if (!res?.conflict)
|
|
306
|
-
return res;
|
|
307
|
-
}
|
|
308
|
-
return res;
|
|
309
|
-
}
|
|
310
|
-
/** cognee DataPoint normalization: lowercase, spaces→_, strip apostrophes. */
|
|
311
|
-
export function normalizeName(name) {
|
|
312
|
-
return String(name)
|
|
313
|
-
.trim()
|
|
314
|
-
.toLowerCase()
|
|
315
|
-
.replace(/['’]/g, '')
|
|
316
|
-
.replace(/\s+/g, '_')
|
|
317
|
-
.slice(0, MAX_NAME_LEN);
|
|
318
|
-
}
|
|
319
|
-
/** Edge identity: the endpoint IDs (already collision-free) plus the relation,
|
|
320
|
-
* hashed injectively so a relation containing the old `|` separator can no
|
|
321
|
-
* longer forge a different edge's key. */
|
|
322
|
-
function edgeKey(srcKey, relation, dstKey) {
|
|
323
|
-
return `e:${hashTuple([String(KG_ID_VERSION), srcKey, canonicalName(relation), dstKey])}`;
|
|
324
|
-
}
|
|
325
|
-
/** The pre-`KG_ID_VERSION` edge key, for the same in-place adoption as
|
|
326
|
-
* `legacyNodeKey`. Composed from the RESOLVED endpoint keys, so it names the
|
|
327
|
-
* legacy edge exactly when both endpoints are themselves legacy rows. */
|
|
328
|
-
function legacyEdgeKey(srcKey, relation, dstKey) {
|
|
329
|
-
return `e:${srcKey}|${normalizeName(relation)}|${dstKey}`;
|
|
330
|
-
}
|
|
331
|
-
/** Rule identity: the full rule text. The old key truncated the normalized rule
|
|
332
|
-
* at 120 characters, so two rules sharing a long preamble were one rule. */
|
|
333
|
-
function ruleKey(rule) {
|
|
334
|
-
return `rule:${hashTuple([String(KG_ID_VERSION), canonicalName(rule)])}`;
|
|
335
|
-
}
|
|
336
|
-
function legacyRuleKey(rule) {
|
|
337
|
-
return `rule:${normalizeName(rule).slice(0, 120)}`;
|
|
338
|
-
}
|
|
339
|
-
/** The namespaces a scope owns. Every read and write in this module resolves
|
|
340
|
-
* through here, which is what makes ownership enforced rather than advisory:
|
|
341
|
-
* there is no code path that reaches an org's facts without naming that org,
|
|
342
|
-
* and none that reaches every org at once. */
|
|
343
|
-
export function kgNamespaces(scope) {
|
|
344
|
-
const org = scope?.org?.trim();
|
|
345
|
-
if (!org)
|
|
346
|
-
return {
|
|
347
|
-
nodes: KG_NODES_NS,
|
|
348
|
-
edges: KG_EDGES_NS,
|
|
349
|
-
rules: RULES_NS,
|
|
350
|
-
names: KG_NAMES_NS,
|
|
351
|
-
adj: KG_ADJ_NS,
|
|
352
|
-
originIdx: KG_ORIGIN_IDX_NS,
|
|
353
|
-
indexStatus: KG_INDEX_STATUS_NS,
|
|
354
|
-
};
|
|
355
|
-
const suffix = `:org:${normalizeName(org)}`;
|
|
356
|
-
return {
|
|
357
|
-
nodes: KG_NODES_NS + suffix,
|
|
358
|
-
edges: KG_EDGES_NS + suffix,
|
|
359
|
-
rules: RULES_NS + suffix,
|
|
360
|
-
names: KG_NAMES_NS + suffix,
|
|
361
|
-
adj: KG_ADJ_NS + suffix,
|
|
362
|
-
originIdx: KG_ORIGIN_IDX_NS + suffix,
|
|
363
|
-
indexStatus: KG_INDEX_STATUS_NS + suffix,
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
/** Stamp the asserting org onto a provenance ref, so a claim's origin says WHO
|
|
367
|
-
* asserted it and not merely which run id — `run:m4x2` alone is ambiguous
|
|
368
|
-
* across orgs, and a promoted claim in the shared graph would otherwise carry
|
|
369
|
-
* an origin no one owns.
|
|
370
|
-
*
|
|
371
|
-
* Applied by the ingest/rollback entry points rather than by callers: a
|
|
372
|
-
* caller that forgets is exactly how ownership stopped being enforced. */
|
|
373
|
-
export function kgQualifyOrigin(originRef, scope) {
|
|
374
|
-
const org = scope?.org?.trim();
|
|
375
|
-
return org ? `org:${normalizeName(org)}/${originRef}` : originRef;
|
|
376
|
-
}
|
|
377
|
-
function claimsOf(md) {
|
|
378
|
-
const raw = md.claims;
|
|
379
|
-
if (Array.isArray(raw))
|
|
380
|
-
return raw
|
|
381
|
-
.filter((c) => c && typeof c.origin === 'string')
|
|
382
|
-
.map((c) => ({
|
|
383
|
-
origin: c.origin,
|
|
384
|
-
description: typeof c.description === 'string' ? c.description : '',
|
|
385
|
-
at: typeof c.at === 'number' ? c.at : 0,
|
|
386
|
-
...(c.method === 'asserted' || c.method === 'heuristic' ? { method: c.method } : {}),
|
|
387
|
-
}));
|
|
388
|
-
// Pre-ledger row (including every entry written before KG_ID_VERSION 2):
|
|
389
|
-
// seed one contribution per recorded origin, all carrying the one description
|
|
390
|
-
// the merge left behind. That is the honest reconstruction — the old merge
|
|
391
|
-
// destroyed which origin said what — and it preserves current rollback
|
|
392
|
-
// behaviour exactly: withdrawing one of several origins leaves the summary
|
|
393
|
-
// unchanged, withdrawing the last deletes the element.
|
|
394
|
-
const origins = Array.isArray(md.origin_refs) ? md.origin_refs : [];
|
|
395
|
-
const description = typeof md.description === 'string' ? md.description : '';
|
|
396
|
-
const at = typeof md.valid_from === 'number' ? md.valid_from : 0;
|
|
397
|
-
return origins.map((origin) => ({ origin, description, at }));
|
|
398
|
-
}
|
|
399
|
-
/** Add or replace `origin`'s contribution and re-derive everything that hangs
|
|
400
|
-
* off the ledger. */
|
|
401
|
-
function applyClaim(md, origin, description, now, method = 'asserted') {
|
|
402
|
-
const existing = claimsOf(md);
|
|
403
|
-
const dropped = typeof md.origins_dropped === 'number' ? md.origins_dropped : 0;
|
|
404
|
-
// An EMPTY description is support without content — "this entity exists",
|
|
405
|
-
// which is what naming an edge endpoint asserts. It must never overwrite what
|
|
406
|
-
// the same origin already said, or an edge listing a node the same payload
|
|
407
|
-
// described would blank that description out.
|
|
408
|
-
const prior = existing.find((c) => c.origin === origin);
|
|
409
|
-
if (!description.trim() && prior?.description.trim())
|
|
410
|
-
return deriveClaims(existing, dropped);
|
|
411
|
-
const claims = existing.filter((c) => c.origin !== origin);
|
|
412
|
-
claims.push({ origin, description, at: now, method });
|
|
413
|
-
return deriveClaims(claims, dropped);
|
|
414
|
-
}
|
|
415
|
-
/** The element's standing, from its surviving claims.
|
|
416
|
-
*
|
|
417
|
-
* One asserted claim outranks any number of co-occurrence guesses: a fact
|
|
418
|
-
* someone stated does not become less stated because a heuristic also stumbled
|
|
419
|
-
* onto it. A claim with no recorded method makes the whole projection UNKNOWN
|
|
420
|
-
* rather than voting — an old row cannot be read as evidence either way. */
|
|
421
|
-
function deriveMethod(claims) {
|
|
422
|
-
if (!claims.length || claims.some((c) => !c.method))
|
|
423
|
-
return undefined;
|
|
424
|
-
return claims.some((c) => c.method === 'asserted') ? 'asserted' : 'heuristic';
|
|
425
|
-
}
|
|
426
|
-
function deriveClaims(claims, alreadyDropped) {
|
|
427
|
-
let dropped = alreadyDropped;
|
|
428
|
-
if (claims.length > MAX_CLAIMS) {
|
|
429
|
-
// Oldest support goes first, and the loss is RECORDED. The scheme this
|
|
430
|
-
// replaced did the same drop silently and left the entry claiming a
|
|
431
|
-
// complete history it no longer had.
|
|
432
|
-
dropped += claims.length - MAX_CLAIMS;
|
|
433
|
-
claims = claims.slice(-MAX_CLAIMS);
|
|
434
|
-
}
|
|
435
|
-
return {
|
|
436
|
-
claims,
|
|
437
|
-
description: currentDescription(claims),
|
|
438
|
-
origin_refs: claims.map((c) => c.origin),
|
|
439
|
-
provenance_complete: dropped === 0,
|
|
440
|
-
origins_dropped: dropped,
|
|
441
|
-
conflict: new Set(claims.map((c) => c.description.trim()).filter(Boolean)).size > 1,
|
|
442
|
-
method: deriveMethod(claims),
|
|
443
|
-
};
|
|
444
|
-
}
|
|
445
|
-
/** The current summary: the most recent contribution that actually says
|
|
446
|
-
* something. LATEST wins, not longest — length measures verbosity, and
|
|
447
|
-
* "longest wins" is why re-ingesting a corrected "Now PostgreSQL" left the
|
|
448
|
-
* stale MySQL blurb in place. */
|
|
449
|
-
function currentDescription(claims) {
|
|
450
|
-
let best;
|
|
451
|
-
for (const c of claims) {
|
|
452
|
-
if (!c.description.trim())
|
|
453
|
-
continue;
|
|
454
|
-
if (!best || c.at >= best.at)
|
|
455
|
-
best = c;
|
|
456
|
-
}
|
|
457
|
-
return best?.description ?? '';
|
|
458
|
-
}
|
|
459
|
-
/** Withdraw one origin. Returns null when nothing supports the element any
|
|
460
|
-
* more (the caller deletes it); otherwise the RE-DERIVED state, which is what
|
|
461
|
-
* restores a previous correct description after a bad update is rolled back. */
|
|
462
|
-
function withoutOrigin(md, origin) {
|
|
463
|
-
const remaining = claimsOf(md).filter((c) => c.origin !== origin);
|
|
464
|
-
if (!remaining.length)
|
|
465
|
-
return null;
|
|
466
|
-
return deriveClaims(remaining, typeof md.origins_dropped === 'number' ? md.origins_dropped : 0);
|
|
467
|
-
}
|
|
468
|
-
function nameIndexKey(name) {
|
|
469
|
-
return `nm:${hashTuple([String(KG_ID_VERSION), canonicalName(name)])}`;
|
|
470
|
-
}
|
|
471
|
-
/** A coarser, additive index alongside the exact one: bucketed by `mergeKey`
|
|
472
|
-
* (case/separator/plural-insensitive), so "Node.js" and "nodejs" land in the
|
|
473
|
-
* same bucket even though their exact `nameIndexKey`s differ. Purely a hint
|
|
474
|
-
* like the exact index — `resolveEntity` still applies the same (type, name)
|
|
475
|
-
* merge rules to whatever it finds here, so a bucket hit is a candidate,
|
|
476
|
-
* never an automatic cross-type merge. */
|
|
477
|
-
function mergeIndexKey(name) {
|
|
478
|
-
return `nmk:${hashTuple([String(KG_ID_VERSION), mergeKey(name)])}`;
|
|
479
|
-
}
|
|
480
|
-
/** @returns the indexed entities, or null when the backend could not be read.
|
|
481
|
-
*
|
|
482
|
-
* The null is load-bearing: an unreadable index looks exactly like an empty
|
|
483
|
-
* one, and treating a read failure as "no entities with this name" would make
|
|
484
|
-
* the next write REPLACE the row with a single entry, erasing every other
|
|
485
|
-
* same-name entity from the index. A later generic assertion would then see one
|
|
486
|
-
* candidate and adopt it — a silent wrong merge caused by a read hiccup. */
|
|
487
|
-
async function readNameIndex(name, ns, dbPath) {
|
|
488
|
-
const res = await bridgeGetEntry({ key: nameIndexKey(name), namespace: ns.names, dbPath });
|
|
489
|
-
if (!res)
|
|
490
|
-
return null;
|
|
491
|
-
const raw = res.entry?.metadata?.entities;
|
|
492
|
-
if (!Array.isArray(raw))
|
|
493
|
-
return [];
|
|
494
|
-
return raw.filter((c) => c && typeof c.id === 'string');
|
|
495
|
-
}
|
|
496
|
-
/** Persist (or clear) the index row for one name.
|
|
497
|
-
*
|
|
498
|
-
* The index is a HINT, not a source of truth: it is written after the entity
|
|
499
|
-
* write lands, and a row that goes stale — pointing at an entity a rollback
|
|
500
|
-
* removed — self-heals, because resolving onto a missing id makes the next
|
|
501
|
-
* ingest create that entity fresh with an empty claim ledger, which is exactly
|
|
502
|
-
* what a new entity is. Nothing reads the index to decide what the graph
|
|
503
|
-
* contains; `kgStats`, `kgSearch` and `kgRollback` all read the claim
|
|
504
|
-
* namespaces. */
|
|
505
|
-
async function writeNameIndex(name, entities, ns, dbPath, failures) {
|
|
506
|
-
const key = nameIndexKey(name);
|
|
507
|
-
if (!entities.length) {
|
|
508
|
-
await bridgeDeleteEntry({ key, namespace: ns.names, dbPath });
|
|
509
|
-
return;
|
|
510
|
-
}
|
|
511
|
-
const res = await bridgeStoreEntry({
|
|
512
|
-
key,
|
|
513
|
-
value: name,
|
|
514
|
-
namespace: ns.names,
|
|
515
|
-
dbPath,
|
|
516
|
-
upsert: true,
|
|
517
|
-
generateEmbeddingFlag: false,
|
|
518
|
-
tags: ['kg', 'name-index'],
|
|
519
|
-
metadata: { kg: 'name', name, entities },
|
|
520
|
-
});
|
|
521
|
-
failures.add(res, `name index ${key}`);
|
|
522
|
-
await writeMergeIndex(name, entities, ns, dbPath, failures);
|
|
523
|
-
}
|
|
524
|
-
/** Add `entities` into the merge-key bucket for `name`, unioned with whatever
|
|
525
|
-
* other exact spellings already sharing that bucket contributed. Additive
|
|
526
|
-
* only — a name written with fewer entities than before (e.g. a rollback's
|
|
527
|
-
* survivor list) never removes another spelling's candidates from the shared
|
|
528
|
-
* bucket. That is a smaller staleness cost than the exact index already
|
|
529
|
-
* accepts, and self-heals the same way: a stale candidate that no longer
|
|
530
|
-
* resolves to a live entity simply is not adopted. */
|
|
531
|
-
async function writeMergeIndex(name, entities, ns, dbPath, failures) {
|
|
532
|
-
const key = mergeIndexKey(name);
|
|
533
|
-
const existing = await bridgeGetEntry({ key, namespace: ns.names, dbPath });
|
|
534
|
-
const prior = existing?.entry?.metadata?.entities;
|
|
535
|
-
const merged = new Map();
|
|
536
|
-
if (Array.isArray(prior))
|
|
537
|
-
for (const c of prior)
|
|
538
|
-
if (c && typeof c.id === 'string')
|
|
539
|
-
merged.set(c.id, c);
|
|
540
|
-
for (const c of entities)
|
|
541
|
-
merged.set(c.id, c);
|
|
542
|
-
const res = await bridgeStoreEntry({
|
|
543
|
-
key,
|
|
544
|
-
value: mergeKey(name),
|
|
545
|
-
namespace: ns.names,
|
|
546
|
-
dbPath,
|
|
547
|
-
upsert: true,
|
|
548
|
-
generateEmbeddingFlag: false,
|
|
549
|
-
tags: ['kg', 'name-merge-index'],
|
|
550
|
-
metadata: { kg: 'name-merge', mergeKey: mergeKey(name), entities: [...merged.values()] },
|
|
551
|
-
});
|
|
552
|
-
failures.add(res, `merge index ${key}`);
|
|
553
|
-
}
|
|
554
|
-
/** @returns candidates from the merge-key bucket, or null on a read failure —
|
|
555
|
-
* same null-is-load-bearing contract as `readNameIndex`. */
|
|
556
|
-
async function readMergeIndex(name, ns, dbPath) {
|
|
557
|
-
const res = await bridgeGetEntry({ key: mergeIndexKey(name), namespace: ns.names, dbPath });
|
|
558
|
-
if (!res)
|
|
559
|
-
return null;
|
|
560
|
-
const raw = res.entry?.metadata?.entities;
|
|
561
|
-
if (!Array.isArray(raw))
|
|
562
|
-
return [];
|
|
563
|
-
return raw.filter((c) => c && typeof c.id === 'string');
|
|
564
|
-
}
|
|
565
|
-
/**
|
|
566
|
-
* Resolve (type, name) to the entity that should carry the assertion.
|
|
567
|
-
*
|
|
568
|
-
* Identity is the (type, name) tuple, so distinct types are distinct entities.
|
|
569
|
-
* The one thing name-only identity got right — a generic label must not fork an
|
|
570
|
-
* entity away from its typed self — is preserved here instead of in the key:
|
|
571
|
-
*
|
|
572
|
-
* - exact type-bucket match → that entity
|
|
573
|
-
* - generic assertion, exactly one same-name entity → adopt it
|
|
574
|
-
* - typed assertion, exactly one same-name entity and it is untyped → adopt it
|
|
575
|
-
* and promote its type (its ID does not change, so references stay valid)
|
|
576
|
-
* - anything else → a new entity, with the alternatives as candidates
|
|
577
|
-
*
|
|
578
|
-
* @returns null when the name index could not be read — the caller must refuse
|
|
579
|
-
* the assertion rather than resolve it against an index it could not see.
|
|
580
|
-
*/
|
|
581
|
-
async function resolveEntity(name, type, ns, dbPath) {
|
|
582
|
-
const bucket = typeBucket(type);
|
|
583
|
-
let known = await readNameIndex(name, ns, dbPath);
|
|
584
|
-
if (known === null)
|
|
585
|
-
return null;
|
|
586
|
-
/** True when the index row already holds exactly `known` — the only case in
|
|
587
|
-
* which an unchanged resolution needs no index write. */
|
|
588
|
-
let indexed = known.length > 0;
|
|
589
|
-
if (!indexed) {
|
|
590
|
-
// Nothing indexed. Before minting, probe the pre-KG_ID_VERSION key: an
|
|
591
|
-
// existing graph's rows are adopted IN PLACE (keeping their key, and so
|
|
592
|
-
// keeping the edges whose keys embed it) rather than re-keyed or orphaned.
|
|
593
|
-
const legacyKey = legacyNodeKey(name);
|
|
594
|
-
const legacy = await bridgeGetEntry({ key: legacyKey, namespace: ns.nodes, dbPath });
|
|
595
|
-
if (legacy?.found && legacy.entry) {
|
|
596
|
-
const md = legacy.entry.metadata;
|
|
597
|
-
known = [{ id: legacyKey, type: typeBucket(typeof md.type === 'string' ? md.type : '') }];
|
|
598
|
-
indexed = false;
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
if (!indexed && known.length === 0) {
|
|
602
|
-
// Still nothing under this exact spelling — check for a spelling variant
|
|
603
|
-
// (case, separators, a plain plural) already known under the same
|
|
604
|
-
// merge-key bucket. A hit is only a CANDIDATE: it still goes through the
|
|
605
|
-
// exact/promote/mint rules below, so it can never merge across types.
|
|
606
|
-
const merged = await readMergeIndex(name, ns, dbPath);
|
|
607
|
-
if (merged && merged.length > 0)
|
|
608
|
-
known = merged;
|
|
609
|
-
}
|
|
610
|
-
const exact = known.find((c) => c.type === bucket);
|
|
611
|
-
if (exact)
|
|
612
|
-
return { id: exact.id, candidates: others(known, exact.id), index: indexed ? null : known };
|
|
613
|
-
// A generic assertion adopts the one entity that bears this name; a typed
|
|
614
|
-
// assertion adopts a lone UNTYPED entity and PROMOTES it — same id, so every
|
|
615
|
-
// edge and returned reference to it stays valid. A typed assertion never
|
|
616
|
-
// absorbs a differently-typed entity, and neither adopts when the name is
|
|
617
|
-
// already ambiguous.
|
|
618
|
-
if (known.length === 1 && (!bucket || !known[0].type)) {
|
|
619
|
-
const only = known[0];
|
|
620
|
-
const promoted = { id: only.id, type: bucket || only.type };
|
|
621
|
-
return { id: only.id, candidates: [], index: [promoted] };
|
|
622
|
-
}
|
|
623
|
-
const minted = { id: mintEntityId(type, name, known), type: bucket };
|
|
624
|
-
return { id: minted.id, candidates: known, index: [...known, minted] };
|
|
625
|
-
}
|
|
626
|
-
/** An ID for a new entity that no entity under this name already uses.
|
|
627
|
-
*
|
|
628
|
-
* `nodeKey(type, name)` alone is not sufficient, because promotion decouples
|
|
629
|
-
* an entity's ID from its current type: an entity minted untyped keeps the
|
|
630
|
-
* `''`-bucket ID after a typed assertion promotes it, so a LATER untyped
|
|
631
|
-
* assertion would re-derive that same ID and write its claims into the
|
|
632
|
-
* promoted entity — while simultaneously reporting that entity as one it had
|
|
633
|
-
* "kept separate from". The discriminator is bumped only on collision, so the
|
|
634
|
-
* first entity of a (type, name) still gets exactly `nodeKey(type, name)`. */
|
|
635
|
-
function mintEntityId(type, name, known) {
|
|
636
|
-
const taken = new Set(known.map((c) => c.id));
|
|
637
|
-
let id = nodeKey(type, name);
|
|
638
|
-
for (let n = 1; taken.has(id); n++)
|
|
639
|
-
id = `n:${hashTuple([String(KG_ID_VERSION), typeBucket(type), canonicalName(name), String(n)])}`;
|
|
640
|
-
return id;
|
|
641
|
-
}
|
|
642
|
-
function others(known, id) {
|
|
643
|
-
return known.filter((c) => c.id !== id);
|
|
644
|
-
}
|
|
645
|
-
// ── Ingest ──────────────────────────────────────────────────────────
|
|
646
|
-
/** Merge extracted nodes/edges into the KG.
|
|
647
|
-
*
|
|
648
|
-
* Identity is the (type, name) tuple resolved through the name index, so an
|
|
649
|
-
* entity is idempotent under re-extraction but two different things sharing a
|
|
650
|
-
* name stay two things. Each write adds this origin's CONTRIBUTION to the
|
|
651
|
-
* element's claim ledger, from which the description and `origin_refs` are
|
|
652
|
-
* derived — which is what lets a later ingest correct an earlier one and lets
|
|
653
|
-
* rollback put the earlier one back.
|
|
654
|
-
*
|
|
655
|
-
* The COMPLETE payload is validated before anything is written, and every edge
|
|
656
|
-
* endpoint is made to exist (as a placeholder entity when the caller named one
|
|
657
|
-
* that does not) before the edge lands. An edge whose endpoint could not be
|
|
658
|
-
* created is rejected rather than written: retrieval is seeded from nodes, so
|
|
659
|
-
* an edge with a missing endpoint is a fact that cannot be found through
|
|
660
|
-
* either of the things it is about.
|
|
661
|
-
*
|
|
662
|
-
* NOT ATOMIC — the memory bridge has no transaction primitive, so a failure
|
|
663
|
-
* part-way through leaves earlier writes persisted. The counters therefore
|
|
664
|
-
* report only what actually landed, `failures` lists what did not, and
|
|
665
|
-
* `success` is false whenever anything was refused. A partial ingest is safe
|
|
666
|
-
* to retry: every write is a keyed upsert. */
|
|
667
|
-
export async function kgIngest(options) {
|
|
668
|
-
const { dbPath } = options;
|
|
669
|
-
const method = options.method ?? 'asserted';
|
|
670
|
-
const ns = kgNamespaces(options.scope);
|
|
671
|
-
const originRef = kgQualifyOrigin(options.originRef, options.scope);
|
|
672
|
-
const failures = new FailureLog();
|
|
673
|
-
const report = new IngestReport();
|
|
674
|
-
let nodesAdded = 0, nodesMerged = 0, edgesAdded = 0, edgesMerged = 0;
|
|
675
|
-
try {
|
|
676
|
-
// ── Validate the COMPLETE payload before mutating anything ──
|
|
677
|
-
// An invalid item that surfaces halfway through leaves the graph holding
|
|
678
|
-
// part of a payload the caller was told was rejected.
|
|
679
|
-
const allNodes = options.nodes ?? [];
|
|
680
|
-
const allEdges = options.edges ?? [];
|
|
681
|
-
const nodes = allNodes.slice(0, MAX_NODES_PER_CALL);
|
|
682
|
-
const edges = allEdges.slice(0, MAX_EDGES_PER_CALL);
|
|
683
|
-
report.nodesTruncated = allNodes.length - nodes.length;
|
|
684
|
-
report.edgesTruncated = allEdges.length - edges.length;
|
|
685
|
-
const validNodes = [];
|
|
686
|
-
nodes.forEach((n, i) => {
|
|
687
|
-
if (!n?.name?.trim())
|
|
688
|
-
return report.rejectNode(`node[${i}]: missing name`);
|
|
689
|
-
validNodes.push({
|
|
690
|
-
input: n,
|
|
691
|
-
type: n.type?.trim() || 'entity',
|
|
692
|
-
desc: (n.description ?? '').slice(0, MAX_DESC_LEN),
|
|
693
|
-
});
|
|
694
|
-
});
|
|
695
|
-
const validEdges = [];
|
|
696
|
-
edges.forEach((e, i) => {
|
|
697
|
-
const missing = !e?.source?.trim()
|
|
698
|
-
? 'source'
|
|
699
|
-
: !e?.target?.trim()
|
|
700
|
-
? 'target'
|
|
701
|
-
: !e?.relation?.trim()
|
|
702
|
-
? 'relation'
|
|
703
|
-
: '';
|
|
704
|
-
if (missing)
|
|
705
|
-
return report.rejectEdge(`edge[${i}]: missing ${missing}`);
|
|
706
|
-
validEdges.push(e);
|
|
707
|
-
});
|
|
708
|
-
/** Entity IDs this call has already resolved AND confirmed to exist, so an
|
|
709
|
-
* edge does not re-resolve an endpoint the node loop just wrote. */
|
|
710
|
-
const resolved = new Map();
|
|
711
|
-
// Length-prefix the type so the split point is unambiguous regardless of
|
|
712
|
-
// what characters `name` contains (it can hold arbitrary text, including
|
|
713
|
-
// whatever separator a naive join might pick) — same discipline
|
|
714
|
-
// hashTuple() above uses for the same reason. A bare `\0`-joined string
|
|
715
|
-
// here was written as an actual NUL byte, not the literal two-character
|
|
716
|
-
// escape text, which is harmless at runtime (this key is an in-memory
|
|
717
|
-
// Map key only, never persisted) but made git treat this file as binary.
|
|
718
|
-
const memoKey = (name, type) => {
|
|
719
|
-
const t = typeBucket(type);
|
|
720
|
-
return `${t.length}:${t}${canonicalName(name)}`;
|
|
721
|
-
};
|
|
722
|
-
for (const { input: n, type, desc } of validNodes) {
|
|
723
|
-
const target = await resolveEntity(n.name, type, ns, dbPath);
|
|
724
|
-
if (!target) {
|
|
725
|
-
failures.note(`node ${n.name}: name index unreadable`);
|
|
726
|
-
continue;
|
|
727
|
-
}
|
|
728
|
-
report.noteAmbiguity(n.name, target.candidates);
|
|
729
|
-
const wrote = await writeEntity({
|
|
730
|
-
id: target.id,
|
|
731
|
-
name: n.name,
|
|
732
|
-
type,
|
|
733
|
-
description: desc,
|
|
734
|
-
nodeSet: n.nodeSet,
|
|
735
|
-
originRef,
|
|
736
|
-
method,
|
|
737
|
-
ns,
|
|
738
|
-
dbPath,
|
|
739
|
-
failures,
|
|
740
|
-
report,
|
|
741
|
-
});
|
|
742
|
-
if (wrote === null)
|
|
743
|
-
continue;
|
|
744
|
-
if (wrote)
|
|
745
|
-
nodesAdded++;
|
|
746
|
-
else
|
|
747
|
-
nodesMerged++;
|
|
748
|
-
resolved.set(memoKey(n.name, type), target.id);
|
|
749
|
-
if (target.index)
|
|
750
|
-
await writeNameIndex(n.name, target.index, ns, dbPath, failures);
|
|
751
|
-
await onEntrySupported(ns, ns.nodes, target.id, originRef, dbPath);
|
|
752
|
-
}
|
|
753
|
-
/** Resolve an edge endpoint, creating an explicit placeholder entity when
|
|
754
|
-
* the caller named something that does not exist. Returns null when the
|
|
755
|
-
* endpoint could not be made to exist — the edge is then rejected rather
|
|
756
|
-
* than written without it. */
|
|
757
|
-
const endpoint = async (name, type) => {
|
|
758
|
-
const memo = memoKey(name, type ?? 'entity');
|
|
759
|
-
const hit = resolved.get(memo);
|
|
760
|
-
if (hit)
|
|
761
|
-
return hit;
|
|
762
|
-
const target = await resolveEntity(name, type ?? 'entity', ns, dbPath);
|
|
763
|
-
if (!target) {
|
|
764
|
-
failures.note(`endpoint ${name}: name index unreadable`);
|
|
765
|
-
return null;
|
|
766
|
-
}
|
|
767
|
-
report.noteAmbiguity(name, target.candidates);
|
|
768
|
-
// Always write: naming an endpoint IS an assertion that it exists, so
|
|
769
|
-
// this origin joins the entity's support either way. A write that CREATES
|
|
770
|
-
// the entity is the placeholder case worth reporting.
|
|
771
|
-
const wrote = await writeEntity({
|
|
772
|
-
id: target.id,
|
|
773
|
-
name,
|
|
774
|
-
type: type ?? 'entity',
|
|
775
|
-
description: '',
|
|
776
|
-
placeholder: true,
|
|
777
|
-
originRef,
|
|
778
|
-
method,
|
|
779
|
-
ns,
|
|
780
|
-
dbPath,
|
|
781
|
-
failures,
|
|
782
|
-
report,
|
|
783
|
-
});
|
|
784
|
-
if (wrote === null)
|
|
785
|
-
return null;
|
|
786
|
-
if (wrote)
|
|
787
|
-
report.placeholders++;
|
|
788
|
-
if (target.index)
|
|
789
|
-
await writeNameIndex(name, target.index, ns, dbPath, failures);
|
|
790
|
-
resolved.set(memo, target.id);
|
|
791
|
-
await onEntrySupported(ns, ns.nodes, target.id, originRef, dbPath);
|
|
792
|
-
return target.id;
|
|
793
|
-
};
|
|
794
|
-
for (const e of validEdges) {
|
|
795
|
-
const srcKey = await endpoint(e.source, e.sourceType);
|
|
796
|
-
const dstKey = await endpoint(e.target, e.targetType);
|
|
797
|
-
if (!srcKey || !dstKey) {
|
|
798
|
-
report.rejectEdge(`edge ${e.source}-${e.relation}->${e.target}: endpoint not persisted`);
|
|
799
|
-
continue;
|
|
800
|
-
}
|
|
801
|
-
const desc = (e.description ?? '').slice(0, MAX_DESC_LEN);
|
|
802
|
-
const fallbackFact = `${e.source} ${e.relation} ${e.target}`;
|
|
803
|
-
// New key first, then the pre-KG_ID_VERSION shape: a legacy edge between
|
|
804
|
-
// two adopted legacy endpoints keeps its own key rather than being
|
|
805
|
-
// duplicated under a new one. Resolved once — a CAS retry re-reads this
|
|
806
|
-
// same key, it does not re-run legacy resolution.
|
|
807
|
-
let key = edgeKey(srcKey, e.relation, dstKey);
|
|
808
|
-
const probe = await bridgeGetEntry({ key, namespace: ns.edges, dbPath });
|
|
809
|
-
if (!probe?.found) {
|
|
810
|
-
const legacy = legacyEdgeKey(srcKey, e.relation, dstKey);
|
|
811
|
-
const hit = await bridgeGetEntry({ key: legacy, namespace: ns.edges, dbPath });
|
|
812
|
-
if (hit?.found && hit.entry)
|
|
813
|
-
key = legacy;
|
|
814
|
-
}
|
|
815
|
-
// Re-read and re-merge on every CAS attempt (see withCasRetry): a stale
|
|
816
|
-
// `md`/`derived` retried against a fresh row would silently re-lose
|
|
817
|
-
// whatever a concurrent writer just added (K5).
|
|
818
|
-
let md = {};
|
|
819
|
-
let derived;
|
|
820
|
-
let isNew = false;
|
|
821
|
-
const res = await withCasRetry(async () => {
|
|
822
|
-
const existing = await bridgeGetEntry({ key, namespace: ns.edges, dbPath });
|
|
823
|
-
isNew = !(existing?.found && existing.entry);
|
|
824
|
-
md = (existing?.entry?.metadata ?? {});
|
|
825
|
-
derived = applyClaim(md, originRef, desc, Date.now(), method);
|
|
826
|
-
const ver = existing?.entry?.version;
|
|
827
|
-
return bridgeStoreEntry({
|
|
828
|
-
key,
|
|
829
|
-
value: derived.description || fallbackFact,
|
|
830
|
-
namespace: ns.edges,
|
|
831
|
-
dbPath,
|
|
832
|
-
upsert: true,
|
|
833
|
-
generateEmbeddingFlag: false,
|
|
834
|
-
tags: ['kg', normalizeName(e.relation)],
|
|
835
|
-
metadata: {
|
|
836
|
-
...md,
|
|
837
|
-
kg: 'edge',
|
|
838
|
-
id_version: KG_ID_VERSION,
|
|
839
|
-
src: srcKey,
|
|
840
|
-
dst: dstKey,
|
|
841
|
-
relation: normalizeName(e.relation),
|
|
842
|
-
source_name: e.source,
|
|
843
|
-
target_name: e.target,
|
|
844
|
-
...derived,
|
|
845
|
-
valid_from: md.valid_from ?? Date.now(),
|
|
846
|
-
valid_to: null,
|
|
847
|
-
},
|
|
848
|
-
ifVersion: isNew ? 'absent' : typeof ver === 'number' ? ver : undefined,
|
|
849
|
-
});
|
|
850
|
-
});
|
|
851
|
-
if (failures.add(res, `edge ${key}`))
|
|
852
|
-
continue;
|
|
853
|
-
report.noteProvenanceLoss(md, derived);
|
|
854
|
-
if (isNew)
|
|
855
|
-
edgesAdded++;
|
|
856
|
-
else
|
|
857
|
-
edgesMerged++;
|
|
858
|
-
await onEntrySupported(ns, ns.edges, key, originRef, dbPath);
|
|
859
|
-
if (isNew)
|
|
860
|
-
await onEdgeWritten(ns, key, srcKey, dstKey, dbPath);
|
|
861
|
-
}
|
|
862
|
-
return {
|
|
863
|
-
success: !failures.failed,
|
|
864
|
-
nodesAdded,
|
|
865
|
-
nodesMerged,
|
|
866
|
-
edgesAdded,
|
|
867
|
-
edgesMerged,
|
|
868
|
-
...(failures.failed ? { failures: failures.messages, error: failures.summary() } : {}),
|
|
869
|
-
...report.fields(),
|
|
870
|
-
};
|
|
871
|
-
}
|
|
872
|
-
catch (err) {
|
|
873
|
-
return {
|
|
874
|
-
success: false,
|
|
875
|
-
nodesAdded,
|
|
876
|
-
nodesMerged,
|
|
877
|
-
edgesAdded,
|
|
878
|
-
edgesMerged,
|
|
879
|
-
...(failures.messages.length ? { failures: failures.messages } : {}),
|
|
880
|
-
...report.fields(),
|
|
881
|
-
error: err instanceof Error ? err.message : String(err),
|
|
882
|
-
};
|
|
883
|
-
}
|
|
884
|
-
}
|
|
885
|
-
/** Write one entity's claim contribution.
|
|
886
|
-
*
|
|
887
|
-
* @returns true when the entity was created, false when an existing one was
|
|
888
|
-
* merged into, and null when the bridge refused the write (the caller must not
|
|
889
|
-
* count it, and must not treat the entity as existing). */
|
|
890
|
-
async function writeEntity(o) {
|
|
891
|
-
// Re-read and re-merge on every CAS attempt (see withCasRetry) rather than
|
|
892
|
-
// once up front: a stale `md`/`derived` retried against a fresh row would
|
|
893
|
-
// silently re-lose whatever a concurrent writer just added (K5).
|
|
894
|
-
let md = {};
|
|
895
|
-
let derived;
|
|
896
|
-
let isNew = false;
|
|
897
|
-
const res = await withCasRetry(async () => {
|
|
898
|
-
const existing = await bridgeGetEntry({ key: o.id, namespace: o.ns.nodes, dbPath: o.dbPath });
|
|
899
|
-
isNew = !(existing?.found && existing.entry);
|
|
900
|
-
md = (existing?.entry?.metadata ?? {});
|
|
901
|
-
derived = applyClaim(md, o.originRef, o.description, Date.now(), o.method);
|
|
902
|
-
// Keep the most specific type: a generic heuristic 'entity' never
|
|
903
|
-
// overwrites an LLM-assigned one, and a specific one promotes an untyped
|
|
904
|
-
// entity.
|
|
905
|
-
const prevType = typeof md.type === 'string' ? md.type : '';
|
|
906
|
-
const bestType = typeBucket(prevType) ? prevType : o.type;
|
|
907
|
-
const nodeSet = o.nodeSet ?? (typeof md.node_set === 'string' ? md.node_set : null);
|
|
908
|
-
const ver = existing?.entry?.version;
|
|
909
|
-
return bridgeStoreEntry({
|
|
910
|
-
key: o.id,
|
|
911
|
-
value: `${o.name} — ${derived.description || bestType}`,
|
|
912
|
-
namespace: o.ns.nodes,
|
|
913
|
-
dbPath: o.dbPath,
|
|
914
|
-
upsert: true,
|
|
915
|
-
tags: ['kg', normalizeName(bestType), ...(nodeSet ? [normalizeName(nodeSet)] : [])],
|
|
916
|
-
metadata: {
|
|
917
|
-
...md,
|
|
918
|
-
kg: 'node',
|
|
919
|
-
id_version: KG_ID_VERSION,
|
|
920
|
-
type: bestType,
|
|
921
|
-
name: o.name,
|
|
922
|
-
node_set: nodeSet,
|
|
923
|
-
...derived,
|
|
924
|
-
// A placeholder stops being one the moment a real assertion describes it.
|
|
925
|
-
placeholder: o.placeholder === true && !derived.description ? true : undefined,
|
|
926
|
-
version: (typeof md.version === 'number' ? md.version : 0) + 1,
|
|
927
|
-
valid_from: md.valid_from ?? Date.now(),
|
|
928
|
-
valid_to: null,
|
|
929
|
-
},
|
|
930
|
-
// isNew ⇒ this row must not already exist; otherwise it must still be at
|
|
931
|
-
// the version we just read. Absent when the loaded backend/test double
|
|
932
|
-
// does not report a version, so behaviour is unchanged there (see
|
|
933
|
-
// bridgeGetEntry's `version` field).
|
|
934
|
-
ifVersion: isNew ? 'absent' : typeof ver === 'number' ? ver : undefined,
|
|
935
|
-
});
|
|
936
|
-
});
|
|
937
|
-
// Name first: the ID is a digest, and a failure message a human cannot map
|
|
938
|
-
// back to the thing that failed is not a diagnosis.
|
|
939
|
-
if (o.failures.add(res, `node ${o.name} (${o.id})`))
|
|
940
|
-
return null;
|
|
941
|
-
// Only after the write landed: a refused write dropped no provenance, because
|
|
942
|
-
// it changed nothing.
|
|
943
|
-
o.report.noteProvenanceLoss(md, derived);
|
|
944
|
-
return isNew;
|
|
945
|
-
}
|
|
946
|
-
/** The accepted/rejected/truncated bookkeeping a caller needs in order to know
|
|
947
|
-
* that what it sent is what the graph holds. */
|
|
948
|
-
class IngestReport {
|
|
949
|
-
nodesRejected = 0;
|
|
950
|
-
edgesRejected = 0;
|
|
951
|
-
nodesTruncated = 0;
|
|
952
|
-
edgesTruncated = 0;
|
|
953
|
-
placeholders = 0;
|
|
954
|
-
provenanceTruncated = 0;
|
|
955
|
-
rejections = [];
|
|
956
|
-
ambiguities = [];
|
|
957
|
-
rejectNode(message) {
|
|
958
|
-
this.nodesRejected++;
|
|
959
|
-
if (this.rejections.length < MAX_FAILURES)
|
|
960
|
-
this.rejections.push(message);
|
|
961
|
-
}
|
|
962
|
-
rejectEdge(message) {
|
|
963
|
-
this.edgesRejected++;
|
|
964
|
-
if (this.rejections.length < MAX_FAILURES)
|
|
965
|
-
this.rejections.push(message);
|
|
966
|
-
}
|
|
967
|
-
noteAmbiguity(name, candidates) {
|
|
968
|
-
if (!candidates.length || this.ambiguities.length >= MAX_FAILURES)
|
|
969
|
-
return;
|
|
970
|
-
const types = candidates.map((c) => c.type || 'untyped').join(', ');
|
|
971
|
-
this.ambiguities.push(`${name}: kept separate from ${candidates.length} same-name (${types})`);
|
|
972
|
-
}
|
|
973
|
-
noteProvenanceLoss(md, derived) {
|
|
974
|
-
const before = typeof md.origins_dropped === 'number' ? md.origins_dropped : 0;
|
|
975
|
-
if (derived.origins_dropped > before)
|
|
976
|
-
this.provenanceTruncated++;
|
|
977
|
-
}
|
|
978
|
-
/** Only non-zero counters are emitted, so a clean payload yields a clean
|
|
979
|
-
* result and the existing `KgIngestResult` shape is unchanged for callers
|
|
980
|
-
* that construct it themselves. */
|
|
981
|
-
fields() {
|
|
982
|
-
return {
|
|
983
|
-
...(this.nodesRejected ? { nodesRejected: this.nodesRejected } : {}),
|
|
984
|
-
...(this.edgesRejected ? { edgesRejected: this.edgesRejected } : {}),
|
|
985
|
-
...(this.nodesTruncated ? { nodesTruncated: this.nodesTruncated } : {}),
|
|
986
|
-
...(this.edgesTruncated ? { edgesTruncated: this.edgesTruncated } : {}),
|
|
987
|
-
...(this.rejections.length ? { rejections: this.rejections } : {}),
|
|
988
|
-
...(this.placeholders ? { placeholders: this.placeholders } : {}),
|
|
989
|
-
...(this.ambiguities.length ? { ambiguities: this.ambiguities } : {}),
|
|
990
|
-
...(this.provenanceTruncated ? { provenanceTruncated: this.provenanceTruncated } : {}),
|
|
991
|
-
};
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
/** Stage-2 of cognee's curator/writer distillation: the CALLER (an LLM agent)
|
|
995
|
-
* proposes candidate rules; this accepts each unless a semantically
|
|
996
|
-
* near-identical rule exists (embedding dedup — deterministic keys can't
|
|
997
|
-
* collapse paraphrases). Accepted rules are stored both as KG nodes
|
|
998
|
-
* (node_set=rules) and as plain `rules`-namespace entries so the existing
|
|
999
|
-
* injection/search surfaces pick them up with zero new plumbing.
|
|
1000
|
-
*
|
|
1001
|
-
* A candidate that dedups against an existing rule still ADDS its origin to
|
|
1002
|
-
* that rule's support set: two independent runs asserting the same rule mean
|
|
1003
|
-
* the rule survives either one being rolled back. Dropping the second origin
|
|
1004
|
-
* (as this used to) made rollback of the FIRST run delete knowledge the
|
|
1005
|
-
* second run independently vouched for.
|
|
1006
|
-
*
|
|
1007
|
-
* Like `kgIngest`, NOT atomic — see that function's note. */
|
|
1008
|
-
export async function kgIngestRules(options) {
|
|
1009
|
-
const verdicts = [];
|
|
1010
|
-
const threshold = options.dedupThreshold ?? 0.78;
|
|
1011
|
-
const failures = new FailureLog();
|
|
1012
|
-
const ns = kgNamespaces(options.scope);
|
|
1013
|
-
// Direct writes here store the qualified ref; nested kgIngest calls get the
|
|
1014
|
-
// RAW ref plus the scope and qualify it themselves, so it is stamped once.
|
|
1015
|
-
const originRef = kgQualifyOrigin(options.originRef, options.scope);
|
|
1016
|
-
let accepted = 0;
|
|
1017
|
-
const all = options.rules ?? [];
|
|
1018
|
-
const batch = all.slice(0, MAX_RULES_PER_CALL);
|
|
1019
|
-
const rulesTruncated = all.length - batch.length;
|
|
1020
|
-
try {
|
|
1021
|
-
for (const r of batch) {
|
|
1022
|
-
const rule = r?.rule?.trim();
|
|
1023
|
-
if (!rule || rule.length < 8) {
|
|
1024
|
-
verdicts.push({ rule: r?.rule ?? '', verdict: 'invalid' });
|
|
1025
|
-
continue;
|
|
1026
|
-
}
|
|
1027
|
-
const similar = await bridgeSearchEntries({
|
|
1028
|
-
query: rule,
|
|
1029
|
-
namespace: ns.rules,
|
|
1030
|
-
limit: 1,
|
|
1031
|
-
threshold,
|
|
1032
|
-
dbPath: options.dbPath,
|
|
1033
|
-
});
|
|
1034
|
-
const top = similar?.results?.[0];
|
|
1035
|
-
// Issue #111: FTS5 keyword scores are min-max normalized per batch.
|
|
1036
|
-
// With limit:1, the sole result always scores 1.0 — any rule sharing
|
|
1037
|
-
// even one keyword was falsely marked as a duplicate.
|
|
1038
|
-
//
|
|
1039
|
-
// Only trust embedding-backed (semantic) cosine scores for dedup.
|
|
1040
|
-
// Keyword-only matches can't distinguish paraphrases from merely
|
|
1041
|
-
// overlapping vocabulary — fall back to exact-text comparison.
|
|
1042
|
-
// (Key-based upsert on store already handles identical keys.)
|
|
1043
|
-
const provenance = top?.provenance;
|
|
1044
|
-
let isDuplicate = false;
|
|
1045
|
-
if (top && provenance?.startsWith('semantic:')) {
|
|
1046
|
-
const rawCosine = parseFloat(provenance.slice('semantic:'.length));
|
|
1047
|
-
isDuplicate = rawCosine >= threshold;
|
|
1048
|
-
}
|
|
1049
|
-
else if (top) {
|
|
1050
|
-
// Keyword-only: only suppress true near-exact duplicates.
|
|
1051
|
-
const existing = (top.content || '').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
1052
|
-
const candidate = rule.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
1053
|
-
isDuplicate = existing === candidate;
|
|
1054
|
-
}
|
|
1055
|
-
if (isDuplicate && top?.key) {
|
|
1056
|
-
// `already_known` is a claim that this origin now supports the existing
|
|
1057
|
-
// rule. If reinforcement did not land, it does not — and a later
|
|
1058
|
-
// rollback of the OTHER origin would delete a rule this run believes it
|
|
1059
|
-
// vouched for.
|
|
1060
|
-
const reinforced = await reinforceRuleOrigin(top.key, top.content, options.originRef, options.scope, options.dbPath, failures);
|
|
1061
|
-
verdicts.push({
|
|
1062
|
-
rule,
|
|
1063
|
-
verdict: reinforced ? 'already_known' : 'failed',
|
|
1064
|
-
similarTo: top.key,
|
|
1065
|
-
});
|
|
1066
|
-
continue;
|
|
1067
|
-
}
|
|
1068
|
-
// Identity is the full rule text. The old key truncated at 120 normalized
|
|
1069
|
-
// characters, so two rules sharing a long preamble were the same rule.
|
|
1070
|
-
// A rule stored under that scheme is adopted in place rather than
|
|
1071
|
-
// duplicated under the new key.
|
|
1072
|
-
let key = ruleKey(rule);
|
|
1073
|
-
let priorMd = {};
|
|
1074
|
-
const current = await bridgeGetEntry({ key, namespace: ns.rules, dbPath: options.dbPath });
|
|
1075
|
-
if (current?.found && current.entry)
|
|
1076
|
-
priorMd = current.entry.metadata;
|
|
1077
|
-
else {
|
|
1078
|
-
const legacy = legacyRuleKey(rule);
|
|
1079
|
-
const hit = await bridgeGetEntry({
|
|
1080
|
-
key: legacy,
|
|
1081
|
-
namespace: ns.rules,
|
|
1082
|
-
dbPath: options.dbPath,
|
|
1083
|
-
});
|
|
1084
|
-
if (hit?.found && hit.entry) {
|
|
1085
|
-
key = legacy;
|
|
1086
|
-
priorMd = hit.entry.metadata;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
// The mirrored KG node is named by the FULL rule text. Truncating it to
|
|
1090
|
-
// 200 characters put two rules sharing a long preamble on one node — the
|
|
1091
|
-
// same collision `ruleKey` was just widened to prevent, reintroduced one
|
|
1092
|
-
// namespace over.
|
|
1093
|
-
const ruleName = rule;
|
|
1094
|
-
const stored = await bridgeStoreEntry({
|
|
1095
|
-
key,
|
|
1096
|
-
value: rule + (r.context ? `\n(context: ${r.context.slice(0, 500)})` : ''),
|
|
1097
|
-
namespace: ns.rules,
|
|
1098
|
-
dbPath: options.dbPath,
|
|
1099
|
-
upsert: true,
|
|
1100
|
-
tags: ['rule'],
|
|
1101
|
-
metadata: {
|
|
1102
|
-
...priorMd,
|
|
1103
|
-
...applyClaim(priorMd, originRef, rule, Date.now()),
|
|
1104
|
-
id_version: KG_ID_VERSION,
|
|
1105
|
-
derived_from: originRef,
|
|
1106
|
-
// Lets the dedup path reinforce the matching KG node without having
|
|
1107
|
-
// to re-derive the node name from the stored value (which may carry
|
|
1108
|
-
// an appended context block).
|
|
1109
|
-
rule: ruleName,
|
|
1110
|
-
},
|
|
1111
|
-
});
|
|
1112
|
-
const nodeRes = await kgIngest({
|
|
1113
|
-
nodes: [{ name: ruleName, type: 'Rule', description: rule, nodeSet: 'rules' }],
|
|
1114
|
-
originRef: options.originRef,
|
|
1115
|
-
scope: options.scope,
|
|
1116
|
-
dbPath: options.dbPath,
|
|
1117
|
-
});
|
|
1118
|
-
const ruleFailed = failures.add(stored, `rule ${key}`);
|
|
1119
|
-
if (!ruleFailed)
|
|
1120
|
-
await onEntrySupported(ns, ns.rules, key, originRef, options.dbPath);
|
|
1121
|
-
if (nodeRes.failures?.length)
|
|
1122
|
-
for (const m of nodeRes.failures)
|
|
1123
|
-
failures.note(m);
|
|
1124
|
-
// Only count a rule as accepted when BOTH of its writes landed; a rule
|
|
1125
|
-
// present in one namespace only is not the state the caller was told
|
|
1126
|
-
// about. The per-rule verdict now says the same thing — it read
|
|
1127
|
-
// `accepted` unconditionally, so the two halves of one result disagreed.
|
|
1128
|
-
const landed = !ruleFailed && nodeRes.success;
|
|
1129
|
-
if (landed)
|
|
1130
|
-
accepted++;
|
|
1131
|
-
verdicts.push({ rule, verdict: landed ? 'accepted' : 'failed' });
|
|
1132
|
-
}
|
|
1133
|
-
return {
|
|
1134
|
-
success: !failures.failed,
|
|
1135
|
-
verdicts,
|
|
1136
|
-
accepted,
|
|
1137
|
-
...(failures.failed ? { failures: failures.messages, error: failures.summary() } : {}),
|
|
1138
|
-
...(rulesTruncated ? { rulesTruncated } : {}),
|
|
1139
|
-
};
|
|
1140
|
-
}
|
|
1141
|
-
catch (err) {
|
|
1142
|
-
return {
|
|
1143
|
-
success: false,
|
|
1144
|
-
verdicts,
|
|
1145
|
-
accepted,
|
|
1146
|
-
...(failures.messages.length ? { failures: failures.messages } : {}),
|
|
1147
|
-
...(rulesTruncated ? { rulesTruncated } : {}),
|
|
1148
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1149
|
-
};
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
/** Add `originRef` to an already-stored rule's support set — both the
|
|
1153
|
-
* `rules`-namespace entry and its `node_set=rules` KG node. Idempotent in
|
|
1154
|
-
* effect: re-asserting an origin the rule already carries leaves the support
|
|
1155
|
-
* set unchanged.
|
|
1156
|
-
*
|
|
1157
|
-
* @returns true when the rule genuinely carries this origin afterwards —
|
|
1158
|
-
* including the no-op case where it already did. False means the support set
|
|
1159
|
-
* is not what the caller is about to be told it is. */
|
|
1160
|
-
async function reinforceRuleOrigin(matchedKey, matchedContent,
|
|
1161
|
-
/** RAW ref — qualified here for the rules entry, and passed on unqualified
|
|
1162
|
-
* to `kgIngest`, which qualifies it once with the same scope. */
|
|
1163
|
-
originRef, scope, dbPath, failures) {
|
|
1164
|
-
const ns = kgNamespaces(scope);
|
|
1165
|
-
const qualified = kgQualifyOrigin(originRef, scope);
|
|
1166
|
-
const existing = await bridgeGetEntry({ key: matchedKey, namespace: ns.rules, dbPath });
|
|
1167
|
-
if (!existing?.found || !existing.entry) {
|
|
1168
|
-
// The dedup hit came from search; if the entry can't be re-read by key the
|
|
1169
|
-
// support set cannot be updated, and silently proceeding is exactly the
|
|
1170
|
-
// provenance loss this function exists to prevent.
|
|
1171
|
-
failures.note(`rule ${matchedKey}: matched by dedup but not readable by key`);
|
|
1172
|
-
return false;
|
|
1173
|
-
}
|
|
1174
|
-
const entry = existing.entry;
|
|
1175
|
-
const md = entry.metadata;
|
|
1176
|
-
const origins = Array.isArray(md.origin_refs) ? md.origin_refs : [];
|
|
1177
|
-
// The KG node's name: recorded at accept time, else the first line of the
|
|
1178
|
-
// stored value (any context block is appended after a newline).
|
|
1179
|
-
const ruleName = typeof md.rule === 'string' && md.rule
|
|
1180
|
-
? md.rule
|
|
1181
|
-
: (matchedContent || entry.content).split('\n')[0];
|
|
1182
|
-
// Both writes are attempted regardless: skipping the node write because the
|
|
1183
|
-
// entry write failed would leave the two halves disagreeing about who
|
|
1184
|
-
// supports this rule. Only the REPORT changes.
|
|
1185
|
-
let ok = true;
|
|
1186
|
-
if (!origins.includes(qualified)) {
|
|
1187
|
-
const res = await bridgeStoreEntry({
|
|
1188
|
-
key: entry.key,
|
|
1189
|
-
value: entry.content,
|
|
1190
|
-
namespace: ns.rules,
|
|
1191
|
-
dbPath,
|
|
1192
|
-
upsert: true,
|
|
1193
|
-
generateEmbeddingFlag: entry.hasEmbedding,
|
|
1194
|
-
tags: entry.tags,
|
|
1195
|
-
metadata: {
|
|
1196
|
-
...md,
|
|
1197
|
-
rule: ruleName,
|
|
1198
|
-
// A dedup hit is SUPPORT, not a correction: the candidate asserts the
|
|
1199
|
-
// rule the entry already holds. Its contribution therefore carries no
|
|
1200
|
-
// description of its own — passing one here (the truncated `rule` name,
|
|
1201
|
-
// as this did) made the newest claim disagree with the stored text and
|
|
1202
|
-
// flagged the rule as conflicted with a truncation of itself.
|
|
1203
|
-
...applyClaim(md, qualified, '', Date.now()),
|
|
1204
|
-
},
|
|
1205
|
-
});
|
|
1206
|
-
if (failures.add(res, `rule ${entry.key}`))
|
|
1207
|
-
ok = false;
|
|
1208
|
-
else
|
|
1209
|
-
await onEntrySupported(ns, ns.rules, entry.key, qualified, dbPath);
|
|
1210
|
-
}
|
|
1211
|
-
// The rule's KG node needs the same origin — rollback walks nodes separately.
|
|
1212
|
-
const nodeRes = await kgIngest({
|
|
1213
|
-
nodes: [{ name: ruleName, type: 'Rule', description: '', nodeSet: 'rules' }],
|
|
1214
|
-
originRef,
|
|
1215
|
-
scope,
|
|
1216
|
-
dbPath,
|
|
1217
|
-
});
|
|
1218
|
-
if (nodeRes.failures?.length)
|
|
1219
|
-
for (const m of nodeRes.failures)
|
|
1220
|
-
failures.note(m);
|
|
1221
|
-
return ok && nodeRes.success;
|
|
1222
|
-
}
|
|
1223
|
-
/** List stored rules (for injection or review). */
|
|
1224
|
-
export async function kgListRules(options) {
|
|
1225
|
-
const res = await bridgeListEntries({
|
|
1226
|
-
namespace: kgNamespaces(options?.scope).rules,
|
|
1227
|
-
limit: options?.limit ?? 50,
|
|
1228
|
-
dbPath: options?.dbPath,
|
|
1229
|
-
});
|
|
1230
|
-
return (res?.entries ?? []).map((e) => ({ rule: e.content, key: e.key }));
|
|
1231
|
-
}
|
|
1232
|
-
/** Seed candidates pulled before filtering and ranking. */
|
|
1233
|
-
const SEARCH_SEED_LIMIT = 15;
|
|
1234
|
-
/** Extra candidates fetched when a `nodeSet` narrows the graph.
|
|
1235
|
-
*
|
|
1236
|
-
* The bridge has no tag filter, so set membership can only be tested after
|
|
1237
|
-
* retrieval. Filtering the unfiltered top-15 meant a node that IS in the set
|
|
1238
|
-
* but ranks 16th overall was missed — the set made results scarcer instead of
|
|
1239
|
-
* more precise. Over-fetching moves the cutoff after the filter. */
|
|
1240
|
-
const SEARCH_NODE_SET_OVERFETCH = 4;
|
|
1241
|
-
/** How far a co-occurrence guess drops below an equally-seeded stated fact.
|
|
1242
|
-
* Enough to lose a tie, not enough to hide it: `mentioned_with` between two
|
|
1243
|
-
* strong seeds is still worth surfacing when nothing better was asserted. */
|
|
1244
|
-
const HEURISTIC_PENALTY = 0.15;
|
|
1245
|
-
/** Live origins disagree about what this edge says. Still returned — a disputed
|
|
1246
|
-
* fact is information — but it does not outrank a settled one. */
|
|
1247
|
-
const CONFLICT_PENALTY = 0.1;
|
|
1248
|
-
/** Seeded retrieval → neighborhood → triplet ranking (cognee's brute-force
|
|
1249
|
-
* triplet search, scaled down). Seed scores already carry the Phase 1 feedback
|
|
1250
|
-
* blend, and the seed retrieval may be vector or keyword — `method` on the
|
|
1251
|
-
* result says which actually ran.
|
|
1252
|
-
*
|
|
1253
|
-
* Ranking weighs exactly two evidence signals, both read off the claim ledger:
|
|
1254
|
-
* extraction method and description conflict. It deliberately does NOT model
|
|
1255
|
-
* source credibility, claim freshness, or whether the relation itself answers
|
|
1256
|
-
* the query — those need an evaluation set to tune against, and guessing at
|
|
1257
|
-
* them would be the same overclaim this weighting exists to correct. */
|
|
1258
|
-
export async function kgSearch(options) {
|
|
1259
|
-
try {
|
|
1260
|
-
const limit = options.limit ?? 8;
|
|
1261
|
-
const ns = kgNamespaces(options.scope);
|
|
1262
|
-
const seedsRes = await bridgeSearchEntries({
|
|
1263
|
-
query: options.query,
|
|
1264
|
-
namespace: ns.nodes,
|
|
1265
|
-
// Over-fetch when a set filter follows, so the cutoff lands AFTER it.
|
|
1266
|
-
limit: options.nodeSet ? SEARCH_SEED_LIMIT * SEARCH_NODE_SET_OVERFETCH : SEARCH_SEED_LIMIT,
|
|
1267
|
-
threshold: 0.25,
|
|
1268
|
-
dbPath: options.dbPath,
|
|
1269
|
-
});
|
|
1270
|
-
// What the retrieval actually was, carried on every return below: a keyword
|
|
1271
|
-
// fallback presented as vector-seeded search is the overclaim B5 names.
|
|
1272
|
-
const retrieval = {
|
|
1273
|
-
...(seedsRes?.searchMethod ? { method: seedsRes.searchMethod } : {}),
|
|
1274
|
-
...(seedsRes?.fallbackReason ? { fallbackReason: seedsRes.fallbackReason } : {}),
|
|
1275
|
-
};
|
|
1276
|
-
let seedResults = seedsRes?.results ?? [];
|
|
1277
|
-
if (options.nodeSet) {
|
|
1278
|
-
const setTag = normalizeName(options.nodeSet);
|
|
1279
|
-
seedResults = seedResults.filter((r) => (r.tags ?? []).includes(setTag));
|
|
1280
|
-
}
|
|
1281
|
-
seedResults = seedResults.slice(0, SEARCH_SEED_LIMIT);
|
|
1282
|
-
if (!seedResults.length)
|
|
1283
|
-
return { success: true, context: '', triplets: [], seeds: [], ...retrieval };
|
|
1284
|
-
const seedScore = new Map();
|
|
1285
|
-
for (const s of seedResults)
|
|
1286
|
-
seedScore.set(s.key, s.score);
|
|
1287
|
-
const triplets = [];
|
|
1288
|
-
let scannedEdges = 0;
|
|
1289
|
-
let truncated = false;
|
|
1290
|
-
/** Score one edge against the seeded entities and, if relevant, push its
|
|
1291
|
-
* triplet — shared by the indexed and exhaustive gathering paths below
|
|
1292
|
-
* so ranking never depends on which one ran (K7). */
|
|
1293
|
-
const considerEdge = (e) => {
|
|
1294
|
-
scannedEdges++;
|
|
1295
|
-
const md = (e.metadata ?? {});
|
|
1296
|
-
if (md?.kg !== 'edge' || md.valid_to != null)
|
|
1297
|
-
return;
|
|
1298
|
-
const src = String(md.src ?? '');
|
|
1299
|
-
const dst = String(md.dst ?? '');
|
|
1300
|
-
const sSrc = seedScore.get(src) ?? 0;
|
|
1301
|
-
const sDst = seedScore.get(dst) ?? 0;
|
|
1302
|
-
if (sSrc === 0 && sDst === 0)
|
|
1303
|
-
return;
|
|
1304
|
-
// Both endpoints seeded beats one; the unseeded endpoint contributes a
|
|
1305
|
-
// neutral 0.35 so bridging edges from a strong seed still surface.
|
|
1306
|
-
const relevance = (Math.max(sSrc, 0.35) + Math.max(sDst, 0.35)) / 2 + (sSrc > 0 && sDst > 0 ? 0.1 : 0);
|
|
1307
|
-
// Evidence, from the claim ledger. An edge whose method was never
|
|
1308
|
-
// recorded is left at its relevance score — unknown is not evidence
|
|
1309
|
-
// against it, and penalizing it would demote the entire pre-existing
|
|
1310
|
-
// graph relative to anything written today.
|
|
1311
|
-
const method = md.method === 'asserted' || md.method === 'heuristic'
|
|
1312
|
-
? md.method
|
|
1313
|
-
: undefined;
|
|
1314
|
-
const conflict = md.conflict === true;
|
|
1315
|
-
const score = Math.max(0, relevance -
|
|
1316
|
-
(method === 'heuristic' ? HEURISTIC_PENALTY : 0) -
|
|
1317
|
-
(conflict ? CONFLICT_PENALTY : 0));
|
|
1318
|
-
triplets.push({
|
|
1319
|
-
source: String(md.source_name ?? src),
|
|
1320
|
-
relation: String(md.relation ?? 'related_to'),
|
|
1321
|
-
target: String(md.target_name ?? dst),
|
|
1322
|
-
fact: e.content,
|
|
1323
|
-
score,
|
|
1324
|
-
...(method ? { method } : {}),
|
|
1325
|
-
...(conflict ? { conflict } : {}),
|
|
1326
|
-
id: e.id,
|
|
1327
|
-
key: e.key,
|
|
1328
|
-
});
|
|
1329
|
-
};
|
|
1330
|
-
// Scores are per-edge, so pruning to the running top-`limit` after every
|
|
1331
|
-
// batch yields exactly the same result as sorting the whole set at the end.
|
|
1332
|
-
const pruneToLimit = () => {
|
|
1333
|
-
if (triplets.length > limit) {
|
|
1334
|
-
triplets.sort((a, b) => b.score - a.score);
|
|
1335
|
-
triplets.length = limit;
|
|
1336
|
-
}
|
|
1337
|
-
};
|
|
1338
|
-
// K7: gather candidate edges via each seed's adjacency entry — O(seeds ×
|
|
1339
|
-
// degree) instead of a full namespace scan — when the scope's index is
|
|
1340
|
-
// ready and every seed's adjacency entry resolves. Any seed that misses
|
|
1341
|
-
// (index not ready, or an unresolvable ref) falls the WHOLE query back to
|
|
1342
|
-
// the exhaustive scan rather than silently searching only some seeds.
|
|
1343
|
-
let covered = false;
|
|
1344
|
-
let usedIndex = false;
|
|
1345
|
-
if ((await readIndexStatus(ns, options.dbPath)).state === 'ready') {
|
|
1346
|
-
const candidateKeys = new Set();
|
|
1347
|
-
let indexOk = true;
|
|
1348
|
-
for (const s of seedResults) {
|
|
1349
|
-
const adj = await readAdj(ns, s.key, options.dbPath);
|
|
1350
|
-
if (adj === null) {
|
|
1351
|
-
indexOk = false;
|
|
1352
|
-
break;
|
|
1353
|
-
}
|
|
1354
|
-
for (const key of adj.edgeKeys)
|
|
1355
|
-
candidateKeys.add(key);
|
|
1356
|
-
}
|
|
1357
|
-
if (indexOk) {
|
|
1358
|
-
for (const key of candidateKeys) {
|
|
1359
|
-
const res = await bridgeGetEntry({ key, namespace: ns.edges, dbPath: options.dbPath });
|
|
1360
|
-
if (res?.found && res.entry)
|
|
1361
|
-
considerEdge(res.entry);
|
|
1362
|
-
}
|
|
1363
|
-
pruneToLimit();
|
|
1364
|
-
usedIndex = true;
|
|
1365
|
-
covered = true;
|
|
1366
|
-
}
|
|
1367
|
-
}
|
|
1368
|
-
// Paged edge scan (see monolean note in module header). Each page is folded
|
|
1369
|
-
// into the running top-`limit` immediately, so memory stays at one page
|
|
1370
|
-
// regardless of how many edges the namespace holds.
|
|
1371
|
-
if (!usedIndex) {
|
|
1372
|
-
covered = await scanNamespace(ns.edges, options.dbPath, (page) => {
|
|
1373
|
-
for (const e of page)
|
|
1374
|
-
considerEdge(e);
|
|
1375
|
-
pruneToLimit();
|
|
1376
|
-
if (scannedEdges >= SEARCH_EDGE_SCAN_MAX) {
|
|
1377
|
-
truncated = true;
|
|
1378
|
-
return false;
|
|
1379
|
-
}
|
|
1380
|
-
return true;
|
|
1381
|
-
});
|
|
1382
|
-
}
|
|
1383
|
-
// An unreadable namespace is an incomplete answer, not an empty graph.
|
|
1384
|
-
if (!covered)
|
|
1385
|
-
truncated = true;
|
|
1386
|
-
triplets.sort((a, b) => b.score - a.score);
|
|
1387
|
-
const seeds = seedResults.slice(0, limit).map((s) => {
|
|
1388
|
-
// metadata is not in search results; parse from rendered content "name — description"
|
|
1389
|
-
const dash = s.content.indexOf(' — ');
|
|
1390
|
-
return {
|
|
1391
|
-
name: dash > 0 ? s.content.slice(0, dash) : s.key,
|
|
1392
|
-
// Tags carry the stored type (`['kg', <type>, …]`). The key never did:
|
|
1393
|
-
// reading `key.split(':')[1]` reported the type of `n:shared_service`
|
|
1394
|
-
// as `shared_service`, and under hashed IDs would report a digest.
|
|
1395
|
-
type: (s.tags ?? [])[1] ?? 'entity',
|
|
1396
|
-
description: dash > 0 ? s.content.slice(dash + 3) : s.content,
|
|
1397
|
-
score: s.score,
|
|
1398
|
-
id: s.id,
|
|
1399
|
-
};
|
|
1400
|
-
});
|
|
1401
|
-
const context = [
|
|
1402
|
-
...triplets.map((t) => `${t.source} —${t.relation}→ ${t.target}${t.fact && t.fact !== `${t.source} ${t.relation} ${t.target}` ? ` (${t.fact})` : ''}`),
|
|
1403
|
-
...(triplets.length ? [] : seeds.map((s) => `${s.name}: ${s.description}`)),
|
|
1404
|
-
].join('\n');
|
|
1405
|
-
return {
|
|
1406
|
-
success: true,
|
|
1407
|
-
context,
|
|
1408
|
-
triplets,
|
|
1409
|
-
seeds,
|
|
1410
|
-
scannedEdges,
|
|
1411
|
-
...(truncated && { truncated }),
|
|
1412
|
-
...retrieval,
|
|
1413
|
-
};
|
|
1414
|
-
}
|
|
1415
|
-
catch (err) {
|
|
1416
|
-
return {
|
|
1417
|
-
success: false,
|
|
1418
|
-
context: '',
|
|
1419
|
-
triplets: [],
|
|
1420
|
-
seeds: [],
|
|
1421
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1422
|
-
};
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
// ── Glossary (anti-duplicate-entity injection for extraction prompts) ──
|
|
1426
|
-
export async function kgGlossary(options) {
|
|
1427
|
-
const limit = options?.limit ?? 40;
|
|
1428
|
-
// Running top-`limit` by rank, deduplicated by normalized name. Folding each
|
|
1429
|
-
// page in and pruning keeps the whole node namespace in scope without ever
|
|
1430
|
-
// holding more than a page plus `limit` names.
|
|
1431
|
-
let top = [];
|
|
1432
|
-
await scanNamespace(kgNamespaces(options?.scope).nodes, options?.dbPath, (page) => {
|
|
1433
|
-
for (const e of page) {
|
|
1434
|
-
const md = e.metadata;
|
|
1435
|
-
// Glossary is for ENTITY name reuse — rule prose and extraction-source
|
|
1436
|
-
// Session nodes would drown it.
|
|
1437
|
-
const t = String(md?.type ?? '').toLowerCase();
|
|
1438
|
-
if (md?.node_set === 'rules' || t === 'rule' || t === 'session')
|
|
1439
|
-
continue;
|
|
1440
|
-
const fw = typeof md.feedback_weight === 'number' ? md.feedback_weight : 0.5;
|
|
1441
|
-
const freq = typeof md.frequency_weight === 'number' ? md.frequency_weight : 0;
|
|
1442
|
-
const version = typeof md.version === 'number' ? md.version : 1;
|
|
1443
|
-
const name = String(md.name ?? e.key);
|
|
1444
|
-
top.push({ name, norm: normalizeName(name), rank: version + freq + fw });
|
|
1445
|
-
}
|
|
1446
|
-
top.sort((a, b) => b.rank - a.rank);
|
|
1447
|
-
const seen = new Set();
|
|
1448
|
-
const pruned = [];
|
|
1449
|
-
for (const n of top) {
|
|
1450
|
-
if (seen.has(n.norm))
|
|
1451
|
-
continue;
|
|
1452
|
-
seen.add(n.norm);
|
|
1453
|
-
pruned.push(n);
|
|
1454
|
-
if (pruned.length >= limit)
|
|
1455
|
-
break;
|
|
1456
|
-
}
|
|
1457
|
-
top = pruned;
|
|
1458
|
-
});
|
|
1459
|
-
return top.map((n) => n.name);
|
|
1460
|
-
}
|
|
1461
|
-
// ── Rollback (per-origin bad-ingest recovery) ───────────────────────
|
|
1462
|
-
function originsOf(entry) {
|
|
1463
|
-
const origins = (entry.metadata ?? {}).origin_refs;
|
|
1464
|
-
return Array.isArray(origins) ? origins : [];
|
|
1465
|
-
}
|
|
1466
|
-
/** Every entry in `namespace` that `originRef` supports, collected by an
|
|
1467
|
-
* EXHAUSTIVE paged scan. `covered` is false when the backend became
|
|
1468
|
-
* unreadable partway — an incomplete answer, never an empty one.
|
|
1469
|
-
*
|
|
1470
|
-
* Collecting rather than streaming is required by the callers that mutate:
|
|
1471
|
-
* deleting a row pulls later rows back, so an advancing offset would skip
|
|
1472
|
-
* whatever slid into the gap. Only origin-carrying entries are retained, so
|
|
1473
|
-
* memory tracks the operation's own footprint, not the namespace size. */
|
|
1474
|
-
async function collectByOrigin(namespace, originRef, dbPath) {
|
|
1475
|
-
const entries = [];
|
|
1476
|
-
const covered = await scanNamespace(namespace, dbPath, (page) => {
|
|
1477
|
-
for (const e of page)
|
|
1478
|
-
if (originsOf(e).includes(originRef))
|
|
1479
|
-
entries.push(e);
|
|
1480
|
-
});
|
|
1481
|
-
return { entries, covered };
|
|
1482
|
-
}
|
|
1483
|
-
/** Withdraw `originRef`'s support from the graph: remove it from every
|
|
1484
|
-
* node/edge/rule it backs, and delete the element once no origin remains.
|
|
1485
|
-
*
|
|
1486
|
-
* The withdrawn ref is REWRITTEN out of the surviving elements' origin lists,
|
|
1487
|
-
* not left behind. Retaining it (as this used to) meant a second rollback saw
|
|
1488
|
-
* a two-entry list and retained again — so an element could outlive the
|
|
1489
|
-
* withdrawal of every origin that ever supported it.
|
|
1490
|
-
*
|
|
1491
|
-
* The scan is EXHAUSTIVE: it pages each namespace to the end rather than
|
|
1492
|
-
* reading one capped list. A capped scan let an element past the cap keep a
|
|
1493
|
-
* withdrawn origin while the caller was told the rollback succeeded.
|
|
1494
|
-
*
|
|
1495
|
-
* Collect-then-mutate is deliberate: deleting a row pulls every later row back
|
|
1496
|
-
* one position, so a delete inside the page loop would make an advancing
|
|
1497
|
-
* offset skip whatever slid into the gap. (Rewrites are no longer a hazard —
|
|
1498
|
-
* the bridge's upsert reuses the existing entry id and preserves `createdAt`
|
|
1499
|
-
* rather than re-inserting at the head of the default ordering — but this
|
|
1500
|
-
* function deletes as well as rewrites.) Only origin-carrying entries are
|
|
1501
|
-
* retained during the scan, so memory tracks the rollback's own footprint, not
|
|
1502
|
-
* the namespace size. */
|
|
1503
|
-
export async function kgRollback(options) {
|
|
1504
|
-
const failures = new FailureLog();
|
|
1505
|
-
const namespaces = kgNamespaces(options.scope);
|
|
1506
|
-
const originRef = kgQualifyOrigin(options.originRef, options.scope);
|
|
1507
|
-
let deleted = 0, retained = 0;
|
|
1508
|
-
/** Entity IDs this rollback removed, and the names that pointed at them. */
|
|
1509
|
-
const removedIds = new Set();
|
|
1510
|
-
const removedNames = new Set();
|
|
1511
|
-
try {
|
|
1512
|
-
// K7: one indexed read across all three namespaces, tried before the
|
|
1513
|
-
// exhaustive per-namespace scan. Only trusted when the scope's index is
|
|
1514
|
-
// `ready` AND every ref it names still resolves — `kgIndexedByOrigin`
|
|
1515
|
-
// returns null otherwise, and this falls straight back to the scan.
|
|
1516
|
-
let indexedByNs = null;
|
|
1517
|
-
if ((await readIndexStatus(namespaces, options.dbPath)).state === 'ready') {
|
|
1518
|
-
const indexed = await kgIndexedByOrigin(namespaces, originRef, options.dbPath);
|
|
1519
|
-
if (indexed !== null) {
|
|
1520
|
-
indexedByNs = new Map([
|
|
1521
|
-
[namespaces.nodes, []],
|
|
1522
|
-
[namespaces.edges, []],
|
|
1523
|
-
[namespaces.rules, []],
|
|
1524
|
-
]);
|
|
1525
|
-
for (const { ns, entry } of indexed)
|
|
1526
|
-
indexedByNs.get(ns)?.push(entry);
|
|
1527
|
-
}
|
|
1528
|
-
}
|
|
1529
|
-
for (const ns of [namespaces.nodes, namespaces.edges, namespaces.rules]) {
|
|
1530
|
-
const found = indexedByNs
|
|
1531
|
-
? { entries: indexedByNs.get(ns) ?? [], covered: true }
|
|
1532
|
-
: await collectByOrigin(ns, originRef, options.dbPath);
|
|
1533
|
-
// A partial scan cannot be reported as a completed withdrawal.
|
|
1534
|
-
if (!found.covered) {
|
|
1535
|
-
failures.note(`${ns}: memory backend unavailable`);
|
|
1536
|
-
continue;
|
|
1537
|
-
}
|
|
1538
|
-
const withdrawn = found.entries.map((e) => ({
|
|
1539
|
-
entry: e,
|
|
1540
|
-
// Re-derive from the claim ledger rather than editing an origin list:
|
|
1541
|
-
// this is what puts back the description a withdrawn origin overwrote.
|
|
1542
|
-
remaining: withoutOrigin((e.metadata ?? {}), originRef),
|
|
1543
|
-
}));
|
|
1544
|
-
for (const { entry: e, remaining } of withdrawn) {
|
|
1545
|
-
const md = (e.metadata ?? {});
|
|
1546
|
-
if (remaining === null) {
|
|
1547
|
-
const del = await bridgeDeleteEntry({ id: e.id, namespace: ns, dbPath: options.dbPath });
|
|
1548
|
-
if (del?.deleted) {
|
|
1549
|
-
deleted++;
|
|
1550
|
-
await onOriginWithdrawn(namespaces, ns, e.key, originRef, options.dbPath);
|
|
1551
|
-
await onEntryDeleted(namespaces, e.key, options.dbPath, ns === namespaces.edges
|
|
1552
|
-
? { src: String(md.src ?? ''), dst: String(md.dst ?? '') }
|
|
1553
|
-
: undefined);
|
|
1554
|
-
}
|
|
1555
|
-
else
|
|
1556
|
-
failures.note(`${ns}/${e.key}: delete failed`);
|
|
1557
|
-
if (ns === namespaces.nodes && del?.deleted) {
|
|
1558
|
-
removedIds.add(e.key);
|
|
1559
|
-
if (typeof md.name === 'string')
|
|
1560
|
-
removedNames.add(md.name);
|
|
1561
|
-
}
|
|
1562
|
-
continue;
|
|
1563
|
-
}
|
|
1564
|
-
const store = await bridgeStoreEntry({
|
|
1565
|
-
key: e.key,
|
|
1566
|
-
value: rerender(e.content, md, remaining.description, ns === namespaces.rules),
|
|
1567
|
-
namespace: ns,
|
|
1568
|
-
dbPath: options.dbPath,
|
|
1569
|
-
upsert: true,
|
|
1570
|
-
// Edges are stored without embeddings; re-deriving one here would
|
|
1571
|
-
// silently change how the entry behaves in search.
|
|
1572
|
-
generateEmbeddingFlag: e.hasEmbedding,
|
|
1573
|
-
tags: e.tags,
|
|
1574
|
-
metadata: { ...md, ...remaining },
|
|
1575
|
-
});
|
|
1576
|
-
if (!failures.add(store, `${ns}/${e.key}: origin withdrawal`)) {
|
|
1577
|
-
retained++;
|
|
1578
|
-
await onOriginWithdrawn(namespaces, ns, e.key, originRef, options.dbPath);
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
}
|
|
1582
|
-
// Drop the removed entities out of the name index, so a later ingest of the
|
|
1583
|
-
// same name does not resolve onto an entity that no longer exists.
|
|
1584
|
-
for (const name of removedNames) {
|
|
1585
|
-
const known = await readNameIndex(name, namespaces, options.dbPath);
|
|
1586
|
-
if (known === null) {
|
|
1587
|
-
// An unreadable index must not be rewritten from a guess — doing so
|
|
1588
|
-
// would erase every same-name entity this rollback did NOT remove.
|
|
1589
|
-
failures.note(`${namespaces.names}: name index unreadable for ${name}`);
|
|
1590
|
-
continue;
|
|
1591
|
-
}
|
|
1592
|
-
const survivors = known.filter((c) => !removedIds.has(c.id));
|
|
1593
|
-
if (survivors.length !== known.length)
|
|
1594
|
-
await writeNameIndex(name, survivors, namespaces, options.dbPath, failures);
|
|
1595
|
-
}
|
|
1596
|
-
const danglingEdgesRemoved = removedIds.size
|
|
1597
|
-
? await removeEdgesMissingEndpoints(namespaces, removedIds, options.dbPath, failures)
|
|
1598
|
-
: 0;
|
|
1599
|
-
return {
|
|
1600
|
-
success: !failures.failed,
|
|
1601
|
-
deleted,
|
|
1602
|
-
retained,
|
|
1603
|
-
...(failures.failed ? { failures: failures.messages, error: failures.summary() } : {}),
|
|
1604
|
-
...(danglingEdgesRemoved ? { danglingEdgesRemoved } : {}),
|
|
1605
|
-
};
|
|
1606
|
-
}
|
|
1607
|
-
catch (err) {
|
|
1608
|
-
return {
|
|
1609
|
-
success: false,
|
|
1610
|
-
deleted,
|
|
1611
|
-
retained,
|
|
1612
|
-
...(failures.messages.length ? { failures: failures.messages } : {}),
|
|
1613
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1614
|
-
};
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
/** Re-render a stored value after its derived description changed.
|
|
1618
|
-
*
|
|
1619
|
-
* Rules are left alone: their value is the rule text plus an optional context
|
|
1620
|
-
* block, not a rendering of a description. */
|
|
1621
|
-
function rerender(content, md, description, isRule) {
|
|
1622
|
-
if (isRule || description === md.description)
|
|
1623
|
-
return content;
|
|
1624
|
-
if (md.kg === 'edge')
|
|
1625
|
-
return description || `${md.source_name} ${md.relation} ${md.target_name}`;
|
|
1626
|
-
return `${md.name} — ${description || md.type}`;
|
|
1627
|
-
}
|
|
1628
|
-
/** Delete every edge incident to one of `removedIds`. Collect-then-mutate for
|
|
1629
|
-
* the same reason `kgRollback` does. */
|
|
1630
|
-
async function removeEdgesMissingEndpoints(ns, removedIds, dbPath, failures) {
|
|
1631
|
-
const doomed = [];
|
|
1632
|
-
const covered = await scanNamespace(ns.edges, dbPath, (page) => {
|
|
1633
|
-
for (const e of page) {
|
|
1634
|
-
const md = (e.metadata ?? {});
|
|
1635
|
-
const src = String(md.src ?? '');
|
|
1636
|
-
const dst = String(md.dst ?? '');
|
|
1637
|
-
if (removedIds.has(src) || removedIds.has(dst))
|
|
1638
|
-
doomed.push({ entry: e, src, dst });
|
|
1639
|
-
}
|
|
1640
|
-
});
|
|
1641
|
-
if (!covered) {
|
|
1642
|
-
failures.note(`${ns.edges}: memory backend unavailable during dangling-edge sweep`);
|
|
1643
|
-
return 0;
|
|
1644
|
-
}
|
|
1645
|
-
let removed = 0;
|
|
1646
|
-
for (const { entry: e, src, dst } of doomed) {
|
|
1647
|
-
const del = await bridgeDeleteEntry({ id: e.id, namespace: ns.edges, dbPath });
|
|
1648
|
-
if (del?.deleted) {
|
|
1649
|
-
removed++;
|
|
1650
|
-
// Only the surviving endpoint needs its adjacency entry pruned — the
|
|
1651
|
-
// removed one's own entry is moot, its entity is gone too.
|
|
1652
|
-
await onEntryDeleted(ns, e.key, dbPath, {
|
|
1653
|
-
src: removedIds.has(src) ? dst : src,
|
|
1654
|
-
dst: removedIds.has(src) ? dst : src,
|
|
1655
|
-
});
|
|
1656
|
-
}
|
|
1657
|
-
else
|
|
1658
|
-
failures.note(`${ns.edges}/${e.key}: dangling-edge delete failed`);
|
|
1659
|
-
}
|
|
1660
|
-
return removed;
|
|
1661
|
-
}
|
|
1662
|
-
/** Copy one origin's claims out of an org's scope into project-shared
|
|
1663
|
-
* knowledge.
|
|
1664
|
-
*
|
|
1665
|
-
* Sharing is deliberate, never a side effect of learning: `kgIngest` under a
|
|
1666
|
-
* scope only ever writes that org's namespaces, and this is the one path a
|
|
1667
|
-
* claim takes across the boundary. The org keeps its own copy untouched — the
|
|
1668
|
-
* shared copy is an INDEPENDENT assertion under `promoted:<org-ref>`, so
|
|
1669
|
-
* rolling back either side leaves the other standing, and a shared claim
|
|
1670
|
-
* always names the org that vouched for it.
|
|
1671
|
-
*
|
|
1672
|
-
* Like ingest, NOT atomic: the counters report what actually landed. */
|
|
1673
|
-
export async function kgPromote(options) {
|
|
1674
|
-
const empty = { nodes: 0, edges: 0, rules: 0 };
|
|
1675
|
-
if (!options.from?.org?.trim())
|
|
1676
|
-
return {
|
|
1677
|
-
success: false,
|
|
1678
|
-
...empty,
|
|
1679
|
-
promotedAs: '',
|
|
1680
|
-
error: 'promotion needs an owning org to promote from',
|
|
1681
|
-
};
|
|
1682
|
-
const ns = kgNamespaces(options.from);
|
|
1683
|
-
const sourceRef = kgQualifyOrigin(options.originRef, options.from);
|
|
1684
|
-
const promotedAs = `promoted:${sourceRef}`;
|
|
1685
|
-
const failures = new FailureLog();
|
|
1686
|
-
try {
|
|
1687
|
-
const [nodeHits, edgeHits, ruleHits] = await Promise.all([
|
|
1688
|
-
collectByOrigin(ns.nodes, sourceRef, options.dbPath),
|
|
1689
|
-
collectByOrigin(ns.edges, sourceRef, options.dbPath),
|
|
1690
|
-
collectByOrigin(ns.rules, sourceRef, options.dbPath),
|
|
1691
|
-
]);
|
|
1692
|
-
// A partial read would promote a partial claim set while reporting the
|
|
1693
|
-
// whole origin as shared.
|
|
1694
|
-
for (const [name, hit] of [
|
|
1695
|
-
['nodes', nodeHits],
|
|
1696
|
-
['edges', edgeHits],
|
|
1697
|
-
['rules', ruleHits],
|
|
1698
|
-
])
|
|
1699
|
-
if (!hit.covered)
|
|
1700
|
-
failures.note(`${name}: memory backend unavailable`);
|
|
1701
|
-
if (failures.failed)
|
|
1702
|
-
return {
|
|
1703
|
-
success: false,
|
|
1704
|
-
...empty,
|
|
1705
|
-
promotedAs,
|
|
1706
|
-
failures: failures.messages,
|
|
1707
|
-
error: failures.summary(),
|
|
1708
|
-
};
|
|
1709
|
-
// Rule NODES are re-created by kgIngestRules; promoting them again through
|
|
1710
|
-
// kgIngest would double-count and strip their rules-namespace entry.
|
|
1711
|
-
const nodes = nodeHits.entries
|
|
1712
|
-
.filter((e) => {
|
|
1713
|
-
const md = (e.metadata ?? {});
|
|
1714
|
-
return md.node_set !== 'rules' && String(md.type ?? '').toLowerCase() !== 'rule';
|
|
1715
|
-
})
|
|
1716
|
-
.map((e) => {
|
|
1717
|
-
const md = (e.metadata ?? {});
|
|
1718
|
-
return {
|
|
1719
|
-
name: String(md.name ?? e.key),
|
|
1720
|
-
type: String(md.type ?? 'entity'),
|
|
1721
|
-
description: String(md.description ?? ''),
|
|
1722
|
-
nodeSet: typeof md.node_set === 'string' ? md.node_set : undefined,
|
|
1723
|
-
};
|
|
1724
|
-
});
|
|
1725
|
-
const edges = edgeHits.entries.map((e) => {
|
|
1726
|
-
const md = (e.metadata ?? {});
|
|
1727
|
-
return {
|
|
1728
|
-
source: String(md.source_name ?? md.src ?? ''),
|
|
1729
|
-
target: String(md.target_name ?? md.dst ?? ''),
|
|
1730
|
-
relation: String(md.relation ?? 'related_to'),
|
|
1731
|
-
description: String(md.description ?? ''),
|
|
1732
|
-
};
|
|
1733
|
-
});
|
|
1734
|
-
const rules = ruleHits.entries.map((e) => {
|
|
1735
|
-
const md = (e.metadata ?? {});
|
|
1736
|
-
return { rule: String(md.rule ?? e.content.split('\n')[0]) };
|
|
1737
|
-
});
|
|
1738
|
-
let promotedNodes = 0, promotedEdges = 0, promotedRules = 0;
|
|
1739
|
-
/** A batch that REJECTED part of what it was handed did not promote the
|
|
1740
|
-
* whole origin, so it cannot be reported as a clean share. Rejections are
|
|
1741
|
-
* not in `failures` — they are the caller's payload being refused, not the
|
|
1742
|
-
* bridge failing — so they have to be carried across explicitly. */
|
|
1743
|
-
const carry = (res, what) => {
|
|
1744
|
-
if (res.failures?.length)
|
|
1745
|
-
for (const m of res.failures)
|
|
1746
|
-
failures.note(m);
|
|
1747
|
-
const rejected = (res.nodesRejected ?? 0) + (res.edgesRejected ?? 0);
|
|
1748
|
-
if (rejected)
|
|
1749
|
-
failures.note(`${what}: ${rejected} item(s) rejected — ${res.rejections?.[0]}`);
|
|
1750
|
-
};
|
|
1751
|
-
// kgIngest caps a call at 500 nodes / 1000 edges, so a large origin has to
|
|
1752
|
-
// be promoted in batches rather than silently truncated. Nodes go first so
|
|
1753
|
-
// every edge endpoint already exists when the edges land.
|
|
1754
|
-
for (let i = 0; i < nodes.length; i += MAX_NODES_PER_CALL) {
|
|
1755
|
-
const res = await kgIngest({
|
|
1756
|
-
nodes: nodes.slice(i, i + MAX_NODES_PER_CALL),
|
|
1757
|
-
originRef: promotedAs,
|
|
1758
|
-
dbPath: options.dbPath,
|
|
1759
|
-
});
|
|
1760
|
-
promotedNodes += res.nodesAdded + res.nodesMerged;
|
|
1761
|
-
carry(res, 'promoted nodes');
|
|
1762
|
-
}
|
|
1763
|
-
for (let i = 0; i < edges.length; i += MAX_EDGES_PER_CALL) {
|
|
1764
|
-
const res = await kgIngest({
|
|
1765
|
-
nodes: [],
|
|
1766
|
-
edges: edges.slice(i, i + MAX_EDGES_PER_CALL),
|
|
1767
|
-
originRef: promotedAs,
|
|
1768
|
-
dbPath: options.dbPath,
|
|
1769
|
-
});
|
|
1770
|
-
promotedEdges += res.edgesAdded + res.edgesMerged;
|
|
1771
|
-
carry(res, 'promoted edges');
|
|
1772
|
-
}
|
|
1773
|
-
for (let i = 0; i < rules.length; i += MAX_RULES_PER_CALL) {
|
|
1774
|
-
const res = await kgIngestRules({
|
|
1775
|
-
rules: rules.slice(i, i + MAX_RULES_PER_CALL),
|
|
1776
|
-
originRef: promotedAs,
|
|
1777
|
-
dbPath: options.dbPath,
|
|
1778
|
-
});
|
|
1779
|
-
promotedRules += res.accepted;
|
|
1780
|
-
if (res.failures?.length)
|
|
1781
|
-
for (const m of res.failures)
|
|
1782
|
-
failures.note(m);
|
|
1783
|
-
const invalid = res.verdicts.filter((v) => v.verdict === 'invalid').length;
|
|
1784
|
-
if (invalid)
|
|
1785
|
-
failures.note(`promoted rules: ${invalid} rejected as invalid`);
|
|
1786
|
-
}
|
|
1787
|
-
return {
|
|
1788
|
-
success: !failures.failed,
|
|
1789
|
-
nodes: promotedNodes,
|
|
1790
|
-
edges: promotedEdges,
|
|
1791
|
-
rules: promotedRules,
|
|
1792
|
-
promotedAs,
|
|
1793
|
-
...(failures.failed ? { failures: failures.messages, error: failures.summary() } : {}),
|
|
1794
|
-
};
|
|
1795
|
-
}
|
|
1796
|
-
catch (err) {
|
|
1797
|
-
return {
|
|
1798
|
-
success: false,
|
|
1799
|
-
...empty,
|
|
1800
|
-
promotedAs,
|
|
1801
|
-
...(failures.messages.length ? { failures: failures.messages } : {}),
|
|
1802
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1803
|
-
};
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
/** Entities whose descriptions are stale relative to their connectivity —
|
|
1807
|
-
* the LLM half runs in the LIVE agent: it rewrites each candidate's
|
|
1808
|
-
* description from the neighborhood facts and resubmits via memory_kg_ingest.
|
|
1809
|
-
* A resubmission is a NEW contribution and therefore the current one, so a
|
|
1810
|
-
* shorter but better-supported summary now wins; under "longest description
|
|
1811
|
-
* wins" a consolidation that tightened the prose was silently discarded.
|
|
1812
|
-
* No LLM here (fully local constraint). */
|
|
1813
|
-
export async function kgConsolidateCandidates(options) {
|
|
1814
|
-
const minEdges = options?.minEdges ?? 3;
|
|
1815
|
-
const limit = options?.limit ?? 10;
|
|
1816
|
-
const ns = kgNamespaces(options?.scope);
|
|
1817
|
-
// Degree index over the FULL edge namespace. Only the first 12 facts per node
|
|
1818
|
-
// are kept (that is all the result exposes), so the index costs a bounded
|
|
1819
|
-
// amount per node rather than one string per edge.
|
|
1820
|
-
const degree = new Map();
|
|
1821
|
-
await scanNamespace(ns.edges, options?.dbPath, (page) => {
|
|
1822
|
-
for (const e of page) {
|
|
1823
|
-
const md = e.metadata;
|
|
1824
|
-
for (const end of [String(md.src ?? ''), String(md.dst ?? '')]) {
|
|
1825
|
-
if (!end)
|
|
1826
|
-
continue;
|
|
1827
|
-
const slot = degree.get(end) ?? { count: 0, facts: [] };
|
|
1828
|
-
slot.count++;
|
|
1829
|
-
if (slot.facts.length < 12)
|
|
1830
|
-
slot.facts.push(e.content);
|
|
1831
|
-
degree.set(end, slot);
|
|
1832
|
-
}
|
|
1833
|
-
}
|
|
1834
|
-
});
|
|
1835
|
-
let candidates = [];
|
|
1836
|
-
await scanNamespace(ns.nodes, options?.dbPath, (page) => {
|
|
1837
|
-
for (const n of page) {
|
|
1838
|
-
const md = n.metadata;
|
|
1839
|
-
const slot = degree.get(n.key);
|
|
1840
|
-
if (!slot || slot.count < minEdges)
|
|
1841
|
-
continue;
|
|
1842
|
-
const description = String(md.description ?? '');
|
|
1843
|
-
// Cap the growth target at MAX_DESC_LEN — a very-high-degree node whose
|
|
1844
|
-
// description is already at the cap can never "grow out" of candidacy and
|
|
1845
|
-
// would otherwise permanently occupy a slot.
|
|
1846
|
-
if (description.length >= Math.min(40 * slot.count, MAX_DESC_LEN))
|
|
1847
|
-
continue;
|
|
1848
|
-
candidates.push({
|
|
1849
|
-
name: String(md.name ?? n.key),
|
|
1850
|
-
type: String(md.type ?? 'entity'),
|
|
1851
|
-
description,
|
|
1852
|
-
edgeCount: slot.count,
|
|
1853
|
-
neighborhood: slot.facts,
|
|
1854
|
-
});
|
|
1855
|
-
}
|
|
1856
|
-
// Ranking is per-node, so keeping only the running top-`limit` after each
|
|
1857
|
-
// page gives the same answer as ranking every node at the end.
|
|
1858
|
-
candidates.sort((a, b) => b.edgeCount - a.edgeCount);
|
|
1859
|
-
candidates = candidates.slice(0, limit);
|
|
1860
|
-
});
|
|
1861
|
-
return candidates;
|
|
1862
|
-
}
|
|
1863
|
-
// ── Stats ───────────────────────────────────────────────────────────
|
|
1864
|
-
/** Real counts, not page lengths. `bridgeListEntries.total` reports how many
|
|
1865
|
-
* rows that one call returned, so the old capped list made a 10,001-node graph
|
|
1866
|
-
* report exactly 10,000 forever.
|
|
1867
|
-
*
|
|
1868
|
-
* Tries a real indexed `SELECT COUNT(*) WHERE namespace = ?` first
|
|
1869
|
-
* (`bridgeCountEntries`, K7) — cheap and exact, since every row in a KG
|
|
1870
|
-
* namespace is exactly one node/edge/rule (the name index lives in its own
|
|
1871
|
-
* namespace). Falls back to the paginated scan — one query per 1,000 rows,
|
|
1872
|
-
* still exact — when the loaded bridge predates `bridgeCountEntries`, or a
|
|
1873
|
-
* test double doesn't stub it. */
|
|
1874
|
-
export async function kgStats(options) {
|
|
1875
|
-
const ns = kgNamespaces(options?.scope);
|
|
1876
|
-
const count = async (namespace) => {
|
|
1877
|
-
try {
|
|
1878
|
-
const real = await bridgeCountEntries(namespace, options?.dbPath);
|
|
1879
|
-
if (real !== null)
|
|
1880
|
-
return real;
|
|
1881
|
-
}
|
|
1882
|
-
catch {
|
|
1883
|
-
/* bridgeCountEntries unavailable on this loaded bridge (or a test
|
|
1884
|
-
double that doesn't stub it) — fall back to the exhaustive scan. */
|
|
1885
|
-
}
|
|
1886
|
-
let n = 0;
|
|
1887
|
-
await scanNamespace(namespace, options?.dbPath, (page) => {
|
|
1888
|
-
n += page.length;
|
|
1889
|
-
});
|
|
1890
|
-
return n;
|
|
1891
|
-
};
|
|
1892
|
-
const [nodes, edges, rules] = await Promise.all([
|
|
1893
|
-
count(ns.nodes),
|
|
1894
|
-
count(ns.edges),
|
|
1895
|
-
count(ns.rules),
|
|
1896
|
-
]);
|
|
1897
|
-
return { nodes, edges, rules };
|
|
1898
|
-
}
|
|
1899
|
-
/** Complete, uncapped read of every edge touching `endpointId` (as either
|
|
1900
|
-
* src or dst) and/or asserted by `originRef` — at least one filter is
|
|
1901
|
-
* required. This is the ground truth an indexed adjacency/origin lookup
|
|
1902
|
-
* (K7) must agree with once one exists: an exhaustive paged scan, the same
|
|
1903
|
-
* mechanism `kgRollback` already trusts for origin withdrawal, so it never
|
|
1904
|
-
* depends on an index and never inherits the old first-page cap.
|
|
1905
|
-
*
|
|
1906
|
-
* `truncated: true` means the scan did not finish — an incomplete answer,
|
|
1907
|
-
* never an empty one. Callers comparing this against a future index must
|
|
1908
|
-
* treat a truncated reference read as "unknown", not as "no matches". */
|
|
1909
|
-
export async function kgReferenceEdges(options) {
|
|
1910
|
-
if (!options.endpointId && !options.originRef) {
|
|
1911
|
-
return { success: false, edges: [], error: 'endpointId or originRef is required' };
|
|
1912
|
-
}
|
|
1913
|
-
const ns = kgNamespaces(options.scope);
|
|
1914
|
-
const originRef = options.originRef
|
|
1915
|
-
? kgQualifyOrigin(options.originRef, options.scope)
|
|
1916
|
-
: undefined;
|
|
1917
|
-
const edges = [];
|
|
1918
|
-
try {
|
|
1919
|
-
const covered = await scanNamespace(ns.edges, options.dbPath, (page) => {
|
|
1920
|
-
for (const e of page) {
|
|
1921
|
-
const md = (e.metadata ?? {});
|
|
1922
|
-
if (md.kg !== 'edge')
|
|
1923
|
-
continue;
|
|
1924
|
-
const src = String(md.src ?? '');
|
|
1925
|
-
const dst = String(md.dst ?? '');
|
|
1926
|
-
if (options.endpointId && src !== options.endpointId && dst !== options.endpointId)
|
|
1927
|
-
continue;
|
|
1928
|
-
const originRefs = originsOf(e);
|
|
1929
|
-
if (originRef && !originRefs.includes(originRef))
|
|
1930
|
-
continue;
|
|
1931
|
-
edges.push({
|
|
1932
|
-
key: e.key,
|
|
1933
|
-
src,
|
|
1934
|
-
dst,
|
|
1935
|
-
relation: String(md.relation ?? 'related_to'),
|
|
1936
|
-
originRefs,
|
|
1937
|
-
});
|
|
1938
|
-
}
|
|
1939
|
-
});
|
|
1940
|
-
return { success: true, edges, ...(covered ? {} : { truncated: true }) };
|
|
1941
|
-
}
|
|
1942
|
-
catch (err) {
|
|
1943
|
-
return { success: false, edges, error: err instanceof Error ? err.message : String(err) };
|
|
1944
|
-
}
|
|
1945
|
-
}
|
|
1946
|
-
const KG_INDEX_SCHEMA_VERSION = 1;
|
|
1947
|
-
const INDEX_STATUS_KEY = 'status';
|
|
1948
|
-
/** Edge keys recorded per adjacency entry before the index refuses to grow it
|
|
1949
|
-
* further and fails the SCOPE's index rather than risk an oversized or
|
|
1950
|
-
* silently truncated row. A hub node past this reads via the exhaustive
|
|
1951
|
-
* scan, exactly as before the index existed — this only gates the fast
|
|
1952
|
-
* path, never correctness. */
|
|
1953
|
-
const MAX_ADJ_EDGES_PER_NODE = 2000;
|
|
1954
|
-
/** Refs recorded per origin-support entry before the same refusal applies. */
|
|
1955
|
-
const MAX_ORIGIN_INDEX_REFS = 5000;
|
|
1956
|
-
async function readIndexStatus(ns, dbPath) {
|
|
1957
|
-
const res = await bridgeGetEntry({ key: INDEX_STATUS_KEY, namespace: ns.indexStatus, dbPath });
|
|
1958
|
-
if (!res?.found || !res.entry)
|
|
1959
|
-
return { state: 'absent', schemaVersion: KG_INDEX_SCHEMA_VERSION };
|
|
1960
|
-
const md = (res.entry.metadata ?? {});
|
|
1961
|
-
// An older/foreign schema is not a resumable build — start over rather than
|
|
1962
|
-
// trust rows shaped by a version this code no longer understands.
|
|
1963
|
-
if (md.schemaVersion !== KG_INDEX_SCHEMA_VERSION) {
|
|
1964
|
-
return { state: 'absent', schemaVersion: KG_INDEX_SCHEMA_VERSION };
|
|
1965
|
-
}
|
|
1966
|
-
return { ...md, schemaVersion: KG_INDEX_SCHEMA_VERSION };
|
|
1967
|
-
}
|
|
1968
|
-
async function writeIndexStatus(ns, status, dbPath) {
|
|
1969
|
-
const res = await bridgeStoreEntry({
|
|
1970
|
-
key: INDEX_STATUS_KEY,
|
|
1971
|
-
value: `kg index: ${status.state}`,
|
|
1972
|
-
namespace: ns.indexStatus,
|
|
1973
|
-
dbPath,
|
|
1974
|
-
upsert: true,
|
|
1975
|
-
generateEmbeddingFlag: false,
|
|
1976
|
-
metadata: { ...status, updatedAt: Date.now() },
|
|
1977
|
-
});
|
|
1978
|
-
return Boolean(res?.success);
|
|
1979
|
-
}
|
|
1980
|
-
/** Downgrade a scope's index to `failed` after a dual-write hiccup, so reads
|
|
1981
|
-
* fall back to the exhaustive scan instead of silently drifting. Never
|
|
1982
|
-
* throws — this runs from inside a best-effort maintenance path. */
|
|
1983
|
-
async function markIndexFailed(ns, dbPath, error) {
|
|
1984
|
-
try {
|
|
1985
|
-
const current = await readIndexStatus(ns, dbPath);
|
|
1986
|
-
if (current.state === 'absent' || current.state === 'failed')
|
|
1987
|
-
return; // nothing to protect
|
|
1988
|
-
// Not resumable: the cursor here is stale (either a completed build's end
|
|
1989
|
-
// position, or mid-build), and a dual-write hiccup means something is
|
|
1990
|
-
// missing from an UNKNOWN part of the index — only a fresh rescan finds it.
|
|
1991
|
-
await writeIndexStatus(ns, { ...current, state: 'failed', resumable: false, error }, dbPath);
|
|
1992
|
-
}
|
|
1993
|
-
catch {
|
|
1994
|
-
/* best-effort: if even the downgrade write fails, the next rebuild's
|
|
1995
|
-
validation step still catches an inconsistent index before it is
|
|
1996
|
-
ever trusted for a read. */
|
|
1997
|
-
}
|
|
1998
|
-
}
|
|
1999
|
-
/** This scope's derived-index build/readiness state. `absent` means no one
|
|
2000
|
-
* has ever called `kgRebuildIndex` for it — every read behaves exactly as
|
|
2001
|
-
* it did before this index existed. */
|
|
2002
|
-
export async function kgIndexStatus(options) {
|
|
2003
|
-
return readIndexStatus(kgNamespaces(options?.scope), options?.dbPath);
|
|
2004
|
-
}
|
|
2005
|
-
async function readAdj(ns, entityId, dbPath) {
|
|
2006
|
-
const res = await bridgeGetEntry({ key: entityId, namespace: ns.adj, dbPath });
|
|
2007
|
-
if (!res)
|
|
2008
|
-
return null; // backend unavailable, distinct from "no entry yet"
|
|
2009
|
-
if (!res.found || !res.entry)
|
|
2010
|
-
return { edgeKeys: [] };
|
|
2011
|
-
const md = (res.entry.metadata ?? {});
|
|
2012
|
-
return { edgeKeys: Array.isArray(md.edgeKeys) ? md.edgeKeys : [] };
|
|
2013
|
-
}
|
|
2014
|
-
async function addToAdj(ns, entityId, edgeKey, dbPath) {
|
|
2015
|
-
const current = await readAdj(ns, entityId, dbPath);
|
|
2016
|
-
if (current === null)
|
|
2017
|
-
return false;
|
|
2018
|
-
if (current.edgeKeys.includes(edgeKey))
|
|
2019
|
-
return true; // already present, idempotent
|
|
2020
|
-
if (current.edgeKeys.length >= MAX_ADJ_EDGES_PER_NODE)
|
|
2021
|
-
return false;
|
|
2022
|
-
const res = await bridgeStoreEntry({
|
|
2023
|
-
key: entityId,
|
|
2024
|
-
value: `kg adjacency: ${entityId}`,
|
|
2025
|
-
namespace: ns.adj,
|
|
2026
|
-
dbPath,
|
|
2027
|
-
upsert: true,
|
|
2028
|
-
generateEmbeddingFlag: false,
|
|
2029
|
-
metadata: { edgeKeys: [...current.edgeKeys, edgeKey] },
|
|
2030
|
-
});
|
|
2031
|
-
return Boolean(res?.success);
|
|
2032
|
-
}
|
|
2033
|
-
async function removeFromAdj(ns, entityId, edgeKey, dbPath) {
|
|
2034
|
-
const current = await readAdj(ns, entityId, dbPath);
|
|
2035
|
-
if (current === null)
|
|
2036
|
-
return false;
|
|
2037
|
-
if (!current.edgeKeys.includes(edgeKey))
|
|
2038
|
-
return true; // already absent
|
|
2039
|
-
const res = await bridgeStoreEntry({
|
|
2040
|
-
key: entityId,
|
|
2041
|
-
value: `kg adjacency: ${entityId}`,
|
|
2042
|
-
namespace: ns.adj,
|
|
2043
|
-
dbPath,
|
|
2044
|
-
upsert: true,
|
|
2045
|
-
generateEmbeddingFlag: false,
|
|
2046
|
-
metadata: { edgeKeys: current.edgeKeys.filter((k) => k !== edgeKey) },
|
|
2047
|
-
});
|
|
2048
|
-
return Boolean(res?.success);
|
|
2049
|
-
}
|
|
2050
|
-
async function readOriginIndex(ns, originRef, dbPath) {
|
|
2051
|
-
const res = await bridgeGetEntry({ key: originRef, namespace: ns.originIdx, dbPath });
|
|
2052
|
-
if (!res)
|
|
2053
|
-
return null;
|
|
2054
|
-
if (!res.found || !res.entry)
|
|
2055
|
-
return { refs: [] };
|
|
2056
|
-
const md = (res.entry.metadata ?? {});
|
|
2057
|
-
return { refs: Array.isArray(md.refs) ? md.refs : [] };
|
|
2058
|
-
}
|
|
2059
|
-
async function addToOriginIndex(ns, originRef, ref, dbPath) {
|
|
2060
|
-
const current = await readOriginIndex(ns, originRef, dbPath);
|
|
2061
|
-
if (current === null)
|
|
2062
|
-
return false;
|
|
2063
|
-
if (current.refs.some((r) => r.ns === ref.ns && r.key === ref.key))
|
|
2064
|
-
return true;
|
|
2065
|
-
if (current.refs.length >= MAX_ORIGIN_INDEX_REFS)
|
|
2066
|
-
return false;
|
|
2067
|
-
const res = await bridgeStoreEntry({
|
|
2068
|
-
key: originRef,
|
|
2069
|
-
value: `kg origin index: ${originRef}`,
|
|
2070
|
-
namespace: ns.originIdx,
|
|
2071
|
-
dbPath,
|
|
2072
|
-
upsert: true,
|
|
2073
|
-
generateEmbeddingFlag: false,
|
|
2074
|
-
metadata: { refs: [...current.refs, ref] },
|
|
2075
|
-
});
|
|
2076
|
-
return Boolean(res?.success);
|
|
2077
|
-
}
|
|
2078
|
-
async function removeFromOriginIndex(ns, originRef, ref, dbPath) {
|
|
2079
|
-
const current = await readOriginIndex(ns, originRef, dbPath);
|
|
2080
|
-
if (current === null)
|
|
2081
|
-
return false;
|
|
2082
|
-
const next = current.refs.filter((r) => !(r.ns === ref.ns && r.key === ref.key));
|
|
2083
|
-
if (next.length === current.refs.length)
|
|
2084
|
-
return true; // already absent
|
|
2085
|
-
const res = await bridgeStoreEntry({
|
|
2086
|
-
key: originRef,
|
|
2087
|
-
value: `kg origin index: ${originRef}`,
|
|
2088
|
-
namespace: ns.originIdx,
|
|
2089
|
-
dbPath,
|
|
2090
|
-
upsert: true,
|
|
2091
|
-
generateEmbeddingFlag: false,
|
|
2092
|
-
metadata: { refs: next },
|
|
2093
|
-
});
|
|
2094
|
-
return Boolean(res?.success);
|
|
2095
|
-
}
|
|
2096
|
-
// ── Dual-write hooks ──────────────────────────────────────────────────
|
|
2097
|
-
//
|
|
2098
|
-
// Best-effort and self-gating: a no-op (one status read) while the scope's
|
|
2099
|
-
// index is `absent`, so ordinary ingest/rollback pay nothing extra until
|
|
2100
|
-
// someone opts in by calling `kgRebuildIndex`. A failure here fails the
|
|
2101
|
-
// INDEX (downgrades the scope to `failed`), never the canonical write or
|
|
2102
|
-
// delete it accompanies — the index is a cache, not a second source of truth.
|
|
2103
|
-
/** After a node/edge/rule write lands, record that `originRef` supports it. */
|
|
2104
|
-
async function onEntrySupported(ns, entryNs, entryKey, originRef, dbPath) {
|
|
2105
|
-
const status = await readIndexStatus(ns, dbPath);
|
|
2106
|
-
if (status.state === 'absent' || status.state === 'failed')
|
|
2107
|
-
return;
|
|
2108
|
-
const ok = await addToOriginIndex(ns, originRef, { ns: entryNs, key: entryKey }, dbPath);
|
|
2109
|
-
if (!ok)
|
|
2110
|
-
await markIndexFailed(ns, dbPath, `origin index write failed for ${originRef}`);
|
|
2111
|
-
}
|
|
2112
|
-
/** After an edge write lands, record it in both endpoints' adjacency. */
|
|
2113
|
-
async function onEdgeWritten(ns, edgeKey, src, dst, dbPath) {
|
|
2114
|
-
const status = await readIndexStatus(ns, dbPath);
|
|
2115
|
-
if (status.state === 'absent' || status.state === 'failed')
|
|
2116
|
-
return;
|
|
2117
|
-
const okSrc = await addToAdj(ns, src, edgeKey, dbPath);
|
|
2118
|
-
const okDst = src === dst ? true : await addToAdj(ns, dst, edgeKey, dbPath);
|
|
2119
|
-
if (!okSrc || !okDst)
|
|
2120
|
-
await markIndexFailed(ns, dbPath, `adjacency write failed for ${edgeKey}`);
|
|
2121
|
-
}
|
|
2122
|
-
/** After `kgRollback` deletes an entry outright (no origin left to support
|
|
2123
|
-
* it), remove it from every index it could appear in. */
|
|
2124
|
-
async function onEntryDeleted(ns, entryKey, dbPath, endpoints) {
|
|
2125
|
-
const status = await readIndexStatus(ns, dbPath);
|
|
2126
|
-
if (status.state === 'absent' || status.state === 'failed')
|
|
2127
|
-
return;
|
|
2128
|
-
let ok = true;
|
|
2129
|
-
if (endpoints) {
|
|
2130
|
-
ok = (await removeFromAdj(ns, endpoints.src, entryKey, dbPath)) && ok;
|
|
2131
|
-
if (endpoints.dst !== endpoints.src)
|
|
2132
|
-
ok = (await removeFromAdj(ns, endpoints.dst, entryKey, dbPath)) && ok;
|
|
2133
|
-
}
|
|
2134
|
-
if (!ok)
|
|
2135
|
-
await markIndexFailed(ns, dbPath, `adjacency removal failed for ${entryKey}`);
|
|
2136
|
-
}
|
|
2137
|
-
/** After `kgRollback` withdraws one origin's support from an entry that
|
|
2138
|
-
* survives (another origin still supports it), drop just that origin's ref. */
|
|
2139
|
-
async function onOriginWithdrawn(ns, entryNs, entryKey, originRef, dbPath) {
|
|
2140
|
-
const status = await readIndexStatus(ns, dbPath);
|
|
2141
|
-
if (status.state === 'absent' || status.state === 'failed')
|
|
2142
|
-
return;
|
|
2143
|
-
const ok = await removeFromOriginIndex(ns, originRef, { ns: entryNs, key: entryKey }, dbPath);
|
|
2144
|
-
if (!ok)
|
|
2145
|
-
await markIndexFailed(ns, dbPath, `origin index removal failed for ${originRef}`);
|
|
2146
|
-
}
|
|
2147
|
-
// ── Indexed reads (used by kgSearch/kgRollback only when state === 'ready') ─
|
|
2148
|
-
/** The indexed equivalent of `kgReferenceEdges({ endpointId })`: edges
|
|
2149
|
-
* touching one entity, read via its adjacency entry instead of a namespace
|
|
2150
|
-
* scan. Returns `null` when any edge key it names cannot be resolved — a
|
|
2151
|
-
* torn index must never be presented as a complete answer. */
|
|
2152
|
-
async function kgIndexedEdgesByEndpoint(ns, entityId, dbPath) {
|
|
2153
|
-
const adj = await readAdj(ns, entityId, dbPath);
|
|
2154
|
-
if (adj === null)
|
|
2155
|
-
return null;
|
|
2156
|
-
const edges = [];
|
|
2157
|
-
for (const key of adj.edgeKeys) {
|
|
2158
|
-
const res = await bridgeGetEntry({ key, namespace: ns.edges, dbPath });
|
|
2159
|
-
if (!res?.found || !res.entry)
|
|
2160
|
-
return null; // stale ref — do not half-answer
|
|
2161
|
-
const md = (res.entry.metadata ?? {});
|
|
2162
|
-
edges.push({
|
|
2163
|
-
key,
|
|
2164
|
-
src: String(md.src ?? ''),
|
|
2165
|
-
dst: String(md.dst ?? ''),
|
|
2166
|
-
relation: String(md.relation ?? 'related_to'),
|
|
2167
|
-
originRefs: originsOf(res.entry),
|
|
2168
|
-
});
|
|
2169
|
-
}
|
|
2170
|
-
return edges;
|
|
2171
|
-
}
|
|
2172
|
-
/** The indexed equivalent of `collectByOrigin`: every {namespace,key} entry
|
|
2173
|
-
* one origin supports, read via its origin-index entry instead of scanning
|
|
2174
|
-
* every namespace. Returns `null` on any unresolvable ref, same reasoning
|
|
2175
|
-
* as `kgIndexedEdgesByEndpoint`. */
|
|
2176
|
-
/** Returns each entry paired with the namespace it was read from (from the
|
|
2177
|
-
* index's own ref, not trusted from the entry itself) so a caller can sort
|
|
2178
|
-
* results back into per-namespace buckets without guessing. */
|
|
2179
|
-
async function kgIndexedByOrigin(ns, originRef, dbPath) {
|
|
2180
|
-
const idx = await readOriginIndex(ns, originRef, dbPath);
|
|
2181
|
-
if (idx === null)
|
|
2182
|
-
return null;
|
|
2183
|
-
const entries = [];
|
|
2184
|
-
for (const ref of idx.refs) {
|
|
2185
|
-
const res = await bridgeGetEntry({ key: ref.key, namespace: ref.ns, dbPath });
|
|
2186
|
-
if (!res?.found || !res.entry)
|
|
2187
|
-
return null;
|
|
2188
|
-
entries.push({ ns: ref.ns, entry: res.entry });
|
|
2189
|
-
}
|
|
2190
|
-
return entries;
|
|
2191
|
-
}
|
|
2192
|
-
// ── Rebuild (K7): the one function that builds/repairs the derived index ──
|
|
2193
|
-
/** Entities/origins sampled for post-build validation. A scan that saw fewer
|
|
2194
|
-
* than this many distinct entities AND origins gets FULL validation, not a
|
|
2195
|
-
* sample — most real scopes will. `KgRebuildResult.status.validation` says
|
|
2196
|
-
* which happened, honestly, rather than letting "validated" imply "all". */
|
|
2197
|
-
const VALIDATE_SAMPLE = 200;
|
|
2198
|
-
function phaseOrder(phase) {
|
|
2199
|
-
return phase === 'edges' ? 1 : phase === 'rules' ? 2 : 0;
|
|
2200
|
-
}
|
|
2201
|
-
function sameEdgeKeySet(a, b) {
|
|
2202
|
-
const ak = new Set(a.map((e) => e.key));
|
|
2203
|
-
const bk = new Set(b.map((e) => e.key));
|
|
2204
|
-
if (ak.size !== bk.size)
|
|
2205
|
-
return false;
|
|
2206
|
-
for (const k of ak)
|
|
2207
|
-
if (!bk.has(k))
|
|
2208
|
-
return false;
|
|
2209
|
-
return true;
|
|
2210
|
-
}
|
|
2211
|
-
/** (Re)build a scope's derived index from canonical data — nodes, then
|
|
2212
|
-
* edges, then rules, that fixed order, resuming from the last checkpointed
|
|
2213
|
-
* `{phase, offset}` rather than restarting when a prior call was
|
|
2214
|
-
* interrupted mid-build. Every write here (`addToAdj`/`addToOriginIndex`)
|
|
2215
|
-
* is idempotent, so a page reprocessed after an interruption cannot
|
|
2216
|
-
* duplicate an entry.
|
|
2217
|
-
*
|
|
2218
|
-
* A concurrent ingest/rollback during the build is safe, not just tolerated:
|
|
2219
|
-
* the dual-write hooks run whenever state is not `absent`/`failed`, so a
|
|
2220
|
-
* write made mid-build is captured whether or not the scan has reached that
|
|
2221
|
-
* row yet — at worst twice, which idempotency absorbs for free. The one
|
|
2222
|
-
* residual race (a row deleted between the scan reading it and the scan's
|
|
2223
|
-
* own write landing) can leave a dangling ref in the index; it is never
|
|
2224
|
-
* observable as wrong data, because every indexed READ
|
|
2225
|
-
* (`kgIndexedEdgesByEndpoint`/`kgIndexedByOrigin`) returns `null` — and the
|
|
2226
|
-
* caller falls back to the exhaustive scan — the instant it cannot resolve
|
|
2227
|
-
* a ref it holds. This is the backend's real capability (single-row CAS,
|
|
2228
|
-
* no cross-row transaction), used honestly rather than claiming atomicity
|
|
2229
|
-
* it cannot provide.
|
|
2230
|
-
*
|
|
2231
|
-
* Ends in `validating`: samples up to `VALIDATE_SAMPLE` of the entities and
|
|
2232
|
-
* origins the scan actually saw, and re-reads them through the just-built
|
|
2233
|
-
* index, comparing against a FRESH, independent reference read
|
|
2234
|
-
* (`kgReferenceEdges`/`collectByOrigin`) — not the in-memory data the build
|
|
2235
|
-
* itself computed, which would only prove the build agrees with itself. A
|
|
2236
|
-
* write that silently failed, or a concurrent change the dual-write hooks
|
|
2237
|
-
* missed, is exactly what this catches before the index is ever trusted. */
|
|
2238
|
-
export async function kgRebuildIndex(options) {
|
|
2239
|
-
const ns = kgNamespaces(options?.scope);
|
|
2240
|
-
const dbPath = options?.dbPath;
|
|
2241
|
-
let status = await readIndexStatus(ns, dbPath);
|
|
2242
|
-
// Resume from the checkpointed cursor for an interrupted build or a
|
|
2243
|
-
// resumable (build-phase) failure; a fresh scan for everything else —
|
|
2244
|
-
// `absent`, `ready` (this call IS the deliberate re-verify), and a
|
|
2245
|
-
// validation-phase failure, where something already written was wrong.
|
|
2246
|
-
const resume = status.state === 'building' || (status.state === 'failed' && status.resumable);
|
|
2247
|
-
if (!resume) {
|
|
2248
|
-
status = {
|
|
2249
|
-
state: 'building',
|
|
2250
|
-
schemaVersion: KG_INDEX_SCHEMA_VERSION,
|
|
2251
|
-
cursor: { phase: 'nodes', offset: 0 },
|
|
2252
|
-
counts: { nodes: 0, edges: 0, rules: 0 },
|
|
2253
|
-
startedAt: Date.now(),
|
|
2254
|
-
};
|
|
2255
|
-
if (!(await writeIndexStatus(ns, status, dbPath))) {
|
|
2256
|
-
return { success: false, status, error: 'could not persist initial build status' };
|
|
2257
|
-
}
|
|
2258
|
-
}
|
|
2259
|
-
const seenEntities = [];
|
|
2260
|
-
const seenOrigins = [];
|
|
2261
|
-
const noteEntity = (id) => {
|
|
2262
|
-
if (id && !seenEntities.includes(id) && seenEntities.length < VALIDATE_SAMPLE) {
|
|
2263
|
-
seenEntities.push(id);
|
|
2264
|
-
}
|
|
2265
|
-
};
|
|
2266
|
-
const noteOrigin = (ref) => {
|
|
2267
|
-
if (!seenOrigins.includes(ref) && seenOrigins.length < VALIDATE_SAMPLE)
|
|
2268
|
-
seenOrigins.push(ref);
|
|
2269
|
-
};
|
|
2270
|
-
// `addToAdj`/`addToOriginIndex` only APPEND onto whatever is already
|
|
2271
|
-
// there. That's exactly right for a RESUME (everything present was
|
|
2272
|
-
// written earlier in this same build attempt), but wrong for a FRESH
|
|
2273
|
-
// build: a stale or corrupted entry left over from a PRIOR build (e.g.
|
|
2274
|
-
// the exact thing a failed validation just caught) would never be
|
|
2275
|
-
// cleared, only added to. So a fresh build clears both derived-index
|
|
2276
|
-
// namespaces up front, once, before touching any phase — every write for
|
|
2277
|
-
// the rest of THIS build attempt, in this call or a later one resuming
|
|
2278
|
-
// it, can then safely append, because the namespace is known to hold only
|
|
2279
|
-
// rows this attempt wrote.
|
|
2280
|
-
if (!resume) {
|
|
2281
|
-
const adjCleared = await clearNamespace(ns.adj, dbPath);
|
|
2282
|
-
const originCleared = await clearNamespace(ns.originIdx, dbPath);
|
|
2283
|
-
if (!adjCleared || !originCleared) {
|
|
2284
|
-
status = { ...status, state: 'failed', error: 'could not clear prior index before rebuild' };
|
|
2285
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2286
|
-
return { success: false, status, error: status.error };
|
|
2287
|
-
}
|
|
2288
|
-
}
|
|
2289
|
-
const phases = [
|
|
2290
|
-
{ phase: 'nodes', namespace: ns.nodes },
|
|
2291
|
-
{ phase: 'edges', namespace: ns.edges },
|
|
2292
|
-
{ phase: 'rules', namespace: ns.rules },
|
|
2293
|
-
];
|
|
2294
|
-
try {
|
|
2295
|
-
for (const { phase, namespace } of phases) {
|
|
2296
|
-
if (phaseOrder(status.cursor?.phase) > phaseOrder(phase))
|
|
2297
|
-
continue; // already scanned
|
|
2298
|
-
let offset = status.cursor?.phase === phase ? (status.cursor.offset ?? 0) : 0;
|
|
2299
|
-
let count = status.counts?.[phase] ?? 0;
|
|
2300
|
-
for (;;) {
|
|
2301
|
-
const page = await bridgeListEntries({ namespace, limit: SCAN_PAGE, offset, dbPath });
|
|
2302
|
-
if (!page) {
|
|
2303
|
-
// The cursor stays exactly where it was: everything indexed before
|
|
2304
|
-
// this page is still correct, only incomplete, so a retry resumes.
|
|
2305
|
-
status = {
|
|
2306
|
-
...status,
|
|
2307
|
-
state: 'failed',
|
|
2308
|
-
resumable: true,
|
|
2309
|
-
error: `${namespace}: backend unavailable during rebuild`,
|
|
2310
|
-
};
|
|
2311
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2312
|
-
return { success: false, status, error: status.error };
|
|
2313
|
-
}
|
|
2314
|
-
for (const e of page.entries) {
|
|
2315
|
-
const md = (e.metadata ?? {});
|
|
2316
|
-
count++;
|
|
2317
|
-
if (phase === 'edges' && md.kg === 'edge') {
|
|
2318
|
-
const src = String(md.src ?? '');
|
|
2319
|
-
const dst = String(md.dst ?? '');
|
|
2320
|
-
if (src) {
|
|
2321
|
-
await addToAdj(ns, src, e.key, dbPath);
|
|
2322
|
-
noteEntity(src);
|
|
2323
|
-
}
|
|
2324
|
-
if (dst && dst !== src) {
|
|
2325
|
-
await addToAdj(ns, dst, e.key, dbPath);
|
|
2326
|
-
noteEntity(dst);
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
if (phase === 'nodes')
|
|
2330
|
-
noteEntity(e.key);
|
|
2331
|
-
for (const originRef of originsOf(e)) {
|
|
2332
|
-
await addToOriginIndex(ns, originRef, { ns: namespace, key: e.key }, dbPath);
|
|
2333
|
-
noteOrigin(originRef);
|
|
2334
|
-
}
|
|
2335
|
-
}
|
|
2336
|
-
offset += page.entries.length;
|
|
2337
|
-
status = {
|
|
2338
|
-
...status,
|
|
2339
|
-
cursor: { phase, offset },
|
|
2340
|
-
counts: { ...status.counts, [phase]: count },
|
|
2341
|
-
};
|
|
2342
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2343
|
-
if (page.entries.length < SCAN_PAGE)
|
|
2344
|
-
break; // last page of this namespace
|
|
2345
|
-
}
|
|
2346
|
-
}
|
|
2347
|
-
status = { ...status, state: 'validating' };
|
|
2348
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2349
|
-
const mismatches = [];
|
|
2350
|
-
for (const id of seenEntities) {
|
|
2351
|
-
const indexed = await kgIndexedEdgesByEndpoint(ns, id, dbPath);
|
|
2352
|
-
const reference = await kgReferenceEdges({ endpointId: id, scope: options?.scope, dbPath });
|
|
2353
|
-
if (indexed === null || !reference.success || reference.truncated) {
|
|
2354
|
-
mismatches.push(`entity ${id}: reference read incomplete`);
|
|
2355
|
-
continue;
|
|
2356
|
-
}
|
|
2357
|
-
if (!sameEdgeKeySet(indexed, reference.edges))
|
|
2358
|
-
mismatches.push(`entity ${id}: adjacency mismatch`);
|
|
2359
|
-
}
|
|
2360
|
-
for (const originRef of seenOrigins) {
|
|
2361
|
-
const indexed = await kgIndexedByOrigin(ns, originRef, dbPath);
|
|
2362
|
-
if (indexed === null) {
|
|
2363
|
-
mismatches.push(`origin ${originRef}: index unreadable`);
|
|
2364
|
-
continue;
|
|
2365
|
-
}
|
|
2366
|
-
const refA = await collectByOrigin(ns.nodes, originRef, dbPath);
|
|
2367
|
-
const refB = await collectByOrigin(ns.edges, originRef, dbPath);
|
|
2368
|
-
const refC = await collectByOrigin(ns.rules, originRef, dbPath);
|
|
2369
|
-
if (!refA.covered || !refB.covered || !refC.covered) {
|
|
2370
|
-
mismatches.push(`origin ${originRef}: reference scan incomplete`);
|
|
2371
|
-
continue;
|
|
2372
|
-
}
|
|
2373
|
-
const expected = new Set([
|
|
2374
|
-
...refA.entries.map((e) => `${ns.nodes}|${e.key}`),
|
|
2375
|
-
...refB.entries.map((e) => `${ns.edges}|${e.key}`),
|
|
2376
|
-
...refC.entries.map((e) => `${ns.rules}|${e.key}`),
|
|
2377
|
-
]);
|
|
2378
|
-
const indexedKeys = new Set(indexed.map((e) => `${e.ns}|${e.entry.key}`));
|
|
2379
|
-
if (expected.size !== indexedKeys.size || [...expected].some((k) => !indexedKeys.has(k))) {
|
|
2380
|
-
mismatches.push(`origin ${originRef}: support-index mismatch`);
|
|
2381
|
-
}
|
|
2382
|
-
}
|
|
2383
|
-
const full = seenEntities.length < VALIDATE_SAMPLE && seenOrigins.length < VALIDATE_SAMPLE;
|
|
2384
|
-
const validation = {
|
|
2385
|
-
sampledEntities: seenEntities.length,
|
|
2386
|
-
sampledOrigins: seenOrigins.length,
|
|
2387
|
-
full,
|
|
2388
|
-
};
|
|
2389
|
-
if (mismatches.length) {
|
|
2390
|
-
const summary = mismatches.slice(0, 5).join('; ');
|
|
2391
|
-
status = {
|
|
2392
|
-
...status,
|
|
2393
|
-
state: 'failed',
|
|
2394
|
-
error: `validation failed: ${summary}${mismatches.length > 5 ? ` (+${mismatches.length - 5} more)` : ''}`,
|
|
2395
|
-
};
|
|
2396
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2397
|
-
return { success: false, status, validation, error: status.error };
|
|
2398
|
-
}
|
|
2399
|
-
status = { ...status, state: 'ready', error: undefined };
|
|
2400
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2401
|
-
return { success: true, status, validation };
|
|
2402
|
-
}
|
|
2403
|
-
catch (err) {
|
|
2404
|
-
status = {
|
|
2405
|
-
...status,
|
|
2406
|
-
state: 'failed',
|
|
2407
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2408
|
-
};
|
|
2409
|
-
try {
|
|
2410
|
-
await writeIndexStatus(ns, status, dbPath);
|
|
2411
|
-
}
|
|
2412
|
-
catch {
|
|
2413
|
-
/* best-effort — see markIndexFailed */
|
|
2414
|
-
}
|
|
2415
|
-
return { success: false, status, error: status.error };
|
|
2416
|
-
}
|
|
2417
|
-
}
|
|
2418
|
-
/** Verify that every edge's endpoints exist.
|
|
2419
|
-
*
|
|
2420
|
-
* `kgIngest` now creates missing endpoints before writing an edge and
|
|
2421
|
-
* `kgRollback` removes edges whose endpoints it deleted, so a healthy graph
|
|
2422
|
-
* reports nothing. This exists for graphs written before either rule, and as
|
|
2423
|
-
* the check that says so rather than assuming it. */
|
|
2424
|
-
export async function kgIntegrityCheck(options) {
|
|
2425
|
-
const ns = kgNamespaces(options?.scope);
|
|
2426
|
-
const limit = options?.limit ?? 100;
|
|
2427
|
-
const dangling = [];
|
|
2428
|
-
let edges = 0;
|
|
2429
|
-
try {
|
|
2430
|
-
/** Endpoint id → exists. One keyed lookup per DISTINCT endpoint. */
|
|
2431
|
-
const seen = new Map();
|
|
2432
|
-
const exists = async (id) => {
|
|
2433
|
-
const hit = seen.get(id);
|
|
2434
|
-
if (hit !== undefined)
|
|
2435
|
-
return hit;
|
|
2436
|
-
const res = await bridgeGetEntry({ key: id, namespace: ns.nodes, dbPath: options?.dbPath });
|
|
2437
|
-
const found = Boolean(res?.found && res.entry);
|
|
2438
|
-
seen.set(id, found);
|
|
2439
|
-
return found;
|
|
2440
|
-
};
|
|
2441
|
-
// Collect first, then probe: the probes are reads, but interleaving many
|
|
2442
|
-
// per page would hold the page open far longer than the scan needs.
|
|
2443
|
-
const rows = [];
|
|
2444
|
-
const covered = await scanNamespace(ns.edges, options?.dbPath, (page) => {
|
|
2445
|
-
for (const e of page) {
|
|
2446
|
-
const md = (e.metadata ?? {});
|
|
2447
|
-
if (md.kg !== 'edge')
|
|
2448
|
-
continue;
|
|
2449
|
-
edges++;
|
|
2450
|
-
rows.push({ key: e.key, src: String(md.src ?? ''), dst: String(md.dst ?? '') });
|
|
2451
|
-
}
|
|
2452
|
-
});
|
|
2453
|
-
for (const row of rows) {
|
|
2454
|
-
if (dangling.length >= limit)
|
|
2455
|
-
break;
|
|
2456
|
-
const missing = [];
|
|
2457
|
-
if (!row.src || !(await exists(row.src)))
|
|
2458
|
-
missing.push(row.src || '<no src>');
|
|
2459
|
-
if (!row.dst || !(await exists(row.dst)))
|
|
2460
|
-
missing.push(row.dst || '<no dst>');
|
|
2461
|
-
if (missing.length)
|
|
2462
|
-
dangling.push({ key: row.key, missing });
|
|
2463
|
-
}
|
|
2464
|
-
return {
|
|
2465
|
-
success: true,
|
|
2466
|
-
edges,
|
|
2467
|
-
dangling,
|
|
2468
|
-
...(covered && dangling.length < limit ? {} : { truncated: true }),
|
|
2469
|
-
};
|
|
2470
|
-
}
|
|
2471
|
-
catch (err) {
|
|
2472
|
-
return {
|
|
2473
|
-
success: false,
|
|
2474
|
-
edges,
|
|
2475
|
-
dangling,
|
|
2476
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2477
|
-
};
|
|
2478
|
-
}
|
|
2479
|
-
}
|
|
91
|
+
import { normalizeName } from './memory-kg-model.js';
|
|
92
|
+
export { kgIndexStatus } from './memory-kg-index.js';
|
|
93
|
+
export { kgIngest } from './memory-kg-ingest.js';
|
|
94
|
+
export { KG_ADJ_NS, KG_EDGES_NS, KG_ID_VERSION, KG_INDEX_STATUS_NS, KG_NAMES_NS, KG_NODES_NS, KG_ORIGIN_IDX_NS, kgNamespaces, kgQualifyOrigin, nodeKey, normalizeName, RULES_NS, } from './memory-kg-model.js';
|
|
95
|
+
export { kgConsolidateCandidates, kgPromote, kgStats } from './memory-kg-promote.js';
|
|
96
|
+
export { kgIntegrityCheck, kgRebuildIndex } from './memory-kg-rebuild.js';
|
|
97
|
+
export { kgRollback } from './memory-kg-rollback.js';
|
|
98
|
+
export { kgIngestRules, kgListRules } from './memory-kg-rules.js';
|
|
99
|
+
export { kgReferenceEdges } from './memory-kg-scan.js';
|
|
100
|
+
export { kgGlossary, kgSearch } from './memory-kg-search.js';
|
|
2480
101
|
// ── Heuristic extraction (LLM-less fallback) ────────────────────────
|
|
2481
102
|
/** Regex extraction for when no LLM is in the loop (memory-palace lineage):
|
|
2482
103
|
* proper-noun phrases and `code identifiers` become entities, sentence
|