@cortexkit/opencode-magic-context 0.46.1 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/magic-context-prompt.d.ts +1 -1
- package/dist/agents/magic-context-prompt.d.ts.map +1 -1
- package/dist/{announcement-y35wzg2r.js → announcement-fp14dw0n.js} +3 -2
- package/dist/auto-search-worker.js +112 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/live-run-config.d.ts.map +1 -1
- package/dist/config/migrate-config-location.d.ts.map +1 -1
- package/dist/config/mural-model-deprecation.d.ts +15 -0
- package/dist/config/mural-model-deprecation.d.ts.map +1 -0
- package/dist/config/project-security.d.ts +0 -3
- package/dist/config/project-security.d.ts.map +1 -1
- package/dist/config/schema/magic-context.d.ts +21 -9
- package/dist/config/schema/magic-context.d.ts.map +1 -1
- package/dist/{conflict-warning-hook-7fj2b9jw.js → conflict-warning-hook-w8cykrnf.js} +12 -9
- package/dist/{context-sh2m2vde.js → context-m0cy262m.js} +1785 -913
- package/dist/{derive-budgets-7nj7r1rv.js → derive-budgets-sg0xsq7s.js} +4 -3
- package/dist/embedding-activity-g1dhes6g.js +10 -0
- package/dist/embedding-backfill-9rcm1mfy.js +21 -0
- package/dist/embedding-cache-mgyr73ef.js +16 -0
- package/dist/embedding-worker.js +11 -9
- package/dist/features/magic-context/compaction-marker.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/evaluate-smart-notes.d.ts +1 -0
- package/dist/features/magic-context/dreamer/evaluate-smart-notes.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-config.d.ts +3 -3
- package/dist/features/magic-context/dreamer/task-config.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-executor.d.ts +1 -1
- package/dist/features/magic-context/dreamer/task-executor.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-gates.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-registry.d.ts +1 -0
- package/dist/features/magic-context/dreamer/task-registry.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/verify.d.ts.map +1 -1
- package/dist/features/magic-context/message-index-async.d.ts +16 -3
- package/dist/features/magic-context/message-index-async.d.ts.map +1 -1
- package/dist/features/magic-context/message-index-bounded.d.ts +17 -0
- package/dist/features/magic-context/message-index-bounded.d.ts.map +1 -0
- package/dist/features/magic-context/message-index.d.ts +7 -1
- package/dist/features/magic-context/message-index.d.ts.map +1 -1
- package/dist/features/magic-context/mural/render-trigger.d.ts +7 -4
- package/dist/features/magic-context/mural/render-trigger.d.ts.map +1 -1
- package/dist/features/magic-context/overflow-detection.d.ts +2 -0
- package/dist/features/magic-context/overflow-detection.d.ts.map +1 -1
- package/dist/features/magic-context/pi-content-decisions.d.ts +7 -1
- package/dist/features/magic-context/pi-content-decisions.d.ts.map +1 -1
- package/dist/features/magic-context/project-embedding-registry.d.ts +7 -0
- package/dist/features/magic-context/project-embedding-registry.d.ts.map +1 -1
- package/dist/features/magic-context/search.d.ts +31 -0
- package/dist/features/magic-context/search.d.ts.map +1 -1
- package/dist/features/magic-context/session-activity.d.ts.map +1 -1
- package/dist/features/magic-context/session-project-storage.d.ts +8 -2
- package/dist/features/magic-context/session-project-storage.d.ts.map +1 -1
- package/dist/features/magic-context/smart-notes/runner.d.ts +1 -0
- package/dist/features/magic-context/smart-notes/runner.d.ts.map +1 -1
- package/dist/features/magic-context/storage-db.d.ts.map +1 -1
- package/dist/features/magic-context/storage-historian-runs.d.ts +2 -5
- package/dist/features/magic-context/storage-historian-runs.d.ts.map +1 -1
- package/dist/features/magic-context/storage-meta-persisted.d.ts +12 -0
- package/dist/features/magic-context/storage-meta-persisted.d.ts.map +1 -1
- package/dist/features/magic-context/storage-meta-session.d.ts +6 -0
- package/dist/features/magic-context/storage-meta-session.d.ts.map +1 -1
- package/dist/features/magic-context/storage-tag-identity-revision.d.ts +30 -0
- package/dist/features/magic-context/storage-tag-identity-revision.d.ts.map +1 -0
- package/dist/features/magic-context/storage-tags.d.ts +148 -3
- package/dist/features/magic-context/storage-tags.d.ts.map +1 -1
- package/dist/features/magic-context/storage.d.ts +1 -0
- package/dist/features/magic-context/storage.d.ts.map +1 -1
- package/dist/features/magic-context/subagent-token-capture.d.ts.map +1 -1
- package/dist/features/magic-context/transform-decision-log.d.ts +4 -2
- package/dist/features/magic-context/transform-decision-log.d.ts.map +1 -1
- package/dist/harness-e4avh192.js +10 -0
- package/dist/hooks/magic-context/active-anthropic-turn.d.ts +10 -0
- package/dist/hooks/magic-context/active-anthropic-turn.d.ts.map +1 -0
- package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-blocking.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/auto-search-blocking.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-deadline.d.ts +7 -0
- package/dist/hooks/magic-context/auto-search-deadline.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-host-review.fixture.d.ts +3 -0
- package/dist/hooks/magic-context/auto-search-host-review.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-lock-review.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/auto-search-lock-review.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-long-lock-review.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/auto-search-long-lock-review.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-omp-host-review.fixture.d.ts +4 -0
- package/dist/hooks/magic-context/auto-search-omp-host-review.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-omp-loader.fixture.d.ts +3 -0
- package/dist/hooks/magic-context/auto-search-omp-loader.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-runner.d.ts +1 -1
- package/dist/hooks/magic-context/auto-search-runner.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-snapshot.fixture.d.ts +4 -0
- package/dist/hooks/magic-context/auto-search-snapshot.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts +98 -0
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts.map +1 -0
- package/dist/hooks/magic-context/auto-search-worker.d.ts +2 -0
- package/dist/hooks/magic-context/auto-search-worker.d.ts.map +1 -0
- package/dist/hooks/magic-context/caveman-cleanup.d.ts.map +1 -1
- package/dist/hooks/magic-context/command-handler.d.ts +1 -0
- package/dist/hooks/magic-context/command-handler.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-incremental.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-recomp.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-types.d.ts +3 -5
- package/dist/hooks/magic-context/compartment-runner-types.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner.d.ts +2 -1
- package/dist/hooks/magic-context/compartment-runner.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts +3 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
- package/dist/hooks/magic-context/event-handler.d.ts +1 -1
- package/dist/hooks/magic-context/event-handler.d.ts.map +1 -1
- package/dist/hooks/magic-context/event-resolvers.d.ts.map +1 -1
- package/dist/hooks/magic-context/execute-status.d.ts.map +1 -1
- package/dist/hooks/magic-context/final-wire-token-estimate.d.ts +2 -0
- package/dist/hooks/magic-context/final-wire-token-estimate.d.ts.map +1 -1
- package/dist/hooks/magic-context/heuristic-cleanup.d.ts.map +1 -1
- package/dist/hooks/magic-context/historian-drain-gate.d.ts +11 -0
- package/dist/hooks/magic-context/historian-drain-gate.d.ts.map +1 -0
- package/dist/hooks/magic-context/hook.d.ts +3 -2
- package/dist/hooks/magic-context/hook.d.ts.map +1 -1
- package/dist/hooks/magic-context/inject-compartments.d.ts.map +1 -1
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts +25 -0
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts.map +1 -0
- package/dist/hooks/magic-context/latest-thinking-recovery.d.ts +61 -0
- package/dist/hooks/magic-context/latest-thinking-recovery.d.ts.map +1 -0
- package/dist/hooks/magic-context/lkg-persist.d.ts +4 -3
- package/dist/hooks/magic-context/lkg-persist.d.ts.map +1 -1
- package/dist/hooks/magic-context/lkg-replay.d.ts +13 -8
- package/dist/hooks/magic-context/lkg-replay.d.ts.map +1 -1
- package/dist/hooks/magic-context/lkg-slot.d.ts +139 -2
- package/dist/hooks/magic-context/lkg-slot.d.ts.map +1 -1
- package/dist/hooks/magic-context/message-search-accepting-silent-worker.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/message-search-accepting-silent-worker.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/module-wire.d.ts.map +1 -1
- package/dist/hooks/magic-context/oc-stall-r2-warmup-worker.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/oc-stall-r2-warmup-worker.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/oc-stall-search-review.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/oc-stall-search-review.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/opencode-proven-floor.d.ts +6 -0
- package/dist/hooks/magic-context/opencode-proven-floor.d.ts.map +1 -0
- package/dist/hooks/magic-context/opencode1-to-model-messages.fixture.d.ts +63 -0
- package/dist/hooks/magic-context/opencode1-to-model-messages.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/pass-outcome.d.ts.map +1 -1
- package/dist/hooks/magic-context/protected-tail-boundary.d.ts.map +1 -1
- package/dist/hooks/magic-context/provider-visible-parts.d.ts +45 -0
- package/dist/hooks/magic-context/provider-visible-parts.d.ts.map +1 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup-silent-worker.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup-silent-worker.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup-worker.d.ts +2 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup-worker.d.ts.map +1 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup.d.ts +87 -0
- package/dist/hooks/magic-context/raw-ordinal-warmup.d.ts.map +1 -0
- package/dist/hooks/magic-context/read-session-chunk.d.ts +12 -0
- package/dist/hooks/magic-context/read-session-chunk.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-db.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-formatting.d.ts +5 -0
- package/dist/hooks/magic-context/read-session-formatting.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-raw.d.ts +83 -1
- package/dist/hooks/magic-context/read-session-raw.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-true-raw-tokens.d.ts.map +1 -1
- package/dist/hooks/magic-context/reasoning-budget-status.d.ts +15 -0
- package/dist/hooks/magic-context/reasoning-budget-status.d.ts.map +1 -0
- package/dist/hooks/magic-context/reasoning-budget.d.ts +40 -0
- package/dist/hooks/magic-context/reasoning-budget.d.ts.map +1 -0
- package/dist/hooks/magic-context/reasoning-removal.d.ts +6 -6
- package/dist/hooks/magic-context/reasoning-removal.d.ts.map +1 -1
- package/dist/hooks/magic-context/rust-mode-transform.d.ts.map +1 -1
- package/dist/hooks/magic-context/sentinel.d.ts +45 -1
- package/dist/hooks/magic-context/sentinel.d.ts.map +1 -1
- package/dist/hooks/magic-context/strip-content.d.ts +21 -5
- package/dist/hooks/magic-context/strip-content.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-prompt-hash.d.ts +1 -0
- package/dist/hooks/magic-context/system-prompt-hash.d.ts.map +1 -1
- package/dist/hooks/magic-context/tag-messages.d.ts +21 -0
- package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
- package/dist/hooks/magic-context/tag-owner-warmup.d.ts +6 -0
- package/dist/hooks/magic-context/tag-owner-warmup.d.ts.map +1 -0
- package/dist/hooks/magic-context/tail-hygiene-walk.d.ts.map +1 -1
- package/dist/hooks/magic-context/token-count-exact.d.ts +35 -0
- package/dist/hooks/magic-context/token-count-exact.d.ts.map +1 -0
- package/dist/hooks/magic-context/tool-drop-target.d.ts +5 -0
- package/dist/hooks/magic-context/tool-drop-target.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-compartment-phase.d.ts +3 -28
- package/dist/hooks/magic-context/transform-compartment-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +28 -6
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform.d.ts +11 -3
- package/dist/hooks/magic-context/transform.d.ts.map +1 -1
- package/dist/hooks/magic-context/wrapup-orchestrator.d.ts.map +1 -1
- package/dist/{index-hpecdzec.js → index-0an7742d.js} +6820 -6587
- package/dist/index-0qhce371.js +120 -0
- package/dist/{index-q6hef7fh.js → index-0sahzkwr.js} +6 -19
- package/dist/{index-k5bdx1bx.js → index-0w56v4fm.js} +3 -5
- package/dist/index-14cjxt20.js +819 -0
- package/dist/{v2/server-51dv1pr5.js → index-1qb0z7wa.js} +301 -27
- package/dist/index-2bvekkfy.js +11 -0
- package/dist/{index-sj29b3z2.js → index-2y0s2hep.js} +3 -3
- package/dist/index-4dhv7mqz.js +131 -0
- package/dist/{index-b35pxds2.js → index-58e3y90n.js} +1 -18
- package/dist/index-5zygecde.js +152 -0
- package/dist/{index-sjthx78w.js → index-692jzy0p.js} +16879 -22907
- package/dist/index-72nhv486.js +2307 -0
- package/dist/index-7jzrrxap.js +931 -0
- package/dist/index-7mx8v0pw.js +3591 -0
- package/dist/index-8ajstryz.js +18 -0
- package/dist/{index-sfpzqvdz.js → index-8cd2j99z.js} +2 -3
- package/dist/{index-hybnrwq0.js → index-8j7gqj40.js} +131 -128
- package/dist/index-bdckkfw6.js +3261 -0
- package/dist/{index-8x2v7gg9.js → index-bhvjj0ys.js} +158 -2455
- package/dist/index-c43kr12f.js +3030 -0
- package/dist/index-cb7emxgj.js +984 -0
- package/dist/index-d9vfe7xq.js +16 -0
- package/dist/{index-5gq3b2ya.js → index-db7zwa3v.js} +147 -209
- package/dist/index-dhwzyd54.js +1142 -0
- package/dist/{index-8da2zjhz.js → index-eeezr5zz.js} +9 -7
- package/dist/{index-pkbcrdqs.js → index-f0h6xwqm.js} +3412 -4552
- package/dist/index-fb3zxn06.js +13 -0
- package/dist/index-fbv1dn35.js +171 -0
- package/dist/index-fr0wbhdn.js +43 -0
- package/dist/{index-mc22ayvf.js → index-ghc9f0n7.js} +11 -9
- package/dist/{index-fzbvakh0.js → index-jcbd3znx.js} +3 -3
- package/dist/{index-5p8jdswx.js → index-k1xb52kd.js} +20 -13
- package/dist/index-k9nnwad5.js +3532 -0
- package/dist/{v2/server-fg34g6a5.js → index-kb9nn0a0.js} +309 -52
- package/dist/index-knh4y330.js +566 -0
- package/dist/{index-q4yp4b52.js → index-mfkzp7qj.js} +28 -28
- package/dist/{index-sgthv3f7.js → index-mmyh6d8f.js} +4 -4
- package/dist/index-pavzsdc6.js +17 -0
- package/dist/{index-0d3g3461.js → index-pdecwzrd.js} +19 -16
- package/dist/index-pg50n748.js +1060 -0
- package/dist/index-q2tqeqde.js +2106 -0
- package/dist/index-q7h7csab.js +3469 -0
- package/dist/index-qt1qqasc.js +22 -0
- package/dist/{index-majxda46.js → index-ty64eqcn.js} +28 -18
- package/dist/{index-7qw742ev.js → index-yq8n9qcb.js} +25 -44
- package/dist/{index-sfnj6h8h.js → index-z8e0psj8.js} +20 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1061 -260
- package/dist/logger-d3wex3dz.js +20 -0
- package/dist/message-index-bounded-7ezbgddp.js +124 -0
- package/dist/{v2/message-time-backfill-r6qkem1x.js → message-time-backfill-brbd63hv.js} +8 -6
- package/dist/migration-worker.js +25 -16
- package/dist/{model-resolution-xhfmyyvy.js → model-resolution-nx97z3x7.js} +1 -1
- package/dist/plugin/dream-timer.d.ts +1 -1
- package/dist/plugin/dream-timer.d.ts.map +1 -1
- package/dist/plugin/hooks/create-session-hooks.d.ts +3 -1
- package/dist/plugin/hooks/create-session-hooks.d.ts.map +1 -1
- package/dist/plugin/host-stall-profiler-report.d.ts +2 -0
- package/dist/plugin/host-stall-profiler-report.d.ts.map +1 -0
- package/dist/plugin/host-stall-profiler.d.ts +3 -0
- package/dist/plugin/host-stall-profiler.d.ts.map +1 -0
- package/dist/plugin/messages-transform.d.ts +21 -19
- package/dist/plugin/messages-transform.d.ts.map +1 -1
- package/dist/plugin/rpc-handlers.d.ts.map +1 -1
- package/dist/plugin/tool-registry.d.ts.map +1 -1
- package/dist/project-embedding-registry-7qzptcw7.js +88 -0
- package/dist/raw-ordinal-warmup-worker.js +60 -0
- package/dist/{read-session-chunk-egz7r68d.js → read-session-chunk-6k83m4fj.js} +15 -9
- package/dist/read-session-raw-3g9qy1xa.js +58 -0
- package/dist/recomp-orchestrator-d3fp1cy0.js +57 -0
- package/dist/{v2/safe-notification-target-eb1n1vfb.js → safe-notification-target-5ffqq32m.js} +4 -2
- package/dist/search-epax1ecv.js +45 -0
- package/dist/{v2/send-session-notification-qs0j78v6.js → send-session-notification-erecz4y4.js} +10 -6
- package/dist/shared/auto-search-lifecycle.d.ts +7 -0
- package/dist/shared/auto-search-lifecycle.d.ts.map +1 -0
- package/dist/shared/conflict-detector.d.ts.map +1 -1
- package/dist/shared/historian-tool-expansions.d.ts +1 -1
- package/dist/shared/historian-tool-expansions.d.ts.map +1 -1
- package/dist/shared/historian-tool-template.d.ts +2 -2
- package/dist/shared/historian-tool-template.d.ts.map +1 -1
- package/dist/shared/host-stall-profiler-bun.d.ts +46 -0
- package/dist/shared/host-stall-profiler-bun.d.ts.map +1 -0
- package/dist/shared/host-stall-profiler-common.d.ts +58 -0
- package/dist/shared/host-stall-profiler-common.d.ts.map +1 -0
- package/dist/shared/host-stall-profiler-node.d.ts +36 -0
- package/dist/shared/host-stall-profiler-node.d.ts.map +1 -0
- package/dist/shared/host-stall-profiler-report.d.ts +68 -0
- package/dist/shared/host-stall-profiler-report.d.ts.map +1 -0
- package/dist/shared/host-stall-profiler.d.ts +5 -0
- package/dist/shared/host-stall-profiler.d.ts.map +1 -0
- package/dist/shared/model-resolution.d.ts +2 -3
- package/dist/shared/model-resolution.d.ts.map +1 -1
- package/dist/shared/sqlite.d.ts +46 -1
- package/dist/shared/sqlite.d.ts.map +1 -1
- package/dist/shared/stable-json.d.ts.map +1 -1
- package/dist/shared/status-view.d.ts.map +1 -1
- package/dist/shared/user-facing-codes.d.ts +0 -5
- package/dist/shared/user-facing-codes.d.ts.map +1 -1
- package/dist/shared/window-geometry.d.ts +4 -0
- package/dist/shared/window-geometry.d.ts.map +1 -1
- package/dist/sqlite-context-pragmas-mv9jqwjh.js +6 -0
- package/dist/sqlite-mhjkerzq.js +46 -0
- package/dist/storage-memory-07fx57k6.js +71 -0
- package/dist/storage-tags-d3b0v2ss.js +131 -0
- package/dist/{task-config-v81k8jvk.js → task-config-cgpg0nvh.js} +2 -2
- package/dist/tools/ctx-expand/render.d.ts.map +1 -1
- package/dist/tools/ctx-expand/tools.d.ts.map +1 -1
- package/dist/tools/ctx-note/constants.d.ts +4 -0
- package/dist/tools/ctx-note/constants.d.ts.map +1 -1
- package/dist/tools/ctx-note/tools.d.ts +1 -0
- package/dist/tools/ctx-note/tools.d.ts.map +1 -1
- package/dist/tools/ctx-search/tools.d.ts.map +1 -1
- package/dist/tools/parameter-descriptions.d.ts +2 -2
- package/dist/tools/parameter-descriptions.d.ts.map +1 -1
- package/dist/{transform-postprocess-phase-bqhz6vdr.js → transform-postprocess-phase-spcy2qre.js} +35 -17
- package/dist/transformers-node-wasm.js +18415 -18415
- package/dist/v2/auto-search-worker.js +112 -0
- package/dist/v2/{context-zecmzk5s.js → context-sf98kstd.js} +8378 -8530
- package/dist/v2/{derive-budgets-kdgp4s3r.js → derive-budgets-3vc0y84t.js} +4 -3
- package/dist/v2/embedding-activity-g1dhes6g.js +10 -0
- package/dist/v2/embedding-backfill-vn8g1nh1.js +21 -0
- package/dist/v2/embedding-cache-mgyr73ef.js +16 -0
- package/dist/v2/embedding-worker.js +12 -9
- package/dist/v2/fold/host-media.d.ts +4 -0
- package/dist/v2/fold/host-media.d.ts.map +1 -1
- package/dist/v2/fold/memory-cache.d.ts +70 -0
- package/dist/v2/fold/memory-cache.d.ts.map +1 -0
- package/dist/v2/fold/native-codec.d.ts +10 -0
- package/dist/v2/fold/native-codec.d.ts.map +1 -0
- package/dist/v2/fold/native-replay.d.ts +58 -0
- package/dist/v2/fold/native-replay.d.ts.map +1 -0
- package/dist/v2/fold/policy.d.ts +24 -0
- package/dist/v2/fold/policy.d.ts.map +1 -0
- package/dist/v2/harness-e4avh192.js +10 -0
- package/dist/v2/hidden-completion.d.ts.map +1 -1
- package/dist/v2/hooks/context.d.ts +2 -2
- package/dist/v2/hooks/context.d.ts.map +1 -1
- package/dist/v2/hooks/dream-manual.d.ts +1 -1
- package/dist/v2/hooks/dream-manual.d.ts.map +1 -1
- package/dist/v2/hooks/dream-timer.d.ts.map +1 -1
- package/dist/v2/hooks/dream-trigger.d.ts +1 -2
- package/dist/v2/hooks/dream-trigger.d.ts.map +1 -1
- package/dist/v2/hooks/payload.d.ts.map +1 -1
- package/dist/v2/hooks/tools.d.ts.map +1 -1
- package/dist/v2/hooks/types.d.ts +6 -0
- package/dist/v2/hooks/types.d.ts.map +1 -1
- package/dist/v2/logger-mxd0ym09.js +19 -0
- package/dist/v2/message-index-bounded-dy3s2k1e.js +123 -0
- package/dist/{message-time-backfill-68gj6t3e.js → v2/message-time-backfill-gtx1pwk1.js} +7 -7
- package/dist/v2/migration-worker.js +24 -15
- package/dist/v2/{model-resolution-xhfmyyvy.js → model-resolution-nx97z3x7.js} +1 -1
- package/dist/v2/project-embedding-registry-1tgba5dv.js +86 -0
- package/dist/v2/raw-ordinal-warmup-worker.js +60 -0
- package/dist/v2/{read-session-chunk-8vjp6581.js → read-session-chunk-rh4khdkv.js} +14 -8
- package/dist/v2/read-session-raw-bsbmf0vj.js +57 -0
- package/dist/v2/recomp-orchestrator-az61rfcg.js +57 -0
- package/dist/{safe-notification-target-43r8jsxw.js → v2/safe-notification-target-73ax1p5v.js} +3 -3
- package/dist/v2/search-nzkwrc6n.js +45 -0
- package/dist/{send-session-notification-dyw42dtr.js → v2/send-session-notification-6km439kn.js} +9 -7
- package/dist/{index-5eqk6z3s.js → v2/server-1r6pmz4f.js} +301 -27
- package/dist/v2/server-2bvekkfy.js +11 -0
- package/dist/v2/server-2fbhp79z.js +3205 -0
- package/dist/v2/{server-ajzeteqw.js → server-2ra6pwf3.js} +1 -18
- package/dist/v2/server-4dhv7mqz.js +131 -0
- package/dist/v2/{server-xa4xe3rv.js → server-4k748ke6.js} +11 -1053
- package/dist/v2/{server-vs5gpnd4.js → server-52awr2mw.js} +81 -145
- package/dist/v2/{server-1qbnzc80.js → server-5m3dt5r1.js} +136 -148
- package/dist/v2/server-686jg6eb.js +3508 -0
- package/dist/v2/server-7jzrrxap.js +931 -0
- package/dist/v2/server-7wwc2nga.js +3023 -0
- package/dist/v2/server-8ajstryz.js +18 -0
- package/dist/v2/{server-sfpzqvdz.js → server-8cd2j99z.js} +2 -3
- package/dist/v2/server-8pyt7hev.js +3408 -0
- package/dist/v2/server-8tqanswc.js +984 -0
- package/dist/v2/{server-14vq4kpm.js → server-aan4q8mx.js} +3 -3
- package/dist/v2/server-d9vfe7xq.js +16 -0
- package/dist/v2/server-e0vdaae2.js +3591 -0
- package/dist/v2/server-ezhze64f.js +1048 -0
- package/dist/v2/server-fb3zxn06.js +13 -0
- package/dist/v2/server-fr0wbhdn.js +43 -0
- package/dist/v2/server-g5r0n3b6.js +1137 -0
- package/dist/v2/server-gjcr4w98.js +154 -0
- package/dist/v2/server-k228xhjg.js +171 -0
- package/dist/v2/{server-jc7y02q0.js → server-k8x2rbxc.js} +5119 -10948
- package/dist/{index-fg34g6a5.js → v2/server-kb9nn0a0.js} +309 -52
- package/dist/v2/{server-g0fjrkta.js → server-mfkzp7qj.js} +28 -28
- package/dist/v2/server-pavzsdc6.js +17 -0
- package/dist/v2/server-psqc7e9n.js +2106 -0
- package/dist/v2/server-qt1qqasc.js +22 -0
- package/dist/v2/server-qye6ykf7.js +120 -0
- package/dist/v2/{server-qdjnb2z1.js → server-r5p3b1wy.js} +11 -9
- package/dist/v2/server-sggvvzhb.js +759 -0
- package/dist/v2/{server-y8maxeft.js → server-v06jcv0w.js} +16 -13
- package/dist/v2/server-vqpf3167.js +1048 -0
- package/dist/v2/{server-c5gz8wph.js → server-x2g40a71.js} +2048 -2059
- package/dist/v2/{server-wvjvfhck.js → server-xdfwt6kh.js} +20 -16
- package/dist/v2/{server-mcfawjgc.js → server-xxm59eww.js} +25 -44
- package/dist/v2/server-yczc713m.js +564 -0
- package/dist/v2/{server-3n4q9xme.js → server-z2037m6z.js} +4 -4
- package/dist/v2/{server-e29d7p7c.js → server-zm570tr2.js} +166 -160
- package/dist/v2/{server-6r257mf5.js → server-zxhgvsf9.js} +3 -3
- package/dist/v2/server.js +16 -14
- package/dist/v2/sqlite-context-pragmas-mv9jqwjh.js +6 -0
- package/dist/v2/sqlite-mhjkerzq.js +46 -0
- package/dist/v2/storage-memory-07fx57k6.js +71 -0
- package/dist/v2/storage-tags-w4tv77sn.js +130 -0
- package/dist/v2/store-reader.d.ts +16 -0
- package/dist/v2/store-reader.d.ts.map +1 -1
- package/dist/v2/{task-config-tbndqyny.js → task-config-x0hq3qfd.js} +2 -2
- package/dist/v2/transformers-node-wasm.js +18415 -18415
- package/dist/v2/tui/index.d.ts.map +1 -1
- package/dist/v2/tui/types.d.ts +1 -10
- package/dist/v2/tui/types.d.ts.map +1 -1
- package/dist/v2/{wrapup-orchestrator-0snhvwcv.js → wrapup-orchestrator-mz7efs7w.js} +51 -27
- package/dist/wrapup-orchestrator-amzta1t4.js +44 -0
- package/package.json +4 -3
- package/src/agents/magic-context-prompt.ts +27 -2
- package/src/agents/smart-notes-enabled.test.ts +294 -0
- package/src/agents/smart-notes-on.golden.json +70 -0
- package/src/config/index.test.ts +67 -1
- package/src/config/index.ts +8 -0
- package/src/config/latch-permanence-guard.test.ts +4 -0
- package/src/config/live-run-config.test.ts +37 -1
- package/src/config/live-run-config.ts +28 -0
- package/src/config/migrate-config-location.ts +10 -3
- package/src/config/mural-model-deprecation.ts +24 -0
- package/src/config/project-security.hostile-matrix.test.ts +4 -44
- package/src/config/project-security.test.ts +9 -14
- package/src/config/project-security.ts +13 -55
- package/src/config/schema/magic-context.test.ts +6 -5
- package/src/config/schema/magic-context.ts +34 -16
- package/src/features/magic-context/dreamer/task-registry.ts +6 -0
- package/src/shared/auto-search-lifecycle.ts +27 -0
- package/src/shared/conflict-detector.test.ts +56 -0
- package/src/shared/conflict-detector.ts +16 -5
- package/src/shared/historian-tool-defaults.json +6 -6
- package/src/shared/historian-tool-expansions.ts +65 -8
- package/src/shared/historian-tool-preview-defaults.json +13 -0
- package/src/shared/historian-tool-template.ts +48 -14
- package/src/shared/host-stall-profiler-bun.ts +358 -0
- package/src/shared/host-stall-profiler-common.ts +159 -0
- package/src/shared/host-stall-profiler-node.test.ts +234 -0
- package/src/shared/host-stall-profiler-node.ts +274 -0
- package/src/shared/host-stall-profiler-report.ts +248 -0
- package/src/shared/host-stall-profiler.ts +57 -0
- package/src/shared/model-resolution.test.ts +54 -17
- package/src/shared/model-resolution.ts +6 -17
- package/src/shared/sqlite-640-review-r2.test.ts +136 -0
- package/src/shared/sqlite-640-review.test.ts +183 -0
- package/src/shared/sqlite-acquisition-routing.test.ts +18 -0
- package/src/shared/sqlite.test.ts +52 -0
- package/src/shared/sqlite.ts +423 -37
- package/src/shared/stable-json-linear.test.ts +109 -0
- package/src/shared/stable-json.ts +37 -7
- package/src/shared/status-view.test.ts +6 -1
- package/src/shared/status-view.ts +15 -1
- package/src/shared/tag-transcript.ts +6 -0
- package/src/shared/user-facing-codes.ts +2 -6
- package/src/shared/window-geometry.ts +10 -0
- package/src/shared/write-transaction-attribution.test.ts +2 -2
- package/src/v2/tui/host-contract.test.ts +41 -67
- package/src/v2/tui/index.ts +6 -214
- package/src/v2/tui/types.ts +5 -17
- package/dist/features/magic-context/checkout-claim-fake-subc.test-support.d.ts +0 -45
- package/dist/features/magic-context/checkout-claim-fake-subc.test-support.d.ts.map +0 -1
- package/dist/features/magic-context/checkout-claim.d.ts +0 -155
- package/dist/features/magic-context/checkout-claim.d.ts.map +0 -1
- package/dist/index-3h0ybwah.js +0 -6540
- package/dist/index-r5hmbx3t.js +0 -7773
- package/dist/recomp-orchestrator-y15j0rb9.js +0 -39
- package/dist/v2/recomp-orchestrator-8ejxvrzz.js +0 -39
- package/dist/v2/server-frbm99q5.js +0 -7698
- package/dist/v2/server-qp5hx1ad.js +0 -6366
- package/dist/wrapup-orchestrator-39amhv9s.js +0 -26
- /package/dist/{index-en1k8h8m.js → index-2pp04pmg.js} +0 -0
- /package/dist/v2/{index-en1k8h8m.js → index-2pp04pmg.js} +0 -0
|
@@ -0,0 +1,3030 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getHarness2
|
|
3
|
+
} from "./index-pavzsdc6.js";
|
|
4
|
+
import {
|
|
5
|
+
sanitizeDiagnosticText,
|
|
6
|
+
log2
|
|
7
|
+
} from "./index-8j7gqj40.js";
|
|
8
|
+
import {
|
|
9
|
+
isUserHomeDirectory,
|
|
10
|
+
setBootQuietPeriodForTests
|
|
11
|
+
} from "./index-knh4y330.js";
|
|
12
|
+
import {
|
|
13
|
+
estimateTokens
|
|
14
|
+
} from "./index-1qb0z7wa.js";
|
|
15
|
+
import {
|
|
16
|
+
SYNAPSE_DEFAULT_MODEL,
|
|
17
|
+
isSynapseEmbeddingTruncated,
|
|
18
|
+
getSynapseBatchRequestKey,
|
|
19
|
+
SynapseEmbeddingProvider
|
|
20
|
+
} from "./index-72nhv486.js";
|
|
21
|
+
import {
|
|
22
|
+
withSqliteBackgroundWriter2,
|
|
23
|
+
withoutSqliteTransformPass2,
|
|
24
|
+
isTransientSqliteError2
|
|
25
|
+
} from "./index-kb9nn0a0.js";
|
|
26
|
+
import {
|
|
27
|
+
CHUNK_WINDOW_SAFETY_RATIO,
|
|
28
|
+
normalizeCompartmentChunkMaxInputTokens,
|
|
29
|
+
buildCanonicalChunkTextFromFts,
|
|
30
|
+
buildCompartmentSummaryFallbackText,
|
|
31
|
+
chunkCanonicalText,
|
|
32
|
+
chunkEmbeddingWindowsAreCurrent,
|
|
33
|
+
replaceCompartmentChunkEmbeddings,
|
|
34
|
+
loadUnembeddedShadowChunkCandidates,
|
|
35
|
+
loadUnembeddedCompartmentChunkCandidatesPolite,
|
|
36
|
+
recordChunkEmbedBackoff,
|
|
37
|
+
chunkWindowSourceKey,
|
|
38
|
+
loadUnembeddedSessionChunkCandidatesPolite,
|
|
39
|
+
countSessionCompartmentEmbedCoverage,
|
|
40
|
+
countSessionCompartmentEmbedCoveragePolite,
|
|
41
|
+
countUnembeddedSessionCompartmentsPolite
|
|
42
|
+
} from "./index-pg50n748.js";
|
|
43
|
+
import {
|
|
44
|
+
DEFAULT_LOCAL_EMBEDDING_MODEL,
|
|
45
|
+
embeddingModelsMatch,
|
|
46
|
+
resolveEmbeddingTextPrefixes,
|
|
47
|
+
getEmbeddingProviderIdentity,
|
|
48
|
+
dominantEmbeddingFailure,
|
|
49
|
+
LocalEmbeddingProvider
|
|
50
|
+
} from "./index-bhvjj0ys.js";
|
|
51
|
+
import {
|
|
52
|
+
isEmbeddingHostBusy2
|
|
53
|
+
} from "./index-qt1qqasc.js";
|
|
54
|
+
import {
|
|
55
|
+
saveEmbeddingIfHashMatches,
|
|
56
|
+
getMemoryEmbedCoverage,
|
|
57
|
+
invalidateProject2
|
|
58
|
+
} from "./index-4dhv7mqz.js";
|
|
59
|
+
import {
|
|
60
|
+
logSlowWriteTransaction
|
|
61
|
+
} from "./index-2y0s2hep.js";
|
|
62
|
+
|
|
63
|
+
// src/features/magic-context/session-project-storage.ts
|
|
64
|
+
var SESSION_CHUNK_REPAIR_BATCH_SIZE = 100;
|
|
65
|
+
var upsertSessionProjectStatements = new WeakMap;
|
|
66
|
+
var repairSessionChunkProjectStatements = new WeakMap;
|
|
67
|
+
var misScopedProjectChunkStatements = new WeakMap;
|
|
68
|
+
var MIS_SCOPED_PROJECT_CHUNK_IDS_SQL = `
|
|
69
|
+
SELECT e.id FROM compartment_chunk_embeddings e
|
|
70
|
+
JOIN session_projects sp ON sp.session_id = e.session_id AND sp.harness = e.harness
|
|
71
|
+
WHERE e.project_path = ? AND sp.project_path <> e.project_path
|
|
72
|
+
UNION ALL
|
|
73
|
+
SELECT e.id FROM session_projects sp
|
|
74
|
+
JOIN compartment_chunk_embeddings e ON e.session_id = sp.session_id
|
|
75
|
+
WHERE sp.project_path = ? AND e.harness = sp.harness AND e.project_path <> sp.project_path`;
|
|
76
|
+
function getUpsertSessionProjectStatement(db) {
|
|
77
|
+
let stmt = upsertSessionProjectStatements.get(db);
|
|
78
|
+
if (!stmt) {
|
|
79
|
+
stmt = db.prepare(`INSERT INTO session_projects (session_id, harness, project_path, updated_at)
|
|
80
|
+
VALUES (?, ?, ?, ?)
|
|
81
|
+
ON CONFLICT(session_id, harness) DO UPDATE SET
|
|
82
|
+
project_path = excluded.project_path,
|
|
83
|
+
updated_at = excluded.updated_at
|
|
84
|
+
WHERE session_projects.project_path <> excluded.project_path`);
|
|
85
|
+
upsertSessionProjectStatements.set(db, stmt);
|
|
86
|
+
}
|
|
87
|
+
return stmt;
|
|
88
|
+
}
|
|
89
|
+
function getRepairSessionChunkProjectStatement(db) {
|
|
90
|
+
let stmt = repairSessionChunkProjectStatements.get(db);
|
|
91
|
+
if (!stmt) {
|
|
92
|
+
stmt = db.prepare(`UPDATE compartment_chunk_embeddings
|
|
93
|
+
SET project_path = ?
|
|
94
|
+
WHERE id IN (
|
|
95
|
+
SELECT id
|
|
96
|
+
FROM compartment_chunk_embeddings
|
|
97
|
+
WHERE session_id = ?
|
|
98
|
+
AND harness = ?
|
|
99
|
+
AND project_path <> ?
|
|
100
|
+
LIMIT ?
|
|
101
|
+
)`);
|
|
102
|
+
repairSessionChunkProjectStatements.set(db, stmt);
|
|
103
|
+
}
|
|
104
|
+
return stmt;
|
|
105
|
+
}
|
|
106
|
+
function findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectPath) {
|
|
107
|
+
return db.prepare(`SELECT id FROM (${MIS_SCOPED_PROJECT_CHUNK_IDS_SQL}) LIMIT 25`).all(projectPath, projectPath).map(({ id }) => id);
|
|
108
|
+
}
|
|
109
|
+
function recordSessionProjectIdentity(db, sessionId, projectPath) {
|
|
110
|
+
if (!sessionId || !projectPath)
|
|
111
|
+
return;
|
|
112
|
+
if (!projectPath.startsWith("git:") && !projectPath.startsWith("dir:") && isUserHomeDirectory(projectPath))
|
|
113
|
+
return;
|
|
114
|
+
const harness = getHarness2();
|
|
115
|
+
const now = Date.now();
|
|
116
|
+
let previousProjectPath;
|
|
117
|
+
db.transaction(() => {
|
|
118
|
+
const existing = db.prepare("SELECT project_path FROM session_projects WHERE session_id = ? AND harness = ?").get(sessionId, harness);
|
|
119
|
+
previousProjectPath = existing?.project_path;
|
|
120
|
+
getUpsertSessionProjectStatement(db).run(sessionId, harness, projectPath, now);
|
|
121
|
+
getRepairSessionChunkProjectStatement(db).run(projectPath, sessionId, harness, projectPath, SESSION_CHUNK_REPAIR_BATCH_SIZE);
|
|
122
|
+
}).immediate();
|
|
123
|
+
return previousProjectPath === projectPath ? undefined : { previousProjectPath, projectPath };
|
|
124
|
+
}
|
|
125
|
+
function hasRecordedSessionProjectIdentity(db, sessionId) {
|
|
126
|
+
const row = db.prepare("SELECT 1 AS found FROM session_projects WHERE session_id = ? AND harness = ?").get(sessionId, getHarness2());
|
|
127
|
+
return row?.found === 1;
|
|
128
|
+
}
|
|
129
|
+
function repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectPath, ids = findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectPath)) {
|
|
130
|
+
if (!projectPath || ids.length === 0)
|
|
131
|
+
return 0;
|
|
132
|
+
return db.prepare(`UPDATE compartment_chunk_embeddings
|
|
133
|
+
SET project_path = (SELECT sp.project_path FROM session_projects sp
|
|
134
|
+
WHERE sp.session_id = compartment_chunk_embeddings.session_id
|
|
135
|
+
AND sp.harness = compartment_chunk_embeddings.harness)
|
|
136
|
+
WHERE id IN (${ids.map(() => "?").join(",")}) AND EXISTS (
|
|
137
|
+
SELECT 1 FROM session_projects sp
|
|
138
|
+
WHERE sp.session_id = compartment_chunk_embeddings.session_id
|
|
139
|
+
AND sp.harness = compartment_chunk_embeddings.harness
|
|
140
|
+
AND sp.project_path <> compartment_chunk_embeddings.project_path
|
|
141
|
+
AND (sp.project_path = ? OR compartment_chunk_embeddings.project_path = ?)
|
|
142
|
+
)`).run(...ids, projectPath, projectPath).changes;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// src/features/magic-context/storage-embedding-measurements.ts
|
|
146
|
+
import { createHash } from "node:crypto";
|
|
147
|
+
function normalizedQueryHash(query) {
|
|
148
|
+
const normalized = query.trim().replace(/\s+/g, " ").toLowerCase();
|
|
149
|
+
return createHash("sha256").update(normalized).digest("hex");
|
|
150
|
+
}
|
|
151
|
+
var MEASUREMENT_CORPUS_SESSION_ROW_CAP = 2000;
|
|
152
|
+
function recordEmbeddingMeasurement(db, input, cap = MEASUREMENT_CORPUS_SESSION_ROW_CAP) {
|
|
153
|
+
const queryTextHash = normalizedQueryHash(input.queryText);
|
|
154
|
+
const dedupKey = queryTextHash;
|
|
155
|
+
const result = db.prepare(`INSERT OR IGNORE INTO embedding_measurement_corpus
|
|
156
|
+
(session_id, project_path, dedup_key, cohort_key, query_text_hash,
|
|
157
|
+
primary_result_ids_json, shadow_result_ids_json, primary_latency_ms, shadow_latency_ms,
|
|
158
|
+
primary_failed, shadow_failed, primary_model_id, shadow_model_id,
|
|
159
|
+
primary_fingerprint, shadow_fingerprint, primary_epoch, shadow_epoch,
|
|
160
|
+
corpus_hash, coverage_json, created_at)
|
|
161
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(input.sessionId, input.projectPath, dedupKey, input.cohortKey, queryTextHash, JSON.stringify(input.primaryResultIds.slice(0, 10)), JSON.stringify(input.shadowResultIds.slice(0, 10)), input.primaryLatencyMs, input.shadowLatencyMs, input.primaryFailed ? 1 : 0, input.shadowFailed ? 1 : 0, input.primaryModelId, input.shadowModelId, input.primaryFingerprint, input.shadowFingerprint, input.primaryEpoch, input.shadowEpoch, input.corpusHash, JSON.stringify(input.coverage), Date.now());
|
|
162
|
+
if (result.changes > 0) {
|
|
163
|
+
const rowCount = db.prepare("SELECT COUNT(*) AS count FROM embedding_measurement_corpus WHERE session_id = ?").get(input.sessionId).count;
|
|
164
|
+
const overflow = rowCount - cap;
|
|
165
|
+
if (overflow > 0) {
|
|
166
|
+
db.prepare(`DELETE FROM embedding_measurement_corpus
|
|
167
|
+
WHERE session_id = ?
|
|
168
|
+
AND id IN (
|
|
169
|
+
SELECT id FROM embedding_measurement_corpus
|
|
170
|
+
WHERE session_id = ?
|
|
171
|
+
ORDER BY id ASC
|
|
172
|
+
LIMIT ?
|
|
173
|
+
)`).run(input.sessionId, input.sessionId, overflow);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return result.changes > 0;
|
|
177
|
+
}
|
|
178
|
+
function beginSynapseBatchLedger(db, input, now = Date.now()) {
|
|
179
|
+
db.prepare(`INSERT INTO synapse_batch_ledger
|
|
180
|
+
(session_id, project_path, scope, manifest_json, request_key, status, created_at, updated_at)
|
|
181
|
+
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
|
|
182
|
+
ON CONFLICT(session_id, request_key) DO UPDATE SET
|
|
183
|
+
manifest_json = excluded.manifest_json,
|
|
184
|
+
updated_at = excluded.updated_at`).run(input.sessionId, input.projectPath, input.scope, JSON.stringify(input.manifest), input.requestKey, now, now);
|
|
185
|
+
}
|
|
186
|
+
function finishSynapseBatchLedger(db, sessionId, requestKey, status, now = Date.now()) {
|
|
187
|
+
db.prepare("UPDATE synapse_batch_ledger SET status = ?, updated_at = ? WHERE session_id = ? AND request_key = ?").run(status, now, sessionId, requestKey);
|
|
188
|
+
}
|
|
189
|
+
var SYNAPSE_BATCH_LEDGER_TTL_MS = 14 * 24 * 60 * 60 * 1000;
|
|
190
|
+
function pruneSynapseBatchLedgerForProject(db, projectIdentity, ttlMs = SYNAPSE_BATCH_LEDGER_TTL_MS) {
|
|
191
|
+
const ledgerTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'synapse_batch_ledger'").get();
|
|
192
|
+
if (!ledgerTable)
|
|
193
|
+
return 0;
|
|
194
|
+
const shadow = `shadow:${projectIdentity}`;
|
|
195
|
+
const cutoff = Date.now() - ttlMs;
|
|
196
|
+
const predicate = "session_id IN (?, ?) AND updated_at < ?";
|
|
197
|
+
if (!db.prepare(`SELECT 1 FROM synapse_batch_ledger WHERE ${predicate} LIMIT 1`).get(projectIdentity, shadow, cutoff))
|
|
198
|
+
return 0;
|
|
199
|
+
return db.prepare(`DELETE FROM synapse_batch_ledger WHERE ${predicate}`).run(projectIdentity, shadow, cutoff).changes;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// src/features/magic-context/memory/embedding-ssrf.ts
|
|
203
|
+
var METADATA_HOSTNAMES = new Set(["metadata.google.internal", "metadata.goog"]);
|
|
204
|
+
var IPV6_METADATA_HOSTS = new Set(["fd00:ec2::254"]);
|
|
205
|
+
function isLinkLocalIpv4(host) {
|
|
206
|
+
return /^169\.254\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
207
|
+
}
|
|
208
|
+
function ipv4FromMappedIpv6(host) {
|
|
209
|
+
const m = /^::ffff:(.+)$/.exec(host);
|
|
210
|
+
if (!m)
|
|
211
|
+
return null;
|
|
212
|
+
const tail = m[1];
|
|
213
|
+
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(tail))
|
|
214
|
+
return tail;
|
|
215
|
+
const hex = /^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(tail);
|
|
216
|
+
if (hex) {
|
|
217
|
+
const hi = Number.parseInt(hex[1], 16);
|
|
218
|
+
const lo = Number.parseInt(hex[2], 16);
|
|
219
|
+
if (Number.isNaN(hi) || Number.isNaN(lo))
|
|
220
|
+
return null;
|
|
221
|
+
return `${hi >> 8 & 255}.${hi & 255}.${lo >> 8 & 255}.${lo & 255}`;
|
|
222
|
+
}
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
function blockedEmbeddingEndpointReason(endpoint) {
|
|
226
|
+
const trimmed = endpoint.trim();
|
|
227
|
+
if (trimmed.length === 0)
|
|
228
|
+
return null;
|
|
229
|
+
let url;
|
|
230
|
+
try {
|
|
231
|
+
url = new URL(trimmed);
|
|
232
|
+
} catch {
|
|
233
|
+
return `embedding endpoint is not a valid URL: ${trimmed}`;
|
|
234
|
+
}
|
|
235
|
+
const host = url.hostname.toLowerCase().replace(/^\[/, "").replace(/\]$/, "");
|
|
236
|
+
if (METADATA_HOSTNAMES.has(host)) {
|
|
237
|
+
return `embedding endpoint host ${host} is a cloud metadata service (blocked)`;
|
|
238
|
+
}
|
|
239
|
+
if (IPV6_METADATA_HOSTS.has(host)) {
|
|
240
|
+
return `embedding endpoint host ${host} is the AWS IPv6 metadata service (blocked)`;
|
|
241
|
+
}
|
|
242
|
+
if (isLinkLocalIpv4(host)) {
|
|
243
|
+
return `embedding endpoint host ${host} is link-local / cloud metadata (blocked)`;
|
|
244
|
+
}
|
|
245
|
+
const mappedV4 = ipv4FromMappedIpv6(host);
|
|
246
|
+
if (mappedV4 && isLinkLocalIpv4(mappedV4)) {
|
|
247
|
+
return `embedding endpoint host ${host} (IPv4-mapped ${mappedV4}) is link-local / cloud metadata (blocked)`;
|
|
248
|
+
}
|
|
249
|
+
if (host.startsWith("fe80:")) {
|
|
250
|
+
return `embedding endpoint host ${host} is link-local / cloud metadata (blocked)`;
|
|
251
|
+
}
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// src/features/magic-context/memory/embedding-openai.ts
|
|
256
|
+
function responseSlots(items, inputCount) {
|
|
257
|
+
if (items.every((item) => item?.index === undefined)) {
|
|
258
|
+
return items.map((_, position) => position);
|
|
259
|
+
}
|
|
260
|
+
const slots = [];
|
|
261
|
+
const seen = new Set;
|
|
262
|
+
for (const item of items) {
|
|
263
|
+
const index = item?.index;
|
|
264
|
+
if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= inputCount || seen.has(index)) {
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
seen.add(index);
|
|
268
|
+
slots.push(index);
|
|
269
|
+
}
|
|
270
|
+
return slots;
|
|
271
|
+
}
|
|
272
|
+
function capEmbeddingInput(input, maxInputTokens, purpose) {
|
|
273
|
+
const ratio = purpose === "query" ? CHUNK_WINDOW_SAFETY_RATIO : 1;
|
|
274
|
+
const budget = Math.max(1, Math.floor(maxInputTokens * ratio));
|
|
275
|
+
if (estimateTokens(input) <= budget)
|
|
276
|
+
return input;
|
|
277
|
+
let low = 0;
|
|
278
|
+
let high = input.length;
|
|
279
|
+
while (low < high) {
|
|
280
|
+
const mid = Math.ceil((low + high) / 2);
|
|
281
|
+
if (estimateTokens(input.slice(0, mid)) <= budget)
|
|
282
|
+
low = mid;
|
|
283
|
+
else
|
|
284
|
+
high = mid - 1;
|
|
285
|
+
}
|
|
286
|
+
if (low > 0 && low < input.length) {
|
|
287
|
+
const last = input.charCodeAt(low - 1);
|
|
288
|
+
const next = input.charCodeAt(low);
|
|
289
|
+
if (last >= 55296 && last <= 56319 && next >= 56320 && next <= 57343)
|
|
290
|
+
low -= 1;
|
|
291
|
+
}
|
|
292
|
+
return input.slice(0, low);
|
|
293
|
+
}
|
|
294
|
+
function normalizeEndpoint(endpoint) {
|
|
295
|
+
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
296
|
+
}
|
|
297
|
+
var FAILURE_THRESHOLD = 3;
|
|
298
|
+
var FAILURE_WINDOW_MS = 60000;
|
|
299
|
+
var OPEN_DURATION_MS = 5 * 60000;
|
|
300
|
+
var FETCH_TIMEOUT_MS = 30000;
|
|
301
|
+
|
|
302
|
+
class OpenAICompatibleEmbeddingProvider {
|
|
303
|
+
modelId;
|
|
304
|
+
maxInputTokens;
|
|
305
|
+
endpoint;
|
|
306
|
+
model;
|
|
307
|
+
apiKey;
|
|
308
|
+
inputType;
|
|
309
|
+
queryInputType;
|
|
310
|
+
queryPrefix;
|
|
311
|
+
documentPrefix;
|
|
312
|
+
truncate;
|
|
313
|
+
initialized = false;
|
|
314
|
+
failureTimes = [];
|
|
315
|
+
circuitOpenUntil = 0;
|
|
316
|
+
openLogged = false;
|
|
317
|
+
modelMismatchLogged = false;
|
|
318
|
+
lastFailureReason = null;
|
|
319
|
+
halfOpenProbeInFlight = false;
|
|
320
|
+
queryTruncationLogged = false;
|
|
321
|
+
constructor(options) {
|
|
322
|
+
this.endpoint = normalizeEndpoint(options.endpoint);
|
|
323
|
+
this.model = options.model?.trim() ?? "";
|
|
324
|
+
this.apiKey = options.apiKey?.trim() ?? "";
|
|
325
|
+
this.inputType = options.inputType?.trim() ?? "";
|
|
326
|
+
this.queryInputType = options.queryInputType?.trim() ?? "";
|
|
327
|
+
const prefixes = resolveEmbeddingTextPrefixes(this.model, options.queryInstruction, options.documentPrefix);
|
|
328
|
+
this.queryPrefix = prefixes.queryPrefix;
|
|
329
|
+
this.documentPrefix = prefixes.documentPrefix;
|
|
330
|
+
this.truncate = options.truncate?.trim() ?? "";
|
|
331
|
+
this.maxInputTokens = typeof options.maxInputTokens === "number" && Number.isFinite(options.maxInputTokens) ? Math.max(1, Math.floor(options.maxInputTokens)) : 512;
|
|
332
|
+
this.modelId = getEmbeddingProviderIdentity({
|
|
333
|
+
provider: "openai-compatible",
|
|
334
|
+
endpoint: this.endpoint,
|
|
335
|
+
model: this.model,
|
|
336
|
+
...this.apiKey ? { api_key: this.apiKey } : {},
|
|
337
|
+
...this.inputType ? { input_type: this.inputType } : {},
|
|
338
|
+
...options.documentPrefix !== undefined ? { document_prefix: options.documentPrefix } : {},
|
|
339
|
+
...this.truncate ? { truncate: this.truncate } : {}
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
async initialize() {
|
|
343
|
+
if (this.initialized)
|
|
344
|
+
return true;
|
|
345
|
+
if (!this.endpoint || !this.model) {
|
|
346
|
+
log2("[magic-context] openai-compatible embedding provider is missing endpoint or model");
|
|
347
|
+
this.initialized = false;
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
const blockedReason = blockedEmbeddingEndpointReason(this.endpoint);
|
|
351
|
+
if (blockedReason) {
|
|
352
|
+
log2(`[magic-context] embedding endpoint blocked: ${blockedReason}`);
|
|
353
|
+
this.initialized = false;
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
this.initialized = true;
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
resolveInputTypeForPurpose(purpose = "passage") {
|
|
360
|
+
if (purpose === "query") {
|
|
361
|
+
return this.queryInputType || this.inputType;
|
|
362
|
+
}
|
|
363
|
+
return this.inputType;
|
|
364
|
+
}
|
|
365
|
+
async embed(text, signal, purpose) {
|
|
366
|
+
const [embedding] = await this.embedBatch([text], signal, purpose);
|
|
367
|
+
return embedding ?? null;
|
|
368
|
+
}
|
|
369
|
+
async embedBatch(texts, signal, purpose) {
|
|
370
|
+
if (texts.length === 0) {
|
|
371
|
+
return [];
|
|
372
|
+
}
|
|
373
|
+
const textPrefix = purpose === "query" ? this.queryPrefix : this.documentPrefix;
|
|
374
|
+
const requestTexts = texts.map((text) => {
|
|
375
|
+
const input = `${textPrefix}${text.trim().length === 0 ? " " : text}`;
|
|
376
|
+
const capped = capEmbeddingInput(input, this.maxInputTokens, purpose);
|
|
377
|
+
if (purpose === "query" && capped.length < input.length && !this.queryTruncationLogged) {
|
|
378
|
+
log2(`[magic-context] embedding query truncated from ${input.length} to ${capped.length} characters to fit max_input_tokens`);
|
|
379
|
+
this.queryTruncationLogged = true;
|
|
380
|
+
}
|
|
381
|
+
return capped;
|
|
382
|
+
});
|
|
383
|
+
if (!await this.initialize()) {
|
|
384
|
+
return Array.from({ length: texts.length }, () => null);
|
|
385
|
+
}
|
|
386
|
+
if (signal?.aborted) {
|
|
387
|
+
return Array.from({ length: texts.length }, () => null);
|
|
388
|
+
}
|
|
389
|
+
let isProbe = false;
|
|
390
|
+
let internalController;
|
|
391
|
+
let timeoutHandle;
|
|
392
|
+
let onOuterAbort;
|
|
393
|
+
try {
|
|
394
|
+
const claim = this.claimProbeOrShortCircuit();
|
|
395
|
+
if (claim === "short_circuit") {
|
|
396
|
+
return Array.from({ length: texts.length }, () => null);
|
|
397
|
+
}
|
|
398
|
+
isProbe = claim === "probe";
|
|
399
|
+
internalController = new AbortController;
|
|
400
|
+
timeoutHandle = setTimeout(() => internalController?.abort(), FETCH_TIMEOUT_MS);
|
|
401
|
+
onOuterAbort = () => internalController?.abort();
|
|
402
|
+
if (signal) {
|
|
403
|
+
signal.addEventListener("abort", onOuterAbort, { once: true });
|
|
404
|
+
}
|
|
405
|
+
const inputTypeForRequest = this.resolveInputTypeForPurpose(purpose);
|
|
406
|
+
const response = await fetch(`${this.endpoint}/embeddings`, {
|
|
407
|
+
method: "POST",
|
|
408
|
+
headers: {
|
|
409
|
+
"content-type": "application/json",
|
|
410
|
+
...this.apiKey ? { authorization: `Bearer ${this.apiKey}` } : {}
|
|
411
|
+
},
|
|
412
|
+
body: JSON.stringify({
|
|
413
|
+
model: this.model,
|
|
414
|
+
input: requestTexts,
|
|
415
|
+
...inputTypeForRequest ? { input_type: inputTypeForRequest } : {},
|
|
416
|
+
...this.truncate ? { truncate: this.truncate } : {}
|
|
417
|
+
}),
|
|
418
|
+
redirect: "error",
|
|
419
|
+
signal: internalController.signal
|
|
420
|
+
});
|
|
421
|
+
if (!response.ok) {
|
|
422
|
+
const excerpt = await response.text().catch(() => "");
|
|
423
|
+
const failure = this.failure("http_error", `HTTP ${response.status} from endpoint${this.bodyExcerpt(excerpt)}`, response.status >= 500 || response.status === 408 || response.status === 429);
|
|
424
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
425
|
+
this.recordFailure(isProbe, failure);
|
|
426
|
+
return Array.from({ length: texts.length }, () => null);
|
|
427
|
+
}
|
|
428
|
+
const rawBody = await response.text();
|
|
429
|
+
if (rawBody.trim().length === 0) {
|
|
430
|
+
const failure = this.failure("invalid_envelope", "response body was empty", false);
|
|
431
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
432
|
+
this.recordFailure(isProbe, failure);
|
|
433
|
+
return Array.from({ length: texts.length }, () => null);
|
|
434
|
+
}
|
|
435
|
+
let body;
|
|
436
|
+
try {
|
|
437
|
+
body = JSON.parse(rawBody);
|
|
438
|
+
} catch {
|
|
439
|
+
const failure = this.failure("invalid_envelope", `response body was not valid JSON${this.bodyExcerpt(rawBody)}`, false);
|
|
440
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
441
|
+
this.recordFailure(isProbe, failure);
|
|
442
|
+
return Array.from({ length: texts.length }, () => null);
|
|
443
|
+
}
|
|
444
|
+
const servedModel = typeof body.model === "string" ? body.model : "";
|
|
445
|
+
if (this.model && servedModel && !embeddingModelsMatch(servedModel, this.model)) {
|
|
446
|
+
if (!this.modelMismatchLogged) {
|
|
447
|
+
log2(`[magic-context] embedding endpoint served a DIFFERENT model than requested — refusing the substituted vectors (they have the wrong dimensions/space). requested="${sanitizeDiagnosticText(this.model)}" served="${sanitizeDiagnosticText(servedModel)}". Check that the endpoint serves the requested model; variant suffixes and vendor prefixes are matched automatically.`);
|
|
448
|
+
this.modelMismatchLogged = true;
|
|
449
|
+
}
|
|
450
|
+
this.recordFailure(isProbe, this.failure("substitution_rejected", `served model '${sanitizeDiagnosticText(servedModel)}' does not match requested '${sanitizeDiagnosticText(this.model)}' (substitution guard)`, false));
|
|
451
|
+
return Array.from({ length: texts.length }, () => null);
|
|
452
|
+
}
|
|
453
|
+
const responseKeys = Object.keys(body).sort();
|
|
454
|
+
if (!Array.isArray(body.data)) {
|
|
455
|
+
const failure = this.failure("invalid_envelope", `response had keys [${responseKeys.join(", ")}] but data[] was absent`, false);
|
|
456
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
457
|
+
this.recordFailure(isProbe, failure);
|
|
458
|
+
return Array.from({ length: texts.length }, () => null);
|
|
459
|
+
}
|
|
460
|
+
if (body.data.length === 0) {
|
|
461
|
+
const failure = this.failure("empty_result", "response data[] was empty", true);
|
|
462
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
463
|
+
this.recordFailure(isProbe, failure);
|
|
464
|
+
return Array.from({ length: texts.length }, () => null);
|
|
465
|
+
}
|
|
466
|
+
const slots = responseSlots(body.data, texts.length);
|
|
467
|
+
if (!slots) {
|
|
468
|
+
const failure = this.failure("invalid_envelope", "response data[].index values were missing, duplicated or out of range", false);
|
|
469
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
470
|
+
this.recordFailure(isProbe, failure);
|
|
471
|
+
return Array.from({ length: texts.length }, () => null);
|
|
472
|
+
}
|
|
473
|
+
const results = Array.from({ length: texts.length }, () => null);
|
|
474
|
+
body.data.forEach((item, position) => {
|
|
475
|
+
const embedding = item?.embedding;
|
|
476
|
+
const slot = slots[position];
|
|
477
|
+
if (slot !== undefined && slot < results.length && Array.isArray(embedding)) {
|
|
478
|
+
results[slot] = Float32Array.from(embedding);
|
|
479
|
+
}
|
|
480
|
+
});
|
|
481
|
+
if (results.every((r) => r === null)) {
|
|
482
|
+
const failure = this.failure("invalid_envelope", `response had keys [${responseKeys.join(", ")}] but data[].embedding was absent`, false);
|
|
483
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
484
|
+
this.recordFailure(isProbe, failure);
|
|
485
|
+
} else {
|
|
486
|
+
this.recordSuccess();
|
|
487
|
+
}
|
|
488
|
+
return results;
|
|
489
|
+
} catch (error) {
|
|
490
|
+
const isAbort = error instanceof Error && (error.name === "AbortError" || error.message.includes("aborted"));
|
|
491
|
+
if (isAbort) {
|
|
492
|
+
if (signal?.aborted) {} else {
|
|
493
|
+
log2(`[magic-context] openai-compatible embedding request timed out after ${FETCH_TIMEOUT_MS}ms`);
|
|
494
|
+
const failure = this.failure("transport_error", `request timed out after ${FETCH_TIMEOUT_MS}ms`, true);
|
|
495
|
+
this.recordFailure(isProbe, failure);
|
|
496
|
+
}
|
|
497
|
+
} else {
|
|
498
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
499
|
+
const failure = this.failure("transport_error", `transport error: ${sanitizeDiagnosticText(detail)}`, true);
|
|
500
|
+
log2(`[magic-context] openai-compatible embedding request failed: ${failure.reason}`);
|
|
501
|
+
this.recordFailure(isProbe, failure);
|
|
502
|
+
}
|
|
503
|
+
return Array.from({ length: texts.length }, () => null);
|
|
504
|
+
} finally {
|
|
505
|
+
if (timeoutHandle !== undefined) {
|
|
506
|
+
clearTimeout(timeoutHandle);
|
|
507
|
+
}
|
|
508
|
+
if (signal && onOuterAbort) {
|
|
509
|
+
signal.removeEventListener("abort", onOuterAbort);
|
|
510
|
+
}
|
|
511
|
+
if (isProbe) {
|
|
512
|
+
this.halfOpenProbeInFlight = false;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
async dispose() {
|
|
517
|
+
this.initialized = false;
|
|
518
|
+
}
|
|
519
|
+
isLoaded() {
|
|
520
|
+
return this.initialized;
|
|
521
|
+
}
|
|
522
|
+
claimProbeOrShortCircuit() {
|
|
523
|
+
if (this.circuitOpenUntil === 0) {
|
|
524
|
+
return "allow";
|
|
525
|
+
}
|
|
526
|
+
if (Date.now() < this.circuitOpenUntil) {
|
|
527
|
+
return "short_circuit";
|
|
528
|
+
}
|
|
529
|
+
if (this.halfOpenProbeInFlight) {
|
|
530
|
+
return "short_circuit";
|
|
531
|
+
}
|
|
532
|
+
this.halfOpenProbeInFlight = true;
|
|
533
|
+
log2(`[magic-context] openai-compatible embedding: circuit half-open, probing endpoint after ${this.lastFailureReason?.reason ?? "unknown failure"}`);
|
|
534
|
+
return "probe";
|
|
535
|
+
}
|
|
536
|
+
failure(failureClass, reason, retryable) {
|
|
537
|
+
return { class: failureClass, reason, retryable };
|
|
538
|
+
}
|
|
539
|
+
bodyExcerpt(body) {
|
|
540
|
+
const excerpt = sanitizeDiagnosticText(body).replace(/\s+/g, " ").trim().slice(0, 200);
|
|
541
|
+
return excerpt ? `: ${excerpt}` : "";
|
|
542
|
+
}
|
|
543
|
+
recordFailure(isProbe, failure) {
|
|
544
|
+
if (failure)
|
|
545
|
+
this.lastFailureReason = failure;
|
|
546
|
+
if (isProbe) {
|
|
547
|
+
this.circuitOpenUntil = Date.now() + OPEN_DURATION_MS;
|
|
548
|
+
if (!this.openLogged) {
|
|
549
|
+
log2(`[magic-context] openai-compatible embedding: probe failed (${failure?.reason ?? this.lastFailureReason?.reason ?? "unknown failure"}), re-opening circuit for ${OPEN_DURATION_MS / 60000}min`);
|
|
550
|
+
this.openLogged = true;
|
|
551
|
+
}
|
|
552
|
+
this.failureTimes = [];
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
const now = Date.now();
|
|
556
|
+
const cutoff = now - FAILURE_WINDOW_MS;
|
|
557
|
+
this.failureTimes = this.failureTimes.filter((t) => t > cutoff);
|
|
558
|
+
this.failureTimes.push(now);
|
|
559
|
+
if (this.failureTimes.length >= FAILURE_THRESHOLD) {
|
|
560
|
+
this.circuitOpenUntil = now + OPEN_DURATION_MS;
|
|
561
|
+
if (!this.openLogged) {
|
|
562
|
+
log2(`[magic-context] openai-compatible embedding: opening circuit for ${OPEN_DURATION_MS / 60000}min after ${this.failureTimes.length} failures in ${FAILURE_WINDOW_MS / 1000}s (${failure?.reason ?? this.lastFailureReason?.reason ?? "unknown failure"})`);
|
|
563
|
+
this.openLogged = true;
|
|
564
|
+
}
|
|
565
|
+
this.failureTimes = [];
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
recordSuccess() {
|
|
569
|
+
if (this.failureTimes.length > 0 || this.circuitOpenUntil > 0 || this.openLogged) {
|
|
570
|
+
log2("[magic-context] openai-compatible embedding: endpoint recovered, circuit closed");
|
|
571
|
+
}
|
|
572
|
+
this.failureTimes = [];
|
|
573
|
+
this.circuitOpenUntil = 0;
|
|
574
|
+
this.openLogged = false;
|
|
575
|
+
this.lastFailureReason = null;
|
|
576
|
+
}
|
|
577
|
+
getLastFailureReason() {
|
|
578
|
+
return this.lastFailureReason;
|
|
579
|
+
}
|
|
580
|
+
_getCircuitState() {
|
|
581
|
+
if (this.circuitOpenUntil === 0)
|
|
582
|
+
return "closed";
|
|
583
|
+
if (Date.now() < this.circuitOpenUntil) {
|
|
584
|
+
return this.halfOpenProbeInFlight ? "half_open" : "open";
|
|
585
|
+
}
|
|
586
|
+
return "half_open";
|
|
587
|
+
}
|
|
588
|
+
_getFailureCount() {
|
|
589
|
+
return this.failureTimes.length;
|
|
590
|
+
}
|
|
591
|
+
_resetCircuit() {
|
|
592
|
+
this.failureTimes = [];
|
|
593
|
+
this.circuitOpenUntil = 0;
|
|
594
|
+
this.openLogged = false;
|
|
595
|
+
this.halfOpenProbeInFlight = false;
|
|
596
|
+
this.lastFailureReason = null;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// src/features/magic-context/project-embedding-registry.ts
|
|
601
|
+
import { createHash as createHash2, randomUUID } from "node:crypto";
|
|
602
|
+
|
|
603
|
+
// src/shared/background-batch-drain.ts
|
|
604
|
+
async function drainBackgroundBatches(step, options = {}) {
|
|
605
|
+
const now = options.now ?? (() => performance.now());
|
|
606
|
+
const deadline = now() + (options.budgetMs ?? 2000);
|
|
607
|
+
const yieldFn = options.yieldFn ?? (() => new Promise((resolve) => setTimeout(resolve, 0)));
|
|
608
|
+
let batches = 0;
|
|
609
|
+
while (now() < deadline) {
|
|
610
|
+
try {
|
|
611
|
+
if (!withoutSqliteTransformPass2(() => withSqliteBackgroundWriter2(step)))
|
|
612
|
+
break;
|
|
613
|
+
} catch (error) {
|
|
614
|
+
if (isTransientSqliteError2(error))
|
|
615
|
+
break;
|
|
616
|
+
throw error;
|
|
617
|
+
}
|
|
618
|
+
batches++;
|
|
619
|
+
await yieldFn();
|
|
620
|
+
}
|
|
621
|
+
return batches;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// src/features/magic-context/git-commits/storage-git-commit-embeddings.ts
|
|
625
|
+
var saveStatements = new WeakMap;
|
|
626
|
+
var loadProjectStatements = new WeakMap;
|
|
627
|
+
var loadUnembeddedStatements = new WeakMap;
|
|
628
|
+
var countEmbeddedStatements = new WeakMap;
|
|
629
|
+
function getSaveStatement(db) {
|
|
630
|
+
let stmt = saveStatements.get(db);
|
|
631
|
+
if (!stmt) {
|
|
632
|
+
stmt = db.prepare(`INSERT INTO git_commit_embeddings (sha, embedding, model_id, created_at)
|
|
633
|
+
VALUES (?, ?, ?, ?)
|
|
634
|
+
ON CONFLICT(sha, model_id) DO UPDATE SET
|
|
635
|
+
embedding = excluded.embedding,
|
|
636
|
+
created_at = excluded.created_at`);
|
|
637
|
+
saveStatements.set(db, stmt);
|
|
638
|
+
}
|
|
639
|
+
return stmt;
|
|
640
|
+
}
|
|
641
|
+
function getLoadProjectStatement(db) {
|
|
642
|
+
let stmt = loadProjectStatements.get(db);
|
|
643
|
+
if (!stmt) {
|
|
644
|
+
stmt = db.prepare(`SELECT e.sha AS sha, e.embedding AS embedding, e.model_id AS model_id
|
|
645
|
+
FROM git_commit_embeddings e
|
|
646
|
+
JOIN git_commits c ON c.sha = e.sha
|
|
647
|
+
WHERE c.project_path = ? AND e.model_id = ?`);
|
|
648
|
+
loadProjectStatements.set(db, stmt);
|
|
649
|
+
}
|
|
650
|
+
return stmt;
|
|
651
|
+
}
|
|
652
|
+
function getLoadUnembeddedStatement(db) {
|
|
653
|
+
let stmt = loadUnembeddedStatements.get(db);
|
|
654
|
+
if (!stmt) {
|
|
655
|
+
stmt = db.prepare(`SELECT c.sha AS sha, c.message AS message
|
|
656
|
+
FROM git_commits c
|
|
657
|
+
LEFT JOIN git_commit_embeddings e ON c.sha = e.sha AND e.model_id = ?
|
|
658
|
+
WHERE c.project_path = ? AND e.sha IS NULL
|
|
659
|
+
ORDER BY c.committed_at DESC
|
|
660
|
+
LIMIT ?`);
|
|
661
|
+
loadUnembeddedStatements.set(db, stmt);
|
|
662
|
+
}
|
|
663
|
+
return stmt;
|
|
664
|
+
}
|
|
665
|
+
function getCountEmbeddedStatement(db) {
|
|
666
|
+
let stmt = countEmbeddedStatements.get(db);
|
|
667
|
+
if (!stmt) {
|
|
668
|
+
stmt = db.prepare(`SELECT COUNT(*) AS count FROM git_commit_embeddings e
|
|
669
|
+
JOIN git_commits c ON c.sha = e.sha WHERE c.project_path = ? AND e.model_id = ?`);
|
|
670
|
+
countEmbeddedStatements.set(db, stmt);
|
|
671
|
+
}
|
|
672
|
+
return stmt;
|
|
673
|
+
}
|
|
674
|
+
function saveCommitEmbedding(db, sha, embedding, modelId) {
|
|
675
|
+
const bytes = new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
676
|
+
getSaveStatement(db).run(sha, bytes, modelId, Date.now());
|
|
677
|
+
}
|
|
678
|
+
function loadProjectCommitEmbeddings(db, projectPath, modelId) {
|
|
679
|
+
const rows = getLoadProjectStatement(db).all(projectPath, modelId);
|
|
680
|
+
const map = new Map;
|
|
681
|
+
for (const row of rows) {
|
|
682
|
+
const buffer = row.embedding.buffer.slice(row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength);
|
|
683
|
+
map.set(row.sha, new Float32Array(buffer));
|
|
684
|
+
}
|
|
685
|
+
return map;
|
|
686
|
+
}
|
|
687
|
+
function loadUnembeddedCommits(db, projectPath, modelId, limit) {
|
|
688
|
+
return getLoadUnembeddedStatement(db).all(modelId, projectPath, limit);
|
|
689
|
+
}
|
|
690
|
+
function countEmbeddedCommits(db, projectPath, modelId) {
|
|
691
|
+
const row = getCountEmbeddedStatement(db).get(projectPath, modelId);
|
|
692
|
+
return row?.count ?? 0;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
// src/features/magic-context/git-commits/storage-git-commits.ts
|
|
696
|
+
var insertStatements = new WeakMap;
|
|
697
|
+
var existingShasStatements = new WeakMap;
|
|
698
|
+
var projectCountStatements = new WeakMap;
|
|
699
|
+
var evictOverflowStatements = new WeakMap;
|
|
700
|
+
var latestCommitTimeStatements = new WeakMap;
|
|
701
|
+
var latestCommitShaStatements = new WeakMap;
|
|
702
|
+
function getInsertStatement(db) {
|
|
703
|
+
let stmt = insertStatements.get(db);
|
|
704
|
+
if (!stmt) {
|
|
705
|
+
stmt = db.prepare(`INSERT INTO git_commits (sha, project_path, short_sha, message, author, committed_at, indexed_at)
|
|
706
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
707
|
+
ON CONFLICT(sha) DO UPDATE SET
|
|
708
|
+
project_path = excluded.project_path,
|
|
709
|
+
short_sha = excluded.short_sha,
|
|
710
|
+
message = excluded.message,
|
|
711
|
+
author = excluded.author,
|
|
712
|
+
committed_at = excluded.committed_at,
|
|
713
|
+
indexed_at = excluded.indexed_at
|
|
714
|
+
WHERE git_commits.message != excluded.message`);
|
|
715
|
+
insertStatements.set(db, stmt);
|
|
716
|
+
}
|
|
717
|
+
return stmt;
|
|
718
|
+
}
|
|
719
|
+
function getExistingShasStatement(db) {
|
|
720
|
+
let stmt = existingShasStatements.get(db);
|
|
721
|
+
if (!stmt) {
|
|
722
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ?");
|
|
723
|
+
existingShasStatements.set(db, stmt);
|
|
724
|
+
}
|
|
725
|
+
return stmt;
|
|
726
|
+
}
|
|
727
|
+
function getProjectCountStatement(db) {
|
|
728
|
+
let stmt = projectCountStatements.get(db);
|
|
729
|
+
if (!stmt) {
|
|
730
|
+
stmt = db.prepare("SELECT COUNT(*) AS count FROM git_commits WHERE project_path = ?");
|
|
731
|
+
projectCountStatements.set(db, stmt);
|
|
732
|
+
}
|
|
733
|
+
return stmt;
|
|
734
|
+
}
|
|
735
|
+
function getLatestCommitShaStatement(db) {
|
|
736
|
+
let stmt = latestCommitShaStatements.get(db);
|
|
737
|
+
if (!stmt) {
|
|
738
|
+
stmt = db.prepare("SELECT sha FROM git_commits WHERE project_path = ? ORDER BY committed_at DESC, sha DESC LIMIT 1");
|
|
739
|
+
latestCommitShaStatements.set(db, stmt);
|
|
740
|
+
}
|
|
741
|
+
return stmt;
|
|
742
|
+
}
|
|
743
|
+
function getEvictOverflowStatement(db) {
|
|
744
|
+
let stmt = evictOverflowStatements.get(db);
|
|
745
|
+
if (!stmt) {
|
|
746
|
+
stmt = db.prepare(`DELETE FROM git_commits
|
|
747
|
+
WHERE rowid IN (
|
|
748
|
+
SELECT rowid FROM git_commits
|
|
749
|
+
WHERE project_path = ?
|
|
750
|
+
ORDER BY committed_at DESC, sha DESC
|
|
751
|
+
LIMIT -1 OFFSET ?
|
|
752
|
+
)`);
|
|
753
|
+
evictOverflowStatements.set(db, stmt);
|
|
754
|
+
}
|
|
755
|
+
return stmt;
|
|
756
|
+
}
|
|
757
|
+
function upsertCommits(db, projectPath, commits) {
|
|
758
|
+
if (commits.length === 0)
|
|
759
|
+
return { inserted: 0, updated: 0 };
|
|
760
|
+
const existing = new Set;
|
|
761
|
+
for (const row of getExistingShasStatement(db).all(projectPath)) {
|
|
762
|
+
existing.add(row.sha);
|
|
763
|
+
}
|
|
764
|
+
let inserted = 0;
|
|
765
|
+
let updated = 0;
|
|
766
|
+
const now = Date.now();
|
|
767
|
+
const insertStmt = getInsertStatement(db);
|
|
768
|
+
db.transaction(() => {
|
|
769
|
+
for (const commit of commits) {
|
|
770
|
+
const result = insertStmt.run(commit.sha, projectPath, commit.shortSha, commit.message, commit.author, commit.committedAtMs, now);
|
|
771
|
+
if (result.changes > 0) {
|
|
772
|
+
if (existing.has(commit.sha)) {
|
|
773
|
+
updated++;
|
|
774
|
+
} else {
|
|
775
|
+
inserted++;
|
|
776
|
+
existing.add(commit.sha);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}).immediate();
|
|
781
|
+
return { inserted, updated };
|
|
782
|
+
}
|
|
783
|
+
function getCommitCount(db, projectPath) {
|
|
784
|
+
const row = getProjectCountStatement(db).get(projectPath);
|
|
785
|
+
return row?.count ?? 0;
|
|
786
|
+
}
|
|
787
|
+
function getLatestIndexedCommitSha(db, projectPath) {
|
|
788
|
+
const row = getLatestCommitShaStatement(db).get(projectPath);
|
|
789
|
+
return row?.sha ?? null;
|
|
790
|
+
}
|
|
791
|
+
function enforceProjectCap(db, projectPath, maxCommits) {
|
|
792
|
+
if (maxCommits <= 0)
|
|
793
|
+
return 0;
|
|
794
|
+
const count = getCommitCount(db, projectPath);
|
|
795
|
+
if (count <= maxCommits)
|
|
796
|
+
return 0;
|
|
797
|
+
getEvictOverflowStatement(db).run(projectPath, maxCommits);
|
|
798
|
+
const after = getCommitCount(db, projectPath);
|
|
799
|
+
const evicted = Math.max(0, count - after);
|
|
800
|
+
if (evicted > 0) {
|
|
801
|
+
log2(`[git-commits] evicted ${evicted} oldest commits for project ${projectPath} (cap=${maxCommits}, was=${count})`);
|
|
802
|
+
}
|
|
803
|
+
return evicted;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// src/features/magic-context/git-commits/sweep-coordinator.ts
|
|
807
|
+
var GIT_SWEEP_COOLDOWN_MS = 10 * 60 * 1000;
|
|
808
|
+
var GIT_SWEEP_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
809
|
+
var GIT_SWEEP_NON_INDEXABLE_REPROBE_MS = 24 * 60 * 60 * 1000;
|
|
810
|
+
var GIT_SWEEP_LEASE_RENEWAL_MS = 60 * 1000;
|
|
811
|
+
function runImmediate(db, body) {
|
|
812
|
+
db.exec("BEGIN IMMEDIATE");
|
|
813
|
+
const transactionStartedAt = performance.now();
|
|
814
|
+
let committed = false;
|
|
815
|
+
try {
|
|
816
|
+
const result = body();
|
|
817
|
+
db.exec("COMMIT");
|
|
818
|
+
committed = true;
|
|
819
|
+
logSlowWriteTransaction("git_sweep_lease", transactionStartedAt);
|
|
820
|
+
return result;
|
|
821
|
+
} finally {
|
|
822
|
+
if (!committed) {
|
|
823
|
+
try {
|
|
824
|
+
db.exec("ROLLBACK");
|
|
825
|
+
} catch {}
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
function rowToState(row) {
|
|
830
|
+
return {
|
|
831
|
+
projectPath: row.project_path,
|
|
832
|
+
leaseHolder: row.lease_holder,
|
|
833
|
+
leaseExpiresAt: row.lease_expires_at,
|
|
834
|
+
lastSweptAt: row.last_swept_at
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
function getGitSweepCoordinatorState(db, projectPath) {
|
|
838
|
+
const row = db.prepare(`SELECT project_path, lease_holder, lease_expires_at, last_swept_at
|
|
839
|
+
FROM git_sweep_coordinator
|
|
840
|
+
WHERE project_path = ?`).get(projectPath);
|
|
841
|
+
return row ? rowToState(row) : null;
|
|
842
|
+
}
|
|
843
|
+
function acquireGitSweepLease(db, projectPath, holderId, options = {}) {
|
|
844
|
+
const cooldownMs = options.cooldownMs ?? GIT_SWEEP_COOLDOWN_MS;
|
|
845
|
+
const leaseTtlMs = options.leaseTtlMs ?? GIT_SWEEP_LEASE_TTL_MS;
|
|
846
|
+
return runImmediate(db, () => {
|
|
847
|
+
const now = Date.now();
|
|
848
|
+
const row = getGitSweepCoordinatorState(db, projectPath);
|
|
849
|
+
if (row?.leaseHolder && row.leaseExpiresAt !== null && row.leaseExpiresAt > now) {
|
|
850
|
+
return {
|
|
851
|
+
acquired: false,
|
|
852
|
+
projectPath,
|
|
853
|
+
reason: "lease_active",
|
|
854
|
+
leaseHolder: row.leaseHolder,
|
|
855
|
+
leaseExpiresAt: row.leaseExpiresAt,
|
|
856
|
+
lastSweptAt: row.lastSweptAt,
|
|
857
|
+
nextAllowedAt: null
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
if (!options.ignoreCooldown && row?.lastSweptAt !== null && row?.lastSweptAt !== undefined) {
|
|
861
|
+
const nextAllowedAt = row.lastSweptAt + cooldownMs;
|
|
862
|
+
if (nextAllowedAt > now) {
|
|
863
|
+
return {
|
|
864
|
+
acquired: false,
|
|
865
|
+
projectPath,
|
|
866
|
+
reason: "cooldown_active",
|
|
867
|
+
leaseHolder: row.leaseHolder,
|
|
868
|
+
leaseExpiresAt: row.leaseExpiresAt,
|
|
869
|
+
lastSweptAt: row.lastSweptAt,
|
|
870
|
+
nextAllowedAt
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
const leaseExpiresAt = now + leaseTtlMs;
|
|
875
|
+
db.prepare(`INSERT INTO git_sweep_coordinator (
|
|
876
|
+
project_path,
|
|
877
|
+
lease_holder,
|
|
878
|
+
lease_expires_at,
|
|
879
|
+
last_swept_at
|
|
880
|
+
) VALUES (?, ?, ?, NULL)
|
|
881
|
+
ON CONFLICT(project_path) DO UPDATE SET
|
|
882
|
+
lease_holder = excluded.lease_holder,
|
|
883
|
+
lease_expires_at = excluded.lease_expires_at`).run(projectPath, holderId, leaseExpiresAt);
|
|
884
|
+
return {
|
|
885
|
+
acquired: true,
|
|
886
|
+
projectPath,
|
|
887
|
+
holderId,
|
|
888
|
+
acquiredAt: now,
|
|
889
|
+
leaseExpiresAt
|
|
890
|
+
};
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
function renewGitSweepLease(db, projectPath, holderId, leaseTtlMs = GIT_SWEEP_LEASE_TTL_MS) {
|
|
894
|
+
return runImmediate(db, () => {
|
|
895
|
+
const now = Date.now();
|
|
896
|
+
const leaseExpiresAt = now + leaseTtlMs;
|
|
897
|
+
const result = db.prepare(`UPDATE git_sweep_coordinator
|
|
898
|
+
SET lease_expires_at = ?
|
|
899
|
+
WHERE project_path = ?
|
|
900
|
+
AND lease_holder = ?
|
|
901
|
+
AND lease_expires_at > ?`).run(leaseExpiresAt, projectPath, holderId, now);
|
|
902
|
+
return result.changes === 1;
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
function markGitSweepSuccessAndRelease(db, projectPath, holderId) {
|
|
906
|
+
return runImmediate(db, () => {
|
|
907
|
+
const now = Date.now();
|
|
908
|
+
const result = db.prepare(`UPDATE git_sweep_coordinator
|
|
909
|
+
SET lease_holder = NULL,
|
|
910
|
+
lease_expires_at = NULL,
|
|
911
|
+
last_swept_at = ?
|
|
912
|
+
WHERE project_path = ?
|
|
913
|
+
AND lease_holder = ?
|
|
914
|
+
AND lease_expires_at > ?`).run(now, projectPath, holderId, now);
|
|
915
|
+
return result.changes === 1;
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
function parkGitSweepNonIndexable(db, projectPath, holderId, reprobeMs = GIT_SWEEP_NON_INDEXABLE_REPROBE_MS) {
|
|
919
|
+
return runImmediate(db, () => {
|
|
920
|
+
const now = Date.now();
|
|
921
|
+
const sweptAt = now + reprobeMs - GIT_SWEEP_COOLDOWN_MS;
|
|
922
|
+
const result = db.prepare(`UPDATE git_sweep_coordinator
|
|
923
|
+
SET lease_holder = NULL,
|
|
924
|
+
lease_expires_at = NULL,
|
|
925
|
+
last_swept_at = ?
|
|
926
|
+
WHERE project_path = ?
|
|
927
|
+
AND lease_holder = ?
|
|
928
|
+
AND lease_expires_at > ?`).run(sweptAt, projectPath, holderId, now);
|
|
929
|
+
return result.changes === 1;
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
function releaseGitSweepLease(db, projectPath, holderId) {
|
|
933
|
+
runImmediate(db, () => {
|
|
934
|
+
db.prepare(`UPDATE git_sweep_coordinator
|
|
935
|
+
SET lease_holder = NULL,
|
|
936
|
+
lease_expires_at = NULL
|
|
937
|
+
WHERE project_path = ?
|
|
938
|
+
AND lease_holder = ?`).run(projectPath, holderId);
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
// src/features/magic-context/shadow-backfill-state.ts
|
|
943
|
+
var SHADOW_BACKFILL_PROVENANCE_KEY = "_magicContextShadowBackfill";
|
|
944
|
+
function parseJsonRecord(value) {
|
|
945
|
+
try {
|
|
946
|
+
const parsed = JSON.parse(value);
|
|
947
|
+
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : {};
|
|
948
|
+
} catch {
|
|
949
|
+
return {};
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
function parsePersistedShadowBackfillState(provenanceJson) {
|
|
953
|
+
const raw = parseJsonRecord(provenanceJson)[SHADOW_BACKFILL_PROVENANCE_KEY];
|
|
954
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw))
|
|
955
|
+
return;
|
|
956
|
+
const state = raw;
|
|
957
|
+
if (state.version !== 1)
|
|
958
|
+
return;
|
|
959
|
+
return state;
|
|
960
|
+
}
|
|
961
|
+
function hasShadowEmbeddingRegistrationsTable(db) {
|
|
962
|
+
return Boolean(db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get());
|
|
963
|
+
}
|
|
964
|
+
function describeShadowBackfillWriteRefusal2(reason) {
|
|
965
|
+
switch (reason) {
|
|
966
|
+
case "provider_returned_no_vectors":
|
|
967
|
+
return "the provider returned no vectors";
|
|
968
|
+
case "memory_hash_guard_rejected":
|
|
969
|
+
return "the memory normalized-hash guard rejected vectors because content changed in flight";
|
|
970
|
+
case "candidate_rows_changed":
|
|
971
|
+
return "the selected source rows changed before the writer loaded them";
|
|
972
|
+
case "registration_retired_during_embed":
|
|
973
|
+
return "the shadow registration was retired or replaced while the provider call was in flight, so the vectors were discarded";
|
|
974
|
+
case "chunk_fts_mapping_incomplete":
|
|
975
|
+
return "the chunk writer refused rows whose transcript ordinals are not fully mapped in FTS";
|
|
976
|
+
case "chunk_empty_canonical_text":
|
|
977
|
+
return "the chunk writer produced no canonical windows";
|
|
978
|
+
case "chunk_partial_vector_set":
|
|
979
|
+
return "the chunk writer refused a partial provider result so it would not replace a compartment incompletely";
|
|
980
|
+
case "chunk_window_contract_mismatch":
|
|
981
|
+
return "written chunk window keys or hashes did not satisfy the selector's window contract";
|
|
982
|
+
case "duplicate_submission_budget":
|
|
983
|
+
return "the same content batch was already submitted within the one-hour provider budget";
|
|
984
|
+
default:
|
|
985
|
+
return "the write completed without satisfying the selected candidate";
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
function listShadowBackfillStalls2(db, projectIdentity) {
|
|
989
|
+
if (!hasShadowEmbeddingRegistrationsTable(db))
|
|
990
|
+
return [];
|
|
991
|
+
const rows = db.prepare(`SELECT project_path AS projectIdentity, scope, model_id AS modelId,
|
|
992
|
+
provenance_json AS provenanceJson
|
|
993
|
+
FROM shadow_embedding_registrations
|
|
994
|
+
${projectIdentity ? "WHERE project_path = ?" : ""}
|
|
995
|
+
ORDER BY updated_at DESC, generation DESC`).all(...projectIdentity ? [projectIdentity] : []);
|
|
996
|
+
const latestScopes = new Set;
|
|
997
|
+
const stalls = [];
|
|
998
|
+
for (const row of rows) {
|
|
999
|
+
const scopeKey = `${row.projectIdentity}:${row.scope}`;
|
|
1000
|
+
if (latestScopes.has(scopeKey))
|
|
1001
|
+
continue;
|
|
1002
|
+
latestScopes.add(scopeKey);
|
|
1003
|
+
const state = parsePersistedShadowBackfillState(row.provenanceJson);
|
|
1004
|
+
if (state?.stopReason !== "stalled_no_progress")
|
|
1005
|
+
continue;
|
|
1006
|
+
stalls.push({
|
|
1007
|
+
projectIdentity: row.projectIdentity,
|
|
1008
|
+
scope: row.scope,
|
|
1009
|
+
modelId: row.modelId,
|
|
1010
|
+
candidateSignature: state.candidateSignature ?? "unknown",
|
|
1011
|
+
writeRefusalReason: state.writeRefusalReason ?? "unknown_write_rejection",
|
|
1012
|
+
stoppedAt: state.stoppedAt ?? 0
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
return stalls;
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
// src/features/magic-context/project-embedding-registry.ts
|
|
1019
|
+
var OFF_PROVIDER_IDENTITY = "embedding-provider:off";
|
|
1020
|
+
var SWEEP_MAX_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
1021
|
+
var SWEEP_MAX_CONSECUTIVE_EMPTY = 3;
|
|
1022
|
+
var COMMIT_DRAIN_BATCH_SIZE = 16;
|
|
1023
|
+
var COMMIT_DRAIN_MAX_PER_SWEEP = 500;
|
|
1024
|
+
var CHUNK_DRAIN_BATCH_SIZE = 8;
|
|
1025
|
+
var CHUNK_DRAIN_MAX_PER_SWEEP = CHUNK_DRAIN_BATCH_SIZE;
|
|
1026
|
+
var EMBEDDING_IDENTITY_GC_GRACE_MS = 14 * 24 * 60 * 60 * 1000;
|
|
1027
|
+
var STALE_EMBEDDING_GC_BATCH_SIZE = 25;
|
|
1028
|
+
var MAX_WINDOWS_PER_EMBED_CALL = 2;
|
|
1029
|
+
var SESSION_EMBED_LEASE_RENEWAL_MS = 60 * 1000;
|
|
1030
|
+
var EMBED_SLICE_RETRY_ATTEMPTS = 3;
|
|
1031
|
+
var EMBED_SLICE_RETRY_BASE_MS = 250;
|
|
1032
|
+
var EMBED_SLOW_FAILURE_NO_RETRY_MS = 1e4;
|
|
1033
|
+
var MAX_CONSECUTIVE_FAILED_BATCHES = 3;
|
|
1034
|
+
var projectRegistrations = new Map;
|
|
1035
|
+
var shadowRegistrations = new Map;
|
|
1036
|
+
var shadowQueue = [];
|
|
1037
|
+
var shadowWorker = null;
|
|
1038
|
+
var SHADOW_MAX_ITEMS_PER_TICK = 64;
|
|
1039
|
+
var SHADOW_MAX_BYTES_PER_TICK = 512 * 1024;
|
|
1040
|
+
var SHADOW_MAX_WALL_CLOCK_MS = 2000;
|
|
1041
|
+
var SHADOW_RESUBMIT_WINDOW_MS = 60 * 60 * 1000;
|
|
1042
|
+
var SHADOW_BACKFILL_PROVENANCE_KEY2 = "_magicContextShadowBackfill";
|
|
1043
|
+
var pendingShadowBackfills = new Map;
|
|
1044
|
+
var shadowBackfillLastIds = new Map;
|
|
1045
|
+
var shadowBackfillStopReasons = new Map;
|
|
1046
|
+
var shadowBackfillLastWriteOutcomes = new Map;
|
|
1047
|
+
var shadowBackfillNow = () => Date.now();
|
|
1048
|
+
function getShadowBackfillStopReason2(projectIdentity, scope) {
|
|
1049
|
+
return shadowBackfillStopReasons.get(`${projectIdentity}:${scope}`);
|
|
1050
|
+
}
|
|
1051
|
+
var loadUnembeddedMemoriesStatements = new WeakMap;
|
|
1052
|
+
var upsertActiveIdentityStatements = new WeakMap;
|
|
1053
|
+
var backfillActiveIdentityStatements = new Map;
|
|
1054
|
+
var staleIdentityStatements = new Map;
|
|
1055
|
+
var deleteActiveIdentityStatements = new WeakMap;
|
|
1056
|
+
var globalRegistrationGeneration = 0;
|
|
1057
|
+
var untrustedLoadProjects = new Set;
|
|
1058
|
+
function markProjectLoadUntrusted2(projectIdentity) {
|
|
1059
|
+
untrustedLoadProjects.add(projectIdentity);
|
|
1060
|
+
}
|
|
1061
|
+
var projectSweepInProgress = false;
|
|
1062
|
+
var testProviderFactory = null;
|
|
1063
|
+
|
|
1064
|
+
class TestProviderFactoryRequiredError extends Error {
|
|
1065
|
+
constructor() {
|
|
1066
|
+
super("test constructed a network-capable embedding provider without a test factory — install _setTestProviderFactoryForProject or set embedding.provider off in the fixture");
|
|
1067
|
+
this.name = "TestProviderFactoryRequiredError";
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
function synapseDescriptorFromConfig(config) {
|
|
1071
|
+
const raw = config;
|
|
1072
|
+
const candidate = raw.synapse_descriptor;
|
|
1073
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate))
|
|
1074
|
+
return;
|
|
1075
|
+
const descriptor = candidate;
|
|
1076
|
+
if (typeof descriptor.lane !== "string" || descriptor.lane.length === 0 || typeof descriptor.max_tokens !== "number" || !Number.isInteger(descriptor.max_tokens) || descriptor.max_tokens <= 0 || descriptor.max_tokens_source !== "runtime_bucket" && descriptor.max_tokens_source !== "worker_bucket" && descriptor.max_tokens_source !== "catalog" && descriptor.max_tokens_source !== "catalog_unloaded" || typeof descriptor.warm !== "boolean") {
|
|
1077
|
+
return;
|
|
1078
|
+
}
|
|
1079
|
+
return descriptor;
|
|
1080
|
+
}
|
|
1081
|
+
function synapseConfigFields(config) {
|
|
1082
|
+
const raw = config;
|
|
1083
|
+
const descriptor = synapseDescriptorFromConfig(config);
|
|
1084
|
+
const rawProvenance = raw.synapse_provenance;
|
|
1085
|
+
const provenance = descriptor ? {
|
|
1086
|
+
...rawProvenance && typeof rawProvenance === "object" && !Array.isArray(rawProvenance) ? rawProvenance : rawProvenance === undefined ? {} : { synapse_provenance: rawProvenance },
|
|
1087
|
+
capability_descriptor: descriptor
|
|
1088
|
+
} : rawProvenance;
|
|
1089
|
+
return {
|
|
1090
|
+
...typeof raw.model === "string" ? { model: raw.model } : {},
|
|
1091
|
+
...typeof raw.synapse_fingerprint === "string" ? { fingerprint: raw.synapse_fingerprint } : {},
|
|
1092
|
+
...typeof raw.synapse_table_epoch === "number" ? { tableEpoch: raw.synapse_table_epoch } : {},
|
|
1093
|
+
...typeof raw.synapse_dims === "number" ? { dims: raw.synapse_dims } : {},
|
|
1094
|
+
...provenance !== undefined ? { provenance } : {},
|
|
1095
|
+
...descriptor ? { descriptor } : {}
|
|
1096
|
+
};
|
|
1097
|
+
}
|
|
1098
|
+
function parseJsonRecord2(value) {
|
|
1099
|
+
try {
|
|
1100
|
+
const parsed = JSON.parse(value);
|
|
1101
|
+
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : { synapse_provenance: parsed };
|
|
1102
|
+
} catch {
|
|
1103
|
+
return {};
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
function hasShadowEmbeddingRegistrationsTable2(db) {
|
|
1107
|
+
return Boolean(db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get());
|
|
1108
|
+
}
|
|
1109
|
+
function getPersistedShadowBackfillState(db, projectIdentity, scope, modelId) {
|
|
1110
|
+
if (!hasShadowEmbeddingRegistrationsTable2(db))
|
|
1111
|
+
return;
|
|
1112
|
+
const row = db.prepare(`SELECT provenance_json AS provenanceJson
|
|
1113
|
+
FROM shadow_embedding_registrations
|
|
1114
|
+
WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
|
|
1115
|
+
return typeof row?.provenanceJson === "string" ? parsePersistedShadowBackfillState(row.provenanceJson) : undefined;
|
|
1116
|
+
}
|
|
1117
|
+
function updatePersistedShadowBackfillState(db, projectIdentity, scope, modelId, update) {
|
|
1118
|
+
if (!hasShadowEmbeddingRegistrationsTable2(db))
|
|
1119
|
+
return;
|
|
1120
|
+
const row = db.prepare(`SELECT provenance_json AS provenanceJson
|
|
1121
|
+
FROM shadow_embedding_registrations
|
|
1122
|
+
WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
|
|
1123
|
+
if (typeof row?.provenanceJson !== "string")
|
|
1124
|
+
return;
|
|
1125
|
+
const provenance = parseJsonRecord2(row.provenanceJson);
|
|
1126
|
+
const current = parsePersistedShadowBackfillState(row.provenanceJson) ?? { version: 1 };
|
|
1127
|
+
provenance[SHADOW_BACKFILL_PROVENANCE_KEY2] = update(current);
|
|
1128
|
+
db.prepare(`UPDATE shadow_embedding_registrations
|
|
1129
|
+
SET provenance_json = ?
|
|
1130
|
+
WHERE project_path = ? AND scope = ? AND model_id = ?`).run(JSON.stringify(provenance), projectIdentity, scope, modelId);
|
|
1131
|
+
}
|
|
1132
|
+
function shadowDescriptorProvenanceJson(db, projectIdentity, scope, modelId, synapseProvenance) {
|
|
1133
|
+
const provenance = typeof synapseProvenance === "object" && synapseProvenance !== null && !Array.isArray(synapseProvenance) ? { ...synapseProvenance } : synapseProvenance === undefined ? {} : { synapse_provenance: synapseProvenance };
|
|
1134
|
+
const existing = getPersistedShadowBackfillState(db, projectIdentity, scope, modelId);
|
|
1135
|
+
if (existing)
|
|
1136
|
+
provenance[SHADOW_BACKFILL_PROVENANCE_KEY2] = existing;
|
|
1137
|
+
return JSON.stringify(provenance);
|
|
1138
|
+
}
|
|
1139
|
+
function persistPrimaryDescriptor(db, registration) {
|
|
1140
|
+
const descriptorTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'embedding_registrations'").get();
|
|
1141
|
+
if (!descriptorTable)
|
|
1142
|
+
return;
|
|
1143
|
+
const fields = synapseConfigFields(registration.config);
|
|
1144
|
+
db.prepare(`INSERT INTO embedding_registrations
|
|
1145
|
+
(project_path, provider_identity, model_id, chunk_model_id, fingerprint, table_epoch, dims, provenance_json, generation, updated_at)
|
|
1146
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1147
|
+
ON CONFLICT(project_path) DO UPDATE SET
|
|
1148
|
+
provider_identity = excluded.provider_identity,
|
|
1149
|
+
model_id = excluded.model_id,
|
|
1150
|
+
chunk_model_id = excluded.chunk_model_id,
|
|
1151
|
+
fingerprint = excluded.fingerprint,
|
|
1152
|
+
table_epoch = excluded.table_epoch,
|
|
1153
|
+
dims = excluded.dims,
|
|
1154
|
+
provenance_json = excluded.provenance_json,
|
|
1155
|
+
generation = excluded.generation,
|
|
1156
|
+
updated_at = excluded.updated_at`).run(registration.projectIdentity, registration.providerIdentity, registration.modelId, registration.chunkModelId, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, JSON.stringify(fields.provenance ?? {}), registration.generation, Date.now());
|
|
1157
|
+
}
|
|
1158
|
+
function persistShadowDescriptor(db, registration) {
|
|
1159
|
+
const descriptorTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'shadow_embedding_registrations'").get();
|
|
1160
|
+
if (!descriptorTable)
|
|
1161
|
+
return;
|
|
1162
|
+
const fields = synapseConfigFields(registration.config);
|
|
1163
|
+
const now = Date.now();
|
|
1164
|
+
db.prepare(`INSERT INTO shadow_embedding_registrations
|
|
1165
|
+
(project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
|
|
1166
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1167
|
+
ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
|
|
1168
|
+
generation = excluded.generation,
|
|
1169
|
+
fingerprint = excluded.fingerprint,
|
|
1170
|
+
table_epoch = excluded.table_epoch,
|
|
1171
|
+
dims = excluded.dims,
|
|
1172
|
+
provenance_json = excluded.provenance_json,
|
|
1173
|
+
updated_at = excluded.updated_at`).run(registration.projectIdentity, "memory", registration.modelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "memory", registration.modelId, fields.provenance), now);
|
|
1174
|
+
db.prepare(`INSERT INTO shadow_embedding_registrations
|
|
1175
|
+
(project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
|
|
1176
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1177
|
+
ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
|
|
1178
|
+
generation = excluded.generation,
|
|
1179
|
+
fingerprint = excluded.fingerprint,
|
|
1180
|
+
table_epoch = excluded.table_epoch,
|
|
1181
|
+
dims = excluded.dims,
|
|
1182
|
+
provenance_json = excluded.provenance_json,
|
|
1183
|
+
updated_at = excluded.updated_at`).run(registration.projectIdentity, "commit", registration.modelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "commit", registration.modelId, fields.provenance), now);
|
|
1184
|
+
db.prepare(`INSERT INTO shadow_embedding_registrations
|
|
1185
|
+
(project_path, scope, model_id, generation, fingerprint, table_epoch, dims, provenance_json, updated_at)
|
|
1186
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1187
|
+
ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
|
|
1188
|
+
generation = excluded.generation,
|
|
1189
|
+
fingerprint = excluded.fingerprint,
|
|
1190
|
+
table_epoch = excluded.table_epoch,
|
|
1191
|
+
dims = excluded.dims,
|
|
1192
|
+
provenance_json = excluded.provenance_json,
|
|
1193
|
+
updated_at = excluded.updated_at`).run(registration.projectIdentity, "chunk", registration.chunkModelId, registration.generation, fields.fingerprint ?? "", fields.tableEpoch ?? 0, fields.dims ?? 0, shadowDescriptorProvenanceJson(db, registration.projectIdentity, "chunk", registration.chunkModelId, fields.provenance), now);
|
|
1194
|
+
}
|
|
1195
|
+
function resolveEmbeddingConfig(config) {
|
|
1196
|
+
if (!config || config.provider === "local") {
|
|
1197
|
+
return {
|
|
1198
|
+
provider: "local",
|
|
1199
|
+
model: config?.model?.trim() || DEFAULT_LOCAL_EMBEDDING_MODEL,
|
|
1200
|
+
local_runtime: config?.local_runtime ?? "auto",
|
|
1201
|
+
...config?.max_input_tokens ? {
|
|
1202
|
+
max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config.max_input_tokens)
|
|
1203
|
+
} : {},
|
|
1204
|
+
...config?.local_dtype ? { local_dtype: config.local_dtype } : {}
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
if (config.provider === "openai-compatible") {
|
|
1208
|
+
const apiKey = config.api_key?.trim();
|
|
1209
|
+
const inputType = config.input_type?.trim();
|
|
1210
|
+
const queryInputType = config.query_input_type?.trim();
|
|
1211
|
+
const truncate = config.truncate?.trim();
|
|
1212
|
+
return {
|
|
1213
|
+
provider: "openai-compatible",
|
|
1214
|
+
model: config.model.trim(),
|
|
1215
|
+
endpoint: config.endpoint.trim(),
|
|
1216
|
+
...apiKey ? { api_key: apiKey } : {},
|
|
1217
|
+
...inputType ? { input_type: inputType } : {},
|
|
1218
|
+
...queryInputType ? { query_input_type: queryInputType } : {},
|
|
1219
|
+
...config.query_instruction !== undefined ? { query_instruction: config.query_instruction } : {},
|
|
1220
|
+
...config.document_prefix !== undefined ? { document_prefix: config.document_prefix } : {},
|
|
1221
|
+
...truncate ? { truncate } : {},
|
|
1222
|
+
...config.max_input_tokens ? {
|
|
1223
|
+
max_input_tokens: normalizeCompartmentChunkMaxInputTokens(config.max_input_tokens)
|
|
1224
|
+
} : {}
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
if (config.provider === "off") {
|
|
1228
|
+
return { provider: "off" };
|
|
1229
|
+
}
|
|
1230
|
+
if (config.provider === "synapse") {
|
|
1231
|
+
const synapse = config;
|
|
1232
|
+
const descriptor = synapseDescriptorFromConfig(config);
|
|
1233
|
+
return {
|
|
1234
|
+
provider: "synapse",
|
|
1235
|
+
model: synapse.model?.trim() || "gte-modernbert-base-f16",
|
|
1236
|
+
max_input_tokens: normalizeCompartmentChunkMaxInputTokens(descriptor?.max_tokens ?? synapse.max_input_tokens),
|
|
1237
|
+
...synapse.synapse_connection_file ? { synapse_connection_file: synapse.synapse_connection_file } : {},
|
|
1238
|
+
...synapse.synapse_fingerprint ? { synapse_fingerprint: synapse.synapse_fingerprint } : {},
|
|
1239
|
+
...typeof synapse.synapse_table_epoch === "number" ? { synapse_table_epoch: synapse.synapse_table_epoch } : {},
|
|
1240
|
+
...typeof synapse.synapse_dims === "number" ? { synapse_dims: synapse.synapse_dims } : {},
|
|
1241
|
+
...typeof synapse.synapse_recommended_batch === "number" ? { synapse_recommended_batch: synapse.synapse_recommended_batch } : {},
|
|
1242
|
+
...typeof synapse.synapse_recommended_token_budget === "number" ? {
|
|
1243
|
+
synapse_recommended_token_budget: synapse.synapse_recommended_token_budget
|
|
1244
|
+
} : {},
|
|
1245
|
+
...descriptor ? { synapse_descriptor: descriptor } : {},
|
|
1246
|
+
...synapse.synapse_provenance !== undefined ? { synapse_provenance: synapse.synapse_provenance } : {}
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
throw new Error("Unknown embedding provider");
|
|
1250
|
+
}
|
|
1251
|
+
function createProvider(config, context) {
|
|
1252
|
+
if (config.provider === "off") {
|
|
1253
|
+
return null;
|
|
1254
|
+
}
|
|
1255
|
+
if (testProviderFactory) {
|
|
1256
|
+
return testProviderFactory(config);
|
|
1257
|
+
}
|
|
1258
|
+
if (process.env.MAGIC_CONTEXT_TEST_DATA_DIR?.trim()) {
|
|
1259
|
+
throw new TestProviderFactoryRequiredError;
|
|
1260
|
+
}
|
|
1261
|
+
if (config.provider === "openai-compatible") {
|
|
1262
|
+
return new OpenAICompatibleEmbeddingProvider({
|
|
1263
|
+
endpoint: config.endpoint,
|
|
1264
|
+
model: config.model,
|
|
1265
|
+
apiKey: config.api_key,
|
|
1266
|
+
inputType: config.input_type,
|
|
1267
|
+
queryInputType: config.query_input_type,
|
|
1268
|
+
queryInstruction: config.query_instruction,
|
|
1269
|
+
documentPrefix: config.document_prefix,
|
|
1270
|
+
truncate: config.truncate,
|
|
1271
|
+
maxInputTokens: config.max_input_tokens
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
if (config.provider === "local") {
|
|
1275
|
+
return new LocalEmbeddingProvider(config.model, config.max_input_tokens, config.local_dtype, config.local_runtime);
|
|
1276
|
+
}
|
|
1277
|
+
if (config.provider === "synapse") {
|
|
1278
|
+
const synapse = config;
|
|
1279
|
+
return new SynapseEmbeddingProvider({
|
|
1280
|
+
connectionFile: synapse.synapse_connection_file ?? "",
|
|
1281
|
+
projectRoot: context?.projectRoot ?? "",
|
|
1282
|
+
session: context?.session ?? "embedding",
|
|
1283
|
+
model: synapse.model,
|
|
1284
|
+
fingerprint: synapse.synapse_fingerprint,
|
|
1285
|
+
tableEpoch: synapse.synapse_table_epoch,
|
|
1286
|
+
dims: synapse.synapse_dims,
|
|
1287
|
+
recommendedBatch: synapse.synapse_recommended_batch,
|
|
1288
|
+
recommendedTokenBudget: synapse.synapse_recommended_token_budget,
|
|
1289
|
+
descriptor: synapse.synapse_descriptor,
|
|
1290
|
+
provenance: synapse.synapse_provenance
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
throw new Error("Unknown embedding provider");
|
|
1294
|
+
}
|
|
1295
|
+
function stableStringify(value) {
|
|
1296
|
+
if (Array.isArray(value)) {
|
|
1297
|
+
return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
|
1298
|
+
}
|
|
1299
|
+
if (value && typeof value === "object") {
|
|
1300
|
+
const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
|
|
1301
|
+
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
|
|
1302
|
+
}
|
|
1303
|
+
return JSON.stringify(value);
|
|
1304
|
+
}
|
|
1305
|
+
function sha256Prefix(value, length = 16) {
|
|
1306
|
+
return createHash2("sha256").update(value).digest("hex").slice(0, length);
|
|
1307
|
+
}
|
|
1308
|
+
function contentSha2562(value) {
|
|
1309
|
+
return createHash2("sha256").update(value).digest("hex");
|
|
1310
|
+
}
|
|
1311
|
+
function getRuntimeFingerprint(config) {
|
|
1312
|
+
if (config.provider === "off") {
|
|
1313
|
+
return OFF_PROVIDER_IDENTITY;
|
|
1314
|
+
}
|
|
1315
|
+
return `${getEmbeddingProviderIdentity(config)}:${sha256Prefix(stableStringify(config))}`;
|
|
1316
|
+
}
|
|
1317
|
+
function getChunkEmbeddingModelId(config, providerIdentity) {
|
|
1318
|
+
if (config.provider === "off") {
|
|
1319
|
+
return OFF_PROVIDER_IDENTITY;
|
|
1320
|
+
}
|
|
1321
|
+
const chunkIdentity = {
|
|
1322
|
+
providerIdentity,
|
|
1323
|
+
chunkerVersion: 2,
|
|
1324
|
+
maxInputTokens: normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in config ? config.max_input_tokens : undefined),
|
|
1325
|
+
truncate: config.provider === "openai-compatible" ? config.truncate ?? "" : ""
|
|
1326
|
+
};
|
|
1327
|
+
return `${providerIdentity}:chunk:${sha256Prefix(stableStringify(chunkIdentity))}`;
|
|
1328
|
+
}
|
|
1329
|
+
function sameFeatures(a, b) {
|
|
1330
|
+
return a.memoryEnabled === b.memoryEnabled && a.gitCommitEnabled === b.gitCommitEnabled;
|
|
1331
|
+
}
|
|
1332
|
+
function snapshotFor(registration) {
|
|
1333
|
+
const providerIsOn = registration.providerIdentity !== OFF_PROVIDER_IDENTITY;
|
|
1334
|
+
const historyEnabled = !registration.observationMode && providerIsOn;
|
|
1335
|
+
const enabled = historyEnabled && registration.features.memoryEnabled;
|
|
1336
|
+
const gitCommitEnabled = !registration.observationMode && providerIsOn && registration.features.gitCommitEnabled;
|
|
1337
|
+
const configuredModel = "model" in registration.config && typeof registration.config.model === "string" ? registration.config.model.trim() : "";
|
|
1338
|
+
const synapseDescriptor = !registration.observationMode && registration.config.provider === "synapse" ? synapseDescriptorFromConfig(registration.config) : undefined;
|
|
1339
|
+
return {
|
|
1340
|
+
projectIdentity: registration.projectIdentity,
|
|
1341
|
+
sourceDirectory: registration.sourceDirectory,
|
|
1342
|
+
providerIdentity: registration.providerIdentity,
|
|
1343
|
+
runtimeFingerprint: registration.runtimeFingerprint,
|
|
1344
|
+
generation: registration.generation,
|
|
1345
|
+
features: { ...registration.features },
|
|
1346
|
+
enabled,
|
|
1347
|
+
historyEnabled,
|
|
1348
|
+
gitCommitEnabled,
|
|
1349
|
+
modelId: registration.observationMode || !providerIsOn ? "off" : registration.modelId,
|
|
1350
|
+
chunkModelId: registration.observationMode || !providerIsOn ? "off" : registration.chunkModelId,
|
|
1351
|
+
model: registration.observationMode || !providerIsOn ? "off" : configuredModel ? configuredModel : registration.modelId,
|
|
1352
|
+
provider: registration.observationMode || !providerIsOn ? "off" : registration.config.provider ?? "local",
|
|
1353
|
+
...synapseDescriptor ? { synapseDescriptor } : {}
|
|
1354
|
+
};
|
|
1355
|
+
}
|
|
1356
|
+
function disposeProvider(provider) {
|
|
1357
|
+
if (!provider)
|
|
1358
|
+
return;
|
|
1359
|
+
provider.dispose().catch((error) => {
|
|
1360
|
+
log2("[magic-context] embedding provider dispose failed:", error);
|
|
1361
|
+
});
|
|
1362
|
+
}
|
|
1363
|
+
function getUpsertActiveIdentityStatement(db) {
|
|
1364
|
+
let stmt = upsertActiveIdentityStatements.get(db);
|
|
1365
|
+
if (!stmt) {
|
|
1366
|
+
stmt = db.prepare(`INSERT INTO embedding_identity_active (project_path, scope, model_id, last_active_at)
|
|
1367
|
+
VALUES (?, ?, ?, ?)
|
|
1368
|
+
ON CONFLICT(project_path, scope, model_id) DO UPDATE SET
|
|
1369
|
+
last_active_at = excluded.last_active_at`);
|
|
1370
|
+
upsertActiveIdentityStatements.set(db, stmt);
|
|
1371
|
+
}
|
|
1372
|
+
return stmt;
|
|
1373
|
+
}
|
|
1374
|
+
function statementMapFor(maps, key) {
|
|
1375
|
+
let map = maps.get(key);
|
|
1376
|
+
if (!map) {
|
|
1377
|
+
map = new WeakMap;
|
|
1378
|
+
maps.set(key, map);
|
|
1379
|
+
}
|
|
1380
|
+
return map;
|
|
1381
|
+
}
|
|
1382
|
+
function getBackfillActiveIdentityStatement(db, scope) {
|
|
1383
|
+
const map = statementMapFor(backfillActiveIdentityStatements, scope);
|
|
1384
|
+
let stmt = map.get(db);
|
|
1385
|
+
if (!stmt) {
|
|
1386
|
+
const selectByScope = {
|
|
1387
|
+
memory: `SELECT DISTINCT e.model_id AS model_id
|
|
1388
|
+
FROM memory_embeddings e
|
|
1389
|
+
JOIN memories m ON m.id = e.memory_id
|
|
1390
|
+
WHERE m.project_path = ?`,
|
|
1391
|
+
commit: `SELECT DISTINCT e.model_id AS model_id
|
|
1392
|
+
FROM git_commit_embeddings e
|
|
1393
|
+
JOIN git_commits c ON c.sha = e.sha
|
|
1394
|
+
WHERE c.project_path = ?`,
|
|
1395
|
+
chunk: `SELECT DISTINCT e.model_id AS model_id
|
|
1396
|
+
FROM compartment_chunk_embeddings e
|
|
1397
|
+
WHERE e.project_path = ?`
|
|
1398
|
+
};
|
|
1399
|
+
stmt = db.prepare(`SELECT model_id FROM (${selectByScope[scope]}) legacy
|
|
1400
|
+
WHERE model_id IS NOT NULL AND NOT EXISTS (
|
|
1401
|
+
SELECT 1 FROM embedding_identity_active active
|
|
1402
|
+
WHERE active.project_path = ? AND active.scope = ?
|
|
1403
|
+
AND active.model_id = legacy.model_id
|
|
1404
|
+
) LIMIT 25`);
|
|
1405
|
+
map.set(db, stmt);
|
|
1406
|
+
}
|
|
1407
|
+
return stmt;
|
|
1408
|
+
}
|
|
1409
|
+
function recordScopeActiveIdentity(db, projectIdentity, scope, modelId, now) {
|
|
1410
|
+
getUpsertActiveIdentityStatement(db).run(projectIdentity, scope, modelId, now);
|
|
1411
|
+
}
|
|
1412
|
+
var legacyDiscoveryComplete = new WeakMap;
|
|
1413
|
+
function recordActiveEmbeddingIdentity(db, projectIdentity, currentProviderIdentity, currentChunkIdentity, features) {
|
|
1414
|
+
if (currentProviderIdentity === OFF_PROVIDER_IDENTITY) {
|
|
1415
|
+
return false;
|
|
1416
|
+
}
|
|
1417
|
+
const scopes = [["chunk", currentChunkIdentity]];
|
|
1418
|
+
if (features.memoryEnabled)
|
|
1419
|
+
scopes.push(["memory", currentProviderIdentity]);
|
|
1420
|
+
if (features.gitCommitEnabled)
|
|
1421
|
+
scopes.push(["commit", currentProviderIdentity]);
|
|
1422
|
+
const active = db.prepare("SELECT 1 FROM embedding_identity_active WHERE project_path = ? AND scope = ? AND model_id = ?");
|
|
1423
|
+
let discovered = legacyDiscoveryComplete.get(db);
|
|
1424
|
+
if (!discovered) {
|
|
1425
|
+
discovered = new Set;
|
|
1426
|
+
legacyDiscoveryComplete.set(db, discovered);
|
|
1427
|
+
}
|
|
1428
|
+
const discoveryKey = (scope, model) => JSON.stringify([projectIdentity, scope, model]);
|
|
1429
|
+
const legacy = scopes.filter(([scope, model]) => !discovered.has(discoveryKey(scope, model)) || !active.get(projectIdentity, scope, model)).map(([scope, model]) => ({
|
|
1430
|
+
scope,
|
|
1431
|
+
model,
|
|
1432
|
+
rows: getBackfillActiveIdentityStatement(db, scope).all(projectIdentity, projectIdentity, scope)
|
|
1433
|
+
}));
|
|
1434
|
+
const repairIds = findMisScopedCompartmentChunkEmbeddingIdsForProject(db, projectIdentity);
|
|
1435
|
+
if (legacy.every(({ rows }) => rows.length === 0) && scopes.every(([scope, model]) => active.get(projectIdentity, scope, model)) && repairIds.length === 0) {
|
|
1436
|
+
for (const { scope, model } of legacy)
|
|
1437
|
+
discovered.add(discoveryKey(scope, model));
|
|
1438
|
+
return false;
|
|
1439
|
+
}
|
|
1440
|
+
const now = Date.now();
|
|
1441
|
+
db.exec("BEGIN IMMEDIATE");
|
|
1442
|
+
const transactionStartedAt = performance.now();
|
|
1443
|
+
try {
|
|
1444
|
+
if (features.memoryEnabled) {
|
|
1445
|
+
recordScopeActiveIdentity(db, projectIdentity, "memory", currentProviderIdentity, now);
|
|
1446
|
+
}
|
|
1447
|
+
if (features.gitCommitEnabled) {
|
|
1448
|
+
recordScopeActiveIdentity(db, projectIdentity, "commit", currentProviderIdentity, now);
|
|
1449
|
+
}
|
|
1450
|
+
for (const { scope, rows } of legacy) {
|
|
1451
|
+
for (const { model_id } of rows) {
|
|
1452
|
+
db.prepare(`INSERT OR IGNORE INTO embedding_identity_active
|
|
1453
|
+
(project_path, scope, model_id, last_active_at) VALUES (?, ?, ?, ?)`).run(projectIdentity, scope, model_id, now);
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectIdentity, repairIds);
|
|
1457
|
+
recordScopeActiveIdentity(db, projectIdentity, "chunk", currentChunkIdentity, now);
|
|
1458
|
+
db.exec("COMMIT");
|
|
1459
|
+
logSlowWriteTransaction("embedding_identity_record", transactionStartedAt);
|
|
1460
|
+
for (const { scope, model, rows } of legacy) {
|
|
1461
|
+
if (rows.length < 25)
|
|
1462
|
+
discovered.add(discoveryKey(scope, model));
|
|
1463
|
+
}
|
|
1464
|
+
return true;
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
try {
|
|
1467
|
+
db.exec("ROLLBACK");
|
|
1468
|
+
} catch {}
|
|
1469
|
+
throw error;
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
function getStaleIdentityStatement(db, scope) {
|
|
1473
|
+
const map = statementMapFor(staleIdentityStatements, scope);
|
|
1474
|
+
let stmt = map.get(db);
|
|
1475
|
+
if (!stmt) {
|
|
1476
|
+
stmt = db.prepare(`SELECT model_id AS modelId
|
|
1477
|
+
FROM embedding_identity_active
|
|
1478
|
+
WHERE project_path = ?
|
|
1479
|
+
AND scope = ?
|
|
1480
|
+
AND model_id <> ?
|
|
1481
|
+
AND last_active_at < ?`);
|
|
1482
|
+
map.set(db, stmt);
|
|
1483
|
+
}
|
|
1484
|
+
return stmt;
|
|
1485
|
+
}
|
|
1486
|
+
function getDeleteActiveIdentityStatement(db) {
|
|
1487
|
+
let stmt = deleteActiveIdentityStatements.get(db);
|
|
1488
|
+
if (!stmt) {
|
|
1489
|
+
stmt = db.prepare(`DELETE FROM embedding_identity_active
|
|
1490
|
+
WHERE project_path = ? AND scope = ? AND model_id = ?`);
|
|
1491
|
+
deleteActiveIdentityStatements.set(db, stmt);
|
|
1492
|
+
}
|
|
1493
|
+
return stmt;
|
|
1494
|
+
}
|
|
1495
|
+
function staleModelsForScope(db, projectIdentity, scope, currentModelId, cutoff, protectedModelIds = new Set([currentModelId])) {
|
|
1496
|
+
const rows = getStaleIdentityStatement(db, scope).all(projectIdentity, scope, currentModelId, cutoff);
|
|
1497
|
+
return rows.map((row) => row.modelId).filter((modelId) => typeof modelId === "string" && !protectedModelIds.has(modelId));
|
|
1498
|
+
}
|
|
1499
|
+
function deleteStaleEmbeddingBatch(db, scope, projectIdentity, modelId, limit) {
|
|
1500
|
+
if (scope === "memory") {
|
|
1501
|
+
return db.prepare(`DELETE FROM memory_embeddings
|
|
1502
|
+
WHERE rowid IN (
|
|
1503
|
+
SELECT me.rowid
|
|
1504
|
+
FROM memory_embeddings me
|
|
1505
|
+
WHERE me.memory_id IN (SELECT id FROM memories WHERE project_path = ?)
|
|
1506
|
+
AND me.model_id = ?
|
|
1507
|
+
LIMIT ?
|
|
1508
|
+
)`).run(projectIdentity, modelId, limit).changes;
|
|
1509
|
+
}
|
|
1510
|
+
if (scope === "commit") {
|
|
1511
|
+
return db.prepare(`DELETE FROM git_commit_embeddings
|
|
1512
|
+
WHERE rowid IN (
|
|
1513
|
+
SELECT gce.rowid
|
|
1514
|
+
FROM git_commit_embeddings gce
|
|
1515
|
+
WHERE gce.sha IN (SELECT sha FROM git_commits WHERE project_path = ?)
|
|
1516
|
+
AND gce.model_id = ?
|
|
1517
|
+
LIMIT ?
|
|
1518
|
+
)`).run(projectIdentity, modelId, limit).changes;
|
|
1519
|
+
}
|
|
1520
|
+
return db.prepare(`DELETE FROM compartment_chunk_embeddings
|
|
1521
|
+
WHERE id IN (
|
|
1522
|
+
SELECT id
|
|
1523
|
+
FROM compartment_chunk_embeddings
|
|
1524
|
+
WHERE project_path = ? AND model_id = ?
|
|
1525
|
+
LIMIT ?
|
|
1526
|
+
)`).run(projectIdentity, modelId, limit).changes;
|
|
1527
|
+
}
|
|
1528
|
+
function hasStaleEmbeddingRows(db, scope, projectIdentity, modelId) {
|
|
1529
|
+
if (scope === "memory") {
|
|
1530
|
+
return Boolean(db.prepare(`SELECT 1
|
|
1531
|
+
FROM memory_embeddings me
|
|
1532
|
+
WHERE me.memory_id IN (SELECT id FROM memories WHERE project_path = ?)
|
|
1533
|
+
AND me.model_id = ?
|
|
1534
|
+
LIMIT 1`).get(projectIdentity, modelId));
|
|
1535
|
+
}
|
|
1536
|
+
if (scope === "commit") {
|
|
1537
|
+
return Boolean(db.prepare(`SELECT 1
|
|
1538
|
+
FROM git_commit_embeddings gce
|
|
1539
|
+
WHERE gce.sha IN (SELECT sha FROM git_commits WHERE project_path = ?)
|
|
1540
|
+
AND gce.model_id = ?
|
|
1541
|
+
LIMIT 1`).get(projectIdentity, modelId));
|
|
1542
|
+
}
|
|
1543
|
+
return Boolean(db.prepare(`SELECT 1
|
|
1544
|
+
FROM compartment_chunk_embeddings
|
|
1545
|
+
WHERE project_path = ? AND model_id = ?
|
|
1546
|
+
LIMIT 1`).get(projectIdentity, modelId));
|
|
1547
|
+
}
|
|
1548
|
+
function sweepStaleEmbeddingIdentitiesForProject(db, projectIdentity, now = Date.now()) {
|
|
1549
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
1550
|
+
const result = {
|
|
1551
|
+
memoryRowsDeleted: 0,
|
|
1552
|
+
commitRowsDeleted: 0,
|
|
1553
|
+
chunkRowsDeleted: 0,
|
|
1554
|
+
trackingRowsDeleted: 0
|
|
1555
|
+
};
|
|
1556
|
+
if (!snapshot)
|
|
1557
|
+
return result;
|
|
1558
|
+
if (untrustedLoadProjects.has(projectIdentity))
|
|
1559
|
+
return result;
|
|
1560
|
+
const cutoff = now - EMBEDDING_IDENTITY_GC_GRACE_MS;
|
|
1561
|
+
const deleteTracking = getDeleteActiveIdentityStatement(db);
|
|
1562
|
+
const shadow = shadowRegistrations.get(projectIdentity);
|
|
1563
|
+
const protectedModels = {
|
|
1564
|
+
memory: new Set([snapshot.modelId, ...shadow ? [shadow.modelId] : []]),
|
|
1565
|
+
commit: new Set([snapshot.modelId, ...shadow ? [shadow.modelId] : []]),
|
|
1566
|
+
chunk: new Set([snapshot.chunkModelId, ...shadow ? [shadow.chunkModelId] : []])
|
|
1567
|
+
};
|
|
1568
|
+
const scopes = [
|
|
1569
|
+
{
|
|
1570
|
+
scope: "memory",
|
|
1571
|
+
enabled: snapshot.enabled && snapshot.modelId !== "off",
|
|
1572
|
+
currentModelId: snapshot.modelId
|
|
1573
|
+
},
|
|
1574
|
+
{
|
|
1575
|
+
scope: "commit",
|
|
1576
|
+
enabled: snapshot.gitCommitEnabled && snapshot.modelId !== "off",
|
|
1577
|
+
currentModelId: snapshot.modelId
|
|
1578
|
+
},
|
|
1579
|
+
{
|
|
1580
|
+
scope: "chunk",
|
|
1581
|
+
enabled: snapshot.historyEnabled && snapshot.chunkModelId !== "off",
|
|
1582
|
+
currentModelId: snapshot.chunkModelId
|
|
1583
|
+
}
|
|
1584
|
+
];
|
|
1585
|
+
let remainingBudget = STALE_EMBEDDING_GC_BATCH_SIZE;
|
|
1586
|
+
const candidates = scopes.flatMap(({ scope, enabled, currentModelId }) => enabled ? staleModelsForScope(db, projectIdentity, scope, currentModelId, cutoff, protectedModels[scope]).slice(0, STALE_EMBEDDING_GC_BATCH_SIZE).map((modelId) => ({ scope, modelId })) : []).slice(0, STALE_EMBEDDING_GC_BATCH_SIZE);
|
|
1587
|
+
if (candidates.length === 0)
|
|
1588
|
+
return result;
|
|
1589
|
+
db.exec("BEGIN IMMEDIATE");
|
|
1590
|
+
const transactionStartedAt = performance.now();
|
|
1591
|
+
try {
|
|
1592
|
+
for (const { scope, modelId } of candidates) {
|
|
1593
|
+
if (remainingBudget === 0)
|
|
1594
|
+
break;
|
|
1595
|
+
const marker = db.prepare(`SELECT last_active_at FROM embedding_identity_active
|
|
1596
|
+
WHERE project_path = ? AND scope = ? AND model_id = ?`).get(projectIdentity, scope, modelId);
|
|
1597
|
+
if (!marker || marker.last_active_at >= cutoff)
|
|
1598
|
+
continue;
|
|
1599
|
+
const deleted = deleteStaleEmbeddingBatch(db, scope, projectIdentity, modelId, remainingBudget);
|
|
1600
|
+
remainingBudget -= deleted;
|
|
1601
|
+
if (scope === "memory")
|
|
1602
|
+
result.memoryRowsDeleted += deleted;
|
|
1603
|
+
else if (scope === "commit")
|
|
1604
|
+
result.commitRowsDeleted += deleted;
|
|
1605
|
+
else
|
|
1606
|
+
result.chunkRowsDeleted += deleted;
|
|
1607
|
+
if (!hasStaleEmbeddingRows(db, scope, projectIdentity, modelId)) {
|
|
1608
|
+
result.trackingRowsDeleted += deleteTracking.run(projectIdentity, scope, modelId).changes;
|
|
1609
|
+
} else if (deleted === 0) {
|
|
1610
|
+
remainingBudget = 0;
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
db.exec("COMMIT");
|
|
1614
|
+
logSlowWriteTransaction("embedding_stale_gc", transactionStartedAt);
|
|
1615
|
+
} catch (error) {
|
|
1616
|
+
try {
|
|
1617
|
+
db.exec("ROLLBACK");
|
|
1618
|
+
} catch {}
|
|
1619
|
+
throw error;
|
|
1620
|
+
}
|
|
1621
|
+
if (result.memoryRowsDeleted > 0)
|
|
1622
|
+
invalidateProject2(projectIdentity);
|
|
1623
|
+
return result;
|
|
1624
|
+
}
|
|
1625
|
+
async function drainProjectEmbeddingIdentityMaintenance2(db, projectIdentity, budgetMs = 2000) {
|
|
1626
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
1627
|
+
if (!snapshot || untrustedLoadProjects.has(projectIdentity))
|
|
1628
|
+
return 0;
|
|
1629
|
+
return drainBackgroundBatches(() => recordActiveEmbeddingIdentity(db, projectIdentity, snapshot.modelId, snapshot.chunkModelId, { memoryEnabled: snapshot.enabled, gitCommitEnabled: snapshot.gitCommitEnabled }), { budgetMs });
|
|
1630
|
+
}
|
|
1631
|
+
async function drainStaleEmbeddingIdentitiesForProject2(db, projectIdentity, budgetMs = 2000) {
|
|
1632
|
+
const total = {
|
|
1633
|
+
memoryRowsDeleted: 0,
|
|
1634
|
+
commitRowsDeleted: 0,
|
|
1635
|
+
chunkRowsDeleted: 0,
|
|
1636
|
+
trackingRowsDeleted: 0
|
|
1637
|
+
};
|
|
1638
|
+
await drainBackgroundBatches(() => {
|
|
1639
|
+
const result = sweepStaleEmbeddingIdentitiesForProject(db, projectIdentity);
|
|
1640
|
+
total.memoryRowsDeleted += result.memoryRowsDeleted;
|
|
1641
|
+
total.commitRowsDeleted += result.commitRowsDeleted;
|
|
1642
|
+
total.chunkRowsDeleted += result.chunkRowsDeleted;
|
|
1643
|
+
total.trackingRowsDeleted += result.trackingRowsDeleted;
|
|
1644
|
+
return result.memoryRowsDeleted + result.commitRowsDeleted + result.chunkRowsDeleted + result.trackingRowsDeleted > 0;
|
|
1645
|
+
}, { budgetMs });
|
|
1646
|
+
return total;
|
|
1647
|
+
}
|
|
1648
|
+
function registerProjectEmbedding2(db, projectIdentity, config, features, sourceDirectory) {
|
|
1649
|
+
const resolvedConfig = resolveEmbeddingConfig(config);
|
|
1650
|
+
const providerIdentity = getEmbeddingProviderIdentity(resolvedConfig);
|
|
1651
|
+
const runtimeFingerprint = getRuntimeFingerprint(resolvedConfig);
|
|
1652
|
+
const chunkModelId = getChunkEmbeddingModelId(resolvedConfig, providerIdentity);
|
|
1653
|
+
const prior = projectRegistrations.get(projectIdentity);
|
|
1654
|
+
const canReuseProvider = prior !== undefined && !prior.observationMode && prior.runtimeFingerprint === runtimeFingerprint && prior.providerIdentity === providerIdentity;
|
|
1655
|
+
recordActiveEmbeddingIdentity(db, projectIdentity, providerIdentity, chunkModelId, features);
|
|
1656
|
+
pruneSynapseBatchLedgerForProject(db, projectIdentity);
|
|
1657
|
+
untrustedLoadProjects.delete(projectIdentity);
|
|
1658
|
+
const generationChanged = prior === undefined || prior.observationMode || prior.runtimeFingerprint !== runtimeFingerprint || prior.chunkModelId !== chunkModelId || !sameFeatures(prior.features, features);
|
|
1659
|
+
const generation = generationChanged ? ++globalRegistrationGeneration : prior.generation;
|
|
1660
|
+
const registration = {
|
|
1661
|
+
projectIdentity,
|
|
1662
|
+
sourceDirectory,
|
|
1663
|
+
config: resolvedConfig,
|
|
1664
|
+
providerIdentity,
|
|
1665
|
+
runtimeFingerprint,
|
|
1666
|
+
provider: canReuseProvider ? prior.provider : null,
|
|
1667
|
+
generation,
|
|
1668
|
+
features: { ...features },
|
|
1669
|
+
modelId: providerIdentity === OFF_PROVIDER_IDENTITY ? "off" : providerIdentity,
|
|
1670
|
+
chunkModelId: providerIdentity === OFF_PROVIDER_IDENTITY ? "off" : chunkModelId,
|
|
1671
|
+
observationMode: false
|
|
1672
|
+
};
|
|
1673
|
+
projectRegistrations.set(projectIdentity, registration);
|
|
1674
|
+
if (generationChanged || !db.prepare("SELECT 1 FROM embedding_registrations WHERE project_path = ? AND provider_identity = ? AND chunk_model_id = ? AND generation = ?").get(projectIdentity, providerIdentity, registration.chunkModelId, generation)) {
|
|
1675
|
+
persistPrimaryDescriptor(db, registration);
|
|
1676
|
+
}
|
|
1677
|
+
if (!canReuseProvider) {
|
|
1678
|
+
disposeProvider(prior?.provider ?? null);
|
|
1679
|
+
}
|
|
1680
|
+
return snapshotFor(registration);
|
|
1681
|
+
}
|
|
1682
|
+
function registerProjectShadowEmbedding2(db, projectIdentity, config, sourceDirectory, options = {}) {
|
|
1683
|
+
const resolvedConfig = resolveEmbeddingConfig(config);
|
|
1684
|
+
if (resolvedConfig.provider !== "synapse") {
|
|
1685
|
+
throw new Error("Shadow embedding registration requires the synapse provider");
|
|
1686
|
+
}
|
|
1687
|
+
const providerIdentity = getEmbeddingProviderIdentity(resolvedConfig);
|
|
1688
|
+
const chunkModelId = getChunkEmbeddingModelId(resolvedConfig, providerIdentity);
|
|
1689
|
+
const provider = createProvider(resolvedConfig, {
|
|
1690
|
+
projectRoot: sourceDirectory,
|
|
1691
|
+
session: `shadow:${projectIdentity}`
|
|
1692
|
+
});
|
|
1693
|
+
if (!provider)
|
|
1694
|
+
return null;
|
|
1695
|
+
const prior = shadowRegistrations.get(projectIdentity);
|
|
1696
|
+
if (prior && prior.providerIdentity === providerIdentity) {
|
|
1697
|
+
provider.dispose();
|
|
1698
|
+
dbForShadowQueue.set(projectIdentity, db);
|
|
1699
|
+
persistShadowDescriptor(db, prior);
|
|
1700
|
+
const backfillAlreadyArmed = hasPendingShadowBackfill(projectIdentity) || shadowQueue.some((item) => item.projectIdentity === projectIdentity);
|
|
1701
|
+
if (!backfillAlreadyArmed || options.manualBackfill === true) {
|
|
1702
|
+
maybeArmShadowBackfill(db, projectIdentity, prior, options.manualBackfill === true);
|
|
1703
|
+
}
|
|
1704
|
+
return {
|
|
1705
|
+
...snapshotFor({
|
|
1706
|
+
projectIdentity,
|
|
1707
|
+
sourceDirectory,
|
|
1708
|
+
config: prior.config,
|
|
1709
|
+
providerIdentity: prior.providerIdentity,
|
|
1710
|
+
runtimeFingerprint: `shadow:${prior.providerIdentity}`,
|
|
1711
|
+
provider: prior.provider,
|
|
1712
|
+
generation: prior.generation,
|
|
1713
|
+
features: { memoryEnabled: true, gitCommitEnabled: true },
|
|
1714
|
+
modelId: prior.modelId,
|
|
1715
|
+
chunkModelId: prior.chunkModelId,
|
|
1716
|
+
observationMode: false
|
|
1717
|
+
}),
|
|
1718
|
+
provider: "synapse"
|
|
1719
|
+
};
|
|
1720
|
+
}
|
|
1721
|
+
const generation = ++globalRegistrationGeneration;
|
|
1722
|
+
const registration = {
|
|
1723
|
+
projectIdentity,
|
|
1724
|
+
sourceDirectory,
|
|
1725
|
+
config: resolvedConfig,
|
|
1726
|
+
provider,
|
|
1727
|
+
providerIdentity,
|
|
1728
|
+
modelId: providerIdentity,
|
|
1729
|
+
chunkModelId,
|
|
1730
|
+
generation
|
|
1731
|
+
};
|
|
1732
|
+
shadowRegistrations.set(projectIdentity, registration);
|
|
1733
|
+
dbForShadowQueue.set(projectIdentity, db);
|
|
1734
|
+
if (prior) {
|
|
1735
|
+
disposeProvider(prior.provider);
|
|
1736
|
+
for (const scope of ["memory", "commit", "chunk"]) {
|
|
1737
|
+
const scopeKey = `${projectIdentity}:${scope}`;
|
|
1738
|
+
shadowBackfillLastIds.delete(scopeKey);
|
|
1739
|
+
shadowBackfillStopReasons.delete(scopeKey);
|
|
1740
|
+
shadowBackfillLastWriteOutcomes.delete(scopeKey);
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
db.transaction(() => {
|
|
1744
|
+
const now = Date.now();
|
|
1745
|
+
recordScopeActiveIdentity(db, projectIdentity, "memory", registration.modelId, now);
|
|
1746
|
+
recordScopeActiveIdentity(db, projectIdentity, "commit", registration.modelId, now);
|
|
1747
|
+
recordScopeActiveIdentity(db, projectIdentity, "chunk", registration.chunkModelId, now);
|
|
1748
|
+
persistShadowDescriptor(db, registration);
|
|
1749
|
+
}).immediate();
|
|
1750
|
+
maybeArmShadowBackfill(db, projectIdentity, registration, options.manualBackfill === true);
|
|
1751
|
+
return {
|
|
1752
|
+
projectIdentity,
|
|
1753
|
+
sourceDirectory,
|
|
1754
|
+
providerIdentity,
|
|
1755
|
+
runtimeFingerprint: `shadow:${providerIdentity}`,
|
|
1756
|
+
generation,
|
|
1757
|
+
features: { memoryEnabled: true, gitCommitEnabled: true },
|
|
1758
|
+
enabled: true,
|
|
1759
|
+
historyEnabled: true,
|
|
1760
|
+
gitCommitEnabled: true,
|
|
1761
|
+
modelId: registration.modelId,
|
|
1762
|
+
chunkModelId: registration.chunkModelId,
|
|
1763
|
+
model: "model" in resolvedConfig && typeof resolvedConfig.model === "string" ? resolvedConfig.model : registration.modelId,
|
|
1764
|
+
provider: "synapse",
|
|
1765
|
+
...synapseDescriptorFromConfig(resolvedConfig) ? { synapseDescriptor: synapseDescriptorFromConfig(resolvedConfig) } : {}
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
function startShadowWorker() {
|
|
1769
|
+
if (shadowWorker)
|
|
1770
|
+
return;
|
|
1771
|
+
shadowWorker = runShadowWorker().finally(() => {
|
|
1772
|
+
shadowWorker = null;
|
|
1773
|
+
if (shadowQueue.length > 0 || hasPendingShadowBackfill())
|
|
1774
|
+
startShadowWorker();
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
function getShadowEmbeddingMeasurementCohort2(projectIdentity) {
|
|
1778
|
+
const registration = shadowRegistrations.get(projectIdentity);
|
|
1779
|
+
if (!registration)
|
|
1780
|
+
return null;
|
|
1781
|
+
const fields = synapseConfigFields(registration.config);
|
|
1782
|
+
return {
|
|
1783
|
+
modelId: registration.modelId,
|
|
1784
|
+
chunkModelId: registration.chunkModelId,
|
|
1785
|
+
fingerprint: fields.fingerprint ?? "",
|
|
1786
|
+
epoch: fields.tableEpoch ?? 0,
|
|
1787
|
+
dims: fields.dims ?? 0
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
function getPrimaryEmbeddingMeasurementCohort2(projectIdentity) {
|
|
1791
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
1792
|
+
if (!registration)
|
|
1793
|
+
return null;
|
|
1794
|
+
const fields = synapseConfigFields(registration.config);
|
|
1795
|
+
return {
|
|
1796
|
+
modelId: registration.modelId,
|
|
1797
|
+
chunkModelId: registration.chunkModelId,
|
|
1798
|
+
fingerprint: fields.fingerprint ?? "",
|
|
1799
|
+
epoch: fields.tableEpoch ?? 0,
|
|
1800
|
+
dims: fields.dims ?? 0
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
async function embedShadowTextForProject2(projectIdentity, text, signal) {
|
|
1804
|
+
const registration = shadowRegistrations.get(projectIdentity);
|
|
1805
|
+
if (!registration)
|
|
1806
|
+
return null;
|
|
1807
|
+
try {
|
|
1808
|
+
const vector = await registration.provider.embed(text, signal, "query");
|
|
1809
|
+
return vector && !isSynapseEmbeddingTruncated(vector) ? vector : null;
|
|
1810
|
+
} catch (error) {
|
|
1811
|
+
log2("[magic-context] Synapse shadow query failed:", error);
|
|
1812
|
+
return null;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
function enqueueShadowEmbeddingItems2(projectIdentity, scope, ids) {
|
|
1816
|
+
if (ids.length === 0 || !shadowRegistrations.has(projectIdentity))
|
|
1817
|
+
return;
|
|
1818
|
+
shadowQueue.push({ projectIdentity, scope, ids: [...ids] });
|
|
1819
|
+
startShadowWorker();
|
|
1820
|
+
}
|
|
1821
|
+
function shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity) {
|
|
1822
|
+
if (scope === "memory") {
|
|
1823
|
+
return {
|
|
1824
|
+
sql: `SELECT m.id AS id
|
|
1825
|
+
FROM memories m
|
|
1826
|
+
JOIN memory_embeddings mp ON mp.memory_id = m.id AND mp.model_id = ?
|
|
1827
|
+
LEFT JOIN memory_embeddings ms ON ms.memory_id = m.id AND ms.model_id = ?
|
|
1828
|
+
WHERE m.project_path = ? AND m.status = 'active' AND ms.memory_id IS NULL`,
|
|
1829
|
+
params: [primaryModelId, shadowModelId, projectIdentity],
|
|
1830
|
+
orderBy: " ORDER BY m.id"
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
return {
|
|
1834
|
+
sql: `SELECT gc.sha AS id
|
|
1835
|
+
FROM git_commits gc
|
|
1836
|
+
JOIN git_commit_embeddings gp ON gp.sha = gc.sha AND gp.model_id = ?
|
|
1837
|
+
LEFT JOIN git_commit_embeddings gs ON gs.sha = gc.sha AND gs.model_id = ?
|
|
1838
|
+
WHERE gc.project_path = ? AND gs.sha IS NULL`,
|
|
1839
|
+
params: [primaryModelId, shadowModelId, projectIdentity],
|
|
1840
|
+
orderBy: " ORDER BY gc.committed_at DESC, gc.sha"
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
function shadowBackfillMissingIds(db, projectIdentity, scope, primaryModelId, shadowModelId, limit, shadowMaxInputTokens) {
|
|
1844
|
+
if (scope === "chunk") {
|
|
1845
|
+
return loadUnembeddedShadowChunkCandidates(db, projectIdentity, primaryModelId, shadowModelId, limit, shadowMaxInputTokens).map((candidate) => String(candidate.id));
|
|
1846
|
+
}
|
|
1847
|
+
const { sql, params, orderBy } = shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity);
|
|
1848
|
+
const rows = db.prepare(`${sql}${orderBy} LIMIT ?`).all(...params, limit);
|
|
1849
|
+
return rows.map((row) => String(row.id));
|
|
1850
|
+
}
|
|
1851
|
+
function countShadowBackfillMissing(db, projectIdentity, scope, primaryModelId, shadowModelId, shadowMaxInputTokens) {
|
|
1852
|
+
if (scope === "chunk") {
|
|
1853
|
+
return loadUnembeddedShadowChunkCandidates(db, projectIdentity, primaryModelId, shadowModelId, Number.MAX_SAFE_INTEGER, shadowMaxInputTokens).length;
|
|
1854
|
+
}
|
|
1855
|
+
const { sql, params } = shadowBackfillMissingBase(scope, primaryModelId, shadowModelId, projectIdentity);
|
|
1856
|
+
return db.prepare(`SELECT COUNT(*) AS count FROM (${sql})`).get(...params).count;
|
|
1857
|
+
}
|
|
1858
|
+
function shadowModelIdForScope(registration, scope) {
|
|
1859
|
+
return scope === "chunk" ? registration.chunkModelId : registration.modelId;
|
|
1860
|
+
}
|
|
1861
|
+
function shadowMaxInputTokensFor(registration) {
|
|
1862
|
+
return normalizeCompartmentChunkMaxInputTokens("max_input_tokens" in registration.config ? registration.config.max_input_tokens : undefined);
|
|
1863
|
+
}
|
|
1864
|
+
function shadowBackfillCohortFingerprint(db, projectIdentity, scope, primaryModelId) {
|
|
1865
|
+
if (scope === "memory") {
|
|
1866
|
+
return db.prepare(`SELECT COUNT(*) AS rowCount, MAX(m.id) AS maxId, MAX(m.updated_at) AS maxUpdatedAt
|
|
1867
|
+
FROM memories m
|
|
1868
|
+
JOIN memory_embeddings me ON me.memory_id = m.id AND me.model_id = ?
|
|
1869
|
+
WHERE m.project_path = ? AND m.status = 'active'`).get(primaryModelId, projectIdentity);
|
|
1870
|
+
}
|
|
1871
|
+
if (scope === "commit") {
|
|
1872
|
+
return db.prepare(`SELECT COUNT(*) AS rowCount, MAX(gc.sha) AS maxId, MAX(gc.committed_at) AS maxUpdatedAt
|
|
1873
|
+
FROM git_commits gc
|
|
1874
|
+
JOIN git_commit_embeddings gce ON gce.sha = gc.sha AND gce.model_id = ?
|
|
1875
|
+
WHERE gc.project_path = ?`).get(primaryModelId, projectIdentity);
|
|
1876
|
+
}
|
|
1877
|
+
return db.prepare(`SELECT COUNT(*) AS rowCount,
|
|
1878
|
+
COUNT(DISTINCT compartment_id) AS compartmentCount,
|
|
1879
|
+
MAX(compartment_id) AS maxId,
|
|
1880
|
+
MAX(created_at) AS maxUpdatedAt
|
|
1881
|
+
FROM compartment_chunk_embeddings
|
|
1882
|
+
WHERE project_path = ? AND model_id = ?`).get(projectIdentity, primaryModelId);
|
|
1883
|
+
}
|
|
1884
|
+
function shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, limit) {
|
|
1885
|
+
const shadowModelId = shadowModelIdForScope(shadow, scope);
|
|
1886
|
+
const ids = shadowBackfillMissingIds(db, projectIdentity, scope, primaryModelId, shadowModelId, limit, shadowMaxInputTokensFor(shadow));
|
|
1887
|
+
return {
|
|
1888
|
+
ids,
|
|
1889
|
+
signature: sha256Prefix(stableStringify({
|
|
1890
|
+
scope,
|
|
1891
|
+
primaryModelId,
|
|
1892
|
+
shadowModelId,
|
|
1893
|
+
ids,
|
|
1894
|
+
cohort: shadowBackfillCohortFingerprint(db, projectIdentity, scope, primaryModelId)
|
|
1895
|
+
}), 32)
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
function hasPendingShadowBackfill(projectIdentity) {
|
|
1899
|
+
if (projectIdentity === undefined)
|
|
1900
|
+
return pendingShadowBackfills.size > 0;
|
|
1901
|
+
const scopes = pendingShadowBackfills.get(projectIdentity);
|
|
1902
|
+
return scopes !== undefined && scopes.size > 0;
|
|
1903
|
+
}
|
|
1904
|
+
function pumpShadowBackfill() {
|
|
1905
|
+
for (const [projectIdentity, scopes] of pendingShadowBackfills) {
|
|
1906
|
+
const db = dbForShadowQueue.get(projectIdentity);
|
|
1907
|
+
const shadow = shadowRegistrations.get(projectIdentity);
|
|
1908
|
+
const primary = projectRegistrations.get(projectIdentity);
|
|
1909
|
+
if (!db || !shadow || !primary) {
|
|
1910
|
+
pendingShadowBackfills.delete(projectIdentity);
|
|
1911
|
+
continue;
|
|
1912
|
+
}
|
|
1913
|
+
for (const scope of [...scopes]) {
|
|
1914
|
+
const primaryModelId = shadowModelIdForScope(primary, scope);
|
|
1915
|
+
const shadowModelId = shadowModelIdForScope(shadow, scope);
|
|
1916
|
+
const stallKey = `${projectIdentity}:${scope}`;
|
|
1917
|
+
if (primaryModelId === "off" || shadowModelId === "off") {
|
|
1918
|
+
scopes.delete(scope);
|
|
1919
|
+
shadowBackfillLastIds.delete(stallKey);
|
|
1920
|
+
shadowBackfillLastWriteOutcomes.delete(stallKey);
|
|
1921
|
+
continue;
|
|
1922
|
+
}
|
|
1923
|
+
const batch = shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, SHADOW_MAX_ITEMS_PER_TICK);
|
|
1924
|
+
if (batch.ids.length === 0) {
|
|
1925
|
+
shadowBackfillStopReasons.set(stallKey, "drained");
|
|
1926
|
+
scopes.delete(scope);
|
|
1927
|
+
shadowBackfillLastIds.delete(stallKey);
|
|
1928
|
+
shadowBackfillLastWriteOutcomes.delete(stallKey);
|
|
1929
|
+
updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
|
|
1930
|
+
...state,
|
|
1931
|
+
stopReason: "drained",
|
|
1932
|
+
candidateSignature: undefined,
|
|
1933
|
+
writeRefusalReason: undefined,
|
|
1934
|
+
stoppedAt: shadowBackfillNow()
|
|
1935
|
+
}));
|
|
1936
|
+
continue;
|
|
1937
|
+
}
|
|
1938
|
+
if (shadowBackfillLastIds.get(stallKey) === batch.signature) {
|
|
1939
|
+
const lastOutcome = shadowBackfillLastWriteOutcomes.get(stallKey);
|
|
1940
|
+
const writeRefusalReason = lastOutcome?.refusalReason ?? (lastOutcome && lastOutcome.writes > 0 ? "chunk_window_contract_mismatch" : "unknown_write_rejection");
|
|
1941
|
+
shadowBackfillStopReasons.set(stallKey, "stalled_no_progress");
|
|
1942
|
+
updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
|
|
1943
|
+
...state,
|
|
1944
|
+
stopReason: "stalled_no_progress",
|
|
1945
|
+
candidateSignature: batch.signature,
|
|
1946
|
+
writeRefusalReason,
|
|
1947
|
+
stoppedAt: shadowBackfillNow()
|
|
1948
|
+
}));
|
|
1949
|
+
log2(`[shadow] backfill scope ${scope} for ${projectIdentity} retired without progress — ` + `${describeShadowBackfillWriteRefusal2(writeRefusalReason)}; ` + `${batch.ids.length}+ items remain and automatic registration will not retry unchanged candidates`);
|
|
1950
|
+
scopes.delete(scope);
|
|
1951
|
+
shadowBackfillLastIds.delete(stallKey);
|
|
1952
|
+
shadowBackfillLastWriteOutcomes.delete(stallKey);
|
|
1953
|
+
continue;
|
|
1954
|
+
}
|
|
1955
|
+
shadowBackfillLastIds.set(stallKey, batch.signature);
|
|
1956
|
+
shadowQueue.push({ projectIdentity, scope, ids: batch.ids });
|
|
1957
|
+
}
|
|
1958
|
+
if (scopes.size === 0)
|
|
1959
|
+
pendingShadowBackfills.delete(projectIdentity);
|
|
1960
|
+
}
|
|
1961
|
+
}
|
|
1962
|
+
function maybeArmShadowBackfill(db, projectIdentity, shadow, manualBackfill = false) {
|
|
1963
|
+
if (untrustedLoadProjects.has(projectIdentity))
|
|
1964
|
+
return;
|
|
1965
|
+
const primary = projectRegistrations.get(projectIdentity);
|
|
1966
|
+
if (!primary)
|
|
1967
|
+
return;
|
|
1968
|
+
const pending = new Set;
|
|
1969
|
+
for (const scope of ["memory", "commit", "chunk"]) {
|
|
1970
|
+
const primaryModelId = shadowModelIdForScope(primary, scope);
|
|
1971
|
+
const shadowModelId = shadowModelIdForScope(shadow, scope);
|
|
1972
|
+
const stallKey = `${projectIdentity}:${scope}`;
|
|
1973
|
+
if (primaryModelId === "off" || shadowModelId === "off")
|
|
1974
|
+
continue;
|
|
1975
|
+
if (scope === "memory" && !primary.features.memoryEnabled)
|
|
1976
|
+
continue;
|
|
1977
|
+
const batch = shadowBackfillCandidateBatch(db, projectIdentity, scope, primaryModelId, shadow, SHADOW_MAX_ITEMS_PER_TICK);
|
|
1978
|
+
const persisted = getPersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId);
|
|
1979
|
+
if (batch.ids.length === 0) {
|
|
1980
|
+
if (persisted?.stopReason === "stalled_no_progress") {
|
|
1981
|
+
shadowBackfillStopReasons.set(stallKey, "drained");
|
|
1982
|
+
updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
|
|
1983
|
+
...state,
|
|
1984
|
+
stopReason: "drained",
|
|
1985
|
+
candidateSignature: undefined,
|
|
1986
|
+
writeRefusalReason: undefined,
|
|
1987
|
+
stoppedAt: shadowBackfillNow()
|
|
1988
|
+
}));
|
|
1989
|
+
}
|
|
1990
|
+
continue;
|
|
1991
|
+
}
|
|
1992
|
+
if (!manualBackfill && persisted?.stopReason === "stalled_no_progress" && persisted.candidateSignature === batch.signature) {
|
|
1993
|
+
shadowBackfillStopReasons.set(stallKey, "stalled_no_progress");
|
|
1994
|
+
continue;
|
|
1995
|
+
}
|
|
1996
|
+
shadowBackfillStopReasons.delete(stallKey);
|
|
1997
|
+
shadowBackfillLastIds.delete(stallKey);
|
|
1998
|
+
shadowBackfillLastWriteOutcomes.delete(stallKey);
|
|
1999
|
+
updatePersistedShadowBackfillState(db, projectIdentity, scope, shadowModelId, (state) => ({
|
|
2000
|
+
...state,
|
|
2001
|
+
stopReason: undefined,
|
|
2002
|
+
candidateSignature: undefined,
|
|
2003
|
+
writeRefusalReason: undefined,
|
|
2004
|
+
stoppedAt: undefined
|
|
2005
|
+
}));
|
|
2006
|
+
pending.add(scope);
|
|
2007
|
+
}
|
|
2008
|
+
if (pending.size === 0)
|
|
2009
|
+
return;
|
|
2010
|
+
pendingShadowBackfills.set(projectIdentity, pending);
|
|
2011
|
+
pumpShadowBackfill();
|
|
2012
|
+
startShadowWorker();
|
|
2013
|
+
}
|
|
2014
|
+
function getShadowBackfillRemaining2(db, projectIdentity) {
|
|
2015
|
+
const remaining = { memory: 0, commit: 0, chunk: 0 };
|
|
2016
|
+
const primary = projectRegistrations.get(projectIdentity);
|
|
2017
|
+
const shadow = shadowRegistrations.get(projectIdentity);
|
|
2018
|
+
if (!primary || !shadow)
|
|
2019
|
+
return remaining;
|
|
2020
|
+
for (const scope of ["memory", "commit", "chunk"]) {
|
|
2021
|
+
const primaryModelId = shadowModelIdForScope(primary, scope);
|
|
2022
|
+
const shadowModelId = shadowModelIdForScope(shadow, scope);
|
|
2023
|
+
if (primaryModelId === "off" || shadowModelId === "off")
|
|
2024
|
+
continue;
|
|
2025
|
+
remaining[scope] = countShadowBackfillMissing(db, projectIdentity, scope, primaryModelId, shadowModelId, shadowMaxInputTokensFor(shadow));
|
|
2026
|
+
}
|
|
2027
|
+
return remaining;
|
|
2028
|
+
}
|
|
2029
|
+
async function flushShadowEmbeddingBacklog2(projectIdentity, onSettled) {
|
|
2030
|
+
if (shadowQueue.length > 0 || hasPendingShadowBackfill(projectIdentity)) {
|
|
2031
|
+
startShadowWorker();
|
|
2032
|
+
}
|
|
2033
|
+
for (;; ) {
|
|
2034
|
+
const worker = shadowWorker;
|
|
2035
|
+
if (worker) {
|
|
2036
|
+
await worker;
|
|
2037
|
+
await onSettled?.();
|
|
2038
|
+
continue;
|
|
2039
|
+
}
|
|
2040
|
+
if (shadowQueue.length === 0 && !hasPendingShadowBackfill(projectIdentity))
|
|
2041
|
+
return;
|
|
2042
|
+
startShadowWorker();
|
|
2043
|
+
if (!shadowWorker)
|
|
2044
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
2045
|
+
await onSettled?.();
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
async function embedShadowItems(registration, items, db, scope) {
|
|
2049
|
+
const raw = registration.config;
|
|
2050
|
+
const fingerprint = typeof raw.synapse_fingerprint === "string" ? raw.synapse_fingerprint : "";
|
|
2051
|
+
const tableEpoch = typeof raw.synapse_table_epoch === "number" ? raw.synapse_table_epoch : 0;
|
|
2052
|
+
const requestKey = getSynapseBatchRequestKey({
|
|
2053
|
+
model: typeof raw.model === "string" ? raw.model : SYNAPSE_DEFAULT_MODEL,
|
|
2054
|
+
fingerprint,
|
|
2055
|
+
tableEpoch,
|
|
2056
|
+
items
|
|
2057
|
+
});
|
|
2058
|
+
const sessionId = `shadow:${registration.projectIdentity}`;
|
|
2059
|
+
const now = shadowBackfillNow();
|
|
2060
|
+
const prior = db.prepare(`SELECT updated_at AS updatedAt
|
|
2061
|
+
FROM synapse_batch_ledger
|
|
2062
|
+
WHERE session_id = ? AND request_key = ?`).get(sessionId, requestKey);
|
|
2063
|
+
if (typeof prior?.updatedAt === "number" && now - prior.updatedAt >= 0 && now - prior.updatedAt < SHADOW_RESUBMIT_WINDOW_MS) {
|
|
2064
|
+
const modelId = shadowModelIdForScope(registration, scope);
|
|
2065
|
+
const state = getPersistedShadowBackfillState(db, registration.projectIdentity, scope, modelId);
|
|
2066
|
+
if (state?.budgetLogRequestKey !== requestKey || typeof state.budgetLoggedAt !== "number" || now - state.budgetLoggedAt >= SHADOW_RESUBMIT_WINDOW_MS) {
|
|
2067
|
+
log2(`[shadow] skipped duplicate ${scope} batch for ${registration.projectIdentity}; ` + "the same content was submitted within the one-hour provider budget");
|
|
2068
|
+
updatePersistedShadowBackfillState(db, registration.projectIdentity, scope, modelId, (current) => ({
|
|
2069
|
+
...current,
|
|
2070
|
+
budgetLogRequestKey: requestKey,
|
|
2071
|
+
budgetLoggedAt: now
|
|
2072
|
+
}));
|
|
2073
|
+
}
|
|
2074
|
+
return {
|
|
2075
|
+
vectors: new Map,
|
|
2076
|
+
refusalReason: "duplicate_submission_budget"
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
2079
|
+
beginSynapseBatchLedger(db, {
|
|
2080
|
+
sessionId,
|
|
2081
|
+
projectPath: registration.projectIdentity,
|
|
2082
|
+
scope,
|
|
2083
|
+
manifest: items.map(({ id, contentSha256 }) => ({ id, contentSha256 })),
|
|
2084
|
+
requestKey
|
|
2085
|
+
}, now);
|
|
2086
|
+
try {
|
|
2087
|
+
if (registration.provider.embedItems) {
|
|
2088
|
+
const returned = await registration.provider.embedItems(items);
|
|
2089
|
+
const vectors = new Map([...returned].filter(([, vector]) => !isSynapseEmbeddingTruncated(vector)));
|
|
2090
|
+
finishSynapseBatchLedger(db, sessionId, requestKey, vectors.size === items.length ? "complete" : "partial", shadowBackfillNow());
|
|
2091
|
+
return {
|
|
2092
|
+
vectors,
|
|
2093
|
+
...vectors.size === 0 ? { refusalReason: "provider_returned_no_vectors" } : {}
|
|
2094
|
+
};
|
|
2095
|
+
}
|
|
2096
|
+
const positional = await registration.provider.embedBatch(items.map((item) => item.text));
|
|
2097
|
+
const vectors = new Map(items.flatMap((item, index) => {
|
|
2098
|
+
const vector = positional[index];
|
|
2099
|
+
return vector && !isSynapseEmbeddingTruncated(vector) ? [[item.id, vector]] : [];
|
|
2100
|
+
}));
|
|
2101
|
+
finishSynapseBatchLedger(db, sessionId, requestKey, vectors.size === items.length ? "complete" : "partial", shadowBackfillNow());
|
|
2102
|
+
return {
|
|
2103
|
+
vectors,
|
|
2104
|
+
...vectors.size === 0 ? { refusalReason: "provider_returned_no_vectors" } : {}
|
|
2105
|
+
};
|
|
2106
|
+
} catch (error) {
|
|
2107
|
+
finishSynapseBatchLedger(db, sessionId, requestKey, "failed", shadowBackfillNow());
|
|
2108
|
+
throw error;
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
async function processShadowQueueItem(item) {
|
|
2112
|
+
const registration = shadowRegistrations.get(item.projectIdentity);
|
|
2113
|
+
if (!registration)
|
|
2114
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2115
|
+
const boundedIds = item.ids.slice(0, SHADOW_MAX_ITEMS_PER_TICK);
|
|
2116
|
+
if (item.scope === "memory") {
|
|
2117
|
+
const db = dbForShadowQueue.get(item.projectIdentity);
|
|
2118
|
+
if (!db)
|
|
2119
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2120
|
+
const placeholders = boundedIds.map(() => "?").join(",");
|
|
2121
|
+
const rows = db.prepare(`SELECT id, content, normalized_hash FROM memories
|
|
2122
|
+
WHERE project_path = ? AND id IN (${placeholders}) AND status = 'active'`).all(item.projectIdentity, ...boundedIds.map((id) => Number(id)));
|
|
2123
|
+
if (rows.length === 0)
|
|
2124
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2125
|
+
const embedded = await embedShadowItems(registration, rows.map((row) => ({
|
|
2126
|
+
id: `memory:${row.id}`,
|
|
2127
|
+
text: row.content,
|
|
2128
|
+
contentSha256: contentSha2562(row.content)
|
|
2129
|
+
})), db, "memory");
|
|
2130
|
+
const live = shadowRegistrations.get(item.projectIdentity);
|
|
2131
|
+
if (!live || live.generation !== registration.generation) {
|
|
2132
|
+
return { writes: 0, refusalReason: "registration_retired_during_embed" };
|
|
2133
|
+
}
|
|
2134
|
+
let writes = 0;
|
|
2135
|
+
let hashGuardRejected = false;
|
|
2136
|
+
db.transaction(() => {
|
|
2137
|
+
for (const row of rows) {
|
|
2138
|
+
const vector = embedded.vectors.get(`memory:${row.id}`);
|
|
2139
|
+
if (!vector)
|
|
2140
|
+
continue;
|
|
2141
|
+
if (saveEmbeddingIfHashMatches(db, row.id, vector, registration.modelId, row.normalized_hash)) {
|
|
2142
|
+
writes += 1;
|
|
2143
|
+
} else {
|
|
2144
|
+
hashGuardRejected = true;
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
}).immediate();
|
|
2148
|
+
return {
|
|
2149
|
+
writes,
|
|
2150
|
+
...writes === 0 ? {
|
|
2151
|
+
refusalReason: embedded.refusalReason ?? (hashGuardRejected ? "memory_hash_guard_rejected" : "provider_returned_no_vectors")
|
|
2152
|
+
} : {}
|
|
2153
|
+
};
|
|
2154
|
+
}
|
|
2155
|
+
if (item.scope === "commit") {
|
|
2156
|
+
const db = dbForShadowQueue.get(item.projectIdentity);
|
|
2157
|
+
if (!db)
|
|
2158
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2159
|
+
const placeholders = boundedIds.map(() => "?").join(",");
|
|
2160
|
+
const rows = db.prepare(`SELECT sha, message FROM git_commits WHERE project_path = ? AND sha IN (${placeholders})`).all(item.projectIdentity, ...boundedIds);
|
|
2161
|
+
if (rows.length === 0)
|
|
2162
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2163
|
+
const embedded = await embedShadowItems(registration, rows.map((row) => ({
|
|
2164
|
+
id: `commit:${row.sha}`,
|
|
2165
|
+
text: row.message,
|
|
2166
|
+
contentSha256: contentSha2562(row.message)
|
|
2167
|
+
})), db, "commit");
|
|
2168
|
+
const live = shadowRegistrations.get(item.projectIdentity);
|
|
2169
|
+
if (!live || live.generation !== registration.generation) {
|
|
2170
|
+
return { writes: 0, refusalReason: "registration_retired_during_embed" };
|
|
2171
|
+
}
|
|
2172
|
+
let writes = 0;
|
|
2173
|
+
db.transaction(() => {
|
|
2174
|
+
for (const row of rows) {
|
|
2175
|
+
const vector = embedded.vectors.get(`commit:${row.sha}`);
|
|
2176
|
+
if (!vector)
|
|
2177
|
+
continue;
|
|
2178
|
+
saveCommitEmbedding(db, row.sha, vector, registration.modelId);
|
|
2179
|
+
writes += 1;
|
|
2180
|
+
}
|
|
2181
|
+
}).immediate();
|
|
2182
|
+
return {
|
|
2183
|
+
writes,
|
|
2184
|
+
...writes === 0 ? { refusalReason: embedded.refusalReason ?? "provider_returned_no_vectors" } : {}
|
|
2185
|
+
};
|
|
2186
|
+
}
|
|
2187
|
+
const db = dbForShadowQueue.get(item.projectIdentity);
|
|
2188
|
+
if (!db)
|
|
2189
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2190
|
+
const placeholders = boundedIds.map(() => "?").join(",");
|
|
2191
|
+
const candidates = db.prepare(`SELECT id, session_id, start_message, end_message
|
|
2192
|
+
FROM compartments WHERE id IN (${placeholders})`).all(...boundedIds.map((id) => Number(id)));
|
|
2193
|
+
if (candidates.length === 0) {
|
|
2194
|
+
return { writes: 0, refusalReason: "candidate_rows_changed" };
|
|
2195
|
+
}
|
|
2196
|
+
const prepared = [];
|
|
2197
|
+
let ftsMappingIncomplete = false;
|
|
2198
|
+
let emptyCanonicalText = false;
|
|
2199
|
+
for (const candidate of candidates) {
|
|
2200
|
+
const mappedText = buildCanonicalChunkTextFromFts(db, candidate.session_id, candidate.start_message, candidate.end_message);
|
|
2201
|
+
if (mappedText === null) {
|
|
2202
|
+
ftsMappingIncomplete = true;
|
|
2203
|
+
} else {
|
|
2204
|
+
const text = mappedText || buildCompartmentSummaryFallbackText(db, candidate.id);
|
|
2205
|
+
const shadowMaxInputTokens = shadowMaxInputTokensFor(registration);
|
|
2206
|
+
const windows = chunkCanonicalText(text, candidate.start_message, candidate.end_message, shadowMaxInputTokens);
|
|
2207
|
+
const windowSourceKey = chunkWindowSourceKey(text, candidate.start_message, candidate.end_message, shadowMaxInputTokens);
|
|
2208
|
+
if (windows.length > 0)
|
|
2209
|
+
prepared.push({ candidate, windows, windowSourceKey });
|
|
2210
|
+
else
|
|
2211
|
+
emptyCanonicalText = true;
|
|
2212
|
+
}
|
|
2213
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
2214
|
+
}
|
|
2215
|
+
if (prepared.length === 0) {
|
|
2216
|
+
return {
|
|
2217
|
+
writes: 0,
|
|
2218
|
+
refusalReason: ftsMappingIncomplete ? "chunk_fts_mapping_incomplete" : "chunk_empty_canonical_text"
|
|
2219
|
+
};
|
|
2220
|
+
}
|
|
2221
|
+
const items = prepared.flatMap((item) => item.windows.map((window) => ({
|
|
2222
|
+
id: `chunk:${item.candidate.id}:${window.windowIndex}`,
|
|
2223
|
+
text: window.text,
|
|
2224
|
+
contentSha256: contentSha2562(window.text)
|
|
2225
|
+
})));
|
|
2226
|
+
const embedded = await embedShadowItems(registration, items, db, "chunk");
|
|
2227
|
+
const live = shadowRegistrations.get(item.projectIdentity);
|
|
2228
|
+
if (!live || live.generation !== registration.generation) {
|
|
2229
|
+
return { writes: 0, refusalReason: "registration_retired_during_embed" };
|
|
2230
|
+
}
|
|
2231
|
+
let writes = 0;
|
|
2232
|
+
let partialVectorSet = false;
|
|
2233
|
+
for (const item of prepared) {
|
|
2234
|
+
const rows = item.windows.flatMap((window) => {
|
|
2235
|
+
const vector = embedded.vectors.get(`chunk:${item.candidate.id}:${window.windowIndex}`);
|
|
2236
|
+
return vector ? [
|
|
2237
|
+
{
|
|
2238
|
+
compartmentId: item.candidate.id,
|
|
2239
|
+
sessionId: item.candidate.session_id,
|
|
2240
|
+
projectPath: registration.projectIdentity,
|
|
2241
|
+
window,
|
|
2242
|
+
modelId: registration.chunkModelId,
|
|
2243
|
+
vector
|
|
2244
|
+
}
|
|
2245
|
+
] : [];
|
|
2246
|
+
});
|
|
2247
|
+
if (rows.length === item.windows.length) {
|
|
2248
|
+
replaceCompartmentChunkEmbeddings(db, rows, item.windowSourceKey);
|
|
2249
|
+
writes += 1;
|
|
2250
|
+
} else {
|
|
2251
|
+
partialVectorSet = true;
|
|
2252
|
+
}
|
|
2253
|
+
}
|
|
2254
|
+
return {
|
|
2255
|
+
writes,
|
|
2256
|
+
...writes === 0 ? {
|
|
2257
|
+
refusalReason: embedded.refusalReason ?? (partialVectorSet ? "chunk_partial_vector_set" : ftsMappingIncomplete ? "chunk_fts_mapping_incomplete" : emptyCanonicalText ? "chunk_empty_canonical_text" : "unknown_write_rejection")
|
|
2258
|
+
} : {}
|
|
2259
|
+
};
|
|
2260
|
+
}
|
|
2261
|
+
var dbForShadowQueue = new Map;
|
|
2262
|
+
async function runShadowWorker() {
|
|
2263
|
+
const startedAt = Date.now();
|
|
2264
|
+
let processed = 0;
|
|
2265
|
+
let processedBytes = 0;
|
|
2266
|
+
for (;; ) {
|
|
2267
|
+
if (shadowQueue.length === 0) {
|
|
2268
|
+
pumpShadowBackfill();
|
|
2269
|
+
if (shadowQueue.length === 0)
|
|
2270
|
+
break;
|
|
2271
|
+
}
|
|
2272
|
+
if (processed >= SHADOW_MAX_ITEMS_PER_TICK || Date.now() - startedAt >= SHADOW_MAX_WALL_CLOCK_MS) {
|
|
2273
|
+
break;
|
|
2274
|
+
}
|
|
2275
|
+
const item = shadowQueue.shift();
|
|
2276
|
+
if (!item)
|
|
2277
|
+
break;
|
|
2278
|
+
const itemBytes = item.ids.reduce((total, id) => total + id.length, 0);
|
|
2279
|
+
if (processed > 0 && processedBytes + itemBytes > SHADOW_MAX_BYTES_PER_TICK) {
|
|
2280
|
+
shadowQueue.unshift(item);
|
|
2281
|
+
break;
|
|
2282
|
+
}
|
|
2283
|
+
const generationAtStart = shadowRegistrations.get(item.projectIdentity)?.generation;
|
|
2284
|
+
const isStillCurrent = () => generationAtStart !== undefined && shadowRegistrations.get(item.projectIdentity)?.generation === generationAtStart;
|
|
2285
|
+
try {
|
|
2286
|
+
const outcome = await processShadowQueueItem(item);
|
|
2287
|
+
if (isStillCurrent()) {
|
|
2288
|
+
shadowBackfillLastWriteOutcomes.set(`${item.projectIdentity}:${item.scope}`, outcome);
|
|
2289
|
+
}
|
|
2290
|
+
} catch (error) {
|
|
2291
|
+
if (isStillCurrent()) {
|
|
2292
|
+
shadowBackfillLastWriteOutcomes.set(`${item.projectIdentity}:${item.scope}`, {
|
|
2293
|
+
writes: 0,
|
|
2294
|
+
refusalReason: "provider_returned_no_vectors"
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
log2("[magic-context] Synapse shadow write failed:", error);
|
|
2298
|
+
}
|
|
2299
|
+
processed += item.ids.length;
|
|
2300
|
+
processedBytes += itemBytes;
|
|
2301
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
function registerProjectInObservationMode2(db, projectIdentity, sourceDirectory, failedConfig, failureSummary) {
|
|
2305
|
+
const prior = projectRegistrations.get(projectIdentity);
|
|
2306
|
+
const runtimeFingerprint = `observation:${sha256Prefix(failureSummary)}`;
|
|
2307
|
+
const generation = prior?.runtimeFingerprint === runtimeFingerprint && prior.observationMode ? prior.generation : ++globalRegistrationGeneration;
|
|
2308
|
+
const registration = {
|
|
2309
|
+
projectIdentity,
|
|
2310
|
+
sourceDirectory,
|
|
2311
|
+
config: resolveEmbeddingConfig(failedConfig),
|
|
2312
|
+
providerIdentity: OFF_PROVIDER_IDENTITY,
|
|
2313
|
+
runtimeFingerprint,
|
|
2314
|
+
provider: null,
|
|
2315
|
+
generation,
|
|
2316
|
+
features: { memoryEnabled: false, gitCommitEnabled: false },
|
|
2317
|
+
modelId: "off",
|
|
2318
|
+
chunkModelId: "off",
|
|
2319
|
+
observationMode: true
|
|
2320
|
+
};
|
|
2321
|
+
projectRegistrations.set(projectIdentity, registration);
|
|
2322
|
+
disposeProvider(prior?.provider ?? null);
|
|
2323
|
+
return snapshotFor(registration);
|
|
2324
|
+
}
|
|
2325
|
+
function unregisterProjectShadowEmbedding2(projectIdentity) {
|
|
2326
|
+
const shadow = shadowRegistrations.get(projectIdentity);
|
|
2327
|
+
shadowRegistrations.delete(projectIdentity);
|
|
2328
|
+
dbForShadowQueue.delete(projectIdentity);
|
|
2329
|
+
pendingShadowBackfills.delete(projectIdentity);
|
|
2330
|
+
for (let index = shadowQueue.length - 1;index >= 0; index -= 1) {
|
|
2331
|
+
if (shadowQueue[index].projectIdentity === projectIdentity)
|
|
2332
|
+
shadowQueue.splice(index, 1);
|
|
2333
|
+
}
|
|
2334
|
+
for (const scope of ["memory", "commit", "chunk"]) {
|
|
2335
|
+
const key = `${projectIdentity}:${scope}`;
|
|
2336
|
+
shadowBackfillLastIds.delete(key);
|
|
2337
|
+
shadowBackfillStopReasons.delete(key);
|
|
2338
|
+
shadowBackfillLastWriteOutcomes.delete(key);
|
|
2339
|
+
}
|
|
2340
|
+
const primaryProvider = projectRegistrations.get(projectIdentity)?.provider ?? null;
|
|
2341
|
+
if (shadow?.provider && shadow.provider !== primaryProvider) {
|
|
2342
|
+
disposeProvider(shadow.provider);
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
function unregisterProjectEmbedding2(projectIdentity) {
|
|
2346
|
+
const prior = projectRegistrations.get(projectIdentity);
|
|
2347
|
+
const shadow = shadowRegistrations.get(projectIdentity);
|
|
2348
|
+
if (!prior && !shadow)
|
|
2349
|
+
return;
|
|
2350
|
+
projectRegistrations.delete(projectIdentity);
|
|
2351
|
+
shadowRegistrations.delete(projectIdentity);
|
|
2352
|
+
dbForShadowQueue.delete(projectIdentity);
|
|
2353
|
+
globalRegistrationGeneration += 1;
|
|
2354
|
+
disposeProvider(prior?.provider ?? null);
|
|
2355
|
+
disposeProvider(shadow?.provider ?? null);
|
|
2356
|
+
}
|
|
2357
|
+
function getProjectEmbeddingSnapshot2(projectIdentity) {
|
|
2358
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2359
|
+
return registration ? snapshotFor(registration) : null;
|
|
2360
|
+
}
|
|
2361
|
+
function installProjectEmbeddingSearchBridge2(snapshot, provider) {
|
|
2362
|
+
projectRegistrations.set(snapshot.projectIdentity, {
|
|
2363
|
+
projectIdentity: snapshot.projectIdentity,
|
|
2364
|
+
sourceDirectory: snapshot.sourceDirectory,
|
|
2365
|
+
providerIdentity: snapshot.providerIdentity,
|
|
2366
|
+
runtimeFingerprint: snapshot.runtimeFingerprint,
|
|
2367
|
+
generation: snapshot.generation,
|
|
2368
|
+
features: { ...snapshot.features },
|
|
2369
|
+
modelId: snapshot.modelId,
|
|
2370
|
+
chunkModelId: snapshot.chunkModelId,
|
|
2371
|
+
config: { provider: "off" },
|
|
2372
|
+
provider,
|
|
2373
|
+
observationMode: !snapshot.historyEnabled
|
|
2374
|
+
});
|
|
2375
|
+
}
|
|
2376
|
+
function getProjectChunkEmbeddingModelId2(projectIdentity) {
|
|
2377
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2378
|
+
return registration && !registration.observationMode ? registration.chunkModelId : "off";
|
|
2379
|
+
}
|
|
2380
|
+
function getProjectEmbeddingMaxInputTokens2(projectIdentity) {
|
|
2381
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2382
|
+
const configMax = registration?.config && "max_input_tokens" in registration.config ? registration.config.max_input_tokens : undefined;
|
|
2383
|
+
return normalizeCompartmentChunkMaxInputTokens(registration?.provider?.maxInputTokens ?? configMax);
|
|
2384
|
+
}
|
|
2385
|
+
function getOrCreateProjectProvider(registration) {
|
|
2386
|
+
if (registration.providerIdentity === OFF_PROVIDER_IDENTITY || registration.observationMode) {
|
|
2387
|
+
return null;
|
|
2388
|
+
}
|
|
2389
|
+
if (registration.provider) {
|
|
2390
|
+
return registration.provider;
|
|
2391
|
+
}
|
|
2392
|
+
const provider = createProvider(registration.config, {
|
|
2393
|
+
projectRoot: registration.sourceDirectory,
|
|
2394
|
+
session: `project:${registration.projectIdentity}`
|
|
2395
|
+
});
|
|
2396
|
+
registration.provider = provider;
|
|
2397
|
+
return provider;
|
|
2398
|
+
}
|
|
2399
|
+
async function embedTextForProject2(projectIdentity, text, signal, purpose = "passage") {
|
|
2400
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2401
|
+
if (!registration)
|
|
2402
|
+
return null;
|
|
2403
|
+
const generation = registration.generation;
|
|
2404
|
+
const modelId = registration.modelId;
|
|
2405
|
+
const provider = getOrCreateProjectProvider(registration);
|
|
2406
|
+
if (!provider)
|
|
2407
|
+
return null;
|
|
2408
|
+
const vector = await provider.embed(text, signal, purpose);
|
|
2409
|
+
if (!vector || isSynapseEmbeddingTruncated(vector))
|
|
2410
|
+
return null;
|
|
2411
|
+
const current = projectRegistrations.get(projectIdentity);
|
|
2412
|
+
if (!current || current.generation !== generation || current.runtimeFingerprint !== registration.runtimeFingerprint) {
|
|
2413
|
+
return null;
|
|
2414
|
+
}
|
|
2415
|
+
return { vector, modelId, chunkModelId: registration.chunkModelId, generation };
|
|
2416
|
+
}
|
|
2417
|
+
async function embedBatchForProject2(projectIdentity, texts, signal, purpose = "passage") {
|
|
2418
|
+
if (texts.length === 0) {
|
|
2419
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2420
|
+
if (!registration || registration.observationMode)
|
|
2421
|
+
return null;
|
|
2422
|
+
return { vectors: [], modelId: registration.modelId, generation: registration.generation };
|
|
2423
|
+
}
|
|
2424
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2425
|
+
if (!registration)
|
|
2426
|
+
return null;
|
|
2427
|
+
const generation = registration.generation;
|
|
2428
|
+
const modelId = registration.modelId;
|
|
2429
|
+
const runtimeFingerprint = registration.runtimeFingerprint;
|
|
2430
|
+
const provider = getOrCreateProjectProvider(registration);
|
|
2431
|
+
if (!provider)
|
|
2432
|
+
return null;
|
|
2433
|
+
const vectors = (await provider.embedBatch(texts, signal, purpose)).map((vector) => vector && !isSynapseEmbeddingTruncated(vector) ? vector : null);
|
|
2434
|
+
const current = projectRegistrations.get(projectIdentity);
|
|
2435
|
+
if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
|
|
2436
|
+
return null;
|
|
2437
|
+
}
|
|
2438
|
+
return { vectors, modelId, generation };
|
|
2439
|
+
}
|
|
2440
|
+
async function embedItemsForProject2(projectIdentity, items, signal, db, sessionId = projectIdentity) {
|
|
2441
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2442
|
+
if (!registration || registration.observationMode || items.length === 0)
|
|
2443
|
+
return null;
|
|
2444
|
+
const generation = registration.generation;
|
|
2445
|
+
const modelId = registration.modelId;
|
|
2446
|
+
const runtimeFingerprint = registration.runtimeFingerprint;
|
|
2447
|
+
const provider = getOrCreateProjectProvider(registration);
|
|
2448
|
+
if (!provider)
|
|
2449
|
+
return null;
|
|
2450
|
+
const rawConfig = registration.config;
|
|
2451
|
+
const fingerprint = typeof rawConfig.synapse_fingerprint === "string" ? rawConfig.synapse_fingerprint : "";
|
|
2452
|
+
const tableEpoch = typeof rawConfig.synapse_table_epoch === "number" ? rawConfig.synapse_table_epoch : undefined;
|
|
2453
|
+
const isSynapse = registration.providerIdentity.startsWith("synapse:v1:") && fingerprint && tableEpoch !== undefined;
|
|
2454
|
+
const ledgerKey = isSynapse ? getSynapseBatchRequestKey({
|
|
2455
|
+
model: typeof rawConfig.model === "string" ? rawConfig.model : registration.modelId,
|
|
2456
|
+
fingerprint,
|
|
2457
|
+
tableEpoch,
|
|
2458
|
+
items
|
|
2459
|
+
}) : null;
|
|
2460
|
+
if (db && ledgerKey) {
|
|
2461
|
+
beginSynapseBatchLedger(db, {
|
|
2462
|
+
sessionId,
|
|
2463
|
+
projectPath: projectIdentity,
|
|
2464
|
+
scope: items[0].id.split(":", 1)[0],
|
|
2465
|
+
manifest: items.map(({ id, contentSha256 }) => ({ id, contentSha256 })),
|
|
2466
|
+
requestKey: ledgerKey
|
|
2467
|
+
});
|
|
2468
|
+
}
|
|
2469
|
+
let vectors;
|
|
2470
|
+
try {
|
|
2471
|
+
if (provider.embedItems) {
|
|
2472
|
+
const returned = await provider.embedItems(items, signal);
|
|
2473
|
+
vectors = new Map([...returned].filter(([, vector]) => !isSynapseEmbeddingTruncated(vector)));
|
|
2474
|
+
} else {
|
|
2475
|
+
const positional = await provider.embedBatch(items.map((item) => item.text), signal, "passage");
|
|
2476
|
+
vectors = new Map(items.flatMap((item, index) => {
|
|
2477
|
+
const vector = positional[index];
|
|
2478
|
+
return vector && !isSynapseEmbeddingTruncated(vector) ? [[item.id, vector]] : [];
|
|
2479
|
+
}));
|
|
2480
|
+
}
|
|
2481
|
+
} catch (error) {
|
|
2482
|
+
if (db && ledgerKey)
|
|
2483
|
+
finishSynapseBatchLedger(db, sessionId, ledgerKey, "failed");
|
|
2484
|
+
throw error;
|
|
2485
|
+
}
|
|
2486
|
+
if (db && ledgerKey) {
|
|
2487
|
+
finishSynapseBatchLedger(db, sessionId, ledgerKey, vectors.size === items.length ? "complete" : "partial");
|
|
2488
|
+
}
|
|
2489
|
+
const current = projectRegistrations.get(projectIdentity);
|
|
2490
|
+
if (!current || current.generation !== generation || current.runtimeFingerprint !== runtimeFingerprint) {
|
|
2491
|
+
return null;
|
|
2492
|
+
}
|
|
2493
|
+
return { vectors, modelId, generation };
|
|
2494
|
+
}
|
|
2495
|
+
async function embedItemsWindowBounded(projectIdentity, items, signal, db) {
|
|
2496
|
+
if (items.length <= MAX_WINDOWS_PER_EMBED_CALL) {
|
|
2497
|
+
return embedItemsForProject2(projectIdentity, items, signal, db, projectIdentity);
|
|
2498
|
+
}
|
|
2499
|
+
const vectors = new Map;
|
|
2500
|
+
let modelId = null;
|
|
2501
|
+
let generation = null;
|
|
2502
|
+
for (let start = 0;start < items.length; start += MAX_WINDOWS_PER_EMBED_CALL) {
|
|
2503
|
+
const result = await embedItemsForProject2(projectIdentity, items.slice(start, start + MAX_WINDOWS_PER_EMBED_CALL), signal, db, projectIdentity);
|
|
2504
|
+
if (!result)
|
|
2505
|
+
return null;
|
|
2506
|
+
if (modelId === null) {
|
|
2507
|
+
modelId = result.modelId;
|
|
2508
|
+
generation = result.generation;
|
|
2509
|
+
} else if (modelId !== result.modelId || generation !== result.generation) {
|
|
2510
|
+
return null;
|
|
2511
|
+
}
|
|
2512
|
+
for (const [id, vector] of result.vectors)
|
|
2513
|
+
vectors.set(id, vector);
|
|
2514
|
+
}
|
|
2515
|
+
return modelId === null || generation === null ? null : { vectors, modelId, generation };
|
|
2516
|
+
}
|
|
2517
|
+
function isUnembeddedMemoryRow(row) {
|
|
2518
|
+
if (row === null || typeof row !== "object")
|
|
2519
|
+
return false;
|
|
2520
|
+
const candidate = row;
|
|
2521
|
+
return typeof candidate.id === "number" && typeof candidate.content === "string" && typeof candidate.normalized_hash === "string";
|
|
2522
|
+
}
|
|
2523
|
+
function getLoadUnembeddedMemoriesStatement(db) {
|
|
2524
|
+
let stmt = loadUnembeddedMemoriesStatements.get(db);
|
|
2525
|
+
if (!stmt) {
|
|
2526
|
+
stmt = db.prepare("SELECT m.id AS id, m.content AS content, m.normalized_hash AS normalized_hash FROM memories m LEFT JOIN memory_embeddings me ON m.id = me.memory_id AND me.model_id = ? WHERE m.project_path = ? AND m.status = 'active' AND me.memory_id IS NULL LIMIT ?");
|
|
2527
|
+
loadUnembeddedMemoriesStatements.set(db, stmt);
|
|
2528
|
+
}
|
|
2529
|
+
return stmt;
|
|
2530
|
+
}
|
|
2531
|
+
async function embedUnembeddedMemoriesForProject2(db, projectIdentity, batchSize = 10) {
|
|
2532
|
+
const outcome = await embedUnembeddedMemoriesForProjectOutcome2(db, projectIdentity, batchSize);
|
|
2533
|
+
return outcome.kind === "embedded" ? outcome.count : 0;
|
|
2534
|
+
}
|
|
2535
|
+
async function embedUnembeddedMemoriesForProjectOutcome2(db, projectIdentity, batchSize = 10) {
|
|
2536
|
+
if (isEmbeddingHostBusy2())
|
|
2537
|
+
return { kind: "unavailable" };
|
|
2538
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2539
|
+
if (!snapshot?.enabled)
|
|
2540
|
+
return { kind: "disabled" };
|
|
2541
|
+
const normalizedBatchSize = Math.max(1, Math.floor(batchSize));
|
|
2542
|
+
const memories = getLoadUnembeddedMemoriesStatement(db).all(snapshot.modelId, projectIdentity, normalizedBatchSize).filter(isUnembeddedMemoryRow);
|
|
2543
|
+
if (memories.length === 0)
|
|
2544
|
+
return { kind: "done" };
|
|
2545
|
+
try {
|
|
2546
|
+
const result = await embedItemsForProject2(projectIdentity, memories.map((memory) => ({
|
|
2547
|
+
id: `memory:${memory.id}`,
|
|
2548
|
+
text: memory.content,
|
|
2549
|
+
contentSha256: contentSha2562(memory.content)
|
|
2550
|
+
})), undefined, db, projectIdentity);
|
|
2551
|
+
if (!result)
|
|
2552
|
+
return { kind: "unavailable" };
|
|
2553
|
+
let embeddedCount = 0;
|
|
2554
|
+
db.transaction(() => {
|
|
2555
|
+
for (const memory of memories) {
|
|
2556
|
+
const embedding = result.vectors.get(`memory:${memory.id}`);
|
|
2557
|
+
if (!embedding)
|
|
2558
|
+
continue;
|
|
2559
|
+
if (saveEmbeddingIfHashMatches(db, memory.id, embedding, result.modelId, memory.normalized_hash)) {
|
|
2560
|
+
embeddedCount += 1;
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
}).immediate();
|
|
2564
|
+
enqueueShadowEmbeddingItems2(projectIdentity, "memory", memories.filter((memory) => result.vectors.has(`memory:${memory.id}`)).map((memory) => String(memory.id)));
|
|
2565
|
+
return { kind: "embedded", count: embeddedCount };
|
|
2566
|
+
} catch (error) {
|
|
2567
|
+
log2("[magic-context] failed to proactively embed missing memories:", error);
|
|
2568
|
+
return { kind: "failed", error };
|
|
2569
|
+
}
|
|
2570
|
+
}
|
|
2571
|
+
async function embedCommitBatch(db, projectIdentity, batchSize) {
|
|
2572
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2573
|
+
if (!snapshot?.gitCommitEnabled)
|
|
2574
|
+
return 0;
|
|
2575
|
+
const commits = loadUnembeddedCommits(db, projectIdentity, snapshot.modelId, Math.max(1, Math.floor(batchSize)));
|
|
2576
|
+
if (commits.length === 0)
|
|
2577
|
+
return 0;
|
|
2578
|
+
const result = await embedItemsForProject2(projectIdentity, commits.map((commit) => ({
|
|
2579
|
+
id: `commit:${commit.sha}`,
|
|
2580
|
+
text: commit.message,
|
|
2581
|
+
contentSha256: contentSha2562(commit.message)
|
|
2582
|
+
})), undefined, db, projectIdentity);
|
|
2583
|
+
if (!result)
|
|
2584
|
+
return 0;
|
|
2585
|
+
let embeddedCount = 0;
|
|
2586
|
+
db.transaction(() => {
|
|
2587
|
+
for (const commit of commits) {
|
|
2588
|
+
const embedding = result.vectors.get(`commit:${commit.sha}`);
|
|
2589
|
+
if (!embedding)
|
|
2590
|
+
continue;
|
|
2591
|
+
saveCommitEmbedding(db, commit.sha, embedding, result.modelId);
|
|
2592
|
+
embeddedCount += 1;
|
|
2593
|
+
}
|
|
2594
|
+
}).immediate();
|
|
2595
|
+
enqueueShadowEmbeddingItems2(projectIdentity, "commit", commits.filter((commit) => result.vectors.has(`commit:${commit.sha}`)).map((commit) => commit.sha));
|
|
2596
|
+
return embeddedCount;
|
|
2597
|
+
}
|
|
2598
|
+
async function drainCommitBacklogForProject2(db, projectIdentity, deadline) {
|
|
2599
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2600
|
+
if (!snapshot?.gitCommitEnabled)
|
|
2601
|
+
return 0;
|
|
2602
|
+
const holderId = `embed-sweep-${randomUUID()}`;
|
|
2603
|
+
const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
|
|
2604
|
+
if (!lease.acquired) {
|
|
2605
|
+
return 0;
|
|
2606
|
+
}
|
|
2607
|
+
let total = 0;
|
|
2608
|
+
let leaseLost = false;
|
|
2609
|
+
const renewal = setInterval(() => {
|
|
2610
|
+
try {
|
|
2611
|
+
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
2612
|
+
leaseLost = true;
|
|
2613
|
+
} catch {}
|
|
2614
|
+
}, SESSION_EMBED_LEASE_RENEWAL_MS);
|
|
2615
|
+
renewal.unref?.();
|
|
2616
|
+
try {
|
|
2617
|
+
while (!leaseLost && !isEmbeddingHostBusy2() && Date.now() < deadline && total < COMMIT_DRAIN_MAX_PER_SWEEP) {
|
|
2618
|
+
const embedded = await embedCommitBatch(db, projectIdentity, COMMIT_DRAIN_BATCH_SIZE);
|
|
2619
|
+
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
2620
|
+
leaseLost = true;
|
|
2621
|
+
if (leaseLost || embedded === 0)
|
|
2622
|
+
break;
|
|
2623
|
+
total += embedded;
|
|
2624
|
+
if (embedded < COMMIT_DRAIN_BATCH_SIZE)
|
|
2625
|
+
break;
|
|
2626
|
+
}
|
|
2627
|
+
} finally {
|
|
2628
|
+
clearInterval(renewal);
|
|
2629
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
2630
|
+
}
|
|
2631
|
+
return total;
|
|
2632
|
+
}
|
|
2633
|
+
async function embedCompartmentChunkBatch(db, projectIdentity, batchSize) {
|
|
2634
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2635
|
+
if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off")
|
|
2636
|
+
return 0;
|
|
2637
|
+
repairMisScopedCompartmentChunkEmbeddingsForProject(db, projectIdentity);
|
|
2638
|
+
const candidates = await loadUnembeddedCompartmentChunkCandidatesPolite(db, projectIdentity, snapshot.chunkModelId, batchSize, getProjectEmbeddingMaxInputTokens2(projectIdentity), true);
|
|
2639
|
+
if (candidates.length === 0)
|
|
2640
|
+
return 0;
|
|
2641
|
+
const { embedded } = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates);
|
|
2642
|
+
return embedded;
|
|
2643
|
+
}
|
|
2644
|
+
function getLastEmbeddingFailureForProject(projectIdentity) {
|
|
2645
|
+
const registration = projectRegistrations.get(projectIdentity);
|
|
2646
|
+
return registration?.provider?.getLastFailureReason?.() ?? null;
|
|
2647
|
+
}
|
|
2648
|
+
async function embedCandidateChunkBatch(db, projectIdentity, modelId, candidates, signal) {
|
|
2649
|
+
const noWork = [];
|
|
2650
|
+
const failed = [];
|
|
2651
|
+
const failureReasons = [];
|
|
2652
|
+
if (candidates.length === 0)
|
|
2653
|
+
return { embedded: 0, noWork, failed, failureReasons };
|
|
2654
|
+
const maxInputTokens = getProjectEmbeddingMaxInputTokens2(projectIdentity);
|
|
2655
|
+
const prepared = [];
|
|
2656
|
+
for (const candidate of candidates) {
|
|
2657
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
2658
|
+
const mappedText = buildCanonicalChunkTextFromFts(db, candidate.sessionId, candidate.startMessage, candidate.endMessage);
|
|
2659
|
+
if (mappedText === null)
|
|
2660
|
+
continue;
|
|
2661
|
+
const canonicalText = mappedText || buildCompartmentSummaryFallbackText(db, candidate.id);
|
|
2662
|
+
if (canonicalText.length === 0) {
|
|
2663
|
+
recordChunkEmbedBackoff(db, candidate, projectIdentity, modelId);
|
|
2664
|
+
noWork.push(candidate.id);
|
|
2665
|
+
continue;
|
|
2666
|
+
}
|
|
2667
|
+
const windows = chunkCanonicalText(canonicalText, candidate.startMessage, candidate.endMessage, maxInputTokens);
|
|
2668
|
+
if (windows.length === 0 || chunkEmbeddingWindowsAreCurrent(db, candidate.id, modelId, windows, projectIdentity)) {
|
|
2669
|
+
noWork.push(candidate.id);
|
|
2670
|
+
continue;
|
|
2671
|
+
}
|
|
2672
|
+
prepared.push({
|
|
2673
|
+
candidate,
|
|
2674
|
+
windows,
|
|
2675
|
+
windowSourceKey: chunkWindowSourceKey(canonicalText, candidate.startMessage, candidate.endMessage, maxInputTokens)
|
|
2676
|
+
});
|
|
2677
|
+
}
|
|
2678
|
+
if (prepared.length === 0)
|
|
2679
|
+
return { embedded: 0, noWork, failed, failureReasons };
|
|
2680
|
+
let embedded = 0;
|
|
2681
|
+
let i = 0;
|
|
2682
|
+
while (i < prepared.length) {
|
|
2683
|
+
if (signal?.aborted)
|
|
2684
|
+
break;
|
|
2685
|
+
const slice = [];
|
|
2686
|
+
let windowCount = 0;
|
|
2687
|
+
do {
|
|
2688
|
+
const item = prepared[i];
|
|
2689
|
+
slice.push(item);
|
|
2690
|
+
windowCount += item.windows.length;
|
|
2691
|
+
i += 1;
|
|
2692
|
+
} while (i < prepared.length && windowCount + prepared[i].windows.length <= MAX_WINDOWS_PER_EMBED_CALL);
|
|
2693
|
+
const items = slice.flatMap((item) => item.windows.map((window) => ({
|
|
2694
|
+
id: `chunk:${item.candidate.id}:${window.windowIndex}`,
|
|
2695
|
+
text: window.text,
|
|
2696
|
+
contentSha256: contentSha2562(window.text)
|
|
2697
|
+
})));
|
|
2698
|
+
const persistedIds = new Set;
|
|
2699
|
+
for (let attempt = 0;attempt < EMBED_SLICE_RETRY_ATTEMPTS; attempt++) {
|
|
2700
|
+
if (signal?.aborted)
|
|
2701
|
+
break;
|
|
2702
|
+
await new Promise((resolve) => setTimeout(resolve, 15));
|
|
2703
|
+
let result = null;
|
|
2704
|
+
const attemptStart = Date.now();
|
|
2705
|
+
try {
|
|
2706
|
+
result = await embedItemsWindowBounded(projectIdentity, items, signal, db);
|
|
2707
|
+
} catch (error) {
|
|
2708
|
+
log2("[magic-context] failed to proactively embed compartment chunks:", error);
|
|
2709
|
+
}
|
|
2710
|
+
if (signal?.aborted)
|
|
2711
|
+
break;
|
|
2712
|
+
if (!result || result.vectors.size === 0) {
|
|
2713
|
+
const failure = getLastEmbeddingFailureForProject(projectIdentity);
|
|
2714
|
+
if (failure)
|
|
2715
|
+
failureReasons.push(failure);
|
|
2716
|
+
}
|
|
2717
|
+
if (result) {
|
|
2718
|
+
let writeSliceStarted = performance.now();
|
|
2719
|
+
for (const item of slice) {
|
|
2720
|
+
if (persistedIds.has(item.candidate.id))
|
|
2721
|
+
continue;
|
|
2722
|
+
const vectors = item.windows.map((window) => result.vectors.get(`chunk:${item.candidate.id}:${window.windowIndex}`));
|
|
2723
|
+
if (vectors.length !== item.windows.length || vectors.some((v) => !v)) {
|
|
2724
|
+
continue;
|
|
2725
|
+
}
|
|
2726
|
+
const rows = item.windows.map((window, index) => ({
|
|
2727
|
+
compartmentId: item.candidate.id,
|
|
2728
|
+
sessionId: item.candidate.sessionId,
|
|
2729
|
+
projectPath: projectIdentity,
|
|
2730
|
+
window,
|
|
2731
|
+
modelId,
|
|
2732
|
+
vector: vectors[index]
|
|
2733
|
+
}));
|
|
2734
|
+
replaceCompartmentChunkEmbeddings(db, rows, item.windowSourceKey);
|
|
2735
|
+
persistedIds.add(item.candidate.id);
|
|
2736
|
+
enqueueShadowEmbeddingItems2(projectIdentity, "chunk", [
|
|
2737
|
+
String(item.candidate.id)
|
|
2738
|
+
]);
|
|
2739
|
+
if (performance.now() - writeSliceStarted >= 35) {
|
|
2740
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
2741
|
+
writeSliceStarted = performance.now();
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
}
|
|
2745
|
+
if (persistedIds.size === slice.length)
|
|
2746
|
+
break;
|
|
2747
|
+
if (persistedIds.size > 0)
|
|
2748
|
+
break;
|
|
2749
|
+
if (Date.now() - attemptStart >= EMBED_SLOW_FAILURE_NO_RETRY_MS)
|
|
2750
|
+
break;
|
|
2751
|
+
if (attempt < EMBED_SLICE_RETRY_ATTEMPTS - 1) {
|
|
2752
|
+
await new Promise((resolve) => setTimeout(resolve, EMBED_SLICE_RETRY_BASE_MS * 2 ** attempt));
|
|
2753
|
+
}
|
|
2754
|
+
}
|
|
2755
|
+
embedded += persistedIds.size;
|
|
2756
|
+
if (!signal?.aborted) {
|
|
2757
|
+
for (const item of slice) {
|
|
2758
|
+
if (!persistedIds.has(item.candidate.id)) {
|
|
2759
|
+
recordChunkEmbedBackoff(db, item.candidate, projectIdentity, modelId);
|
|
2760
|
+
failed.push(item.candidate.id);
|
|
2761
|
+
}
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
return { embedded, noWork, failed, failureReasons };
|
|
2766
|
+
}
|
|
2767
|
+
async function drainCompartmentChunkBacklogForProject(db, projectIdentity, deadline) {
|
|
2768
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2769
|
+
if (!snapshot?.historyEnabled)
|
|
2770
|
+
return 0;
|
|
2771
|
+
const holderId = `chunk-embed-sweep-${randomUUID()}`;
|
|
2772
|
+
const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
|
|
2773
|
+
if (!lease.acquired) {
|
|
2774
|
+
return 0;
|
|
2775
|
+
}
|
|
2776
|
+
let total = 0;
|
|
2777
|
+
let leaseLost = false;
|
|
2778
|
+
const renewal = setInterval(() => {
|
|
2779
|
+
try {
|
|
2780
|
+
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
2781
|
+
leaseLost = true;
|
|
2782
|
+
} catch {}
|
|
2783
|
+
}, SESSION_EMBED_LEASE_RENEWAL_MS);
|
|
2784
|
+
renewal.unref?.();
|
|
2785
|
+
try {
|
|
2786
|
+
while (!leaseLost && !isEmbeddingHostBusy2() && Date.now() < deadline && total < CHUNK_DRAIN_MAX_PER_SWEEP) {
|
|
2787
|
+
const embedded = await embedCompartmentChunkBatch(db, projectIdentity, CHUNK_DRAIN_BATCH_SIZE);
|
|
2788
|
+
if (!renewGitSweepLease(db, projectIdentity, holderId))
|
|
2789
|
+
leaseLost = true;
|
|
2790
|
+
if (leaseLost || embedded === 0)
|
|
2791
|
+
break;
|
|
2792
|
+
total += embedded;
|
|
2793
|
+
if (embedded < CHUNK_DRAIN_BATCH_SIZE)
|
|
2794
|
+
break;
|
|
2795
|
+
}
|
|
2796
|
+
} finally {
|
|
2797
|
+
clearInterval(renewal);
|
|
2798
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
2799
|
+
}
|
|
2800
|
+
return total;
|
|
2801
|
+
}
|
|
2802
|
+
async function embedUnembeddedCompartmentChunksForProject2(db, projectIdentity) {
|
|
2803
|
+
return drainCompartmentChunkBacklogForProject(db, projectIdentity, Date.now() + SWEEP_MAX_WALL_CLOCK_MS);
|
|
2804
|
+
}
|
|
2805
|
+
async function embedSessionCompartmentChunks2(db, projectIdentity, sessionId, options) {
|
|
2806
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2807
|
+
if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
|
|
2808
|
+
return { status: "disabled", embedded: 0, total: 0 };
|
|
2809
|
+
}
|
|
2810
|
+
recordSessionProjectIdentity(db, sessionId, projectIdentity);
|
|
2811
|
+
const maxInputTokens = getProjectEmbeddingMaxInputTokens2(projectIdentity);
|
|
2812
|
+
const total = await countUnembeddedSessionCompartmentsPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, maxInputTokens);
|
|
2813
|
+
const holderId = `session-embed-${randomUUID()}`;
|
|
2814
|
+
const lease = acquireGitSweepLease(db, projectIdentity, holderId, { ignoreCooldown: true });
|
|
2815
|
+
if (!lease.acquired)
|
|
2816
|
+
return { status: "busy", embedded: 0, total };
|
|
2817
|
+
let leaseLost = false;
|
|
2818
|
+
const drainAbort = new AbortController;
|
|
2819
|
+
const forwardCallerAbort = () => drainAbort.abort();
|
|
2820
|
+
if (options?.signal?.aborted)
|
|
2821
|
+
drainAbort.abort();
|
|
2822
|
+
else
|
|
2823
|
+
options?.signal?.addEventListener("abort", forwardCallerAbort, { once: true });
|
|
2824
|
+
const renewal = setInterval(() => {
|
|
2825
|
+
try {
|
|
2826
|
+
if (!renewGitSweepLease(db, projectIdentity, holderId)) {
|
|
2827
|
+
leaseLost = true;
|
|
2828
|
+
drainAbort.abort();
|
|
2829
|
+
}
|
|
2830
|
+
} catch {}
|
|
2831
|
+
}, SESSION_EMBED_LEASE_RENEWAL_MS);
|
|
2832
|
+
renewal.unref?.();
|
|
2833
|
+
const batchSize = Math.max(1, options?.batchSize ?? CHUNK_DRAIN_BATCH_SIZE);
|
|
2834
|
+
const skipIds = [];
|
|
2835
|
+
const failedIds = [];
|
|
2836
|
+
let embedded = 0;
|
|
2837
|
+
let aborted = false;
|
|
2838
|
+
let providerDown = false;
|
|
2839
|
+
let consecutiveFailedBatches = 0;
|
|
2840
|
+
const failureReasons = [];
|
|
2841
|
+
try {
|
|
2842
|
+
options?.onProgress?.({ embedded, total });
|
|
2843
|
+
for (;; ) {
|
|
2844
|
+
if (leaseLost || drainAbort.signal.aborted || isEmbeddingHostBusy2()) {
|
|
2845
|
+
aborted = true;
|
|
2846
|
+
break;
|
|
2847
|
+
}
|
|
2848
|
+
const candidates = await loadUnembeddedSessionChunkCandidatesPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, batchSize, [...skipIds, ...failedIds], maxInputTokens, true);
|
|
2849
|
+
if (candidates.length === 0)
|
|
2850
|
+
break;
|
|
2851
|
+
const {
|
|
2852
|
+
embedded: n,
|
|
2853
|
+
noWork,
|
|
2854
|
+
failed,
|
|
2855
|
+
failureReasons: batchFailureReasons
|
|
2856
|
+
} = await embedCandidateChunkBatch(db, projectIdentity, snapshot.chunkModelId, candidates, drainAbort.signal);
|
|
2857
|
+
if (leaseLost || !renewGitSweepLease(db, projectIdentity, holderId)) {
|
|
2858
|
+
leaseLost = true;
|
|
2859
|
+
drainAbort.abort();
|
|
2860
|
+
aborted = true;
|
|
2861
|
+
break;
|
|
2862
|
+
}
|
|
2863
|
+
failureReasons.push(...batchFailureReasons);
|
|
2864
|
+
for (const id of noWork)
|
|
2865
|
+
skipIds.push(id);
|
|
2866
|
+
for (const id of failed)
|
|
2867
|
+
failedIds.push(id);
|
|
2868
|
+
if (n === 0 && noWork.length === 0) {
|
|
2869
|
+
consecutiveFailedBatches += 1;
|
|
2870
|
+
if (consecutiveFailedBatches >= MAX_CONSECUTIVE_FAILED_BATCHES) {
|
|
2871
|
+
providerDown = true;
|
|
2872
|
+
break;
|
|
2873
|
+
}
|
|
2874
|
+
} else {
|
|
2875
|
+
consecutiveFailedBatches = 0;
|
|
2876
|
+
}
|
|
2877
|
+
embedded += n;
|
|
2878
|
+
options?.onProgress?.({ embedded: Math.min(embedded, total), total });
|
|
2879
|
+
await new Promise((resolve) => setTimeout(resolve, 15));
|
|
2880
|
+
}
|
|
2881
|
+
} finally {
|
|
2882
|
+
clearInterval(renewal);
|
|
2883
|
+
options?.signal?.removeEventListener("abort", forwardCallerAbort);
|
|
2884
|
+
try {
|
|
2885
|
+
releaseGitSweepLease(db, projectIdentity, holderId);
|
|
2886
|
+
} catch (error) {
|
|
2887
|
+
log2("[magic-context] embed drain: lease release failed (will TTL-expire):", error);
|
|
2888
|
+
}
|
|
2889
|
+
}
|
|
2890
|
+
if (total === 0)
|
|
2891
|
+
return { status: "nothing", embedded: 0, total: 0 };
|
|
2892
|
+
if (aborted)
|
|
2893
|
+
return { status: "aborted", embedded, total, failed: failedIds.length };
|
|
2894
|
+
if (providerDown || failedIds.length > 0) {
|
|
2895
|
+
const remaining = Math.max(failedIds.length, await countUnembeddedSessionCompartmentsPolite(db, projectIdentity, sessionId, snapshot.chunkModelId, maxInputTokens) - skipIds.length);
|
|
2896
|
+
if (remaining > 0) {
|
|
2897
|
+
return {
|
|
2898
|
+
status: "stalled",
|
|
2899
|
+
embedded,
|
|
2900
|
+
total,
|
|
2901
|
+
remaining,
|
|
2902
|
+
failed: failedIds.length,
|
|
2903
|
+
failure: dominantEmbeddingFailure(failureReasons)
|
|
2904
|
+
};
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
return { status: "done", embedded, total, failed: failedIds.length };
|
|
2908
|
+
}
|
|
2909
|
+
function getEmbeddingCoverageStatus2(db, projectIdentity, sessionId) {
|
|
2910
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2911
|
+
if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
|
|
2912
|
+
return {
|
|
2913
|
+
enabled: false,
|
|
2914
|
+
model: snapshot?.model ?? "off",
|
|
2915
|
+
provider: snapshot?.provider ?? "off",
|
|
2916
|
+
...snapshot?.synapseDescriptor ? { synapseDescriptor: snapshot.synapseDescriptor } : {},
|
|
2917
|
+
session: { embedded: 0, total: 0 },
|
|
2918
|
+
memories: { embedded: 0, total: 0 },
|
|
2919
|
+
commits: { embedded: 0, total: 0, gitEnabled: false },
|
|
2920
|
+
shadowBackfillStalls: listShadowBackfillStalls2(db, projectIdentity)
|
|
2921
|
+
};
|
|
2922
|
+
}
|
|
2923
|
+
const session = countSessionCompartmentEmbedCoverage(db, projectIdentity, sessionId, snapshot.chunkModelId, getProjectEmbeddingMaxInputTokens2(projectIdentity));
|
|
2924
|
+
const memories = snapshot.enabled ? getMemoryEmbedCoverage(db, projectIdentity, snapshot.modelId) : { embedded: 0, total: 0, memoryEnabled: false };
|
|
2925
|
+
const gitEnabled = snapshot.gitCommitEnabled;
|
|
2926
|
+
const commits = gitEnabled ? {
|
|
2927
|
+
embedded: countEmbeddedCommits(db, projectIdentity, snapshot.modelId),
|
|
2928
|
+
total: getCommitCount(db, projectIdentity),
|
|
2929
|
+
gitEnabled: true
|
|
2930
|
+
} : { embedded: 0, total: 0, gitEnabled: false };
|
|
2931
|
+
return {
|
|
2932
|
+
enabled: true,
|
|
2933
|
+
model: snapshot.model,
|
|
2934
|
+
provider: snapshot.provider,
|
|
2935
|
+
...snapshot.synapseDescriptor ? { synapseDescriptor: snapshot.synapseDescriptor } : {},
|
|
2936
|
+
session,
|
|
2937
|
+
memories,
|
|
2938
|
+
commits,
|
|
2939
|
+
shadowBackfillStalls: listShadowBackfillStalls2(db, projectIdentity)
|
|
2940
|
+
};
|
|
2941
|
+
}
|
|
2942
|
+
async function getAutoEmbeddingSessionCoverage2(db, projectIdentity, sessionId) {
|
|
2943
|
+
const snapshot = getProjectEmbeddingSnapshot2(projectIdentity);
|
|
2944
|
+
if (!snapshot?.historyEnabled || snapshot.chunkModelId === "off") {
|
|
2945
|
+
return { enabled: false, embedded: 0, total: 0 };
|
|
2946
|
+
}
|
|
2947
|
+
const coverage = await countSessionCompartmentEmbedCoveragePolite(db, projectIdentity, sessionId, snapshot.chunkModelId, getProjectEmbeddingMaxInputTokens2(projectIdentity));
|
|
2948
|
+
return { enabled: true, ...coverage };
|
|
2949
|
+
}
|
|
2950
|
+
async function sweepAllRegisteredProjects2(db, batchSize = 10) {
|
|
2951
|
+
if (projectSweepInProgress || isEmbeddingHostBusy2()) {
|
|
2952
|
+
log2("[magic-context] project embedding sweep already in progress, skipping this tick");
|
|
2953
|
+
return {
|
|
2954
|
+
memoriesEmbedded: 0,
|
|
2955
|
+
commitsEmbedded: 0,
|
|
2956
|
+
chunksEmbedded: 0,
|
|
2957
|
+
perProject: new Map
|
|
2958
|
+
};
|
|
2959
|
+
}
|
|
2960
|
+
projectSweepInProgress = true;
|
|
2961
|
+
const startedAt = Date.now();
|
|
2962
|
+
const deadline = startedAt + SWEEP_MAX_WALL_CLOCK_MS;
|
|
2963
|
+
const perProject = new Map;
|
|
2964
|
+
let memoriesEmbedded = 0;
|
|
2965
|
+
let commitsEmbedded = 0;
|
|
2966
|
+
let chunksEmbedded = 0;
|
|
2967
|
+
try {
|
|
2968
|
+
for (const projectIdentity of projectRegistrations.keys()) {
|
|
2969
|
+
let memories = 0;
|
|
2970
|
+
let commits = 0;
|
|
2971
|
+
let chunks = 0;
|
|
2972
|
+
let consecutiveEmpty = 0;
|
|
2973
|
+
while (Date.now() < deadline && !isEmbeddingHostBusy2()) {
|
|
2974
|
+
const count = await embedUnembeddedMemoriesForProject2(db, projectIdentity, batchSize);
|
|
2975
|
+
if (count === 0) {
|
|
2976
|
+
consecutiveEmpty += 1;
|
|
2977
|
+
if (consecutiveEmpty >= SWEEP_MAX_CONSECUTIVE_EMPTY)
|
|
2978
|
+
break;
|
|
2979
|
+
break;
|
|
2980
|
+
}
|
|
2981
|
+
consecutiveEmpty = 0;
|
|
2982
|
+
memories += count;
|
|
2983
|
+
memoriesEmbedded += count;
|
|
2984
|
+
if (count < batchSize)
|
|
2985
|
+
break;
|
|
2986
|
+
}
|
|
2987
|
+
if (Date.now() < deadline) {
|
|
2988
|
+
commits = await drainCommitBacklogForProject2(db, projectIdentity, deadline);
|
|
2989
|
+
commitsEmbedded += commits;
|
|
2990
|
+
}
|
|
2991
|
+
if (Date.now() < deadline) {
|
|
2992
|
+
chunks = await drainCompartmentChunkBacklogForProject(db, projectIdentity, deadline);
|
|
2993
|
+
chunksEmbedded += chunks;
|
|
2994
|
+
}
|
|
2995
|
+
perProject.set(projectIdentity, { memories, commits, chunks });
|
|
2996
|
+
if (Date.now() >= deadline)
|
|
2997
|
+
break;
|
|
2998
|
+
}
|
|
2999
|
+
} finally {
|
|
3000
|
+
projectSweepInProgress = false;
|
|
3001
|
+
}
|
|
3002
|
+
return { memoriesEmbedded, commitsEmbedded, chunksEmbedded, perProject };
|
|
3003
|
+
}
|
|
3004
|
+
function _setTestProviderFactoryForProject2(factory) {
|
|
3005
|
+
testProviderFactory = factory;
|
|
3006
|
+
}
|
|
3007
|
+
function _resetProjectEmbeddingRegistryForTests2() {
|
|
3008
|
+
for (const registration of projectRegistrations.values()) {
|
|
3009
|
+
disposeProvider(registration.provider);
|
|
3010
|
+
}
|
|
3011
|
+
for (const registration of shadowRegistrations.values()) {
|
|
3012
|
+
disposeProvider(registration.provider);
|
|
3013
|
+
}
|
|
3014
|
+
projectRegistrations.clear();
|
|
3015
|
+
shadowRegistrations.clear();
|
|
3016
|
+
shadowQueue.length = 0;
|
|
3017
|
+
pendingShadowBackfills.clear();
|
|
3018
|
+
shadowBackfillLastIds.clear();
|
|
3019
|
+
shadowBackfillStopReasons.clear();
|
|
3020
|
+
shadowBackfillLastWriteOutcomes.clear();
|
|
3021
|
+
dbForShadowQueue.clear();
|
|
3022
|
+
untrustedLoadProjects.clear();
|
|
3023
|
+
globalRegistrationGeneration = 0;
|
|
3024
|
+
projectSweepInProgress = false;
|
|
3025
|
+
testProviderFactory = null;
|
|
3026
|
+
shadowBackfillNow = () => Date.now();
|
|
3027
|
+
setBootQuietPeriodForTests(null);
|
|
3028
|
+
}
|
|
3029
|
+
|
|
3030
|
+
export { recordSessionProjectIdentity, hasRecordedSessionProjectIdentity, recordEmbeddingMeasurement, OpenAICompatibleEmbeddingProvider, drainBackgroundBatches, saveCommitEmbedding, loadProjectCommitEmbeddings, loadUnembeddedCommits, countEmbeddedCommits, upsertCommits, getLatestIndexedCommitSha, enforceProjectCap, GIT_SWEEP_LEASE_RENEWAL_MS, acquireGitSweepLease, renewGitSweepLease, markGitSweepSuccessAndRelease, parkGitSweepNonIndexable, releaseGitSweepLease, describeShadowBackfillWriteRefusal2, listShadowBackfillStalls2, getShadowBackfillStopReason2, markProjectLoadUntrusted2, contentSha2562, drainProjectEmbeddingIdentityMaintenance2, drainStaleEmbeddingIdentitiesForProject2, registerProjectEmbedding2, registerProjectShadowEmbedding2, getShadowEmbeddingMeasurementCohort2, getPrimaryEmbeddingMeasurementCohort2, embedShadowTextForProject2, enqueueShadowEmbeddingItems2, getShadowBackfillRemaining2, flushShadowEmbeddingBacklog2, registerProjectInObservationMode2, unregisterProjectShadowEmbedding2, unregisterProjectEmbedding2, getProjectEmbeddingSnapshot2, installProjectEmbeddingSearchBridge2, getProjectChunkEmbeddingModelId2, getProjectEmbeddingMaxInputTokens2, embedTextForProject2, embedBatchForProject2, embedItemsForProject2, embedUnembeddedMemoriesForProject2, embedUnembeddedMemoriesForProjectOutcome2, drainCommitBacklogForProject2, embedUnembeddedCompartmentChunksForProject2, embedSessionCompartmentChunks2, getEmbeddingCoverageStatus2, getAutoEmbeddingSessionCoverage2, sweepAllRegisteredProjects2, _setTestProviderFactoryForProject2, _resetProjectEmbeddingRegistryForTests2 };
|