@cortexkit/opencode-magic-context 0.49.18 → 0.49.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +53 -10
- package/dist/chunk-embedding-write-worker.js +30 -23
- package/dist/{compaction-marker-candidates-n4krhat4.js → compaction-marker-candidates-b2q30cft.js} +3 -3
- package/dist/compaction-marker-candidates-worker.js +2 -2
- package/dist/compartment-chunk-embedding-va090as0.js +65 -0
- package/dist/config/schema/magic-context.d.ts +6 -6
- package/dist/{conflict-warning-hook-4pmz879v.js → conflict-warning-hook-frp3z2wk.js} +9 -8
- package/dist/{context-m897gfp9.js → context-mkece1db.js} +460 -135
- package/dist/{derive-budgets-zn7yzd2d.js → derive-budgets-pqybnx09.js} +2 -2
- package/dist/embedding-backfill-gy3p78py.js +25 -0
- package/dist/embedding-worker-process.js +2 -2
- package/dist/embedding-worker.js +5 -5
- package/dist/features/magic-context/chunk-embedding-writer.d.ts.map +1 -1
- package/dist/features/magic-context/compartment-chunk-embedding.d.ts +44 -1
- package/dist/features/magic-context/compartment-chunk-embedding.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/classify.d.ts +3 -0
- package/dist/features/magic-context/dreamer/classify.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/hidden-single-shot.d.ts +11 -0
- package/dist/features/magic-context/dreamer/hidden-single-shot.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/map-memories.d.ts +3 -0
- package/dist/features/magic-context/dreamer/map-memories.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-config.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-scheduler.d.ts +2 -0
- package/dist/features/magic-context/dreamer/task-scheduler.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/verify.d.ts +3 -0
- package/dist/features/magic-context/dreamer/verify.d.ts.map +1 -1
- package/dist/features/magic-context/memory/embedding-identity.d.ts.map +1 -1
- package/dist/features/magic-context/memory/embedding-local.d.ts +1 -0
- package/dist/features/magic-context/memory/embedding-local.d.ts.map +1 -1
- package/dist/features/magic-context/memory/local-embedding-pooling.d.ts +22 -0
- package/dist/features/magic-context/memory/local-embedding-pooling.d.ts.map +1 -0
- package/dist/features/magic-context/message-index-async.d.ts.map +1 -1
- package/dist/features/magic-context/message-index.d.ts +2 -0
- package/dist/features/magic-context/message-index.d.ts.map +1 -1
- package/dist/features/magic-context/migrations.d.ts.map +1 -1
- package/dist/features/magic-context/mural/compress-cues.d.ts.map +1 -1
- package/dist/features/magic-context/project-embedding-registry.d.ts.map +1 -1
- package/dist/features/magic-context/search.d.ts +27 -0
- package/dist/features/magic-context/search.d.ts.map +1 -1
- package/dist/features/magic-context/smart-notes/compiler.d.ts.map +1 -1
- package/dist/features/magic-context/storage-clone.d.ts.map +1 -1
- package/dist/features/magic-context/storage-meta-persisted.d.ts +14 -0
- package/dist/features/magic-context/storage-meta-persisted.d.ts.map +1 -1
- package/dist/features/magic-context/subagent-token-capture.d.ts +20 -0
- package/dist/features/magic-context/subagent-token-capture.d.ts.map +1 -1
- package/dist/features/magic-context/user-memory/review-user-memories.d.ts.map +1 -1
- package/dist/{history-owner-xp3pngv7.js → history-owner-36wp3mkb.js} +2 -2
- package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-embed-session.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-hint.d.ts +7 -1
- package/dist/hooks/magic-context/auto-search-hint.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-runner.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts +33 -6
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-worker-evaluation.fixture.d.ts +2 -0
- package/dist/hooks/magic-context/auto-search-worker-evaluation.fixture.d.ts.map +1 -0
- package/dist/hooks/magic-context/cache-busting-signals.d.ts +5 -2
- package/dist/hooks/magic-context/cache-busting-signals.d.ts.map +1 -1
- package/dist/hooks/magic-context/channel2-delivery.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-historian.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-types.d.ts +5 -0
- package/dist/hooks/magic-context/compartment-runner-types.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
- package/dist/hooks/magic-context/diagnostic-log-state.d.ts +4 -0
- package/dist/hooks/magic-context/diagnostic-log-state.d.ts.map +1 -0
- package/dist/hooks/magic-context/execute-reclaim.d.ts +35 -0
- package/dist/hooks/magic-context/execute-reclaim.d.ts.map +1 -0
- package/dist/hooks/magic-context/hook-handlers.d.ts.map +1 -1
- package/dist/hooks/magic-context/hook.d.ts +1 -3
- package/dist/hooks/magic-context/hook.d.ts.map +1 -1
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts +8 -0
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts.map +1 -1
- package/dist/hooks/magic-context/lkg-recovery-fence.d.ts +5 -0
- package/dist/hooks/magic-context/lkg-recovery-fence.d.ts.map +1 -1
- package/dist/hooks/magic-context/lkg-recovery-prefix.d.ts +13 -0
- package/dist/hooks/magic-context/lkg-recovery-prefix.d.ts.map +1 -1
- package/dist/hooks/magic-context/nudge-evaluation-log.d.ts +3 -0
- package/dist/hooks/magic-context/nudge-evaluation-log.d.ts.map +1 -0
- package/dist/hooks/magic-context/pending-drop-eligibility.d.ts.map +1 -1
- package/dist/hooks/magic-context/protected-tail-boundary.d.ts +4 -0
- package/dist/hooks/magic-context/protected-tail-boundary.d.ts.map +1 -1
- package/dist/hooks/magic-context/raw-ordinal-warmup.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-date-refresh.d.ts +15 -2
- package/dist/hooks/magic-context/system-date-refresh.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-prompt-hash.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-prompt-sticky-date.d.ts +10 -0
- package/dist/hooks/magic-context/system-prompt-sticky-date.d.ts.map +1 -0
- package/dist/hooks/magic-context/tag-messages.d.ts +2 -0
- package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-compartment-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +23 -6
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform.d.ts.map +1 -1
- package/dist/{index-bjcnntmz.js → index-0wthn7fp.js} +468 -18
- package/dist/{index-kq82mtg0.js → index-11zf2wwc.js} +3 -3
- package/dist/{index-b5vqckbw.js → index-153r2q6z.js} +314 -127
- package/dist/{index-v8awdgvd.js → index-1pefp8ar.js} +5 -5
- package/dist/{v2/server-beht37jy.js → index-1zx2pa5d.js} +1 -1
- package/dist/{index-ze7mvk54.js → index-2yhtnfcf.js} +75 -57
- package/dist/{index-cf74k5ak.js → index-305y133n.js} +4 -4
- package/dist/{index-s734jcsr.js → index-35qc6606.js} +9 -3
- package/dist/{index-x4njr8df.js → index-48stjd56.js} +105 -19
- package/dist/{index-24jbw52d.js → index-4n7w9rad.js} +6 -6
- package/dist/{index-kee4kn5j.js → index-4pnsgvfk.js} +1 -1
- package/dist/{v2/server-8cd2j99z.js → index-5kym00ag.js} +32 -13
- package/dist/{index-4s3t0nd0.js → index-616rsjrv.js} +2 -2
- package/dist/{index-e4xv8890.js → index-67nyj0gy.js} +507 -210
- package/dist/{index-6x57ar45.js → index-adk2z5d8.js} +3 -3
- package/dist/{index-c5pytzne.js → index-b88dwwrh.js} +1 -1
- package/dist/{index-rr2bb45z.js → index-c4b76d4t.js} +2 -2
- package/dist/{index-nj7yykkx.js → index-c7phnbvk.js} +1 -1
- package/dist/index-cpjk5jd2.js +82 -0
- package/dist/{index-6s5941tm.js → index-d6rp4dxh.js} +5 -5
- package/dist/{index-s7j8tvhv.js → index-dc8hy95c.js} +69 -17
- package/dist/{index-j75v2476.js → index-h7tgn49j.js} +12 -15
- package/dist/{index-ke416mdc.js → index-j76ah6d7.js} +2 -2
- package/dist/{index-r4nkrs05.js → index-jfh3cktr.js} +4 -4
- package/dist/{index-a6mc0v4d.js → index-jmycsey7.js} +3 -3
- package/dist/{index-24fezq97.js → index-jr5na10t.js} +4 -4
- package/dist/{index-s74zqahq.js → index-kc9pn495.js} +4 -4
- package/dist/{index-p5y3k4g7.js → index-nda0zgna.js} +2 -1
- package/dist/{index-jn12fvvh.js → index-neb2jaed.js} +3 -3
- package/dist/{index-qn7xstt2.js → index-pnjwa6qa.js} +6 -7
- package/dist/{index-fgncb0ng.js → index-rmd0069m.js} +11 -11
- package/dist/{index-y9f76s3f.js → index-rrv26qjz.js} +1 -1
- package/dist/{index-v10f1rjv.js → index-spwjbkc7.js} +1 -1
- package/dist/{index-k29z4aqh.js → index-t5pck3ss.js} +1 -1
- package/dist/{index-936py2wm.js → index-t7hx3qcb.js} +361 -113
- package/dist/{index-s1ex9y6p.js → index-v7r4td4c.js} +464 -79
- package/dist/{index-2vxtcb95.js → index-vkc1ca9p.js} +2 -2
- package/dist/{index-rmwykbbb.js → index-x4k3f4mf.js} +33 -10
- package/dist/{index-bp5083x2.js → index-x9k3mqrf.js} +1 -1
- package/dist/{index-n37sebck.js → index-y4a3nhtk.js} +1 -1
- package/dist/index.js +57 -50
- package/dist/{logger-jm3sqqzr.js → logger-94kew5wm.js} +1 -1
- package/dist/{message-fts-rowid-map-17s0dfn4.js → message-fts-rowid-map-22tevnnj.js} +6 -5
- package/dist/{message-index-66jv6fh4.js → message-index-5qxcz6h7.js} +16 -15
- package/dist/{message-index-bounded-n1frdn85.js → message-index-bounded-n0m0t7tv.js} +16 -15
- package/dist/{message-time-backfill-73mmm76q.js → message-time-backfill-0myx7fz3.js} +3 -3
- package/dist/migration-worker.js +13 -12
- package/dist/{model-resolution-nx97z3x7.js → model-resolution-280gmqvd.js} +1 -1
- package/dist/plugin/embedding-bootstrap.d.ts +3 -1
- package/dist/plugin/embedding-bootstrap.d.ts.map +1 -1
- package/dist/plugin/hooks/create-session-hooks.d.ts.map +1 -1
- package/dist/plugin/messages-transform.d.ts.map +1 -1
- package/dist/plugin/rpc-handlers.d.ts +8 -0
- package/dist/plugin/rpc-handlers.d.ts.map +1 -1
- package/dist/plugin/subagent-tool-policy.d.ts +9 -0
- package/dist/plugin/subagent-tool-policy.d.ts.map +1 -1
- package/dist/plugin/tool-registry.d.ts.map +1 -1
- package/dist/{project-embedding-registry-z74mhqkf.js → project-embedding-registry-s2pqgec9.js} +13 -12
- package/dist/raw-ordinal-warmup-worker.js +136 -101
- package/dist/{read-session-chunk-v06g3vka.js → read-session-chunk-0a8vdyqp.js} +11 -10
- package/dist/{read-session-raw-rsc8c7qy.js → read-session-raw-y5cszf0b.js} +4 -3
- package/dist/recomp-orchestrator-61vyha9j.js +62 -0
- package/dist/{safe-notification-target-1szv1k5k.js → safe-notification-target-mkr9yzvr.js} +2 -2
- package/dist/{search-archive-4d53y23e.js → search-archive-8q9s49p5.js} +12 -11
- package/dist/{search-nkf271d4.js → search-v1y9ejws.js} +25 -24
- package/dist/{send-session-notification-6ve0e8p4.js → send-session-notification-0k6qzydk.js} +7 -6
- package/dist/shared/logger.d.ts.map +1 -1
- package/dist/shared/model-resolution.d.ts +2 -0
- package/dist/shared/model-resolution.d.ts.map +1 -1
- package/dist/shared/model-suggestion-retry.d.ts +30 -1
- package/dist/shared/model-suggestion-retry.d.ts.map +1 -1
- package/dist/shared/prompt-async-transport.d.ts +3 -0
- package/dist/shared/prompt-async-transport.d.ts.map +1 -1
- package/dist/shared/rpc-types.d.ts +7 -0
- package/dist/shared/rpc-types.d.ts.map +1 -1
- package/dist/shared/sqlite.d.ts +1 -1
- package/dist/shared/sqlite.d.ts.map +1 -1
- package/dist/shared/subagent-runner.d.ts +2 -0
- package/dist/shared/subagent-runner.d.ts.map +1 -1
- package/dist/shared/worker-shutdown.d.ts +68 -0
- package/dist/shared/worker-shutdown.d.ts.map +1 -0
- package/dist/shared/worker-shutdown.fixture.d.ts +2 -0
- package/dist/shared/worker-shutdown.fixture.d.ts.map +1 -0
- package/dist/{sqlite-zf72c52d.js → sqlite-1rpvw2rj.js} +1 -1
- package/dist/{storage-memory-y3qmy00r.js → storage-memory-nm7v82mj.js} +2 -2
- package/dist/{storage-tags-r7k9tjcv.js → storage-tags-r6nk29n4.js} +5 -5
- package/dist/{task-config-x3qmayd2.js → task-config-t6bpyka7.js} +2 -2
- package/dist/tools/ctx-expand/rust-mode-tag.test-support.d.ts +15 -0
- package/dist/tools/ctx-expand/rust-mode-tag.test-support.d.ts.map +1 -0
- package/dist/tools/ctx-expand/tools.d.ts +11 -0
- package/dist/tools/ctx-expand/tools.d.ts.map +1 -1
- package/dist/tools/ctx-search/tools.d.ts.map +1 -1
- package/dist/{transform-postprocess-phase-zpgav0h7.js → transform-postprocess-phase-vtrgb7m6.js} +34 -31
- package/dist/v2/auto-search-worker.js +53 -10
- package/dist/v2/chunk-embedding-write-worker.js +30 -23
- package/dist/v2/{compaction-marker-candidates-ebx3388k.js → compaction-marker-candidates-cfab70cs.js} +3 -3
- package/dist/v2/compaction-marker-candidates-worker.js +2 -2
- package/dist/v2/compartment-chunk-embedding-nbdqx1td.js +64 -0
- package/dist/v2/{context-f2xh7ba4.js → context-cc33tv2z.js} +1259 -404
- package/dist/v2/{derive-budgets-2zh4gm2c.js → derive-budgets-knf763tt.js} +3 -3
- package/dist/v2/embedding-backfill-yxv2h5mc.js +24 -0
- package/dist/v2/embedding-worker-process.js +2 -2
- package/dist/v2/embedding-worker.js +5 -5
- package/dist/v2/hidden-child-record.d.ts +2 -0
- package/dist/v2/hidden-child-record.d.ts.map +1 -1
- package/dist/v2/hidden-completion.d.ts +2 -0
- package/dist/v2/hidden-completion.d.ts.map +1 -1
- package/dist/v2/hidden-model-activity.d.ts +7 -0
- package/dist/v2/hidden-model-activity.d.ts.map +1 -0
- package/dist/v2/{history-owner-42e16w0n.js → history-owner-ad0crnrn.js} +2 -2
- package/dist/v2/hooks/context.d.ts.map +1 -1
- package/dist/v2/hooks/generate.d.ts +36 -1
- package/dist/v2/hooks/generate.d.ts.map +1 -1
- package/dist/v2/hooks/lkg-system.d.ts +35 -4
- package/dist/v2/hooks/lkg-system.d.ts.map +1 -1
- package/dist/v2/hooks/payload.d.ts +16 -1
- package/dist/v2/hooks/payload.d.ts.map +1 -1
- package/dist/v2/hooks/tools.d.ts.map +1 -1
- package/dist/v2/{logger-7qt02bnb.js → logger-p34mjtja.js} +1 -1
- package/dist/v2/{message-fts-rowid-map-wfx67gfd.js → message-fts-rowid-map-dg12wymk.js} +6 -5
- package/dist/v2/{message-index-bounded-w3hfm29z.js → message-index-bounded-f4e01xk0.js} +16 -15
- package/dist/v2/{message-index-48f7hw7a.js → message-index-ydzrp6y6.js} +16 -15
- package/dist/v2/{message-time-backfill-04t0km1z.js → message-time-backfill-ddh8n9gy.js} +3 -3
- package/dist/v2/migration-worker.js +13 -12
- package/dist/v2/{model-resolution-nx97z3x7.js → model-resolution-280gmqvd.js} +1 -1
- package/dist/v2/{project-embedding-registry-gnkj56xp.js → project-embedding-registry-deza9ksw.js} +13 -12
- package/dist/v2/raw-ordinal-warmup-worker.js +136 -101
- package/dist/v2/{read-session-chunk-zpgmxwte.js → read-session-chunk-8cxfagky.js} +11 -10
- package/dist/v2/{read-session-raw-yaegtjzh.js → read-session-raw-1nrrecvx.js} +4 -3
- package/dist/v2/recomp-orchestrator-47m4hv5g.js +61 -0
- package/dist/v2/{safe-notification-target-qg7n2s77.js → safe-notification-target-khnzg57c.js} +2 -2
- package/dist/v2/{search-archive-vv48s2aa.js → search-archive-e80cvc09.js} +12 -11
- package/dist/v2/{search-kza414f0.js → search-fjz746zr.js} +25 -24
- package/dist/v2/{send-session-notification-3pxn658x.js → send-session-notification-nwvrv34d.js} +7 -6
- package/dist/v2/{server-8by8qfec.js → server-0epqaxhd.js} +75 -57
- package/dist/v2/{server-pwv4ezb0.js → server-168pb207.js} +1 -1
- package/dist/v2/{server-h0410h0v.js → server-1bw1wcbp.js} +1 -1
- package/dist/{index-beht37jy.js → v2/server-1zx2pa5d.js} +1 -1
- package/dist/v2/{server-a9xqj88f.js → server-230fnbws.js} +4 -4
- package/dist/v2/{server-tbe1femw.js → server-29qp8nz4.js} +1 -1
- package/dist/v2/{server-tgnbmwah.js → server-3mz2qh5a.js} +1 -1
- package/dist/v2/{server-hy3d3vsx.js → server-4nw4fqhr.js} +2 -2
- package/dist/v2/{server-qv7f0hhe.js → server-4sxjs6hn.js} +5 -5
- package/dist/v2/{server-34kwfsr4.js → server-53eh1r8r.js} +2 -2
- package/dist/v2/{server-xj136kt8.js → server-55rgxfrw.js} +9 -3
- package/dist/v2/{server-nj659894.js → server-5arr9ecn.js} +2 -2
- package/dist/{index-8cd2j99z.js → v2/server-5kym00ag.js} +32 -13
- package/dist/v2/{server-2ehj0bw7.js → server-63fqdghs.js} +106 -20
- package/dist/v2/{server-f2xf5gcb.js → server-6495nmn9.js} +12 -15
- package/dist/v2/{server-qd11g795.js → server-92g7d776.js} +2 -1
- package/dist/v2/{server-gpvay0xz.js → server-9x0kmta4.js} +6 -7
- package/dist/v2/{server-27gq4mfz.js → server-a3dwxfgd.js} +4 -4
- package/dist/v2/{server-6tmj0s7j.js → server-af2pght7.js} +3 -3
- package/dist/v2/{server-1wy7b9de.js → server-br1x0tev.js} +3 -3
- package/dist/v2/{server-n1prf1g6.js → server-brj8j2gk.js} +2 -2
- package/dist/v2/{server-bfpda1tb.js → server-c7dxxsj7.js} +466 -81
- package/dist/v2/server-cpjk5jd2.js +82 -0
- package/dist/v2/{server-g3qma5c0.js → server-dj7tz4ph.js} +2 -2
- package/dist/v2/{server-xhab8jx8.js → server-ej96a39v.js} +33 -10
- package/dist/v2/{server-6pmqg9m4.js → server-gyy4k6mf.js} +1 -1
- package/dist/v2/{server-a2eqhkxr.js → server-hqzyq8n4.js} +11 -11
- package/dist/v2/{server-h1v0pyxb.js → server-j4jnp9b5.js} +468 -18
- package/dist/v2/{server-da07v4xp.js → server-k3t06fxr.js} +1 -1
- package/dist/v2/{server-1nyybqan.js → server-m91f9m5s.js} +69 -17
- package/dist/v2/{server-rmygf5pn.js → server-ne8arh8t.js} +2 -2
- package/dist/v2/{server-my7k4eds.js → server-vn2nwcv6.js} +5 -5
- package/dist/v2/{server-46neh5fa.js → server-wyawsa88.js} +314 -127
- package/dist/v2/{server-zzkj2fch.js → server-xsryvyf6.js} +4 -4
- package/dist/v2/{server-n37sebck.js → server-y4a3nhtk.js} +1 -1
- package/dist/v2/{server-d04vrx0j.js → server-ygtykank.js} +2 -2
- package/dist/v2/server.js +6 -6
- package/dist/v2/{sqlite-zf72c52d.js → sqlite-1rpvw2rj.js} +1 -1
- package/dist/v2/{storage-memory-y3qmy00r.js → storage-memory-nm7v82mj.js} +2 -2
- package/dist/v2/{storage-tags-3wdb8zph.js → storage-tags-ss08kj5r.js} +5 -5
- package/dist/v2/store-reader.d.ts +8 -0
- package/dist/v2/store-reader.d.ts.map +1 -1
- package/dist/v2/{task-config-nvz8mapa.js → task-config-7md27rx8.js} +2 -2
- package/dist/v2/{wrapup-orchestrator-psqz7yh0.js → wrapup-orchestrator-639whfhq.js} +32 -31
- package/dist/wrapup-orchestrator-mnvt2g8m.js +49 -0
- package/package.json +1 -1
- package/src/config/index.test.ts +28 -6
- package/src/shared/logger.test.ts +42 -1
- package/src/shared/logger.ts +11 -13
- package/src/shared/model-resolution.test.ts +67 -12
- package/src/shared/model-resolution.ts +43 -16
- package/src/shared/model-suggestion-retry.test.ts +166 -0
- package/src/shared/model-suggestion-retry.ts +165 -14
- package/src/shared/prompt-async-transport.test.ts +107 -0
- package/src/shared/prompt-async-transport.ts +86 -3
- package/src/shared/rpc-types.ts +7 -0
- package/src/shared/sqlite-busy-budget-review.test.ts +6 -1
- package/src/shared/sqlite.ts +4 -2
- package/src/shared/subagent-runner.ts +2 -0
- package/src/shared/worker-shutdown-fence.test.ts +87 -0
- package/src/shared/worker-shutdown.fixture.ts +25 -0
- package/src/shared/worker-shutdown.test.ts +127 -0
- package/src/shared/worker-shutdown.ts +160 -0
- package/dist/embedding-backfill-fmz50tsy.js +0 -24
- package/dist/recomp-orchestrator-ghpk2dtq.js +0 -61
- package/dist/v2/embedding-backfill-7c7yr69y.js +0 -23
- package/dist/v2/recomp-orchestrator-4vfxej9t.js +0 -60
- package/dist/wrapup-orchestrator-s9zc5b00.js +0 -48
|
@@ -8,13 +8,13 @@ import {
|
|
|
8
8
|
getMagicContextStorageDir,
|
|
9
9
|
importPluginModule,
|
|
10
10
|
log2
|
|
11
|
-
} from "./server-
|
|
11
|
+
} from "./server-6495nmn9.js";
|
|
12
12
|
import {
|
|
13
13
|
getSynapseLaneIdentity
|
|
14
|
-
} from "./server-
|
|
14
|
+
} from "./server-af2pght7.js";
|
|
15
15
|
import {
|
|
16
16
|
inRootAsyncContext2
|
|
17
|
-
} from "./server-
|
|
17
|
+
} from "./server-1zx2pa5d.js";
|
|
18
18
|
import {
|
|
19
19
|
computeNormalizedHash
|
|
20
20
|
} from "./server-2bvekkfy.js";
|
|
@@ -27,9 +27,9 @@ import {
|
|
|
27
27
|
} from "./server-fvnzcnww.js";
|
|
28
28
|
|
|
29
29
|
// src/features/magic-context/memory/embedding-local.ts
|
|
30
|
-
import { readdirSync, statSync } from "node:fs";
|
|
30
|
+
import { readdirSync, statSync as statSync2 } from "node:fs";
|
|
31
31
|
import { stat, unlink } from "node:fs/promises";
|
|
32
|
-
import { dirname, join } from "node:path";
|
|
32
|
+
import { dirname, join as join2 } from "node:path";
|
|
33
33
|
import { pathToFileURL } from "node:url";
|
|
34
34
|
import { workerData } from "node:worker_threads";
|
|
35
35
|
|
|
@@ -200,6 +200,432 @@ function resolveEmbeddingTextPrefixes(model, queryInstruction, documentPrefix) {
|
|
|
200
200
|
};
|
|
201
201
|
}
|
|
202
202
|
|
|
203
|
+
// src/features/magic-context/memory/local-embedding-pooling.ts
|
|
204
|
+
import { createHash } from "node:crypto";
|
|
205
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
206
|
+
import { join } from "node:path";
|
|
207
|
+
var decisionLogged = new Set;
|
|
208
|
+
var unavailableLogged = new Set;
|
|
209
|
+
var metadataByCache = new Map;
|
|
210
|
+
var testMetadataByModel = new Map;
|
|
211
|
+
var testMetadataFallback;
|
|
212
|
+
var UNRESOLVED_LOCAL_EMBEDDING_POOLING = {
|
|
213
|
+
status: "unresolved",
|
|
214
|
+
pooling: "mean",
|
|
215
|
+
normalize: true,
|
|
216
|
+
declared: false,
|
|
217
|
+
reason: "pooling metadata is unavailable; preserving legacy mean pooling and normalization"
|
|
218
|
+
};
|
|
219
|
+
var LEGACY_MEAN_POOLING = {
|
|
220
|
+
status: "ready",
|
|
221
|
+
pooling: "mean",
|
|
222
|
+
normalize: true,
|
|
223
|
+
declared: false,
|
|
224
|
+
reason: "no sentence-transformers Pooling module is declared; preserving legacy mean pooling and normalization"
|
|
225
|
+
};
|
|
226
|
+
function defaultModelCacheDir() {
|
|
227
|
+
return join(getMagicContextStorageDir(), "models");
|
|
228
|
+
}
|
|
229
|
+
function metadataDirectory(model, modelCacheDir) {
|
|
230
|
+
const digest = createHash("sha256").update(model).digest("hex");
|
|
231
|
+
return join(modelCacheDir, ".pooling-metadata", digest);
|
|
232
|
+
}
|
|
233
|
+
function parseJson(text, label) {
|
|
234
|
+
try {
|
|
235
|
+
return JSON.parse(text);
|
|
236
|
+
} catch (error) {
|
|
237
|
+
throw new Error(`${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
function isRecord(value) {
|
|
241
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
242
|
+
}
|
|
243
|
+
function isModelDirectory(model) {
|
|
244
|
+
try {
|
|
245
|
+
return statSync(model).isDirectory();
|
|
246
|
+
} catch {
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function isPoolingModule(value) {
|
|
251
|
+
if (!isRecord(value))
|
|
252
|
+
return false;
|
|
253
|
+
return typeof value.type === "string" && /(?:^|\.)Pooling$/.test(value.type) || typeof value.name === "string" && value.name.toLowerCase() === "pooling";
|
|
254
|
+
}
|
|
255
|
+
function isNormalizeModule(value) {
|
|
256
|
+
if (!isRecord(value))
|
|
257
|
+
return false;
|
|
258
|
+
return typeof value.type === "string" && /(?:^|\.)Normalize$/.test(value.type) || typeof value.name === "string" && value.name.toLowerCase() === "normalize";
|
|
259
|
+
}
|
|
260
|
+
function safeModulePath(value) {
|
|
261
|
+
if (typeof value !== "string" || !value.trim())
|
|
262
|
+
return null;
|
|
263
|
+
const pieces = value.replaceAll("\\", "/").split("/");
|
|
264
|
+
if (pieces.some((piece) => piece === ".." || piece === "."))
|
|
265
|
+
return null;
|
|
266
|
+
return pieces.filter(Boolean).join("/");
|
|
267
|
+
}
|
|
268
|
+
function poolingFromFiles(modulesText, configText, label) {
|
|
269
|
+
if (modulesText === null)
|
|
270
|
+
return LEGACY_MEAN_POOLING;
|
|
271
|
+
const modulesValue = parseJson(modulesText, `${label}/modules.json`);
|
|
272
|
+
if (!Array.isArray(modulesValue)) {
|
|
273
|
+
return {
|
|
274
|
+
status: "unsupported",
|
|
275
|
+
normalize: false,
|
|
276
|
+
declared: true,
|
|
277
|
+
reason: "modules.json is not a module list"
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
const poolingModule = modulesValue.find(isPoolingModule);
|
|
281
|
+
if (!poolingModule)
|
|
282
|
+
return LEGACY_MEAN_POOLING;
|
|
283
|
+
const modulePath = safeModulePath(poolingModule.path);
|
|
284
|
+
if (modulePath === null || configText === null) {
|
|
285
|
+
return {
|
|
286
|
+
status: "unsupported",
|
|
287
|
+
normalize: modulesValue.some(isNormalizeModule),
|
|
288
|
+
declared: true,
|
|
289
|
+
reason: `Pooling module at ${String(poolingModule.path ?? "<missing path>")} has no readable config.json`
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
const config = parseJson(configText, `${label}/${modulePath}/config.json`);
|
|
293
|
+
if (!isRecord(config)) {
|
|
294
|
+
return {
|
|
295
|
+
status: "unsupported",
|
|
296
|
+
normalize: modulesValue.some(isNormalizeModule),
|
|
297
|
+
declared: true,
|
|
298
|
+
reason: `Pooling config at ${modulePath}/config.json is not an object`
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
const enabledModes = [
|
|
302
|
+
["pooling_mode_cls_token", "cls"],
|
|
303
|
+
["pooling_mode_mean_tokens", "mean"],
|
|
304
|
+
["pooling_mode_lasttoken", "last_token"],
|
|
305
|
+
["pooling_mode_max_tokens", undefined],
|
|
306
|
+
["pooling_mode_mean_sqrt_len_tokens", undefined],
|
|
307
|
+
["pooling_mode_weightedmean_tokens", undefined],
|
|
308
|
+
["pooling_mode_mean_last_token", undefined]
|
|
309
|
+
].filter(([key]) => config[key] === true);
|
|
310
|
+
const normalize = modulesValue.some(isNormalizeModule);
|
|
311
|
+
if (enabledModes.length !== 1) {
|
|
312
|
+
return {
|
|
313
|
+
status: "unsupported",
|
|
314
|
+
normalize,
|
|
315
|
+
declared: true,
|
|
316
|
+
reason: enabledModes.length === 0 ? `Pooling config at ${modulePath}/config.json enables no pooling mode` : `Pooling config enables multiple modes: ${enabledModes.map(([key]) => key).join(", ")}`
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
const [flag, method] = enabledModes[0];
|
|
320
|
+
if (!method) {
|
|
321
|
+
return {
|
|
322
|
+
status: "unsupported",
|
|
323
|
+
normalize,
|
|
324
|
+
declared: true,
|
|
325
|
+
reason: `unsupported declared pooling mode ${flag}`
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
status: "ready",
|
|
330
|
+
pooling: method,
|
|
331
|
+
normalize,
|
|
332
|
+
declared: true,
|
|
333
|
+
reason: `modules.json declares ${flag}; ${normalize ? "a Normalize module is present" : "no Normalize module is declared"}`
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
function poolingMetadataCacheKey(model, modelCacheDir) {
|
|
337
|
+
return JSON.stringify([model, modelCacheDir]);
|
|
338
|
+
}
|
|
339
|
+
function cachedMetadataPath(model, modelCacheDir) {
|
|
340
|
+
return join(metadataDirectory(model, modelCacheDir), "resolution.json");
|
|
341
|
+
}
|
|
342
|
+
function parseCachedMetadata(text) {
|
|
343
|
+
try {
|
|
344
|
+
const value = JSON.parse(text);
|
|
345
|
+
if (!isRecord(value) || value.status !== "ready" && value.status !== "unsupported")
|
|
346
|
+
return null;
|
|
347
|
+
if (typeof value.normalize !== "boolean" || typeof value.declared !== "boolean" || typeof value.reason !== "string")
|
|
348
|
+
return null;
|
|
349
|
+
if (value.status === "ready" && value.pooling !== "mean" && value.pooling !== "cls" && value.pooling !== "last_token")
|
|
350
|
+
return null;
|
|
351
|
+
return value;
|
|
352
|
+
} catch {
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
function readDirectoryMetadata(model) {
|
|
357
|
+
const modulesPath = join(model, "modules.json");
|
|
358
|
+
if (!existsSync(modulesPath))
|
|
359
|
+
return LEGACY_MEAN_POOLING;
|
|
360
|
+
const modulesText = readFileSync(modulesPath, "utf8");
|
|
361
|
+
const moduleList = parseJson(modulesText, `${model}/modules.json`);
|
|
362
|
+
if (!Array.isArray(moduleList)) {
|
|
363
|
+
return {
|
|
364
|
+
status: "unsupported",
|
|
365
|
+
normalize: false,
|
|
366
|
+
declared: true,
|
|
367
|
+
reason: "modules.json is not a module list"
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
const poolingModule = moduleList.find(isPoolingModule);
|
|
371
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
372
|
+
const configText = modulePath ? readFileSync(join(model, modulePath, "config.json"), "utf8") : null;
|
|
373
|
+
return poolingFromFiles(modulesText, configText, model);
|
|
374
|
+
}
|
|
375
|
+
function getCachedLocalEmbeddingPooling(model, modelCacheDir = defaultModelCacheDir()) {
|
|
376
|
+
if (testMetadataByModel.has(model))
|
|
377
|
+
return testMetadataByModel.get(model) ?? null;
|
|
378
|
+
const cacheKey = poolingMetadataCacheKey(model, modelCacheDir);
|
|
379
|
+
const cachedInMemory = metadataByCache.get(cacheKey);
|
|
380
|
+
if (cachedInMemory)
|
|
381
|
+
return cachedInMemory;
|
|
382
|
+
if (isModelDirectory(model)) {
|
|
383
|
+
try {
|
|
384
|
+
const metadata = readDirectoryMetadata(model);
|
|
385
|
+
metadataByCache.set(cacheKey, metadata);
|
|
386
|
+
return metadata;
|
|
387
|
+
} catch (error) {
|
|
388
|
+
const metadata = {
|
|
389
|
+
status: "unsupported",
|
|
390
|
+
normalize: false,
|
|
391
|
+
declared: true,
|
|
392
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
393
|
+
};
|
|
394
|
+
metadataByCache.set(cacheKey, metadata);
|
|
395
|
+
return metadata;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
try {
|
|
399
|
+
const cached = parseCachedMetadata(readFileSync(cachedMetadataPath(model, modelCacheDir), "utf8"));
|
|
400
|
+
if (cached) {
|
|
401
|
+
metadataByCache.set(cacheKey, cached);
|
|
402
|
+
return cached;
|
|
403
|
+
}
|
|
404
|
+
} catch {}
|
|
405
|
+
if (testMetadataFallback) {
|
|
406
|
+
metadataByCache.set(cacheKey, testMetadataFallback);
|
|
407
|
+
return testMetadataFallback;
|
|
408
|
+
}
|
|
409
|
+
metadataByCache.set(cacheKey, UNRESOLVED_LOCAL_EMBEDDING_POOLING);
|
|
410
|
+
return UNRESOLVED_LOCAL_EMBEDDING_POOLING;
|
|
411
|
+
}
|
|
412
|
+
function logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger = log2) {
|
|
413
|
+
const decisionKey = `${model}:${metadata.status}:${metadata.pooling ?? "none"}:${metadata.normalize}`;
|
|
414
|
+
if (decisionLogged.has(decisionKey))
|
|
415
|
+
return;
|
|
416
|
+
decisionLogged.add(decisionKey);
|
|
417
|
+
if (metadata.status === "unsupported") {
|
|
418
|
+
logger(`[magic-context] local embedding pooling for ${model} is unsupported: ${metadata.reason}`);
|
|
419
|
+
} else {
|
|
420
|
+
logger(`[magic-context] local embedding pooling for ${model}: ${metadata.pooling} pooling, normalization ${metadata.normalize ? "enabled" : "disabled"} (${metadata.reason})`);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
function logUnavailableOnce(model, reason, logger) {
|
|
424
|
+
if (unavailableLogged.has(model))
|
|
425
|
+
return;
|
|
426
|
+
unavailableLogged.add(model);
|
|
427
|
+
logger(`[magic-context] local embedding pooling metadata for ${model} is not available: ${reason}; using legacy mean pooling and retrying during a later background bootstrap`);
|
|
428
|
+
}
|
|
429
|
+
function remoteHost() {
|
|
430
|
+
const configured = process.env.HF_ENDPOINT?.trim().replace(/\/+$/, "");
|
|
431
|
+
return configured || "https://huggingface.co";
|
|
432
|
+
}
|
|
433
|
+
function safeRepoId(repo) {
|
|
434
|
+
return repo.length > 0 && repo.split("/").every((part) => part && part !== "." && part !== "..");
|
|
435
|
+
}
|
|
436
|
+
function modelFileUrl(host, repo, file) {
|
|
437
|
+
const path = repo.split("/").map(encodeURIComponent).join("/");
|
|
438
|
+
const filePath = file.split("/").map(encodeURIComponent).join("/");
|
|
439
|
+
return `${host}/${path}/resolve/main/${filePath}`;
|
|
440
|
+
}
|
|
441
|
+
async function fetchModelFile(host, repo, file, fetcher) {
|
|
442
|
+
const response = await fetcher(modelFileUrl(host, repo, file));
|
|
443
|
+
if (response.status === 404)
|
|
444
|
+
return { kind: "missing" };
|
|
445
|
+
if (!response.ok)
|
|
446
|
+
throw new Error(`HTTP ${response.status} fetching ${repo}/${file}`);
|
|
447
|
+
return { kind: "found", text: await response.text() };
|
|
448
|
+
}
|
|
449
|
+
async function filesForRepo(host, repo, fetcher) {
|
|
450
|
+
const modules = await fetchModelFile(host, repo, "modules.json", fetcher);
|
|
451
|
+
if (modules.kind === "missing") {
|
|
452
|
+
const rootConfigResult = await fetchModelFile(host, repo, "config.json", fetcher);
|
|
453
|
+
if (rootConfigResult.kind === "missing") {
|
|
454
|
+
return {
|
|
455
|
+
modulesText: null,
|
|
456
|
+
configText: null,
|
|
457
|
+
configPath: null,
|
|
458
|
+
rootConfig: null,
|
|
459
|
+
rootModules: null
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
const rootConfigValue = parseJson(rootConfigResult.text, `${repo}/config.json`);
|
|
463
|
+
const origin = isRecord(rootConfigValue) && typeof rootConfigValue._name_or_path === "string" ? rootConfigValue._name_or_path.trim() : "";
|
|
464
|
+
if (!origin || origin === repo || !safeRepoId(origin)) {
|
|
465
|
+
return {
|
|
466
|
+
modulesText: null,
|
|
467
|
+
configText: null,
|
|
468
|
+
configPath: null,
|
|
469
|
+
rootConfig: rootConfigResult.text,
|
|
470
|
+
rootModules: null
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
const originModules = await fetchModelFile(host, origin, "modules.json", fetcher);
|
|
474
|
+
if (originModules.kind === "missing") {
|
|
475
|
+
return {
|
|
476
|
+
modulesText: null,
|
|
477
|
+
configText: null,
|
|
478
|
+
configPath: null,
|
|
479
|
+
rootConfig: rootConfigResult.text,
|
|
480
|
+
rootModules: null
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
const originModuleList = parseJson(originModules.text, `${origin}/modules.json`);
|
|
484
|
+
if (!Array.isArray(originModuleList)) {
|
|
485
|
+
return {
|
|
486
|
+
modulesText: originModules.text,
|
|
487
|
+
configText: null,
|
|
488
|
+
configPath: null,
|
|
489
|
+
rootConfig: rootConfigResult.text,
|
|
490
|
+
rootModules: originModules.text
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
const poolingModule = originModuleList.find(isPoolingModule);
|
|
494
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
495
|
+
if (!modulePath) {
|
|
496
|
+
return {
|
|
497
|
+
modulesText: originModules.text,
|
|
498
|
+
configText: null,
|
|
499
|
+
configPath: null,
|
|
500
|
+
rootConfig: rootConfigResult.text,
|
|
501
|
+
rootModules: originModules.text
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
const poolingConfig = await fetchModelFile(host, origin, `${modulePath}/config.json`, fetcher);
|
|
505
|
+
if (poolingConfig.kind === "missing") {
|
|
506
|
+
return {
|
|
507
|
+
modulesText: originModules.text,
|
|
508
|
+
configText: null,
|
|
509
|
+
configPath: `${origin}/${modulePath}/config.json`,
|
|
510
|
+
rootConfig: rootConfigResult.text,
|
|
511
|
+
rootModules: originModules.text
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
return {
|
|
515
|
+
modulesText: originModules.text,
|
|
516
|
+
configText: poolingConfig.text,
|
|
517
|
+
configPath: `${origin}/${modulePath}/config.json`,
|
|
518
|
+
rootConfig: rootConfigResult.text,
|
|
519
|
+
rootModules: originModules.text
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
const modulesValue = parseJson(modules.text, `${repo}/modules.json`);
|
|
523
|
+
if (!Array.isArray(modulesValue)) {
|
|
524
|
+
return {
|
|
525
|
+
modulesText: modules.text,
|
|
526
|
+
configText: null,
|
|
527
|
+
configPath: null,
|
|
528
|
+
rootConfig: null,
|
|
529
|
+
rootModules: modules.text
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
const poolingModule = modulesValue.find(isPoolingModule);
|
|
533
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
534
|
+
if (!modulePath) {
|
|
535
|
+
return {
|
|
536
|
+
modulesText: modules.text,
|
|
537
|
+
configText: null,
|
|
538
|
+
configPath: null,
|
|
539
|
+
rootConfig: null,
|
|
540
|
+
rootModules: modules.text
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
const poolingConfig = await fetchModelFile(host, repo, `${modulePath}/config.json`, fetcher);
|
|
544
|
+
return {
|
|
545
|
+
modulesText: modules.text,
|
|
546
|
+
configText: poolingConfig.kind === "found" ? poolingConfig.text : null,
|
|
547
|
+
configPath: `${repo}/${modulePath}/config.json`,
|
|
548
|
+
rootConfig: null,
|
|
549
|
+
rootModules: modules.text
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
async function cacheRemoteFiles(model, modelCacheDir, files) {
|
|
553
|
+
const root = metadataDirectory(model, modelCacheDir);
|
|
554
|
+
const writes = [];
|
|
555
|
+
if (files.rootConfig !== null)
|
|
556
|
+
writes.push(writeStorageFileAsync(join(root, "config.json"), files.rootConfig, true));
|
|
557
|
+
if (files.rootModules !== null)
|
|
558
|
+
writes.push(writeStorageFileAsync(join(root, "modules.json"), files.rootModules, true));
|
|
559
|
+
if (files.configText !== null && files.configPath !== null) {
|
|
560
|
+
const slash = files.configPath.indexOf("/");
|
|
561
|
+
const repo = files.configPath.slice(0, slash);
|
|
562
|
+
const relative = files.configPath.slice(slash + 1);
|
|
563
|
+
writes.push(writeStorageFileAsync(join(root, "source", repo, relative), files.configText, true));
|
|
564
|
+
}
|
|
565
|
+
await Promise.all(writes);
|
|
566
|
+
}
|
|
567
|
+
async function preflightLocalEmbeddingPooling(model, options = {}) {
|
|
568
|
+
const cacheDir = options.modelCacheDir ?? defaultModelCacheDir();
|
|
569
|
+
const logger = options.logger ?? log2;
|
|
570
|
+
const cacheKey = poolingMetadataCacheKey(model, cacheDir);
|
|
571
|
+
const cached = getCachedLocalEmbeddingPooling(model, cacheDir);
|
|
572
|
+
if (cached && cached.status !== "unresolved") {
|
|
573
|
+
logLocalEmbeddingPoolingDecisionOnce(model, cached, logger);
|
|
574
|
+
return cached;
|
|
575
|
+
}
|
|
576
|
+
if (isModelDirectory(model)) {
|
|
577
|
+
try {
|
|
578
|
+
const metadata = readDirectoryMetadata(model);
|
|
579
|
+
metadataByCache.set(cacheKey, metadata);
|
|
580
|
+
testMetadataByModel.delete(model);
|
|
581
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
582
|
+
return metadata;
|
|
583
|
+
} catch (error) {
|
|
584
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
585
|
+
const metadata = {
|
|
586
|
+
status: "unsupported",
|
|
587
|
+
normalize: false,
|
|
588
|
+
declared: true,
|
|
589
|
+
reason
|
|
590
|
+
};
|
|
591
|
+
metadataByCache.set(cacheKey, metadata);
|
|
592
|
+
testMetadataByModel.delete(model);
|
|
593
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
594
|
+
return metadata;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
try {
|
|
598
|
+
const fetcher = options.fetcher ?? ((url) => fetch(url, { signal: AbortSignal.timeout(20000) }));
|
|
599
|
+
const files = await filesForRepo(remoteHost(), model, fetcher);
|
|
600
|
+
let metadata;
|
|
601
|
+
try {
|
|
602
|
+
metadata = poolingFromFiles(files.modulesText, files.configText, model);
|
|
603
|
+
} catch (error) {
|
|
604
|
+
metadata = {
|
|
605
|
+
status: "unsupported",
|
|
606
|
+
normalize: false,
|
|
607
|
+
declared: true,
|
|
608
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
await cacheRemoteFiles(model, cacheDir, files);
|
|
612
|
+
await writeStorageFileAsync(cachedMetadataPath(model, cacheDir), JSON.stringify(metadata), true);
|
|
613
|
+
metadataByCache.set(cacheKey, metadata);
|
|
614
|
+
testMetadataByModel.delete(model);
|
|
615
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
616
|
+
return metadata;
|
|
617
|
+
} catch (error) {
|
|
618
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
619
|
+
metadataByCache.set(cacheKey, UNRESOLVED_LOCAL_EMBEDDING_POOLING);
|
|
620
|
+
testMetadataByModel.delete(model);
|
|
621
|
+
logUnavailableOnce(model, reason, logger);
|
|
622
|
+
return UNRESOLVED_LOCAL_EMBEDDING_POOLING;
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
function logLocalEmbeddingPoolingUnavailableOnce(model, reason, logger = log2) {
|
|
626
|
+
logUnavailableOnce(model, reason, logger);
|
|
627
|
+
}
|
|
628
|
+
|
|
203
629
|
// src/features/magic-context/memory/embedding-identity.ts
|
|
204
630
|
function normalizeEndpoint(endpoint) {
|
|
205
631
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
@@ -221,6 +647,8 @@ function getEmbeddingProviderIdentity(config) {
|
|
|
221
647
|
const truncate = config.provider === "openai-compatible" ? config.truncate?.trim() : undefined;
|
|
222
648
|
const documentPrefix = config.provider === "openai-compatible" ? resolveEmbeddingTextPrefixes(config.model, config.query_instruction, config.document_prefix).documentPrefix : "";
|
|
223
649
|
const localDtype = config.provider === "local" && config.local_dtype && config.local_dtype !== "fp32" ? config.local_dtype : undefined;
|
|
650
|
+
const localPooling = config.provider === "local" ? getCachedLocalEmbeddingPooling(config.model?.trim() || DEFAULT_LOCAL_EMBEDDING_MODEL) : null;
|
|
651
|
+
const localPoolingFingerprint = localPooling?.status === "ready" && (localPooling.pooling !== "mean" || !localPooling.normalize) ? `${localPooling.pooling};normalize=${localPooling.normalize}` : undefined;
|
|
224
652
|
const identityInput = config.provider === "openai-compatible" ? {
|
|
225
653
|
provider: "openai-compatible",
|
|
226
654
|
model: config.model.trim(),
|
|
@@ -235,7 +663,8 @@ function getEmbeddingProviderIdentity(config) {
|
|
|
235
663
|
endpoint: "",
|
|
236
664
|
apiKeyPresent: false,
|
|
237
665
|
runtimeFingerprint: LOCAL_EMBEDDING_RUNTIME_FINGERPRINT,
|
|
238
|
-
...localDtype ? { localDtype } : {}
|
|
666
|
+
...localDtype ? { localDtype } : {},
|
|
667
|
+
...localPoolingFingerprint ? { localPooling: localPoolingFingerprint } : {}
|
|
239
668
|
};
|
|
240
669
|
return `embedding-provider:${computeNormalizedHash(JSON.stringify(identityInput))}`;
|
|
241
670
|
}
|
|
@@ -603,7 +1032,7 @@ var importTransformersNodeWasmFallbackForRuntime = async () => {
|
|
|
603
1032
|
const nodeWasmEntry = new URL(new URL(import.meta.url).pathname.endsWith(".ts") ? "../../../../dist/transformers-node-wasm.js" : "./transformers-node-wasm.js", import.meta.url).href;
|
|
604
1033
|
return await importPluginModule(() => import(nodeWasmEntry));
|
|
605
1034
|
};
|
|
606
|
-
var modelCacheDirForRuntime = () => workerData?.magicContextEmbeddingWorker && typeof workerData.modelCacheDir === "string" ? workerData.modelCacheDir :
|
|
1035
|
+
var modelCacheDirForRuntime = () => workerData?.magicContextEmbeddingWorker && typeof workerData.modelCacheDir === "string" ? workerData.modelCacheDir : join2(getMagicContextStorageDir(), "models");
|
|
607
1036
|
var logForRuntime = log2;
|
|
608
1037
|
var injectWasmOrtForRuntime = injectWasmOrt;
|
|
609
1038
|
var loadedLocalEmbeddingRuntimes = new Map;
|
|
@@ -612,14 +1041,14 @@ function directoryBytes(path) {
|
|
|
612
1041
|
let total = 0;
|
|
613
1042
|
try {
|
|
614
1043
|
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
615
|
-
const entryPath =
|
|
1044
|
+
const entryPath = join2(path, entry.name);
|
|
616
1045
|
if (entry.isDirectory()) {
|
|
617
1046
|
const childBytes = directoryBytes(entryPath);
|
|
618
1047
|
if (childBytes === null)
|
|
619
1048
|
return null;
|
|
620
1049
|
total += childBytes;
|
|
621
1050
|
} else if (entry.isFile()) {
|
|
622
|
-
total +=
|
|
1051
|
+
total += statSync2(entryPath).size;
|
|
623
1052
|
}
|
|
624
1053
|
}
|
|
625
1054
|
return total;
|
|
@@ -845,6 +1274,7 @@ class LocalEmbeddingProvider {
|
|
|
845
1274
|
dtype;
|
|
846
1275
|
runtimePreference;
|
|
847
1276
|
pipeline = null;
|
|
1277
|
+
poolingMetadata = null;
|
|
848
1278
|
workerClient;
|
|
849
1279
|
workerLoaded = false;
|
|
850
1280
|
initPromise = null;
|
|
@@ -922,6 +1352,18 @@ class LocalEmbeddingProvider {
|
|
|
922
1352
|
throw new LocalEmbeddingFsUnavailableError;
|
|
923
1353
|
}
|
|
924
1354
|
const modelCacheDir = modelCacheDirForRuntime();
|
|
1355
|
+
const cachedPoolingMetadata = getCachedLocalEmbeddingPooling(this.model, modelCacheDir);
|
|
1356
|
+
const poolingMetadata = cachedPoolingMetadata?.status === "unresolved" || !cachedPoolingMetadata ? UNRESOLVED_LOCAL_EMBEDDING_POOLING : cachedPoolingMetadata;
|
|
1357
|
+
if (poolingMetadata.status === "unsupported" || !poolingMetadata.pooling) {
|
|
1358
|
+
throw new Error(`unsupported pooling for local model '${this.model}': ${poolingMetadata.reason}`);
|
|
1359
|
+
}
|
|
1360
|
+
if (poolingMetadata.status === "unresolved") {
|
|
1361
|
+
logLocalEmbeddingPoolingUnavailableOnce(this.model, poolingMetadata.reason, logForRuntime);
|
|
1362
|
+
}
|
|
1363
|
+
this.poolingMetadata = poolingMetadata;
|
|
1364
|
+
if (!workerData?.magicContextEmbeddingWorker) {
|
|
1365
|
+
logLocalEmbeddingPoolingDecisionOnce(this.model, poolingMetadata, logForRuntime);
|
|
1366
|
+
}
|
|
925
1367
|
try {
|
|
926
1368
|
ensureStorageDirectorySync(modelCacheDir);
|
|
927
1369
|
env.cacheDir = modelCacheDir;
|
|
@@ -929,7 +1371,7 @@ class LocalEmbeddingProvider {
|
|
|
929
1371
|
log2("[magic-context] could not create model cache dir, using library default");
|
|
930
1372
|
}
|
|
931
1373
|
const createPipeline = transformersModule.pipeline;
|
|
932
|
-
const lockPath =
|
|
1374
|
+
const lockPath = join2(modelCacheDir, ".load.lock");
|
|
933
1375
|
const releaseLock = await acquireModelLoadLock(lockPath);
|
|
934
1376
|
const stopHeartbeat = startLockHeartbeat(lockPath);
|
|
935
1377
|
try {
|
|
@@ -1036,8 +1478,8 @@ class LocalEmbeddingProvider {
|
|
|
1036
1478
|
return null;
|
|
1037
1479
|
}
|
|
1038
1480
|
const result = await withQuietConsole(() => pipeline(text, {
|
|
1039
|
-
pooling: "mean",
|
|
1040
|
-
normalize: true
|
|
1481
|
+
pooling: this.poolingMetadata?.pooling ?? "mean",
|
|
1482
|
+
normalize: this.poolingMetadata?.normalize ?? true
|
|
1041
1483
|
}));
|
|
1042
1484
|
const embedding = extractBatchEmbeddings(result, 1)[0] ?? null;
|
|
1043
1485
|
if (!embedding) {
|
|
@@ -1088,11 +1530,19 @@ class LocalEmbeddingProvider {
|
|
|
1088
1530
|
if (!pipeline) {
|
|
1089
1531
|
return Array.from({ length: texts.length }, () => null);
|
|
1090
1532
|
}
|
|
1091
|
-
const
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1533
|
+
const pooling = this.poolingMetadata?.pooling ?? "mean";
|
|
1534
|
+
const normalize = this.poolingMetadata?.normalize ?? true;
|
|
1535
|
+
let embeddings;
|
|
1536
|
+
if (pooling === "last_token" && texts.length > 1) {
|
|
1537
|
+
embeddings = [];
|
|
1538
|
+
for (const text of texts) {
|
|
1539
|
+
const result = await withQuietConsole(() => pipeline(text, { pooling, normalize }));
|
|
1540
|
+
embeddings.push(extractBatchEmbeddings(result, 1)[0] ?? null);
|
|
1541
|
+
}
|
|
1542
|
+
} else {
|
|
1543
|
+
const result = await withQuietConsole(() => pipeline(texts, { pooling, normalize }));
|
|
1544
|
+
embeddings = extractBatchEmbeddings(result, texts.length);
|
|
1545
|
+
}
|
|
1096
1546
|
if (embeddings.every((embedding) => embedding === null)) {
|
|
1097
1547
|
this.lastFailureReason = classifyLocalEmbeddingFailure(new Error("local embedding pipeline returned no vectors"), { usesWasm: this.usesWasm });
|
|
1098
1548
|
} else {
|
|
@@ -1168,4 +1618,4 @@ class LocalEmbeddingProvider {
|
|
|
1168
1618
|
}
|
|
1169
1619
|
}
|
|
1170
1620
|
|
|
1171
|
-
export { dominantEmbeddingFailure, embeddingModelsMatch, resolveEmbeddingTextPrefixes, getEmbeddingProviderIdentity, resolveLocalEmbeddingWorkerRuntime, getLocalEmbeddingNativeMemoryStats, LocalEmbeddingProvider };
|
|
1621
|
+
export { dominantEmbeddingFailure, embeddingModelsMatch, resolveEmbeddingTextPrefixes, getCachedLocalEmbeddingPooling, logLocalEmbeddingPoolingDecisionOnce, preflightLocalEmbeddingPooling, logLocalEmbeddingPoolingUnavailableOnce, getEmbeddingProviderIdentity, resolveLocalEmbeddingWorkerRuntime, getLocalEmbeddingNativeMemoryStats, LocalEmbeddingProvider };
|