@cortexkit/opencode-magic-context 0.49.19 → 0.49.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-search-worker.js +48 -9
- package/dist/chunk-embedding-write-worker.js +30 -23
- package/dist/{compaction-marker-candidates-n4krhat4.js → compaction-marker-candidates-b2q30cft.js} +3 -3
- package/dist/compaction-marker-candidates-worker.js +2 -2
- package/dist/compartment-chunk-embedding-va090as0.js +65 -0
- package/dist/config/schema/magic-context.d.ts +6 -6
- package/dist/{conflict-warning-hook-4pmz879v.js → conflict-warning-hook-frp3z2wk.js} +9 -8
- package/dist/{context-j31m2dv7.js → context-mkece1db.js} +455 -134
- package/dist/{derive-budgets-zn7yzd2d.js → derive-budgets-pqybnx09.js} +2 -2
- package/dist/embedding-backfill-gy3p78py.js +25 -0
- package/dist/embedding-worker-process.js +2 -2
- package/dist/embedding-worker.js +5 -5
- package/dist/features/magic-context/chunk-embedding-writer.d.ts.map +1 -1
- package/dist/features/magic-context/compartment-chunk-embedding.d.ts +44 -1
- package/dist/features/magic-context/compartment-chunk-embedding.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/hidden-single-shot.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-config.d.ts.map +1 -1
- package/dist/features/magic-context/dreamer/task-scheduler.d.ts +2 -0
- package/dist/features/magic-context/dreamer/task-scheduler.d.ts.map +1 -1
- package/dist/features/magic-context/memory/embedding-identity.d.ts.map +1 -1
- package/dist/features/magic-context/memory/embedding-local.d.ts +1 -0
- package/dist/features/magic-context/memory/embedding-local.d.ts.map +1 -1
- package/dist/features/magic-context/memory/local-embedding-pooling.d.ts +22 -0
- package/dist/features/magic-context/memory/local-embedding-pooling.d.ts.map +1 -0
- package/dist/features/magic-context/message-index-async.d.ts.map +1 -1
- package/dist/features/magic-context/message-index.d.ts +2 -0
- package/dist/features/magic-context/message-index.d.ts.map +1 -1
- package/dist/features/magic-context/mural/compress-cues.d.ts.map +1 -1
- package/dist/features/magic-context/project-embedding-registry.d.ts.map +1 -1
- package/dist/features/magic-context/search.d.ts +27 -0
- package/dist/features/magic-context/search.d.ts.map +1 -1
- package/dist/features/magic-context/smart-notes/compiler.d.ts.map +1 -1
- package/dist/features/magic-context/storage-meta-persisted.d.ts +1 -0
- package/dist/features/magic-context/storage-meta-persisted.d.ts.map +1 -1
- package/dist/features/magic-context/subagent-token-capture.d.ts +20 -0
- package/dist/features/magic-context/subagent-token-capture.d.ts.map +1 -1
- package/dist/features/magic-context/user-memory/review-user-memories.d.ts.map +1 -1
- package/dist/{history-owner-xp3pngv7.js → history-owner-36wp3mkb.js} +2 -2
- package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-embed-session.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-hint.d.ts +7 -1
- package/dist/hooks/magic-context/auto-search-hint.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-runner.d.ts.map +1 -1
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts +33 -6
- package/dist/hooks/magic-context/auto-search-worker-client.d.ts.map +1 -1
- package/dist/hooks/magic-context/cache-busting-signals.d.ts +5 -2
- package/dist/hooks/magic-context/cache-busting-signals.d.ts.map +1 -1
- package/dist/hooks/magic-context/channel2-delivery.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-historian.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-runner-types.d.ts +5 -0
- package/dist/hooks/magic-context/compartment-runner-types.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
- package/dist/hooks/magic-context/diagnostic-log-state.d.ts +4 -0
- package/dist/hooks/magic-context/diagnostic-log-state.d.ts.map +1 -0
- package/dist/hooks/magic-context/execute-reclaim.d.ts +35 -0
- package/dist/hooks/magic-context/execute-reclaim.d.ts.map +1 -0
- package/dist/hooks/magic-context/hook-handlers.d.ts.map +1 -1
- package/dist/hooks/magic-context/hook.d.ts +1 -3
- package/dist/hooks/magic-context/hook.d.ts.map +1 -1
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts +8 -0
- package/dist/hooks/magic-context/latest-assistant-turn.d.ts.map +1 -1
- package/dist/hooks/magic-context/lkg-recovery-fence.d.ts +5 -0
- package/dist/hooks/magic-context/lkg-recovery-fence.d.ts.map +1 -1
- package/dist/hooks/magic-context/nudge-evaluation-log.d.ts +3 -0
- package/dist/hooks/magic-context/nudge-evaluation-log.d.ts.map +1 -0
- package/dist/hooks/magic-context/pending-drop-eligibility.d.ts.map +1 -1
- package/dist/hooks/magic-context/protected-tail-boundary.d.ts +4 -0
- package/dist/hooks/magic-context/protected-tail-boundary.d.ts.map +1 -1
- package/dist/hooks/magic-context/raw-ordinal-warmup.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-date-refresh.d.ts +5 -5
- package/dist/hooks/magic-context/system-prompt-hash.d.ts.map +1 -1
- package/dist/hooks/magic-context/system-prompt-sticky-date.d.ts +10 -0
- package/dist/hooks/magic-context/system-prompt-sticky-date.d.ts.map +1 -0
- package/dist/hooks/magic-context/tag-messages.d.ts +2 -0
- package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-compartment-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +21 -6
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform.d.ts.map +1 -1
- package/dist/{index-bjcnntmz.js → index-0wthn7fp.js} +468 -18
- package/dist/{index-kq82mtg0.js → index-11zf2wwc.js} +3 -3
- package/dist/{index-b5vqckbw.js → index-153r2q6z.js} +314 -127
- package/dist/{index-3h4dp9wb.js → index-1pefp8ar.js} +5 -5
- package/dist/{index-beht37jy.js → index-1zx2pa5d.js} +1 -1
- package/dist/{index-ze7mvk54.js → index-2yhtnfcf.js} +75 -57
- package/dist/{index-cf74k5ak.js → index-305y133n.js} +4 -4
- package/dist/{index-s734jcsr.js → index-35qc6606.js} +9 -3
- package/dist/{index-se4m2g6w.js → index-48stjd56.js} +105 -19
- package/dist/{index-4911rn0b.js → index-4n7w9rad.js} +6 -6
- package/dist/{index-kee4kn5j.js → index-4pnsgvfk.js} +1 -1
- package/dist/{v2/server-8cd2j99z.js → index-5kym00ag.js} +32 -13
- package/dist/{index-4s3t0nd0.js → index-616rsjrv.js} +2 -2
- package/dist/{index-jhghdnsm.js → index-67nyj0gy.js} +485 -202
- package/dist/{index-6x57ar45.js → index-adk2z5d8.js} +3 -3
- package/dist/{index-c5pytzne.js → index-b88dwwrh.js} +1 -1
- package/dist/{index-rr2bb45z.js → index-c4b76d4t.js} +2 -2
- package/dist/{index-nj7yykkx.js → index-c7phnbvk.js} +1 -1
- package/dist/index-cpjk5jd2.js +82 -0
- package/dist/{index-edmpdxcw.js → index-d6rp4dxh.js} +5 -5
- package/dist/{index-dxmq29t1.js → index-dc8hy95c.js} +69 -17
- package/dist/{index-j75v2476.js → index-h7tgn49j.js} +12 -15
- package/dist/{index-ke416mdc.js → index-j76ah6d7.js} +2 -2
- package/dist/{index-c1p88z1y.js → index-jfh3cktr.js} +4 -4
- package/dist/{index-a6mc0v4d.js → index-jmycsey7.js} +3 -3
- package/dist/{index-24fezq97.js → index-jr5na10t.js} +4 -4
- package/dist/{index-s74zqahq.js → index-kc9pn495.js} +4 -4
- package/dist/{index-p5y3k4g7.js → index-nda0zgna.js} +2 -1
- package/dist/{index-jn12fvvh.js → index-neb2jaed.js} +3 -3
- package/dist/{index-63k48rv9.js → index-pnjwa6qa.js} +5 -5
- package/dist/{index-jrr131e0.js → index-rmd0069m.js} +11 -11
- package/dist/{index-y9f76s3f.js → index-rrv26qjz.js} +1 -1
- package/dist/{index-v10f1rjv.js → index-spwjbkc7.js} +1 -1
- package/dist/{index-k29z4aqh.js → index-t5pck3ss.js} +1 -1
- package/dist/{index-3esx8tkf.js → index-t7hx3qcb.js} +318 -104
- package/dist/{index-rpxdjkm0.js → index-v7r4td4c.js} +306 -59
- package/dist/{index-2vxtcb95.js → index-vkc1ca9p.js} +2 -2
- package/dist/{index-9zcsy3hh.js → index-x4k3f4mf.js} +17 -9
- package/dist/{index-bp5083x2.js → index-x9k3mqrf.js} +1 -1
- package/dist/{index-n37sebck.js → index-y4a3nhtk.js} +1 -1
- package/dist/index.js +52 -49
- package/dist/{logger-jm3sqqzr.js → logger-94kew5wm.js} +1 -1
- package/dist/{message-fts-rowid-map-17s0dfn4.js → message-fts-rowid-map-22tevnnj.js} +6 -5
- package/dist/{message-index-gtpqnj8w.js → message-index-5qxcz6h7.js} +16 -15
- package/dist/{message-index-bounded-zyc8h32y.js → message-index-bounded-n0m0t7tv.js} +16 -15
- package/dist/{message-time-backfill-73mmm76q.js → message-time-backfill-0myx7fz3.js} +3 -3
- package/dist/migration-worker.js +13 -12
- package/dist/{model-resolution-nx97z3x7.js → model-resolution-280gmqvd.js} +1 -1
- package/dist/plugin/embedding-bootstrap.d.ts +3 -1
- package/dist/plugin/embedding-bootstrap.d.ts.map +1 -1
- package/dist/plugin/hooks/create-session-hooks.d.ts.map +1 -1
- package/dist/plugin/rpc-handlers.d.ts +8 -0
- package/dist/plugin/rpc-handlers.d.ts.map +1 -1
- package/dist/plugin/subagent-tool-policy.d.ts +9 -0
- package/dist/plugin/subagent-tool-policy.d.ts.map +1 -1
- package/dist/{project-embedding-registry-z74mhqkf.js → project-embedding-registry-s2pqgec9.js} +13 -12
- package/dist/raw-ordinal-warmup-worker.js +136 -101
- package/dist/{read-session-chunk-jjb48bt9.js → read-session-chunk-0a8vdyqp.js} +11 -10
- package/dist/{read-session-raw-rsc8c7qy.js → read-session-raw-y5cszf0b.js} +4 -3
- package/dist/recomp-orchestrator-61vyha9j.js +62 -0
- package/dist/{safe-notification-target-1szv1k5k.js → safe-notification-target-mkr9yzvr.js} +2 -2
- package/dist/{search-archive-6vm0e2f5.js → search-archive-8q9s49p5.js} +12 -11
- package/dist/{search-xsqk1bj4.js → search-v1y9ejws.js} +25 -24
- package/dist/{send-session-notification-6ve0e8p4.js → send-session-notification-0k6qzydk.js} +7 -6
- package/dist/shared/logger.d.ts.map +1 -1
- package/dist/shared/model-resolution.d.ts +2 -0
- package/dist/shared/model-resolution.d.ts.map +1 -1
- package/dist/shared/prompt-async-transport.d.ts +3 -0
- package/dist/shared/prompt-async-transport.d.ts.map +1 -1
- package/dist/shared/rpc-types.d.ts +7 -0
- package/dist/shared/rpc-types.d.ts.map +1 -1
- package/dist/shared/sqlite.d.ts +1 -1
- package/dist/shared/sqlite.d.ts.map +1 -1
- package/dist/shared/subagent-runner.d.ts +2 -0
- package/dist/shared/subagent-runner.d.ts.map +1 -1
- package/dist/shared/worker-shutdown.d.ts +68 -0
- package/dist/shared/worker-shutdown.d.ts.map +1 -0
- package/dist/shared/worker-shutdown.fixture.d.ts +2 -0
- package/dist/shared/worker-shutdown.fixture.d.ts.map +1 -0
- package/dist/{sqlite-zf72c52d.js → sqlite-1rpvw2rj.js} +1 -1
- package/dist/{storage-memory-y3qmy00r.js → storage-memory-nm7v82mj.js} +2 -2
- package/dist/{storage-tags-r7k9tjcv.js → storage-tags-r6nk29n4.js} +5 -5
- package/dist/{task-config-x3qmayd2.js → task-config-t6bpyka7.js} +2 -2
- package/dist/tools/ctx-search/tools.d.ts.map +1 -1
- package/dist/{transform-postprocess-phase-jj99r7z2.js → transform-postprocess-phase-vtrgb7m6.js} +34 -31
- package/dist/v2/auto-search-worker.js +48 -9
- package/dist/v2/chunk-embedding-write-worker.js +30 -23
- package/dist/v2/{compaction-marker-candidates-ebx3388k.js → compaction-marker-candidates-cfab70cs.js} +3 -3
- package/dist/v2/compaction-marker-candidates-worker.js +2 -2
- package/dist/v2/compartment-chunk-embedding-nbdqx1td.js +64 -0
- package/dist/v2/{context-y5yamxak.js → context-cc33tv2z.js} +1189 -386
- package/dist/v2/{derive-budgets-2zh4gm2c.js → derive-budgets-knf763tt.js} +3 -3
- package/dist/v2/embedding-backfill-yxv2h5mc.js +24 -0
- package/dist/v2/embedding-worker-process.js +2 -2
- package/dist/v2/embedding-worker.js +5 -5
- package/dist/v2/hidden-child-record.d.ts +2 -0
- package/dist/v2/hidden-child-record.d.ts.map +1 -1
- package/dist/v2/hidden-completion.d.ts +2 -0
- package/dist/v2/hidden-completion.d.ts.map +1 -1
- package/dist/v2/hidden-model-activity.d.ts +7 -0
- package/dist/v2/hidden-model-activity.d.ts.map +1 -0
- package/dist/v2/{history-owner-42e16w0n.js → history-owner-ad0crnrn.js} +2 -2
- package/dist/v2/hooks/context.d.ts.map +1 -1
- package/dist/v2/hooks/generate.d.ts +36 -1
- package/dist/v2/hooks/generate.d.ts.map +1 -1
- package/dist/v2/hooks/lkg-system.d.ts +35 -4
- package/dist/v2/hooks/lkg-system.d.ts.map +1 -1
- package/dist/v2/hooks/payload.d.ts +16 -1
- package/dist/v2/hooks/payload.d.ts.map +1 -1
- package/dist/v2/{logger-7qt02bnb.js → logger-p34mjtja.js} +1 -1
- package/dist/v2/{message-fts-rowid-map-wfx67gfd.js → message-fts-rowid-map-dg12wymk.js} +6 -5
- package/dist/v2/{message-index-bounded-spd8r7pj.js → message-index-bounded-f4e01xk0.js} +16 -15
- package/dist/v2/{message-index-8vz2pq76.js → message-index-ydzrp6y6.js} +16 -15
- package/dist/v2/{message-time-backfill-04t0km1z.js → message-time-backfill-ddh8n9gy.js} +3 -3
- package/dist/v2/migration-worker.js +13 -12
- package/dist/v2/{model-resolution-nx97z3x7.js → model-resolution-280gmqvd.js} +1 -1
- package/dist/v2/{project-embedding-registry-gnkj56xp.js → project-embedding-registry-deza9ksw.js} +13 -12
- package/dist/v2/raw-ordinal-warmup-worker.js +136 -101
- package/dist/v2/{read-session-chunk-cwkxrmns.js → read-session-chunk-8cxfagky.js} +11 -10
- package/dist/v2/{read-session-raw-yaegtjzh.js → read-session-raw-1nrrecvx.js} +4 -3
- package/dist/v2/recomp-orchestrator-47m4hv5g.js +61 -0
- package/dist/v2/{safe-notification-target-qg7n2s77.js → safe-notification-target-khnzg57c.js} +2 -2
- package/dist/v2/{search-archive-wve3tq1k.js → search-archive-e80cvc09.js} +12 -11
- package/dist/v2/{search-hwg7jfbj.js → search-fjz746zr.js} +25 -24
- package/dist/v2/{send-session-notification-3pxn658x.js → send-session-notification-nwvrv34d.js} +7 -6
- package/dist/v2/{server-8by8qfec.js → server-0epqaxhd.js} +75 -57
- package/dist/v2/{server-pwv4ezb0.js → server-168pb207.js} +1 -1
- package/dist/v2/{server-h0410h0v.js → server-1bw1wcbp.js} +1 -1
- package/dist/v2/{server-beht37jy.js → server-1zx2pa5d.js} +1 -1
- package/dist/v2/{server-a9xqj88f.js → server-230fnbws.js} +4 -4
- package/dist/v2/{server-tbe1femw.js → server-29qp8nz4.js} +1 -1
- package/dist/v2/{server-tgnbmwah.js → server-3mz2qh5a.js} +1 -1
- package/dist/v2/{server-hy3d3vsx.js → server-4nw4fqhr.js} +2 -2
- package/dist/v2/{server-45q69hkq.js → server-4sxjs6hn.js} +5 -5
- package/dist/v2/{server-34kwfsr4.js → server-53eh1r8r.js} +2 -2
- package/dist/v2/{server-xj136kt8.js → server-55rgxfrw.js} +9 -3
- package/dist/v2/{server-nj659894.js → server-5arr9ecn.js} +2 -2
- package/dist/{index-8cd2j99z.js → v2/server-5kym00ag.js} +32 -13
- package/dist/v2/{server-2ar62dyx.js → server-63fqdghs.js} +106 -20
- package/dist/v2/{server-f2xf5gcb.js → server-6495nmn9.js} +12 -15
- package/dist/v2/{server-qd11g795.js → server-92g7d776.js} +2 -1
- package/dist/v2/{server-xq7dfjke.js → server-9x0kmta4.js} +5 -5
- package/dist/v2/{server-27gq4mfz.js → server-a3dwxfgd.js} +4 -4
- package/dist/v2/{server-6tmj0s7j.js → server-af2pght7.js} +3 -3
- package/dist/v2/{server-1wy7b9de.js → server-br1x0tev.js} +3 -3
- package/dist/v2/{server-n1prf1g6.js → server-brj8j2gk.js} +2 -2
- package/dist/v2/{server-6vdzn1td.js → server-c7dxxsj7.js} +308 -61
- package/dist/v2/server-cpjk5jd2.js +82 -0
- package/dist/v2/{server-g3qma5c0.js → server-dj7tz4ph.js} +2 -2
- package/dist/v2/{server-9hx7g5jp.js → server-ej96a39v.js} +17 -9
- package/dist/v2/{server-6pmqg9m4.js → server-gyy4k6mf.js} +1 -1
- package/dist/v2/{server-bmymkx73.js → server-hqzyq8n4.js} +11 -11
- package/dist/v2/{server-h1v0pyxb.js → server-j4jnp9b5.js} +468 -18
- package/dist/v2/{server-da07v4xp.js → server-k3t06fxr.js} +1 -1
- package/dist/v2/{server-gsxjv2q2.js → server-m91f9m5s.js} +69 -17
- package/dist/v2/{server-rmygf5pn.js → server-ne8arh8t.js} +2 -2
- package/dist/v2/{server-hm38q7r0.js → server-vn2nwcv6.js} +5 -5
- package/dist/v2/{server-46neh5fa.js → server-wyawsa88.js} +314 -127
- package/dist/v2/{server-zzkj2fch.js → server-xsryvyf6.js} +4 -4
- package/dist/v2/{server-n37sebck.js → server-y4a3nhtk.js} +1 -1
- package/dist/v2/{server-d04vrx0j.js → server-ygtykank.js} +2 -2
- package/dist/v2/server.js +6 -6
- package/dist/v2/{sqlite-zf72c52d.js → sqlite-1rpvw2rj.js} +1 -1
- package/dist/v2/{storage-memory-y3qmy00r.js → storage-memory-nm7v82mj.js} +2 -2
- package/dist/v2/{storage-tags-3wdb8zph.js → storage-tags-ss08kj5r.js} +5 -5
- package/dist/v2/store-reader.d.ts +8 -0
- package/dist/v2/store-reader.d.ts.map +1 -1
- package/dist/v2/{task-config-nvz8mapa.js → task-config-7md27rx8.js} +2 -2
- package/dist/v2/{wrapup-orchestrator-embdz8gj.js → wrapup-orchestrator-639whfhq.js} +32 -31
- package/dist/wrapup-orchestrator-mnvt2g8m.js +49 -0
- package/package.json +1 -1
- package/src/config/index.test.ts +28 -6
- package/src/shared/logger.test.ts +42 -1
- package/src/shared/logger.ts +11 -13
- package/src/shared/model-resolution.test.ts +67 -12
- package/src/shared/model-resolution.ts +43 -16
- package/src/shared/model-suggestion-retry.ts +1 -1
- package/src/shared/prompt-async-transport.ts +47 -3
- package/src/shared/rpc-types.ts +7 -0
- package/src/shared/sqlite-busy-budget-review.test.ts +6 -1
- package/src/shared/sqlite.ts +4 -2
- package/src/shared/subagent-runner.ts +2 -0
- package/src/shared/worker-shutdown-fence.test.ts +87 -0
- package/src/shared/worker-shutdown.fixture.ts +25 -0
- package/src/shared/worker-shutdown.test.ts +127 -0
- package/src/shared/worker-shutdown.ts +160 -0
- package/dist/embedding-backfill-fmz50tsy.js +0 -24
- package/dist/recomp-orchestrator-999jj4bc.js +0 -61
- package/dist/v2/embedding-backfill-7c7yr69y.js +0 -23
- package/dist/v2/recomp-orchestrator-f6m9jr7p.js +0 -60
- package/dist/wrapup-orchestrator-whhbjmaz.js +0 -48
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
} from "./index-wgypt3cs.js";
|
|
4
4
|
import {
|
|
5
5
|
getSynapseLaneIdentity
|
|
6
|
-
} from "./index-
|
|
6
|
+
} from "./index-adk2z5d8.js";
|
|
7
7
|
import {
|
|
8
8
|
getHarness2
|
|
9
9
|
} from "./index-pavzsdc6.js";
|
|
@@ -13,10 +13,10 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
importPluginModule,
|
|
15
15
|
log2
|
|
16
|
-
} from "./index-
|
|
16
|
+
} from "./index-h7tgn49j.js";
|
|
17
17
|
import {
|
|
18
18
|
inRootAsyncContext2
|
|
19
|
-
} from "./index-
|
|
19
|
+
} from "./index-1zx2pa5d.js";
|
|
20
20
|
import {
|
|
21
21
|
computeNormalizedHash
|
|
22
22
|
} from "./index-2bvekkfy.js";
|
|
@@ -100,6 +100,432 @@ function resolveEmbeddingTextPrefixes(model, queryInstruction, documentPrefix) {
|
|
|
100
100
|
};
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
// src/features/magic-context/memory/local-embedding-pooling.ts
|
|
104
|
+
import { createHash } from "node:crypto";
|
|
105
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
106
|
+
import { join } from "node:path";
|
|
107
|
+
var decisionLogged = new Set;
|
|
108
|
+
var unavailableLogged = new Set;
|
|
109
|
+
var metadataByCache = new Map;
|
|
110
|
+
var testMetadataByModel = new Map;
|
|
111
|
+
var testMetadataFallback;
|
|
112
|
+
var UNRESOLVED_LOCAL_EMBEDDING_POOLING = {
|
|
113
|
+
status: "unresolved",
|
|
114
|
+
pooling: "mean",
|
|
115
|
+
normalize: true,
|
|
116
|
+
declared: false,
|
|
117
|
+
reason: "pooling metadata is unavailable; preserving legacy mean pooling and normalization"
|
|
118
|
+
};
|
|
119
|
+
var LEGACY_MEAN_POOLING = {
|
|
120
|
+
status: "ready",
|
|
121
|
+
pooling: "mean",
|
|
122
|
+
normalize: true,
|
|
123
|
+
declared: false,
|
|
124
|
+
reason: "no sentence-transformers Pooling module is declared; preserving legacy mean pooling and normalization"
|
|
125
|
+
};
|
|
126
|
+
function defaultModelCacheDir() {
|
|
127
|
+
return join(getMagicContextStorageDir(), "models");
|
|
128
|
+
}
|
|
129
|
+
function metadataDirectory(model, modelCacheDir) {
|
|
130
|
+
const digest = createHash("sha256").update(model).digest("hex");
|
|
131
|
+
return join(modelCacheDir, ".pooling-metadata", digest);
|
|
132
|
+
}
|
|
133
|
+
function parseJson(text, label) {
|
|
134
|
+
try {
|
|
135
|
+
return JSON.parse(text);
|
|
136
|
+
} catch (error) {
|
|
137
|
+
throw new Error(`${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function isRecord(value) {
|
|
141
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
142
|
+
}
|
|
143
|
+
function isModelDirectory(model) {
|
|
144
|
+
try {
|
|
145
|
+
return statSync(model).isDirectory();
|
|
146
|
+
} catch {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function isPoolingModule(value) {
|
|
151
|
+
if (!isRecord(value))
|
|
152
|
+
return false;
|
|
153
|
+
return typeof value.type === "string" && /(?:^|\.)Pooling$/.test(value.type) || typeof value.name === "string" && value.name.toLowerCase() === "pooling";
|
|
154
|
+
}
|
|
155
|
+
function isNormalizeModule(value) {
|
|
156
|
+
if (!isRecord(value))
|
|
157
|
+
return false;
|
|
158
|
+
return typeof value.type === "string" && /(?:^|\.)Normalize$/.test(value.type) || typeof value.name === "string" && value.name.toLowerCase() === "normalize";
|
|
159
|
+
}
|
|
160
|
+
function safeModulePath(value) {
|
|
161
|
+
if (typeof value !== "string" || !value.trim())
|
|
162
|
+
return null;
|
|
163
|
+
const pieces = value.replaceAll("\\", "/").split("/");
|
|
164
|
+
if (pieces.some((piece) => piece === ".." || piece === "."))
|
|
165
|
+
return null;
|
|
166
|
+
return pieces.filter(Boolean).join("/");
|
|
167
|
+
}
|
|
168
|
+
function poolingFromFiles(modulesText, configText, label) {
|
|
169
|
+
if (modulesText === null)
|
|
170
|
+
return LEGACY_MEAN_POOLING;
|
|
171
|
+
const modulesValue = parseJson(modulesText, `${label}/modules.json`);
|
|
172
|
+
if (!Array.isArray(modulesValue)) {
|
|
173
|
+
return {
|
|
174
|
+
status: "unsupported",
|
|
175
|
+
normalize: false,
|
|
176
|
+
declared: true,
|
|
177
|
+
reason: "modules.json is not a module list"
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
const poolingModule = modulesValue.find(isPoolingModule);
|
|
181
|
+
if (!poolingModule)
|
|
182
|
+
return LEGACY_MEAN_POOLING;
|
|
183
|
+
const modulePath = safeModulePath(poolingModule.path);
|
|
184
|
+
if (modulePath === null || configText === null) {
|
|
185
|
+
return {
|
|
186
|
+
status: "unsupported",
|
|
187
|
+
normalize: modulesValue.some(isNormalizeModule),
|
|
188
|
+
declared: true,
|
|
189
|
+
reason: `Pooling module at ${String(poolingModule.path ?? "<missing path>")} has no readable config.json`
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const config = parseJson(configText, `${label}/${modulePath}/config.json`);
|
|
193
|
+
if (!isRecord(config)) {
|
|
194
|
+
return {
|
|
195
|
+
status: "unsupported",
|
|
196
|
+
normalize: modulesValue.some(isNormalizeModule),
|
|
197
|
+
declared: true,
|
|
198
|
+
reason: `Pooling config at ${modulePath}/config.json is not an object`
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
const enabledModes = [
|
|
202
|
+
["pooling_mode_cls_token", "cls"],
|
|
203
|
+
["pooling_mode_mean_tokens", "mean"],
|
|
204
|
+
["pooling_mode_lasttoken", "last_token"],
|
|
205
|
+
["pooling_mode_max_tokens", undefined],
|
|
206
|
+
["pooling_mode_mean_sqrt_len_tokens", undefined],
|
|
207
|
+
["pooling_mode_weightedmean_tokens", undefined],
|
|
208
|
+
["pooling_mode_mean_last_token", undefined]
|
|
209
|
+
].filter(([key]) => config[key] === true);
|
|
210
|
+
const normalize = modulesValue.some(isNormalizeModule);
|
|
211
|
+
if (enabledModes.length !== 1) {
|
|
212
|
+
return {
|
|
213
|
+
status: "unsupported",
|
|
214
|
+
normalize,
|
|
215
|
+
declared: true,
|
|
216
|
+
reason: enabledModes.length === 0 ? `Pooling config at ${modulePath}/config.json enables no pooling mode` : `Pooling config enables multiple modes: ${enabledModes.map(([key]) => key).join(", ")}`
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
const [flag, method] = enabledModes[0];
|
|
220
|
+
if (!method) {
|
|
221
|
+
return {
|
|
222
|
+
status: "unsupported",
|
|
223
|
+
normalize,
|
|
224
|
+
declared: true,
|
|
225
|
+
reason: `unsupported declared pooling mode ${flag}`
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
return {
|
|
229
|
+
status: "ready",
|
|
230
|
+
pooling: method,
|
|
231
|
+
normalize,
|
|
232
|
+
declared: true,
|
|
233
|
+
reason: `modules.json declares ${flag}; ${normalize ? "a Normalize module is present" : "no Normalize module is declared"}`
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
function poolingMetadataCacheKey(model, modelCacheDir) {
|
|
237
|
+
return JSON.stringify([model, modelCacheDir]);
|
|
238
|
+
}
|
|
239
|
+
function cachedMetadataPath(model, modelCacheDir) {
|
|
240
|
+
return join(metadataDirectory(model, modelCacheDir), "resolution.json");
|
|
241
|
+
}
|
|
242
|
+
function parseCachedMetadata(text) {
|
|
243
|
+
try {
|
|
244
|
+
const value = JSON.parse(text);
|
|
245
|
+
if (!isRecord(value) || value.status !== "ready" && value.status !== "unsupported")
|
|
246
|
+
return null;
|
|
247
|
+
if (typeof value.normalize !== "boolean" || typeof value.declared !== "boolean" || typeof value.reason !== "string")
|
|
248
|
+
return null;
|
|
249
|
+
if (value.status === "ready" && value.pooling !== "mean" && value.pooling !== "cls" && value.pooling !== "last_token")
|
|
250
|
+
return null;
|
|
251
|
+
return value;
|
|
252
|
+
} catch {
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
function readDirectoryMetadata(model) {
|
|
257
|
+
const modulesPath = join(model, "modules.json");
|
|
258
|
+
if (!existsSync(modulesPath))
|
|
259
|
+
return LEGACY_MEAN_POOLING;
|
|
260
|
+
const modulesText = readFileSync(modulesPath, "utf8");
|
|
261
|
+
const moduleList = parseJson(modulesText, `${model}/modules.json`);
|
|
262
|
+
if (!Array.isArray(moduleList)) {
|
|
263
|
+
return {
|
|
264
|
+
status: "unsupported",
|
|
265
|
+
normalize: false,
|
|
266
|
+
declared: true,
|
|
267
|
+
reason: "modules.json is not a module list"
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
const poolingModule = moduleList.find(isPoolingModule);
|
|
271
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
272
|
+
const configText = modulePath ? readFileSync(join(model, modulePath, "config.json"), "utf8") : null;
|
|
273
|
+
return poolingFromFiles(modulesText, configText, model);
|
|
274
|
+
}
|
|
275
|
+
function getCachedLocalEmbeddingPooling(model, modelCacheDir = defaultModelCacheDir()) {
|
|
276
|
+
if (testMetadataByModel.has(model))
|
|
277
|
+
return testMetadataByModel.get(model) ?? null;
|
|
278
|
+
const cacheKey = poolingMetadataCacheKey(model, modelCacheDir);
|
|
279
|
+
const cachedInMemory = metadataByCache.get(cacheKey);
|
|
280
|
+
if (cachedInMemory)
|
|
281
|
+
return cachedInMemory;
|
|
282
|
+
if (isModelDirectory(model)) {
|
|
283
|
+
try {
|
|
284
|
+
const metadata = readDirectoryMetadata(model);
|
|
285
|
+
metadataByCache.set(cacheKey, metadata);
|
|
286
|
+
return metadata;
|
|
287
|
+
} catch (error) {
|
|
288
|
+
const metadata = {
|
|
289
|
+
status: "unsupported",
|
|
290
|
+
normalize: false,
|
|
291
|
+
declared: true,
|
|
292
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
293
|
+
};
|
|
294
|
+
metadataByCache.set(cacheKey, metadata);
|
|
295
|
+
return metadata;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
try {
|
|
299
|
+
const cached = parseCachedMetadata(readFileSync(cachedMetadataPath(model, modelCacheDir), "utf8"));
|
|
300
|
+
if (cached) {
|
|
301
|
+
metadataByCache.set(cacheKey, cached);
|
|
302
|
+
return cached;
|
|
303
|
+
}
|
|
304
|
+
} catch {}
|
|
305
|
+
if (testMetadataFallback) {
|
|
306
|
+
metadataByCache.set(cacheKey, testMetadataFallback);
|
|
307
|
+
return testMetadataFallback;
|
|
308
|
+
}
|
|
309
|
+
metadataByCache.set(cacheKey, UNRESOLVED_LOCAL_EMBEDDING_POOLING);
|
|
310
|
+
return UNRESOLVED_LOCAL_EMBEDDING_POOLING;
|
|
311
|
+
}
|
|
312
|
+
function logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger = log2) {
|
|
313
|
+
const decisionKey = `${model}:${metadata.status}:${metadata.pooling ?? "none"}:${metadata.normalize}`;
|
|
314
|
+
if (decisionLogged.has(decisionKey))
|
|
315
|
+
return;
|
|
316
|
+
decisionLogged.add(decisionKey);
|
|
317
|
+
if (metadata.status === "unsupported") {
|
|
318
|
+
logger(`[magic-context] local embedding pooling for ${model} is unsupported: ${metadata.reason}`);
|
|
319
|
+
} else {
|
|
320
|
+
logger(`[magic-context] local embedding pooling for ${model}: ${metadata.pooling} pooling, normalization ${metadata.normalize ? "enabled" : "disabled"} (${metadata.reason})`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
function logUnavailableOnce(model, reason, logger) {
|
|
324
|
+
if (unavailableLogged.has(model))
|
|
325
|
+
return;
|
|
326
|
+
unavailableLogged.add(model);
|
|
327
|
+
logger(`[magic-context] local embedding pooling metadata for ${model} is not available: ${reason}; using legacy mean pooling and retrying during a later background bootstrap`);
|
|
328
|
+
}
|
|
329
|
+
function remoteHost() {
|
|
330
|
+
const configured = process.env.HF_ENDPOINT?.trim().replace(/\/+$/, "");
|
|
331
|
+
return configured || "https://huggingface.co";
|
|
332
|
+
}
|
|
333
|
+
function safeRepoId(repo) {
|
|
334
|
+
return repo.length > 0 && repo.split("/").every((part) => part && part !== "." && part !== "..");
|
|
335
|
+
}
|
|
336
|
+
function modelFileUrl(host, repo, file) {
|
|
337
|
+
const path = repo.split("/").map(encodeURIComponent).join("/");
|
|
338
|
+
const filePath = file.split("/").map(encodeURIComponent).join("/");
|
|
339
|
+
return `${host}/${path}/resolve/main/${filePath}`;
|
|
340
|
+
}
|
|
341
|
+
async function fetchModelFile(host, repo, file, fetcher) {
|
|
342
|
+
const response = await fetcher(modelFileUrl(host, repo, file));
|
|
343
|
+
if (response.status === 404)
|
|
344
|
+
return { kind: "missing" };
|
|
345
|
+
if (!response.ok)
|
|
346
|
+
throw new Error(`HTTP ${response.status} fetching ${repo}/${file}`);
|
|
347
|
+
return { kind: "found", text: await response.text() };
|
|
348
|
+
}
|
|
349
|
+
async function filesForRepo(host, repo, fetcher) {
|
|
350
|
+
const modules = await fetchModelFile(host, repo, "modules.json", fetcher);
|
|
351
|
+
if (modules.kind === "missing") {
|
|
352
|
+
const rootConfigResult = await fetchModelFile(host, repo, "config.json", fetcher);
|
|
353
|
+
if (rootConfigResult.kind === "missing") {
|
|
354
|
+
return {
|
|
355
|
+
modulesText: null,
|
|
356
|
+
configText: null,
|
|
357
|
+
configPath: null,
|
|
358
|
+
rootConfig: null,
|
|
359
|
+
rootModules: null
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
const rootConfigValue = parseJson(rootConfigResult.text, `${repo}/config.json`);
|
|
363
|
+
const origin = isRecord(rootConfigValue) && typeof rootConfigValue._name_or_path === "string" ? rootConfigValue._name_or_path.trim() : "";
|
|
364
|
+
if (!origin || origin === repo || !safeRepoId(origin)) {
|
|
365
|
+
return {
|
|
366
|
+
modulesText: null,
|
|
367
|
+
configText: null,
|
|
368
|
+
configPath: null,
|
|
369
|
+
rootConfig: rootConfigResult.text,
|
|
370
|
+
rootModules: null
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
const originModules = await fetchModelFile(host, origin, "modules.json", fetcher);
|
|
374
|
+
if (originModules.kind === "missing") {
|
|
375
|
+
return {
|
|
376
|
+
modulesText: null,
|
|
377
|
+
configText: null,
|
|
378
|
+
configPath: null,
|
|
379
|
+
rootConfig: rootConfigResult.text,
|
|
380
|
+
rootModules: null
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
const originModuleList = parseJson(originModules.text, `${origin}/modules.json`);
|
|
384
|
+
if (!Array.isArray(originModuleList)) {
|
|
385
|
+
return {
|
|
386
|
+
modulesText: originModules.text,
|
|
387
|
+
configText: null,
|
|
388
|
+
configPath: null,
|
|
389
|
+
rootConfig: rootConfigResult.text,
|
|
390
|
+
rootModules: originModules.text
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
const poolingModule = originModuleList.find(isPoolingModule);
|
|
394
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
395
|
+
if (!modulePath) {
|
|
396
|
+
return {
|
|
397
|
+
modulesText: originModules.text,
|
|
398
|
+
configText: null,
|
|
399
|
+
configPath: null,
|
|
400
|
+
rootConfig: rootConfigResult.text,
|
|
401
|
+
rootModules: originModules.text
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
const poolingConfig = await fetchModelFile(host, origin, `${modulePath}/config.json`, fetcher);
|
|
405
|
+
if (poolingConfig.kind === "missing") {
|
|
406
|
+
return {
|
|
407
|
+
modulesText: originModules.text,
|
|
408
|
+
configText: null,
|
|
409
|
+
configPath: `${origin}/${modulePath}/config.json`,
|
|
410
|
+
rootConfig: rootConfigResult.text,
|
|
411
|
+
rootModules: originModules.text
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
return {
|
|
415
|
+
modulesText: originModules.text,
|
|
416
|
+
configText: poolingConfig.text,
|
|
417
|
+
configPath: `${origin}/${modulePath}/config.json`,
|
|
418
|
+
rootConfig: rootConfigResult.text,
|
|
419
|
+
rootModules: originModules.text
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
const modulesValue = parseJson(modules.text, `${repo}/modules.json`);
|
|
423
|
+
if (!Array.isArray(modulesValue)) {
|
|
424
|
+
return {
|
|
425
|
+
modulesText: modules.text,
|
|
426
|
+
configText: null,
|
|
427
|
+
configPath: null,
|
|
428
|
+
rootConfig: null,
|
|
429
|
+
rootModules: modules.text
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
const poolingModule = modulesValue.find(isPoolingModule);
|
|
433
|
+
const modulePath = poolingModule ? safeModulePath(poolingModule.path) : null;
|
|
434
|
+
if (!modulePath) {
|
|
435
|
+
return {
|
|
436
|
+
modulesText: modules.text,
|
|
437
|
+
configText: null,
|
|
438
|
+
configPath: null,
|
|
439
|
+
rootConfig: null,
|
|
440
|
+
rootModules: modules.text
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
const poolingConfig = await fetchModelFile(host, repo, `${modulePath}/config.json`, fetcher);
|
|
444
|
+
return {
|
|
445
|
+
modulesText: modules.text,
|
|
446
|
+
configText: poolingConfig.kind === "found" ? poolingConfig.text : null,
|
|
447
|
+
configPath: `${repo}/${modulePath}/config.json`,
|
|
448
|
+
rootConfig: null,
|
|
449
|
+
rootModules: modules.text
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
async function cacheRemoteFiles(model, modelCacheDir, files) {
|
|
453
|
+
const root = metadataDirectory(model, modelCacheDir);
|
|
454
|
+
const writes = [];
|
|
455
|
+
if (files.rootConfig !== null)
|
|
456
|
+
writes.push(writeStorageFileAsync(join(root, "config.json"), files.rootConfig, true));
|
|
457
|
+
if (files.rootModules !== null)
|
|
458
|
+
writes.push(writeStorageFileAsync(join(root, "modules.json"), files.rootModules, true));
|
|
459
|
+
if (files.configText !== null && files.configPath !== null) {
|
|
460
|
+
const slash = files.configPath.indexOf("/");
|
|
461
|
+
const repo = files.configPath.slice(0, slash);
|
|
462
|
+
const relative = files.configPath.slice(slash + 1);
|
|
463
|
+
writes.push(writeStorageFileAsync(join(root, "source", repo, relative), files.configText, true));
|
|
464
|
+
}
|
|
465
|
+
await Promise.all(writes);
|
|
466
|
+
}
|
|
467
|
+
async function preflightLocalEmbeddingPooling(model, options = {}) {
|
|
468
|
+
const cacheDir = options.modelCacheDir ?? defaultModelCacheDir();
|
|
469
|
+
const logger = options.logger ?? log2;
|
|
470
|
+
const cacheKey = poolingMetadataCacheKey(model, cacheDir);
|
|
471
|
+
const cached = getCachedLocalEmbeddingPooling(model, cacheDir);
|
|
472
|
+
if (cached && cached.status !== "unresolved") {
|
|
473
|
+
logLocalEmbeddingPoolingDecisionOnce(model, cached, logger);
|
|
474
|
+
return cached;
|
|
475
|
+
}
|
|
476
|
+
if (isModelDirectory(model)) {
|
|
477
|
+
try {
|
|
478
|
+
const metadata = readDirectoryMetadata(model);
|
|
479
|
+
metadataByCache.set(cacheKey, metadata);
|
|
480
|
+
testMetadataByModel.delete(model);
|
|
481
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
482
|
+
return metadata;
|
|
483
|
+
} catch (error) {
|
|
484
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
485
|
+
const metadata = {
|
|
486
|
+
status: "unsupported",
|
|
487
|
+
normalize: false,
|
|
488
|
+
declared: true,
|
|
489
|
+
reason
|
|
490
|
+
};
|
|
491
|
+
metadataByCache.set(cacheKey, metadata);
|
|
492
|
+
testMetadataByModel.delete(model);
|
|
493
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
494
|
+
return metadata;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
try {
|
|
498
|
+
const fetcher = options.fetcher ?? ((url) => fetch(url, { signal: AbortSignal.timeout(20000) }));
|
|
499
|
+
const files = await filesForRepo(remoteHost(), model, fetcher);
|
|
500
|
+
let metadata;
|
|
501
|
+
try {
|
|
502
|
+
metadata = poolingFromFiles(files.modulesText, files.configText, model);
|
|
503
|
+
} catch (error) {
|
|
504
|
+
metadata = {
|
|
505
|
+
status: "unsupported",
|
|
506
|
+
normalize: false,
|
|
507
|
+
declared: true,
|
|
508
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
await cacheRemoteFiles(model, cacheDir, files);
|
|
512
|
+
await writeStorageFileAsync(cachedMetadataPath(model, cacheDir), JSON.stringify(metadata), true);
|
|
513
|
+
metadataByCache.set(cacheKey, metadata);
|
|
514
|
+
testMetadataByModel.delete(model);
|
|
515
|
+
logLocalEmbeddingPoolingDecisionOnce(model, metadata, logger);
|
|
516
|
+
return metadata;
|
|
517
|
+
} catch (error) {
|
|
518
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
519
|
+
metadataByCache.set(cacheKey, UNRESOLVED_LOCAL_EMBEDDING_POOLING);
|
|
520
|
+
testMetadataByModel.delete(model);
|
|
521
|
+
logUnavailableOnce(model, reason, logger);
|
|
522
|
+
return UNRESOLVED_LOCAL_EMBEDDING_POOLING;
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
function logLocalEmbeddingPoolingUnavailableOnce(model, reason, logger = log2) {
|
|
526
|
+
logUnavailableOnce(model, reason, logger);
|
|
527
|
+
}
|
|
528
|
+
|
|
103
529
|
// src/features/magic-context/memory/embedding-identity.ts
|
|
104
530
|
function normalizeEndpoint(endpoint) {
|
|
105
531
|
return endpoint?.trim().replace(/\/+$/, "") ?? "";
|
|
@@ -121,6 +547,8 @@ function getEmbeddingProviderIdentity(config) {
|
|
|
121
547
|
const truncate = config.provider === "openai-compatible" ? config.truncate?.trim() : undefined;
|
|
122
548
|
const documentPrefix = config.provider === "openai-compatible" ? resolveEmbeddingTextPrefixes(config.model, config.query_instruction, config.document_prefix).documentPrefix : "";
|
|
123
549
|
const localDtype = config.provider === "local" && config.local_dtype && config.local_dtype !== "fp32" ? config.local_dtype : undefined;
|
|
550
|
+
const localPooling = config.provider === "local" ? getCachedLocalEmbeddingPooling(config.model?.trim() || DEFAULT_LOCAL_EMBEDDING_MODEL) : null;
|
|
551
|
+
const localPoolingFingerprint = localPooling?.status === "ready" && (localPooling.pooling !== "mean" || !localPooling.normalize) ? `${localPooling.pooling};normalize=${localPooling.normalize}` : undefined;
|
|
124
552
|
const identityInput = config.provider === "openai-compatible" ? {
|
|
125
553
|
provider: "openai-compatible",
|
|
126
554
|
model: config.model.trim(),
|
|
@@ -135,15 +563,16 @@ function getEmbeddingProviderIdentity(config) {
|
|
|
135
563
|
endpoint: "",
|
|
136
564
|
apiKeyPresent: false,
|
|
137
565
|
runtimeFingerprint: LOCAL_EMBEDDING_RUNTIME_FINGERPRINT,
|
|
138
|
-
...localDtype ? { localDtype } : {}
|
|
566
|
+
...localDtype ? { localDtype } : {},
|
|
567
|
+
...localPoolingFingerprint ? { localPooling: localPoolingFingerprint } : {}
|
|
139
568
|
};
|
|
140
569
|
return `embedding-provider:${computeNormalizedHash(JSON.stringify(identityInput))}`;
|
|
141
570
|
}
|
|
142
571
|
|
|
143
572
|
// src/features/magic-context/memory/embedding-local.ts
|
|
144
|
-
import { readdirSync, statSync } from "node:fs";
|
|
573
|
+
import { readdirSync, statSync as statSync2 } from "node:fs";
|
|
145
574
|
import { stat, unlink } from "node:fs/promises";
|
|
146
|
-
import { dirname, join } from "node:path";
|
|
575
|
+
import { dirname, join as join2 } from "node:path";
|
|
147
576
|
import { pathToFileURL } from "node:url";
|
|
148
577
|
import { workerData } from "node:worker_threads";
|
|
149
578
|
|
|
@@ -605,7 +1034,7 @@ var importTransformersNodeWasmFallbackForRuntime = async () => {
|
|
|
605
1034
|
const nodeWasmEntry = new URL(new URL(import.meta.url).pathname.endsWith(".ts") ? "../../../../dist/transformers-node-wasm.js" : "./transformers-node-wasm.js", import.meta.url).href;
|
|
606
1035
|
return await importPluginModule(() => import(nodeWasmEntry));
|
|
607
1036
|
};
|
|
608
|
-
var modelCacheDirForRuntime = () => workerData?.magicContextEmbeddingWorker && typeof workerData.modelCacheDir === "string" ? workerData.modelCacheDir :
|
|
1037
|
+
var modelCacheDirForRuntime = () => workerData?.magicContextEmbeddingWorker && typeof workerData.modelCacheDir === "string" ? workerData.modelCacheDir : join2(getMagicContextStorageDir(), "models");
|
|
609
1038
|
var logForRuntime = log2;
|
|
610
1039
|
var injectWasmOrtForRuntime = injectWasmOrt;
|
|
611
1040
|
var loadedLocalEmbeddingRuntimes = new Map;
|
|
@@ -614,14 +1043,14 @@ function directoryBytes(path) {
|
|
|
614
1043
|
let total = 0;
|
|
615
1044
|
try {
|
|
616
1045
|
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
617
|
-
const entryPath =
|
|
1046
|
+
const entryPath = join2(path, entry.name);
|
|
618
1047
|
if (entry.isDirectory()) {
|
|
619
1048
|
const childBytes = directoryBytes(entryPath);
|
|
620
1049
|
if (childBytes === null)
|
|
621
1050
|
return null;
|
|
622
1051
|
total += childBytes;
|
|
623
1052
|
} else if (entry.isFile()) {
|
|
624
|
-
total +=
|
|
1053
|
+
total += statSync2(entryPath).size;
|
|
625
1054
|
}
|
|
626
1055
|
}
|
|
627
1056
|
return total;
|
|
@@ -847,6 +1276,7 @@ class LocalEmbeddingProvider {
|
|
|
847
1276
|
dtype;
|
|
848
1277
|
runtimePreference;
|
|
849
1278
|
pipeline = null;
|
|
1279
|
+
poolingMetadata = null;
|
|
850
1280
|
workerClient;
|
|
851
1281
|
workerLoaded = false;
|
|
852
1282
|
initPromise = null;
|
|
@@ -924,6 +1354,18 @@ class LocalEmbeddingProvider {
|
|
|
924
1354
|
throw new LocalEmbeddingFsUnavailableError;
|
|
925
1355
|
}
|
|
926
1356
|
const modelCacheDir = modelCacheDirForRuntime();
|
|
1357
|
+
const cachedPoolingMetadata = getCachedLocalEmbeddingPooling(this.model, modelCacheDir);
|
|
1358
|
+
const poolingMetadata = cachedPoolingMetadata?.status === "unresolved" || !cachedPoolingMetadata ? UNRESOLVED_LOCAL_EMBEDDING_POOLING : cachedPoolingMetadata;
|
|
1359
|
+
if (poolingMetadata.status === "unsupported" || !poolingMetadata.pooling) {
|
|
1360
|
+
throw new Error(`unsupported pooling for local model '${this.model}': ${poolingMetadata.reason}`);
|
|
1361
|
+
}
|
|
1362
|
+
if (poolingMetadata.status === "unresolved") {
|
|
1363
|
+
logLocalEmbeddingPoolingUnavailableOnce(this.model, poolingMetadata.reason, logForRuntime);
|
|
1364
|
+
}
|
|
1365
|
+
this.poolingMetadata = poolingMetadata;
|
|
1366
|
+
if (!workerData?.magicContextEmbeddingWorker) {
|
|
1367
|
+
logLocalEmbeddingPoolingDecisionOnce(this.model, poolingMetadata, logForRuntime);
|
|
1368
|
+
}
|
|
927
1369
|
try {
|
|
928
1370
|
ensureStorageDirectorySync(modelCacheDir);
|
|
929
1371
|
env.cacheDir = modelCacheDir;
|
|
@@ -931,7 +1373,7 @@ class LocalEmbeddingProvider {
|
|
|
931
1373
|
log2("[magic-context] could not create model cache dir, using library default");
|
|
932
1374
|
}
|
|
933
1375
|
const createPipeline = transformersModule.pipeline;
|
|
934
|
-
const lockPath =
|
|
1376
|
+
const lockPath = join2(modelCacheDir, ".load.lock");
|
|
935
1377
|
const releaseLock = await acquireModelLoadLock(lockPath);
|
|
936
1378
|
const stopHeartbeat = startLockHeartbeat(lockPath);
|
|
937
1379
|
try {
|
|
@@ -1038,8 +1480,8 @@ class LocalEmbeddingProvider {
|
|
|
1038
1480
|
return null;
|
|
1039
1481
|
}
|
|
1040
1482
|
const result = await withQuietConsole(() => pipeline(text, {
|
|
1041
|
-
pooling: "mean",
|
|
1042
|
-
normalize: true
|
|
1483
|
+
pooling: this.poolingMetadata?.pooling ?? "mean",
|
|
1484
|
+
normalize: this.poolingMetadata?.normalize ?? true
|
|
1043
1485
|
}));
|
|
1044
1486
|
const embedding = extractBatchEmbeddings(result, 1)[0] ?? null;
|
|
1045
1487
|
if (!embedding) {
|
|
@@ -1090,11 +1532,19 @@ class LocalEmbeddingProvider {
|
|
|
1090
1532
|
if (!pipeline) {
|
|
1091
1533
|
return Array.from({ length: texts.length }, () => null);
|
|
1092
1534
|
}
|
|
1093
|
-
const
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1535
|
+
const pooling = this.poolingMetadata?.pooling ?? "mean";
|
|
1536
|
+
const normalize = this.poolingMetadata?.normalize ?? true;
|
|
1537
|
+
let embeddings;
|
|
1538
|
+
if (pooling === "last_token" && texts.length > 1) {
|
|
1539
|
+
embeddings = [];
|
|
1540
|
+
for (const text of texts) {
|
|
1541
|
+
const result = await withQuietConsole(() => pipeline(text, { pooling, normalize }));
|
|
1542
|
+
embeddings.push(extractBatchEmbeddings(result, 1)[0] ?? null);
|
|
1543
|
+
}
|
|
1544
|
+
} else {
|
|
1545
|
+
const result = await withQuietConsole(() => pipeline(texts, { pooling, normalize }));
|
|
1546
|
+
embeddings = extractBatchEmbeddings(result, texts.length);
|
|
1547
|
+
}
|
|
1098
1548
|
if (embeddings.every((embedding) => embedding === null)) {
|
|
1099
1549
|
this.lastFailureReason = classifyLocalEmbeddingFailure(new Error("local embedding pipeline returned no vectors"), { usesWasm: this.usesWasm });
|
|
1100
1550
|
} else {
|
|
@@ -1170,4 +1620,4 @@ class LocalEmbeddingProvider {
|
|
|
1170
1620
|
}
|
|
1171
1621
|
}
|
|
1172
1622
|
|
|
1173
|
-
export { embeddingModelsMatch, resolveEmbeddingTextPrefixes, getEmbeddingProviderIdentity, dominantEmbeddingFailure, resolveLocalEmbeddingWorkerRuntime, getLocalEmbeddingNativeMemoryStats, LocalEmbeddingProvider };
|
|
1623
|
+
export { embeddingModelsMatch, resolveEmbeddingTextPrefixes, getCachedLocalEmbeddingPooling, logLocalEmbeddingPoolingDecisionOnce, preflightLocalEmbeddingPooling, logLocalEmbeddingPoolingUnavailableOnce, getEmbeddingProviderIdentity, dominantEmbeddingFailure, resolveLocalEmbeddingWorkerRuntime, getLocalEmbeddingNativeMemoryStats, LocalEmbeddingProvider };
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "./index-pavzsdc6.js";
|
|
5
5
|
import {
|
|
6
6
|
log2
|
|
7
|
-
} from "./index-
|
|
7
|
+
} from "./index-h7tgn49j.js";
|
|
8
8
|
import {
|
|
9
9
|
resolveOpenCodeDbPath,
|
|
10
10
|
assertOpenCodeStoreGeneration,
|
|
@@ -15,11 +15,11 @@ import {
|
|
|
15
15
|
} from "./index-58e3y90n.js";
|
|
16
16
|
import {
|
|
17
17
|
Database2
|
|
18
|
-
} from "./index-
|
|
18
|
+
} from "./index-1zx2pa5d.js";
|
|
19
19
|
import {
|
|
20
20
|
forgetRawSessionOrdinalWatermark2,
|
|
21
21
|
forgetRawSessionSummaryRows2
|
|
22
|
-
} from "./index-
|
|
22
|
+
} from "./index-35qc6606.js";
|
|
23
23
|
import {
|
|
24
24
|
closeQuietly
|
|
25
25
|
} from "./index-8ajstryz.js";
|