kiokuko 1.0.14 → 1.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +16 -4
- package/README.ko.md +16 -4
- package/README.md +16 -4
- package/README.zh-CN.md +16 -4
- package/dist/akinator/agent-task.d.ts +10 -0
- package/dist/akinator/agent-task.d.ts.map +1 -1
- package/dist/akinator/agent-task.js +54 -11
- package/dist/akinator/agent-task.js.map +1 -1
- package/dist/assurance/codex-hooks.js +3 -3
- package/dist/assurance/codex-hooks.js.map +1 -1
- package/dist/assurance/contracts.d.ts +8 -1
- package/dist/assurance/contracts.d.ts.map +1 -1
- package/dist/assurance/contracts.js +2 -1
- package/dist/assurance/contracts.js.map +1 -1
- package/dist/assurance/inspect.d.ts +14 -0
- package/dist/assurance/inspect.d.ts.map +1 -1
- package/dist/assurance/inspect.js +71 -9
- package/dist/assurance/inspect.js.map +1 -1
- package/dist/assurance/refresh.d.ts.map +1 -1
- package/dist/assurance/refresh.js +25 -3
- package/dist/assurance/refresh.js.map +1 -1
- package/dist/assurance/service.d.ts +1 -1
- package/dist/assurance/snapshot.js +16 -2
- package/dist/assurance/snapshot.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +68 -7
- package/dist/cli.js.map +1 -1
- package/dist/commands/export.d.ts.map +1 -1
- package/dist/commands/export.js +3 -0
- package/dist/commands/export.js.map +1 -1
- package/dist/commands/import.d.ts.map +1 -1
- package/dist/commands/import.js +3 -0
- package/dist/commands/import.js.map +1 -1
- package/dist/commands/memory-index.d.ts +6 -0
- package/dist/commands/memory-index.d.ts.map +1 -0
- package/dist/commands/memory-index.js +30 -0
- package/dist/commands/memory-index.js.map +1 -0
- package/dist/commands/reranker.d.ts +9 -0
- package/dist/commands/reranker.d.ts.map +1 -0
- package/dist/commands/reranker.js +52 -0
- package/dist/commands/reranker.js.map +1 -0
- package/dist/config/paths.d.ts +2 -0
- package/dist/config/paths.d.ts.map +1 -1
- package/dist/config/paths.js +10 -0
- package/dist/config/paths.js.map +1 -1
- package/dist/context/broker.d.ts +14 -0
- package/dist/context/broker.d.ts.map +1 -1
- package/dist/context/broker.js +162 -15
- package/dist/context/broker.js.map +1 -1
- package/dist/context/ranking.d.ts +6 -2
- package/dist/context/ranking.d.ts.map +1 -1
- package/dist/context/ranking.js +23 -1
- package/dist/context/ranking.js.map +1 -1
- package/dist/context/scoped-broker.d.ts +20 -0
- package/dist/context/scoped-broker.d.ts.map +1 -1
- package/dist/context/scoped-broker.js +173 -10
- package/dist/context/scoped-broker.js.map +1 -1
- package/dist/context/selection-state.d.ts.map +1 -1
- package/dist/context/selection-state.js +2 -0
- package/dist/context/selection-state.js.map +1 -1
- package/dist/embedding/javascript-backend.d.ts.map +1 -1
- package/dist/embedding/javascript-backend.js +7 -1
- package/dist/embedding/javascript-backend.js.map +1 -1
- package/dist/embedding/model-download.d.ts +2 -2
- package/dist/embedding/model-download.d.ts.map +1 -1
- package/dist/embedding/model-download.js +23 -11
- package/dist/embedding/model-download.js.map +1 -1
- package/dist/embedding/model-installation.d.ts +3 -1
- package/dist/embedding/model-installation.d.ts.map +1 -1
- package/dist/embedding/model-installation.js +16 -9
- package/dist/embedding/model-installation.js.map +1 -1
- package/dist/embedding/model-manifest.d.ts +7 -7
- package/dist/embedding/model-manifest.d.ts.map +1 -1
- package/dist/embedding/model-manifest.js +28 -17
- package/dist/embedding/model-manifest.js.map +1 -1
- package/dist/embedding/presets/manifest.d.ts +20 -7
- package/dist/embedding/presets/manifest.d.ts.map +1 -1
- package/dist/embedding/presets/manifest.js.map +1 -1
- package/dist/embedding/sqlite-vec-backend.d.ts.map +1 -1
- package/dist/embedding/sqlite-vec-backend.js +7 -1
- package/dist/embedding/sqlite-vec-backend.js.map +1 -1
- package/dist/embedding/store.d.ts.map +1 -1
- package/dist/embedding/store.js +2 -0
- package/dist/embedding/store.js.map +1 -1
- package/dist/embedding/types.d.ts +2 -0
- package/dist/embedding/types.d.ts.map +1 -1
- package/dist/ledger/checkpoint-contract.d.ts +6 -6
- package/dist/mcp/chatgpt-contract.d.ts +8 -2
- package/dist/mcp/chatgpt-contract.d.ts.map +1 -1
- package/dist/mcp/request-deadline.d.ts +1 -1
- package/dist/mcp/request-deadline.d.ts.map +1 -1
- package/dist/mcp/request-deadline.js +3 -0
- package/dist/mcp/request-deadline.js.map +1 -1
- package/dist/mcp/runtime-owner.d.ts.map +1 -1
- package/dist/mcp/runtime-owner.js +7 -0
- package/dist/mcp/runtime-owner.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +39 -7
- package/dist/mcp/server.js.map +1 -1
- package/dist/memory/checkpoint-contract.d.ts +9 -9
- package/dist/memory/curator.d.ts.map +1 -1
- package/dist/memory/curator.js +2 -0
- package/dist/memory/curator.js.map +1 -1
- package/dist/memory/entries.d.ts.map +1 -1
- package/dist/memory/entries.js +2 -0
- package/dist/memory/entries.js.map +1 -1
- package/dist/memory/federated-retrieval.d.ts +16 -2
- package/dist/memory/federated-retrieval.d.ts.map +1 -1
- package/dist/memory/federated-retrieval.js +229 -45
- package/dist/memory/federated-retrieval.js.map +1 -1
- package/dist/memory/global-conversation-query.d.ts +5 -0
- package/dist/memory/global-conversation-query.d.ts.map +1 -1
- package/dist/memory/global-conversation-query.js +16 -2
- package/dist/memory/global-conversation-query.js.map +1 -1
- package/dist/memory/hybrid-retrieval.d.ts +18 -2
- package/dist/memory/hybrid-retrieval.d.ts.map +1 -1
- package/dist/memory/hybrid-retrieval.js +180 -9
- package/dist/memory/hybrid-retrieval.js.map +1 -1
- package/dist/memory/index-service.d.ts +103 -0
- package/dist/memory/index-service.d.ts.map +1 -0
- package/dist/memory/index-service.js +196 -0
- package/dist/memory/index-service.js.map +1 -0
- package/dist/memory/index-state.d.ts +56 -0
- package/dist/memory/index-state.d.ts.map +1 -0
- package/dist/memory/index-state.js +100 -0
- package/dist/memory/index-state.js.map +1 -0
- package/dist/memory/interaction-capture.d.ts.map +1 -1
- package/dist/memory/interaction-capture.js +6 -3
- package/dist/memory/interaction-capture.js.map +1 -1
- package/dist/memory/interaction-contract.d.ts +30 -0
- package/dist/memory/interaction-contract.d.ts.map +1 -1
- package/dist/memory/interaction-contract.js +38 -0
- package/dist/memory/interaction-contract.js.map +1 -1
- package/dist/memory/interaction-recall.d.ts +14 -1
- package/dist/memory/interaction-recall.d.ts.map +1 -1
- package/dist/memory/interaction-recall.js +48 -2
- package/dist/memory/interaction-recall.js.map +1 -1
- package/dist/memory/lesson-derivation.d.ts +79 -0
- package/dist/memory/lesson-derivation.d.ts.map +1 -0
- package/dist/memory/lesson-derivation.js +302 -0
- package/dist/memory/lesson-derivation.js.map +1 -0
- package/dist/memory/lesson-reinforcement.d.ts.map +1 -1
- package/dist/memory/lesson-reinforcement.js +7 -4
- package/dist/memory/lesson-reinforcement.js.map +1 -1
- package/dist/memory/lifecycle.d.ts.map +1 -1
- package/dist/memory/lifecycle.js +4 -0
- package/dist/memory/lifecycle.js.map +1 -1
- package/dist/memory/retrieval.d.ts +14 -0
- package/dist/memory/retrieval.d.ts.map +1 -1
- package/dist/memory/retrieval.js +107 -13
- package/dist/memory/retrieval.js.map +1 -1
- package/dist/memory/scoped-memory.d.ts +5 -0
- package/dist/memory/scoped-memory.d.ts.map +1 -1
- package/dist/memory/scoped-memory.js +3 -0
- package/dist/memory/scoped-memory.js.map +1 -1
- package/dist/memory/search-extension-config.d.ts +5 -0
- package/dist/memory/search-extension-config.d.ts.map +1 -0
- package/dist/memory/search-extension-config.js +20 -0
- package/dist/memory/search-extension-config.js.map +1 -0
- package/dist/memory/temporal.d.ts +18 -0
- package/dist/memory/temporal.d.ts.map +1 -0
- package/dist/memory/temporal.js +128 -0
- package/dist/memory/temporal.js.map +1 -0
- package/dist/reranker/config.d.ts +15 -0
- package/dist/reranker/config.d.ts.map +1 -0
- package/dist/reranker/config.js +48 -0
- package/dist/reranker/config.js.map +1 -0
- package/dist/reranker/preset.d.ts +12 -0
- package/dist/reranker/preset.d.ts.map +1 -0
- package/dist/reranker/preset.js +102 -0
- package/dist/reranker/preset.js.map +1 -0
- package/dist/reranker/runtime.d.ts +40 -0
- package/dist/reranker/runtime.d.ts.map +1 -0
- package/dist/reranker/runtime.js +324 -0
- package/dist/reranker/runtime.js.map +1 -0
- package/dist/reranker/service.d.ts +36 -0
- package/dist/reranker/service.d.ts.map +1 -0
- package/dist/reranker/service.js +163 -0
- package/dist/reranker/service.js.map +1 -0
- package/dist/reranker/setup.d.ts +30 -0
- package/dist/reranker/setup.d.ts.map +1 -0
- package/dist/reranker/setup.js +50 -0
- package/dist/reranker/setup.js.map +1 -0
- package/dist/reranker/worker.d.ts +2 -0
- package/dist/reranker/worker.d.ts.map +1 -0
- package/dist/reranker/worker.js +115 -0
- package/dist/reranker/worker.js.map +1 -0
- package/dist/serialization/validate.d.ts.map +1 -1
- package/dist/serialization/validate.js +29 -1
- package/dist/serialization/validate.js.map +1 -1
- package/dist/server/routes/task-assurance.d.ts.map +1 -1
- package/dist/server/routes/task-assurance.js +11 -2
- package/dist/server/routes/task-assurance.js.map +1 -1
- package/dist/web/server.d.ts.map +1 -1
- package/dist/web/server.js +47 -11
- package/dist/web/server.js.map +1 -1
- package/docs/README.ja.md +2 -0
- package/docs/README.md +2 -0
- package/docs/chatgpt.ja.md +1 -0
- package/docs/chatgpt.md +1 -0
- package/docs/memory-assurance.md +20 -0
- package/docs/memory-index-evaluation-2026-09-30.md +50 -0
- package/docs/memory-index.md +138 -0
- package/migrations/007_revision_bound_lesson_sources.sql +33 -0
- package/migrations/008_memory_index.sql +39 -0
- package/package.json +3 -2
- package/skills/kiokuko-soul/SKILL.md +13 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"javascript-backend.js","sourceRoot":"","sources":["../../src/embedding/javascript-backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAClD,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AACrD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"javascript-backend.js","sourceRoot":"","sources":["../../src/embedding/javascript-backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG5E,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAClD,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AACrD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAW3C,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,IAAI,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClH,SAAS,CAAC,mCAAmC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAwB;IAU3C,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrH,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,IAAI;QAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACjI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC;QAAE,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAC/I,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,uBAAuB;QAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAChI,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,KAAK,SAAS;QAC/D,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrN,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE;QACrD,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW,EAAE,eAAe,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC;QACjE,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,SAAS;QACT,kBAAkB;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,IAAe,EAAE,KAAgB;IAC9C,OAAO,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;AAC5G,CAAC;AAED,SAAS,MAAM,CAAC,IAAe,EAAE,KAAgB;IAC/C,OAAO,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;AAC5G,CAAC;AAED,MAAM,cAAc;IAGG;IAFZ,MAAM,GAAgB,EAAE,CAAC;IAElC,YAAqB,KAAa;QAAb,UAAK,GAAL,KAAK,CAAQ;IAAG,CAAC;IAEtC,GAAG,CAAC,GAAc;QAChB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,OAAO;QACvD,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,MAAM;QACJ,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,CAAC;IAEO,MAAM,CAAC,KAAa;QAC1B,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC;gBAAE,MAAM;YACpG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;YACjC,KAAK,GAAG,MAAM,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;YACvB,IAAI,OAAO,GAAG,MAAM,CAAC;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,SAAS,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,EAAE,YAAY,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;YAC5G,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC;gBAAE,OAAO,GAAG,KAAK,CAAC;YACzG,IAAI,OAAO,KAAK,MAAM;gBAAE,OAAO;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO;YACnE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;YACnC,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,6BAA6B;IAC/B,EAAE,GAAG,qBAAqB,CAAC;IAEpC,MAAM,CAAC,QAAwB,EAAE,KAAwB;QACvD,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG;YACd,mBAAmB;YACnB,mBAAmB;YACnB,kCAAkC;YAClC,kCAAkC;YAClC,iBAAiB,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,EAAC,KAAK,CAAC,kBAAkB,IAAE,KAAK,CAAC;SAClF,CAAC;QACF,MAAM,UAAU,GAA2B,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QACzF,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAChC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC;8DAC2C,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1G,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,uBAAuB,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChG,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;;;eAMnB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;;;KAG/B,CAAC,CAAC,GAAG,CAAY,GAAG,UAAU,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,MAAM,GAAG,8BAA8B,EAAE,CAAC;YACjD,MAAM,IAAI,YAAY,CAAC,cAAc,EAAE,0DAA0D,CAAC,CAAC;QACrG,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAG,GAAG,CAAC,eAAe,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAC,GAAG,CAAC,SAAS,EAAC,EAAE,CAAC;gBAAE,SAAS;YACjF,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,KAAK,UAAU,CAAC,UAAU;gBAAE,SAAS,CAAC,0DAA0D,CAAC,CAAC;YAC1J,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACtH,IAAI,MAAoB,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YAC1E,CAAC;YAAC,MAAM,CAAC;gBACP,IAAG,GAAG,CAAC,eAAe;oBAAE,SAAS;gBACjC,SAAS,CAAC,0BAA0B,CAAC,CAAC;YACxC,CAAC;YACD,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,SAAS,CAAC,4BAA4B,CAAC,CAAC;YACxE,IAAI,QAAQ,IAAI,UAAU,CAAC,eAAe;gBAAE,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,mCAAmC,GAAG,GAAwB,EAAE,CAAC,IAAI,6BAA6B,EAAE,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type VerifiedModelManifest } from './model-manifest.js';
|
|
2
|
-
import type {
|
|
2
|
+
import type { ModelArtifactPreset } from './presets/manifest.js';
|
|
3
3
|
export interface ModelDownloadProgress {
|
|
4
4
|
readonly file: string;
|
|
5
5
|
readonly completedBytes: number;
|
|
@@ -10,7 +10,7 @@ export interface DownloadedModel {
|
|
|
10
10
|
readonly manifest: VerifiedModelManifest;
|
|
11
11
|
}
|
|
12
12
|
export interface ModelDownloader {
|
|
13
|
-
download(preset:
|
|
13
|
+
download(preset: ModelArtifactPreset, stagingDirectory: string, options?: {
|
|
14
14
|
readonly signal?: AbortSignal;
|
|
15
15
|
readonly onProgress?: (progress: ModelDownloadProgress) => void;
|
|
16
16
|
}): Promise<DownloadedModel>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-download.d.ts","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"model-download.d.ts","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"AAIA,OAAO,EAA+C,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,CAAC;CAC1C;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CACN,MAAM,EAAE,mBAAmB,EAC3B,gBAAgB,EAAE,MAAM,EACxB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAA;KAAE,GAC3G,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B;AAED,KAAK,eAAe,GAAG,CAAC,MAAM,EAAE;IAC9B,IAAI,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,KAAK,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;AAM3B,wBAAgB,gCAAgC,CAAC,OAAO,GAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,eAAe,CAAA;CAAO,GAAG,eAAe,CA2B3H"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
3
4
|
import { KiokukoError } from '../errors.js';
|
|
4
5
|
import { createModelManifest, validatePresetManifest } from './model-manifest.js';
|
|
5
6
|
function aborted(signal) {
|
|
@@ -16,26 +17,37 @@ export function createHuggingFaceModelDownloader(options = {}) {
|
|
|
16
17
|
let completedBytes = 0;
|
|
17
18
|
for (const file of preset.files) {
|
|
18
19
|
aborted(downloadOptions.signal);
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
xet: true,
|
|
24
|
-
});
|
|
25
|
-
if (blob === null)
|
|
26
|
-
throw new KiokukoError('NOT_FOUND', `Pinned model artifact is unavailable: ${file.path}`);
|
|
27
|
-
const bytes = new Uint8Array(await blob.arrayBuffer());
|
|
20
|
+
const bundled = preset.bundledFiles?.find((item) => item.path === file.path);
|
|
21
|
+
const bytes = bundled === undefined
|
|
22
|
+
? await downloadPinnedFile(preset, file.path, downloadFile, downloadOptions.signal)
|
|
23
|
+
: new TextEncoder().encode(bundled.contents);
|
|
28
24
|
aborted(downloadOptions.signal);
|
|
29
|
-
if (bytes.byteLength !== file.size)
|
|
25
|
+
if (file.size !== undefined && bytes.byteLength !== file.size)
|
|
30
26
|
throw new KiokukoError('INTEGRITY_ERROR', `Downloaded model artifact size mismatch: ${file.path}`);
|
|
27
|
+
if (createHash('sha256').update(bytes).digest('hex') !== file.sha256)
|
|
28
|
+
throw new KiokukoError('SECURITY_REJECTION', `Pinned model artifact hash mismatch: ${file.path}`);
|
|
29
|
+
completedBytes += bytes.byteLength;
|
|
30
|
+
if (completedBytes > (preset.maximumBytes ?? Number.POSITIVE_INFINITY))
|
|
31
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Downloaded model exceeds its configured size limit');
|
|
31
32
|
const target = path.join(stagingDirectory, file.path);
|
|
32
33
|
await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
33
34
|
await writeFile(target, bytes, { mode: 0o600, flag: 'wx' });
|
|
34
|
-
completedBytes += bytes.byteLength;
|
|
35
35
|
downloadOptions.onProgress?.({ file: file.path, completedBytes, totalBytes: manifest.totalBytes });
|
|
36
36
|
}
|
|
37
37
|
return { directory: stagingDirectory, manifest };
|
|
38
38
|
},
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
|
+
async function downloadPinnedFile(preset, filePath, downloadFile, signal) {
|
|
42
|
+
aborted(signal);
|
|
43
|
+
const blob = await downloadFile({
|
|
44
|
+
repo: { type: 'model', name: preset.artifactRepository },
|
|
45
|
+
path: filePath,
|
|
46
|
+
revision: preset.revision,
|
|
47
|
+
xet: true,
|
|
48
|
+
});
|
|
49
|
+
if (blob === null)
|
|
50
|
+
throw new KiokukoError('NOT_FOUND', `Pinned model artifact is unavailable: ${filePath}`);
|
|
51
|
+
return new Uint8Array(await blob.arrayBuffer());
|
|
52
|
+
}
|
|
41
53
|
//# sourceMappingURL=model-download.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-download.js","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAA8B,MAAM,qBAAqB,CAAC;AA8B9G,SAAS,OAAO,CAAC,MAA+B;IAC9C,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,0CAA0C,CAAC,CAAC;AACjH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,UAAuD,EAAE;IACxG,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,eAAe,GAAG,EAAE,EAAE,EAAE;YACjE,sBAAsB,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAA+B,CAAC;YAChH,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,
|
|
1
|
+
{"version":3,"file":"model-download.js","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAA8B,MAAM,qBAAqB,CAAC;AA8B9G,SAAS,OAAO,CAAC,MAA+B;IAC9C,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,0CAA0C,CAAC,CAAC;AACjH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,UAAuD,EAAE;IACxG,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,eAAe,GAAG,EAAE,EAAE,EAAE;YACjE,sBAAsB,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAA+B,CAAC;YAChH,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7E,MAAM,KAAK,GAAG,OAAO,KAAK,SAAS;oBACjC,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,eAAe,CAAC,MAAM,CAAC;oBACnF,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC/C,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,IAAI;oBAAE,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,4CAA4C,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAClK,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxK,cAAc,IAAI,KAAK,CAAC,UAAU,CAAC;gBACnC,IAAI,cAAc,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,iBAAiB,CAAC;oBAAE,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,oDAAoD,CAAC,CAAC;gBACzK,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpE,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5D,eAAe,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrG,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;QACnD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,MAA2B,EAC3B,QAAgB,EAChB,YAA6B,EAC7B,MAA+B;IAE/B,OAAO,CAAC,MAAM,CAAC,CAAC;IAChB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;QAC9B,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,kBAAkB,EAAE;QACxD,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,IAAI;KACV,CAAC,CAAC;IACH,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,yCAAyC,QAAQ,EAAE,CAAC,CAAC;IAC5G,OAAO,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AAClD,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type PathEnvironment } from '../config/paths.js';
|
|
2
2
|
import { type ModelDownloader, type ModelDownloadProgress } from './model-download.js';
|
|
3
|
-
import type { LocalEmbeddingPreset } from './presets/manifest.js';
|
|
3
|
+
import type { LocalEmbeddingPreset, ModelArtifactPreset } from './presets/manifest.js';
|
|
4
4
|
export interface InstalledModel {
|
|
5
5
|
readonly installation: 'installed' | 'reused';
|
|
6
6
|
readonly directory: string;
|
|
@@ -12,6 +12,8 @@ export interface InstallModelOptions extends PathEnvironment {
|
|
|
12
12
|
readonly downloader?: ModelDownloader;
|
|
13
13
|
readonly signal?: AbortSignal;
|
|
14
14
|
readonly onProgress?: (progress: ModelDownloadProgress) => void;
|
|
15
|
+
readonly family?: 'embeddings' | 'rerankers';
|
|
15
16
|
}
|
|
17
|
+
export declare function installLocalModel(preset: ModelArtifactPreset, options?: InstallModelOptions): Promise<InstalledModel>;
|
|
16
18
|
export declare function installEmbeddingModel(preset: LocalEmbeddingPreset, options?: InstallModelOptions): Promise<InstalledModel>;
|
|
17
19
|
//# sourceMappingURL=model-installation.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-installation.d.ts","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAIA,OAAO,
|
|
1
|
+
{"version":3,"file":"model-installation.d.ts","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2D,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAEnH,OAAO,EAAoC,KAAK,eAAe,EAAE,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACzH,OAAO,KAAK,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEvF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,YAAY,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAChE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;CAC9C;AAkBD,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,cAAc,CAAC,CA8CzB;AAED,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,cAAc,CAAC,CAEzB"}
|
|
@@ -2,11 +2,11 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { chmod, lstat, mkdir, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { KiokukoError } from '../errors.js';
|
|
5
|
-
import { getEmbeddingPresetDirectory } from '../config/paths.js';
|
|
5
|
+
import { getEmbeddingPresetDirectory, getRerankerPresetDirectory } from '../config/paths.js';
|
|
6
6
|
import { assertNoUnexpectedModelFiles, MODEL_MANIFEST_FILENAME, serializeModelManifest, verifyModelDirectory } from './model-manifest.js';
|
|
7
7
|
import { createHuggingFaceModelDownloader } from './model-download.js';
|
|
8
|
-
function relativeModelPath(preset) {
|
|
9
|
-
return path.posix.join('models',
|
|
8
|
+
function relativeModelPath(preset, family) {
|
|
9
|
+
return path.posix.join('models', family, preset.id, preset.revision);
|
|
10
10
|
}
|
|
11
11
|
async function existingDirectory(pathname) {
|
|
12
12
|
try {
|
|
@@ -23,13 +23,17 @@ async function existingDirectory(pathname) {
|
|
|
23
23
|
throw error;
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
export async function
|
|
26
|
+
export async function installLocalModel(preset, options = {}) {
|
|
27
27
|
const downloader = options.downloader ?? createHuggingFaceModelDownloader();
|
|
28
|
-
const
|
|
28
|
+
const family = options.family ?? 'embeddings';
|
|
29
|
+
const finalDirectory = family === 'rerankers'
|
|
30
|
+
? getRerankerPresetDirectory(preset.id, preset.revision, options)
|
|
31
|
+
: getEmbeddingPresetDirectory(preset.id, preset.revision, options);
|
|
32
|
+
const relativePath = relativeModelPath(preset, family);
|
|
29
33
|
if (await existingDirectory(finalDirectory)) {
|
|
30
34
|
const manifest = await verifyModelDirectory(finalDirectory, preset);
|
|
31
35
|
await assertNoUnexpectedModelFiles(finalDirectory, preset);
|
|
32
|
-
return { installation: 'reused', directory: finalDirectory, relativePath
|
|
36
|
+
return { installation: 'reused', directory: finalDirectory, relativePath, totalBytes: manifest.totalBytes, manifestHash: manifest.artifactManifestHash };
|
|
33
37
|
}
|
|
34
38
|
const stagingParent = path.dirname(path.dirname(finalDirectory));
|
|
35
39
|
const stagingDirectory = path.join(stagingParent, '.staging', randomUUID());
|
|
@@ -48,17 +52,17 @@ export async function installEmbeddingModel(preset, options = {}) {
|
|
|
48
52
|
const existing = await verifyModelDirectory(finalDirectory, preset);
|
|
49
53
|
if (existing.artifactManifestHash !== manifest.artifactManifestHash)
|
|
50
54
|
throw new KiokukoError('CONFLICT', 'A different model installation already exists for this preset revision');
|
|
51
|
-
return { installation: 'reused', directory: finalDirectory, relativePath
|
|
55
|
+
return { installation: 'reused', directory: finalDirectory, relativePath, totalBytes: existing.totalBytes, manifestHash: existing.artifactManifestHash };
|
|
52
56
|
}
|
|
53
57
|
try {
|
|
54
58
|
await rename(downloaded.directory, finalDirectory);
|
|
55
|
-
return { installation: 'installed', directory: finalDirectory, relativePath
|
|
59
|
+
return { installation: 'installed', directory: finalDirectory, relativePath, totalBytes: manifest.totalBytes, manifestHash: manifest.artifactManifestHash };
|
|
56
60
|
}
|
|
57
61
|
catch (error) {
|
|
58
62
|
if (error instanceof Error && 'code' in error && error.code === 'ENOTEMPTY'
|
|
59
63
|
&& await existingDirectory(finalDirectory)) {
|
|
60
64
|
const existing = await verifyModelDirectory(finalDirectory, preset);
|
|
61
|
-
return { installation: 'reused', directory: finalDirectory, relativePath
|
|
65
|
+
return { installation: 'reused', directory: finalDirectory, relativePath, totalBytes: existing.totalBytes, manifestHash: existing.artifactManifestHash };
|
|
62
66
|
}
|
|
63
67
|
throw error;
|
|
64
68
|
}
|
|
@@ -68,4 +72,7 @@ export async function installEmbeddingModel(preset, options = {}) {
|
|
|
68
72
|
throw error;
|
|
69
73
|
}
|
|
70
74
|
}
|
|
75
|
+
export async function installEmbeddingModel(preset, options = {}) {
|
|
76
|
+
return installLocalModel(preset, { ...options, family: 'embeddings' });
|
|
77
|
+
}
|
|
71
78
|
//# sourceMappingURL=model-installation.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-installation.js","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,2BAA2B,EAAwB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"model-installation.js","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAwB,MAAM,oBAAoB,CAAC;AACnH,OAAO,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC1I,OAAO,EAAE,gCAAgC,EAAoD,MAAM,qBAAqB,CAAC;AAkBzH,SAAS,iBAAiB,CAAC,MAA2B,EAAE,MAAkC;IACxF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wDAAwD,CAAC,CAAC;QACzJ,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY;YAAE,MAAM,KAAK,CAAC;QAC/C,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAClH,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAA2B,EAC3B,UAA+B,EAAE;IAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,gCAAgC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC;IAC9C,MAAM,cAAc,GAAG,MAAM,KAAK,WAAW;QAC3C,CAAC,CAAC,0BAA0B,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC;QACjE,CAAC,CAAC,2BAA2B,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,MAAM,YAAY,GAAG,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvD,IAAI,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,4BAA4B,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;IAC3J,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;IACjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE;YACrE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YACnE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;SAChF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC1E,MAAM,4BAA4B,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzI,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,IAAI,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,oBAAoB,KAAK,QAAQ,CAAC,oBAAoB;gBAAE,MAAM,IAAI,YAAY,CAAC,UAAU,EAAE,wEAAwE,CAAC,CAAC;YAClL,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QAC3J,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;YACnD,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QAC9J,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,WAAW;mBACjG,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;gBACpE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;YAC3J,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpF,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAA4B,EAC5B,UAA+B,EAAE;IAEjC,OAAO,iBAAiB,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;AACzE,CAAC"}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ModelArtifactFile, ModelArtifactPreset } from './presets/manifest.js';
|
|
2
2
|
export declare const MODEL_MANIFEST_FILENAME = "kiokuko-model-manifest.json";
|
|
3
3
|
export declare const MAX_MODEL_BYTES: number;
|
|
4
4
|
export interface VerifiedModelManifest {
|
|
5
5
|
readonly schemaVersion: 1;
|
|
6
|
-
readonly presetId:
|
|
6
|
+
readonly presetId: string;
|
|
7
7
|
readonly repositoryId: string;
|
|
8
8
|
readonly revision: string;
|
|
9
9
|
readonly artifactManifestHash: string;
|
|
10
|
-
readonly files: readonly
|
|
10
|
+
readonly files: readonly ModelArtifactFile[];
|
|
11
11
|
readonly totalBytes: number;
|
|
12
12
|
}
|
|
13
|
-
export declare function validatePresetManifest(preset:
|
|
14
|
-
export declare function createModelManifest(preset:
|
|
13
|
+
export declare function validatePresetManifest(preset: ModelArtifactPreset): void;
|
|
14
|
+
export declare function createModelManifest(preset: ModelArtifactPreset): VerifiedModelManifest;
|
|
15
15
|
export declare function serializeModelManifest(manifest: VerifiedModelManifest): string;
|
|
16
|
-
export declare function verifyModelDirectory(directory: string, preset:
|
|
17
|
-
export declare function assertNoUnexpectedModelFiles(directory: string, preset:
|
|
16
|
+
export declare function verifyModelDirectory(directory: string, preset: ModelArtifactPreset): Promise<VerifiedModelManifest>;
|
|
17
|
+
export declare function assertNoUnexpectedModelFiles(directory: string, preset: ModelArtifactPreset): Promise<void>;
|
|
18
18
|
//# sourceMappingURL=model-manifest.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-manifest.d.ts","sourceRoot":"","sources":["../../src/embedding/model-manifest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"model-manifest.d.ts","sourceRoot":"","sources":["../../src/embedding/model-manifest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAGpF,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,eAAe,QAAoB,CAAC;AAEjD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAUD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA6BxE;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,mBAAmB,GAAG,qBAAqB,CAYtF;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,CAE9E;AAED,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,qBAAqB,CAAC,CAahC;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAWf"}
|
|
@@ -13,34 +13,41 @@ function hashFile(bytes) {
|
|
|
13
13
|
return createHash('sha256').update(bytes).digest('hex');
|
|
14
14
|
}
|
|
15
15
|
export function validatePresetManifest(preset) {
|
|
16
|
-
if (preset.id !== '
|
|
17
|
-
|| preset.
|
|
18
|
-
|| preset.artifactRepository !== '
|
|
16
|
+
if (typeof preset.id !== 'string' || !/^[a-z0-9][a-z0-9-]{0,63}$/u.test(preset.id)
|
|
17
|
+
|| preset.schemaVersion !== 1
|
|
18
|
+
|| typeof preset.artifactRepository !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)+$/u.test(preset.artifactRepository)
|
|
19
19
|
|| !/^[0-9a-f]{40}$/u.test(preset.revision)
|
|
20
|
-
||
|
|
21
|
-
|
|
22
|
-
|| preset.distanceMetric !== 'cosine' || preset.distanceCeiling <= 0 || preset.distanceCeiling >= 2
|
|
23
|
-
|| preset.inputContract !== 'e5-query-passage-v1' || preset.queryPrefix !== 'query: '
|
|
24
|
-
|| preset.documentPrefix !== 'passage: ' || !/^[0-9]+\.[0-9]+\.[0-9]+$/u.test(preset.transformersJsVersion)) {
|
|
25
|
-
invalid('Embedding preset manifest is invalid');
|
|
20
|
+
|| !/^[0-9]+\.[0-9]+\.[0-9]+$/u.test(preset.transformersJsVersion)) {
|
|
21
|
+
invalid('Model artifact preset manifest is invalid');
|
|
26
22
|
}
|
|
27
23
|
const paths = new Set();
|
|
28
24
|
let totalBytes = 0;
|
|
29
25
|
for (const file of preset.files) {
|
|
30
26
|
if (paths.has(file.path) || file.path.length === 0 || file.path.startsWith('/')
|
|
31
27
|
|| file.path.split('/').some((part) => part === '' || part === '.' || part === '..')
|
|
32
|
-
|| !Number.isSafeInteger(file.size) || file.size <= 0
|
|
28
|
+
|| file.size !== undefined && (!Number.isSafeInteger(file.size) || file.size <= 0)
|
|
33
29
|
|| !/^[0-9a-f]{64}$/u.test(file.sha256))
|
|
34
|
-
invalid('
|
|
30
|
+
invalid('Model artifact file manifest is invalid');
|
|
35
31
|
paths.add(file.path);
|
|
36
|
-
totalBytes += file.size;
|
|
32
|
+
totalBytes += file.size ?? 0;
|
|
37
33
|
}
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
const maximumBytes = preset.maximumBytes ?? MAX_MODEL_BYTES;
|
|
35
|
+
const bundledPaths = new Set();
|
|
36
|
+
for (const file of preset.bundledFiles ?? []) {
|
|
37
|
+
if (bundledPaths.has(file.path) || !paths.has(file.path) || typeof file.contents !== 'string') {
|
|
38
|
+
invalid('Bundled model artifact file manifest is invalid');
|
|
39
|
+
}
|
|
40
|
+
bundledPaths.add(file.path);
|
|
41
|
+
}
|
|
42
|
+
if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 1 || maximumBytes > 2 * 1024 * 1024 * 1024
|
|
43
|
+
|| preset.files.length === 0 || totalBytes > maximumBytes)
|
|
44
|
+
invalid('Model artifact file size is unsupported');
|
|
45
|
+
if ((preset.bundledFiles ?? []).some((file) => !bundledPaths.has(file.path)))
|
|
46
|
+
invalid('Bundled model artifact file manifest is invalid');
|
|
40
47
|
}
|
|
41
48
|
export function createModelManifest(preset) {
|
|
42
49
|
validatePresetManifest(preset);
|
|
43
|
-
const totalBytes = preset.files.reduce((total, file) => total + file.size, 0);
|
|
50
|
+
const totalBytes = preset.files.reduce((total, file) => total + (file.size ?? 0), 0);
|
|
44
51
|
return Object.freeze({
|
|
45
52
|
schemaVersion: 1,
|
|
46
53
|
presetId: preset.id,
|
|
@@ -56,17 +63,21 @@ export function serializeModelManifest(manifest) {
|
|
|
56
63
|
}
|
|
57
64
|
export async function verifyModelDirectory(directory, preset) {
|
|
58
65
|
const expected = createModelManifest(preset);
|
|
66
|
+
let totalBytes = 0;
|
|
59
67
|
for (const file of expected.files) {
|
|
60
68
|
const filePath = path.join(directory, file.path);
|
|
61
69
|
const stat = await lstat(filePath);
|
|
62
70
|
if (!stat.isFile() || stat.isSymbolicLink())
|
|
63
71
|
throw new KiokukoError('SECURITY_REJECTION', `Model artifact is not a regular file: ${file.path}`);
|
|
64
|
-
if (stat.size !== file.size)
|
|
72
|
+
if (file.size !== undefined && stat.size !== file.size)
|
|
65
73
|
throw new KiokukoError('INTEGRITY_ERROR', `Model artifact size mismatch: ${file.path}`);
|
|
66
74
|
if (hashFile(await readFile(filePath)) !== file.sha256)
|
|
67
75
|
throw new KiokukoError('SECURITY_REJECTION', `Model artifact hash mismatch: ${file.path}`);
|
|
76
|
+
totalBytes += stat.size;
|
|
68
77
|
}
|
|
69
|
-
|
|
78
|
+
if (totalBytes > (preset.maximumBytes ?? MAX_MODEL_BYTES))
|
|
79
|
+
throw new KiokukoError('SECURITY_REJECTION', 'Model installation exceeds its configured size limit');
|
|
80
|
+
return Object.freeze({ ...expected, totalBytes });
|
|
70
81
|
}
|
|
71
82
|
export async function assertNoUnexpectedModelFiles(directory, preset) {
|
|
72
83
|
const allowed = new Set([...preset.files.map((file) => file.path), MODEL_MANIFEST_FILENAME]);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-manifest.js","sourceRoot":"","sources":["../../src/embedding/model-manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,6BAA6B,CAAC;AACrE,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAYjD,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"model-manifest.js","sourceRoot":"","sources":["../../src/embedding/model-manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,6BAA6B,CAAC;AACrE,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAYjD,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAA2B;IAChE,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;WAC7E,MAAM,CAAC,aAAa,KAAK,CAAC;WAC1B,OAAO,MAAM,CAAC,kBAAkB,KAAK,QAAQ,IAAI,CAAC,gEAAgE,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;WAClJ,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;WACxC,CAAC,2BAA2B,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACrE,OAAO,CAAC,2CAA2C,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;eAC1E,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC;eACjF,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;eAC/E,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,yCAAyC,CAAC,CAAC;QAC9F,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,UAAU,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,eAAe,CAAC;IAC5D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC7C,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC9F,OAAO,CAAC,iDAAiD,CAAC,CAAC;QAC7D,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;WAC/F,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,GAAG,YAAY;QAAE,OAAO,CAAC,yCAAyC,CAAC,CAAC;IAChH,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,iDAAiD,CAAC,CAAC;AAC3I,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAA2B;IAC7D,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,YAAY,EAAE,MAAM,CAAC,kBAAkB;QACvC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,oBAAoB,EAAE,kBAAkB,CAAC,MAAM,CAAC;QAChD,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5E,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAA+B;IACpE,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,SAAiB,EACjB,MAA2B;IAE3B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,yCAAyC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChJ,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,iCAAiC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAChJ,IAAI,QAAQ,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,iCAAiC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACnJ,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC;IAC1B,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,eAAe,CAAC;QAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,sDAAsD,CAAC,CAAC;IAChK,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAAiB,EACjB,MAA2B;IAE3B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,uBAAuB,CAAC,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAG,KAAK,EAAE,OAAe,EAAE,MAAc,EAAiB,EAAE;QACpE,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC9E,IAAI,KAAK,CAAC,cAAc,EAAE;gBAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,6CAA6C,QAAQ,EAAE,CAAC,CAAC;YAClI,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;iBACzE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,gDAAgD,QAAQ,EAAE,CAAC,CAAC;QAC5I,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -1,16 +1,26 @@
|
|
|
1
|
-
export interface
|
|
1
|
+
export interface ModelArtifactFile {
|
|
2
2
|
readonly path: string;
|
|
3
|
-
readonly size
|
|
3
|
+
readonly size?: number;
|
|
4
4
|
readonly sha256: string;
|
|
5
5
|
}
|
|
6
|
-
export interface
|
|
7
|
-
readonly id:
|
|
6
|
+
export interface ModelArtifactPreset {
|
|
7
|
+
readonly id: string;
|
|
8
8
|
readonly schemaVersion: 1;
|
|
9
|
+
readonly artifactRepository: string;
|
|
10
|
+
readonly revision: string;
|
|
11
|
+
readonly transformersJsVersion: string;
|
|
12
|
+
readonly files: readonly ModelArtifactFile[];
|
|
13
|
+
readonly maximumBytes?: number;
|
|
14
|
+
readonly bundledFiles?: readonly {
|
|
15
|
+
readonly path: string;
|
|
16
|
+
readonly contents: string;
|
|
17
|
+
}[];
|
|
18
|
+
}
|
|
19
|
+
export interface LocalEmbeddingPreset extends ModelArtifactPreset {
|
|
20
|
+
readonly id: 'local-small';
|
|
9
21
|
readonly displayName: string;
|
|
10
22
|
readonly sourceModel: 'intfloat/multilingual-e5-small';
|
|
11
23
|
readonly artifactRepository: 'Xenova/multilingual-e5-small';
|
|
12
|
-
readonly revision: string;
|
|
13
|
-
readonly transformersJsVersion: string;
|
|
14
24
|
readonly dimensions: 384;
|
|
15
25
|
readonly maximumTokens: 512;
|
|
16
26
|
readonly dtype: 'q8';
|
|
@@ -23,5 +33,8 @@ export interface LocalEmbeddingPreset {
|
|
|
23
33
|
readonly documentPrefix: 'passage: ';
|
|
24
34
|
readonly files: readonly LocalEmbeddingPresetFile[];
|
|
25
35
|
}
|
|
26
|
-
export
|
|
36
|
+
export interface LocalEmbeddingPresetFile extends ModelArtifactFile {
|
|
37
|
+
readonly size: number;
|
|
38
|
+
}
|
|
39
|
+
export declare function presetManifestHash(preset: ModelArtifactPreset): string;
|
|
27
40
|
//# sourceMappingURL=manifest.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../../src/embedding/presets/manifest.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../../src/embedding/presets/manifest.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAC7C,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACzF;AAED,MAAM,WAAW,oBAAqB,SAAQ,mBAAmB;IAC/D,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,gCAAgC,CAAC;IACvD,QAAQ,CAAC,kBAAkB,EAAE,8BAA8B,CAAC;IAC5D,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,GAAG,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,qBAAqB,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,SAAS,wBAAwB,EAAE,CAAC;CACrD;AAED,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAEtE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../src/embedding/presets/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../src/embedding/presets/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAyCvE,MAAM,UAAU,kBAAkB,CAAC,MAA2B;IAC5D,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite-vec-backend.d.ts","sourceRoot":"","sources":["../../src/embedding/sqlite-vec-backend.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sqlite-vec-backend.d.ts","sourceRoot":"","sources":["../../src/embedding/sqlite-vec-backend.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAKlE,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpF,eAAO,MAAM,qBAAqB,EAAG,YAAqB,CAAC;AA6D3D,qBAAa,4BAA6B,YAAW,mBAAmB;IACtE,QAAQ,CAAC,EAAE,eAAyB;IAEpC,MAAM,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,iBAAiB,GAAG,SAAS,EAAE;CAiExE;AAED,eAAO,MAAM,kCAAkC,QAAO,mBAAyD,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { indexVectorFilter, validIndexVector } from '../memory/index-state.js';
|
|
1
2
|
import { normalizeSubject } from '../memory/interaction-subjects.js';
|
|
2
3
|
import { KiokukoError } from '../errors.js';
|
|
3
4
|
import { requireWorkspace } from '../serialization/validate.js';
|
|
@@ -53,6 +54,7 @@ export class SqliteVecVectorSearchBackend {
|
|
|
53
54
|
'ee.dimensions = ?',
|
|
54
55
|
'ee.revision = e.current_revision',
|
|
55
56
|
'ee.content_hash = r.content_hash',
|
|
57
|
+
indexVectorFilter(input.indexEvaluation ?? false, input.indexOriginalsOnly ?? false),
|
|
56
58
|
];
|
|
57
59
|
const parameters = [normalized.queryBlob, normalized.profileId, normalized.dimensions];
|
|
58
60
|
if (normalized.workspace !== undefined) {
|
|
@@ -72,7 +74,7 @@ export class SqliteVecVectorSearchBackend {
|
|
|
72
74
|
let rows;
|
|
73
75
|
try {
|
|
74
76
|
rows = database.prepare(`
|
|
75
|
-
SELECT ee.entry_id, ee.dimensions, ee.embedding, ee.vector_hash,
|
|
77
|
+
SELECT ee.entry_id, e.workspace, (e.created_by='kiokuko-memory-index' OR json_extract(r.provenance_json,'$.type')='memory_index') AS index_generated, ee.dimensions, ee.embedding, ee.vector_hash,
|
|
76
78
|
vec_distance_cosine(ee.embedding, ?) AS distance
|
|
77
79
|
FROM entry_embeddings AS ee
|
|
78
80
|
JOIN entries AS e
|
|
@@ -93,6 +95,8 @@ export class SqliteVecVectorSearchBackend {
|
|
|
93
95
|
const hits = [];
|
|
94
96
|
for (const row of rows) {
|
|
95
97
|
const id = entryId(row.entry_id);
|
|
98
|
+
if (row.index_generated && !validIndexVector(database, row.workspace, id))
|
|
99
|
+
continue;
|
|
96
100
|
if (typeof row.dimensions !== 'number' || row.dimensions !== normalized.dimensions) {
|
|
97
101
|
integrity('Stored vector dimensions do not match the search profile');
|
|
98
102
|
}
|
|
@@ -101,6 +105,8 @@ export class SqliteVecVectorSearchBackend {
|
|
|
101
105
|
decodeVector(row.embedding, normalized.dimensions, vectorHash);
|
|
102
106
|
}
|
|
103
107
|
catch {
|
|
108
|
+
if (row.index_generated)
|
|
109
|
+
continue;
|
|
104
110
|
integrity('Stored vector is invalid');
|
|
105
111
|
}
|
|
106
112
|
if (typeof row.distance !== 'number' || !Number.isFinite(row.distance))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite-vec-backend.js","sourceRoot":"","sources":["../../src/embedding/sqlite-vec-backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAGlE,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"sqlite-vec-backend.js","sourceRoot":"","sources":["../../src/embedding/sqlite-vec-backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAGlE,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAqB,CAAC;AAY3D,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,IAAI,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClH,SAAS,CAAC,mCAAmC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAwB;IAW3C,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrH,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,IAAI;QAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACjI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC;QAAE,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAC/I,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,uBAAuB;QAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAChI,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,KAAK,SAAS;QAC/D,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrN,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE;QACrD,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,SAAS,EAAE,YAAY,CAAC,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC;QACtD,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,SAAS;QACT,kBAAkB;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,4BAA4B;IAC9B,EAAE,GAAG,qBAAqB,CAAC;IAEpC,MAAM,CAAC,QAAwB,EAAE,KAAwB;QACvD,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG;YACd,mBAAmB;YACnB,mBAAmB;YACnB,kCAAkC;YAClC,kCAAkC;YAClC,iBAAiB,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,EAAC,KAAK,CAAC,kBAAkB,IAAE,KAAK,CAAC;SAClF,CAAC;QACF,MAAM,UAAU,GAAwC,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QAC5H,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAChC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC;8DAC2C,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1G,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,uBAAuB,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChG,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACpD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAElC,IAAI,IAAiB,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;;;;;;iBASb,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;;;OAG/B,CAAC,CAAC,GAAG,CAAY,GAAG,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,qBAAqB,EAAE,yCAAyC,CAAC,CAAC;YACnG,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAgB,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAG,GAAG,CAAC,eAAe,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAC,GAAG,CAAC,SAAS,EAAC,EAAE,CAAC;gBAAE,SAAS;YACjF,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,KAAK,UAAU,CAAC,UAAU,EAAE,CAAC;gBACnF,SAAS,CAAC,0DAA0D,CAAC,CAAC;YACxE,CAAC;YACD,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACtH,IAAI,CAAC;gBACH,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACjE,CAAC;YAAC,MAAM,CAAC;gBACP,IAAG,GAAG,CAAC,eAAe;oBAAE,SAAS;gBACjC,SAAS,CAAC,0BAA0B,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,SAAS,CAAC,4BAA4B,CAAC,CAAC;YAChH,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,eAAe;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,MAAM,CAAC,MAAM,kCAAkC,GAAG,GAAwB,EAAE,CAAC,IAAI,4BAA4B,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/embedding/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAMlE,OAAO,KAAK,EACV,gBAAgB,EAEhB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAYpB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,MAAM,EAAE,CAAC;IAClD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAqQD,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAI9G;AAED,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,cAAc,GAAG,qBAAqB,CAsBzF;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,cAAc,GAAG,sBAAsB,GAAG,IAAI,CAMlG;AAiDD,qGAAqG;AACrG,wBAAgB,yCAAyC,CACvD,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,6BAA6B,GACnC,IAAI,CAQN;AAED,2EAA2E;AAC3E,wBAAgB,6CAA6C,CAC3D,QAAQ,EAAE,cAAc,EACxB,YAAY,CAAC,EAAE,MAAM,EACrB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAoBR;AAED,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAkD5B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAED,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CA8C5B;AAED,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAgBD,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,mBAAmB,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/embedding/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAMlE,OAAO,KAAK,EACV,gBAAgB,EAEhB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAYpB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,MAAM,EAAE,CAAC;IAClD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAqQD,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAI9G;AAED,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,cAAc,GAAG,qBAAqB,CAsBzF;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,cAAc,GAAG,sBAAsB,GAAG,IAAI,CAMlG;AAiDD,qGAAqG;AACrG,wBAAgB,yCAAyC,CACvD,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,6BAA6B,GACnC,IAAI,CAQN;AAED,2EAA2E;AAC3E,wBAAgB,6CAA6C,CAC3D,QAAQ,EAAE,cAAc,EACxB,YAAY,CAAC,EAAE,MAAM,EACrB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAoBR;AAED,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAkD5B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAED,wBAAgB,0CAA0C,CACxD,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CA8C5B;AAED,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,+BAA+B,GACvC,0BAA0B,CAE5B;AAgBD,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAyC5G;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,mBAAmB,GACzB,IAAI,CAEN;AAED,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC5C,oBAAoB,GAAG,SAAS,CAWlC"}
|
package/dist/embedding/store.js
CHANGED
|
@@ -439,6 +439,8 @@ export function upsertEntryEmbeddingInTransaction(database, input) {
|
|
|
439
439
|
if (current.revision !== revision || current.contentHash !== contentHash) {
|
|
440
440
|
conflict('Entry changed before its embedding could be stored');
|
|
441
441
|
}
|
|
442
|
+
if (database.prepare("SELECT 1 AS stale FROM memory_index_artifacts WHERE entry_id=? AND state IN ('stale','unsupported','uncertain')").get(input.entryId))
|
|
443
|
+
conflict('Index source invalidated before embedding completion');
|
|
442
444
|
const bytes = encodeVector(input.vector, profile.identity.dimensions);
|
|
443
445
|
const vectorHash = hashVectorBytes(bytes);
|
|
444
446
|
database.prepare(`
|