@devflow-tools/memory-engine 0.17.7 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +10 -2
- package/dist/consolidator.d.ts.map +1 -1
- package/dist/consolidator.js +2 -0
- package/dist/consolidator.js.map +1 -1
- package/dist/durable-memory-text.js +1 -1
- package/dist/durable-memory-text.js.map +1 -1
- package/dist/embedding-model-store.d.ts +1 -4
- package/dist/embedding-model-store.d.ts.map +1 -1
- package/dist/embedding-model-store.js +3 -74
- package/dist/embedding-model-store.js.map +1 -1
- package/dist/embedding-provider.d.ts +3 -2
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +151 -43
- package/dist/embedding-provider.js.map +1 -1
- package/dist/hybrid-search.d.ts +5 -1
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +92 -56
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/lease-store.d.ts.map +1 -1
- package/dist/lease-store.js +30 -0
- package/dist/lease-store.js.map +1 -1
- package/dist/memory-command-service.d.ts +73 -0
- package/dist/memory-command-service.d.ts.map +1 -0
- package/dist/memory-command-service.js +283 -0
- package/dist/memory-command-service.js.map +1 -0
- package/dist/memory-engine.d.ts +19 -15
- package/dist/memory-engine.d.ts.map +1 -1
- package/dist/memory-engine.js +207 -158
- package/dist/memory-engine.js.map +1 -1
- package/dist/memory-episode-repository.d.ts +77 -0
- package/dist/memory-episode-repository.d.ts.map +1 -0
- package/dist/memory-episode-repository.js +262 -0
- package/dist/memory-episode-repository.js.map +1 -0
- package/dist/memory-fact-provider-port.d.ts +83 -0
- package/dist/memory-fact-provider-port.d.ts.map +1 -0
- package/dist/memory-fact-provider-port.js +22 -0
- package/dist/memory-fact-provider-port.js.map +1 -0
- package/dist/memory-fact-validator.d.ts +16 -0
- package/dist/memory-fact-validator.d.ts.map +1 -0
- package/dist/memory-fact-validator.js +52 -0
- package/dist/memory-fact-validator.js.map +1 -0
- package/dist/memory-gate.d.ts +14 -17
- package/dist/memory-gate.d.ts.map +1 -1
- package/dist/memory-gate.js +377 -320
- package/dist/memory-gate.js.map +1 -1
- package/dist/memory-identity-compiler.d.ts +18 -0
- package/dist/memory-identity-compiler.d.ts.map +1 -0
- package/dist/memory-identity-compiler.js +26 -0
- package/dist/memory-identity-compiler.js.map +1 -0
- package/dist/memory-index-projector.d.ts +22 -0
- package/dist/memory-index-projector.d.ts.map +1 -0
- package/dist/memory-index-projector.js +11 -0
- package/dist/memory-index-projector.js.map +1 -0
- package/dist/memory-lifecycle-reducer.d.ts +21 -0
- package/dist/memory-lifecycle-reducer.d.ts.map +1 -0
- package/dist/memory-lifecycle-reducer.js +34 -0
- package/dist/memory-lifecycle-reducer.js.map +1 -0
- package/dist/memory-maintenance.d.ts +1 -4
- package/dist/memory-maintenance.d.ts.map +1 -1
- package/dist/memory-maintenance.js +3 -20
- package/dist/memory-maintenance.js.map +1 -1
- package/dist/memory-query-service.d.ts +24 -0
- package/dist/memory-query-service.d.ts.map +1 -0
- package/dist/memory-query-service.js +180 -0
- package/dist/memory-query-service.js.map +1 -0
- package/dist/memory-relevance.d.ts +16 -1
- package/dist/memory-relevance.d.ts.map +1 -1
- package/dist/memory-relevance.js +83 -139
- package/dist/memory-relevance.js.map +1 -1
- package/dist/memory-store.d.ts +52 -35
- package/dist/memory-store.d.ts.map +1 -1
- package/dist/memory-store.js +406 -690
- package/dist/memory-store.js.map +1 -1
- package/dist/semantic-memory-compiler.d.ts +0 -2
- package/dist/semantic-memory-compiler.d.ts.map +1 -1
- package/dist/semantic-memory-compiler.js +4 -4
- package/dist/semantic-memory-compiler.js.map +1 -1
- package/dist/session-summarizer.d.ts +3 -0
- package/dist/session-summarizer.d.ts.map +1 -1
- package/dist/session-summarizer.js.map +1 -1
- package/dist/signal-analyzer.d.ts +2 -2
- package/dist/signal-analyzer.d.ts.map +1 -1
- package/dist/signal-analyzer.js +10 -16
- package/dist/signal-analyzer.js.map +1 -1
- package/package.json +14 -5
- package/dist/lease-store.test.d.ts +0 -2
- package/dist/lease-store.test.d.ts.map +0 -1
- package/dist/lease-store.test.js +0 -182
- package/dist/lease-store.test.js.map +0 -1
- package/dist/migrations/v10-to-v11.d.ts +0 -3
- package/dist/migrations/v10-to-v11.d.ts.map +0 -1
- package/dist/migrations/v10-to-v11.js +0 -21
- package/dist/migrations/v10-to-v11.js.map +0 -1
- package/dist/migrations/v11-to-v12.d.ts +0 -66
- package/dist/migrations/v11-to-v12.d.ts.map +0 -1
- package/dist/migrations/v11-to-v12.js +0 -577
- package/dist/migrations/v11-to-v12.js.map +0 -1
- package/dist/migrations/v12-to-v13.d.ts +0 -3
- package/dist/migrations/v12-to-v13.d.ts.map +0 -1
- package/dist/migrations/v12-to-v13.js +0 -50
- package/dist/migrations/v12-to-v13.js.map +0 -1
- package/dist/migrations/v2-to-v3.d.ts +0 -3
- package/dist/migrations/v2-to-v3.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.js +0 -122
- package/dist/migrations/v2-to-v3.js.map +0 -1
- package/dist/migrations/v2-to-v3.test.d.ts +0 -2
- package/dist/migrations/v2-to-v3.test.d.ts.map +0 -1
- package/dist/migrations/v2-to-v3.test.js +0 -88
- package/dist/migrations/v2-to-v3.test.js.map +0 -1
- package/dist/migrations/v3-to-v4.d.ts +0 -3
- package/dist/migrations/v3-to-v4.d.ts.map +0 -1
- package/dist/migrations/v3-to-v4.js +0 -97
- package/dist/migrations/v3-to-v4.js.map +0 -1
- package/dist/migrations/v4-to-v5.d.ts +0 -3
- package/dist/migrations/v4-to-v5.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.js +0 -34
- package/dist/migrations/v4-to-v5.js.map +0 -1
- package/dist/migrations/v4-to-v5.test.d.ts +0 -2
- package/dist/migrations/v4-to-v5.test.d.ts.map +0 -1
- package/dist/migrations/v4-to-v5.test.js +0 -32
- package/dist/migrations/v4-to-v5.test.js.map +0 -1
- package/dist/migrations/v6-to-v7.d.ts +0 -3
- package/dist/migrations/v6-to-v7.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.js +0 -18
- package/dist/migrations/v6-to-v7.js.map +0 -1
- package/dist/migrations/v6-to-v7.test.d.ts +0 -2
- package/dist/migrations/v6-to-v7.test.d.ts.map +0 -1
- package/dist/migrations/v6-to-v7.test.js +0 -33
- package/dist/migrations/v6-to-v7.test.js.map +0 -1
- package/dist/migrations/v7-to-v8.d.ts +0 -3
- package/dist/migrations/v7-to-v8.d.ts.map +0 -1
- package/dist/migrations/v7-to-v8.js +0 -20
- package/dist/migrations/v7-to-v8.js.map +0 -1
- package/dist/migrations/v8-to-v9.d.ts +0 -3
- package/dist/migrations/v8-to-v9.d.ts.map +0 -1
- package/dist/migrations/v8-to-v9.js +0 -18
- package/dist/migrations/v8-to-v9.js.map +0 -1
- package/dist/migrations/v9-to-v10.d.ts +0 -3
- package/dist/migrations/v9-to-v10.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.js +0 -45
- package/dist/migrations/v9-to-v10.js.map +0 -1
- package/dist/migrations/v9-to-v10.test.d.ts +0 -2
- package/dist/migrations/v9-to-v10.test.d.ts.map +0 -1
- package/dist/migrations/v9-to-v10.test.js +0 -40
- package/dist/migrations/v9-to-v10.test.js.map +0 -1
- package/dist/preference-conflicts.test.d.ts +0 -2
- package/dist/preference-conflicts.test.d.ts.map +0 -1
- package/dist/preference-conflicts.test.js +0 -80
- package/dist/preference-conflicts.test.js.map +0 -1
- package/dist/semantic-memory.test.d.ts +0 -2
- package/dist/semantic-memory.test.d.ts.map +0 -1
- package/dist/semantic-memory.test.js +0 -61
- package/dist/semantic-memory.test.js.map +0 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 shilongfeicool
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @devflow-tools/memory-engine
|
|
2
2
|
|
|
3
|
-
> 持久化 session events、结构化 observations 和审核记忆,并提供混合搜索与 embedding 实现。(v0.17.
|
|
3
|
+
> 持久化 session events、结构化 observations 和审核记忆,并提供混合搜索与 embedding 实现。(v0.17.8)
|
|
4
4
|
|
|
5
5
|
[](https://www.npmjs.com/package/@devflow-tools/memory-engine)
|
|
6
6
|
|
|
@@ -29,9 +29,17 @@ console.log(await memory.getProjectContext());
|
|
|
29
29
|
|
|
30
30
|
memory-engine 依赖 SDK 的 embedding contract,并提供具体 provider;宿主模型通过 MCP 提炼事件,不要求本地 LLM 服务。
|
|
31
31
|
|
|
32
|
+
## 宿主模型蒸馏流程
|
|
33
|
+
|
|
34
|
+
1. 宿主模型调用 `memory_request_distill` 获取待处理事件租约,并按 `nextPage` 读取全部分页。
|
|
35
|
+
2. 宿主模型只根据事件证据生成结构化 observations,并为每条 observation 绑定 `sourceEvidenceIds`。
|
|
36
|
+
3. 宿主模型使用同一 `batchId` 调用 `memory_save_distilled`,由 memory-engine 校验 capsule、写入记忆并提交租约。
|
|
37
|
+
|
|
38
|
+
该流程由宿主模型完成语义判断,不要求本地 LLM 服务,也不会在 memory-engine 进程内自行生成 observations。
|
|
39
|
+
|
|
32
40
|
## 版本与兼容性
|
|
33
41
|
|
|
34
|
-
- 当前包版本:`0.17.
|
|
42
|
+
- 当前包版本:`0.17.8`
|
|
35
43
|
- Node.js:未在 package manifest 声明更高版本时,使用 Node.js 18+;原生 SQLite/Playwright 能力请按对应依赖安装
|
|
36
44
|
- workspace 内部包应使用同一 DevFlow minor/patch 版本,避免类型和数据库 schema 不一致
|
|
37
45
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consolidator.d.ts","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;IACR,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,kBAAkB,CAAC,
|
|
1
|
+
{"version":3,"file":"consolidator.d.ts","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;IACR,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,kBAAkB,CAAC,CAkF7B"}
|
package/dist/consolidator.js
CHANGED
|
@@ -40,6 +40,8 @@ export async function consolidate(store, projectRoot, options) {
|
|
|
40
40
|
const memId = `mem:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
|
|
41
41
|
store.add({
|
|
42
42
|
id: memId,
|
|
43
|
+
actorId: 'actor:devflow-consolidator',
|
|
44
|
+
sessionId: best.session_id,
|
|
43
45
|
type: 'semantic',
|
|
44
46
|
content: durableNarrative,
|
|
45
47
|
title: best.title,
|
package/dist/consolidator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consolidator.js","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAS1E,2DAA2D;AAE3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAkB,EAClB,WAAmB,EACnB,OAIC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,OAAO,EAAE,eAAe,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,sBAAsB,CAAC;YACrB,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YAClG,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,iCAAiC,EAAE;SAC/F,CAAC,CAAC;QACH,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAC1F,CAAC;IAED,sBAAsB,CAAC;QACrB,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QAClG,IAAI,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,EAAE;KACnK,CAAC,CAAC;IAEH,kDAAkD;IAClD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;QACpC,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,YAAY,EAAE,CAAC,CAAC,cAAc;aAC3B,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;aACnC,MAAM,CAAC,CAAC,WAAW,EAAkD,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;aAC7F,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,+BAA+B,CAAC,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;KACpG,CAAC,CAAC,CAAC;IAEJ,mFAAmF;IACnF,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChD,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC;QACF,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QACtF,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAEhC,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC7E,KAAK,CAAC,GAAG,CAAC;YACR,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,gBAAgB;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,eAAe;YACvB,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,gBAAgB;SAC/B,CAAC,CAAC;QAEH,KAAK,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9C,KAAK,CAAC,WAAW,CAAC;YAChB,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,oBAAoB,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,yCAAyC,OAAO,CAAC,OAAO,GAAG;SACrE,CAAC,CAAC;QACH,KAAK,CAAC,aAAa,CAAC;YAClB,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,gBAAgB;YAC5B,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QAEH,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACrC,sBAAsB,CAAC;QACrB,KAAK,EAAE,wBAAwB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QAChG,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,EAAE,SAAS,EAAE;KACnG,CAAC,CAAC;IAEH,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACjG,CAAC;AAED,SAAS,+BAA+B,CAAC,SAAiB,EAAE,KAAa;IACvE,OAAO,8BAA8B,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACvE,CAAC"}
|
|
1
|
+
{"version":3,"file":"consolidator.js","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAS1E,2DAA2D;AAE3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAkB,EAClB,WAAmB,EACnB,OAIC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,OAAO,EAAE,eAAe,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,sBAAsB,CAAC;YACrB,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YAClG,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,iCAAiC,EAAE;SAC/F,CAAC,CAAC;QACH,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAC1F,CAAC;IAED,sBAAsB,CAAC;QACrB,KAAK,EAAE,yBAAyB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QAClG,IAAI,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,EAAE;KACnK,CAAC,CAAC;IAEH,kDAAkD;IAClD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;QACpC,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,YAAY,EAAE,CAAC,CAAC,cAAc;aAC3B,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;aACnC,MAAM,CAAC,CAAC,WAAW,EAAkD,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;aAC7F,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,+BAA+B,CAAC,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;KACpG,CAAC,CAAC,CAAC;IAEJ,mFAAmF;IACnF,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChD,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC;QACF,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QACtF,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAEhC,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC7E,KAAK,CAAC,GAAG,CAAC;YACR,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,4BAA4B;YACrC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,gBAAgB;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,eAAe;YACvB,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,WAAW;YAClB,YAAY,EAAE,gBAAgB;SAC/B,CAAC,CAAC;QAEH,KAAK,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9C,KAAK,CAAC,WAAW,CAAC;YAChB,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,oBAAoB,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,yCAAyC,OAAO,CAAC,OAAO,GAAG;SACrE,CAAC,CAAC;QACH,KAAK,CAAC,aAAa,CAAC;YAClB,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,gBAAgB;YAC5B,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QAEH,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACrC,sBAAsB,CAAC;QACrB,KAAK,EAAE,wBAAwB,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QAChG,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,EAAE,SAAS,EAAE;KACnG,CAAC,CAAC;IAEH,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACjG,CAAC;AAED,SAAS,+BAA+B,CAAC,SAAiB,EAAE,KAAa;IACvE,OAAO,8BAA8B,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const EXPLICIT_INTENT_PREFIX = /^(?:(?:\/\S+\s+)?(?:请|麻烦)?(?:帮我)?(?:记住|记录|保存)(?:一下|这条|这个|以下)?\s*[::,,-]?\s*|(?:please\s+)?(?:remember|record|save)\s+(?:that\s+|the\s+following\s+)?)/iu;
|
|
1
|
+
const EXPLICIT_INTENT_PREFIX = /^(?:(?:\/\S+\s+)?(?:请|麻烦)?(?:帮我)?(?:记住|记录|保存)(?:一下|这条|这个|以下)?\s*[::,,-]?\s*|(?:please\s+)?(?:remember|record|save)\s+(?:that\s+|to\s+|the\s+following\s+)?)/iu;
|
|
2
2
|
const EXECUTION_NARRATION = /(?:(?:只|仅|只是|本次|本轮|此次|这个任务).{0,24}(?:记忆|memory).{0,24}(?:保存|写入|记录|提交|commit|receipt|回执)|(?:canonical\s+)?receipt|最终回复|final\s+(?:answer|response)|(?:不要|不需要|无需|不必|do\s+not|don't).{0,20}(?:修改|编辑|创建|删除|modify|edit|create|delete).{0,12}(?:代码|文件|code|files?)|(?:任务|task).{0,16}(?:已完成|完成|结束|completed|done|finished))/iu;
|
|
3
3
|
const META_SUFFIX = new RegExp(String.raw `(?:[,,]\s*)?(?:${EXECUTION_NARRATION.source})[\s\S]*$`, 'iu');
|
|
4
4
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"durable-memory-text.js","sourceRoot":"","sources":["../src/durable-memory-text.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,
|
|
1
|
+
{"version":3,"file":"durable-memory-text.js","sourceRoot":"","sources":["../src/durable-memory-text.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,+JAA+J,CAAC;AAE/L,MAAM,mBAAmB,GAAG,0TAA0T,CAAC;AAEvV,MAAM,WAAW,GAAG,IAAI,MAAM,CAC5B,MAAM,CAAC,GAAG,CAAA,kBAAkB,mBAAmB,CAAC,MAAM,WAAW,EACjE,IAAI,CACL,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAAC,KAAa;IAC1D,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAE3B,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,MAAM,OAAO,GAAG,aAAa;SAC1B,KAAK,CAAC,qBAAqB,CAAC;SAC5B,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC5B,MAAM,CAAC,OAAO,CAAC;SACf,OAAO,CAAC,MAAM,CAAC,EAAE;QAChB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC;YAAE,OAAO,EAAE,CAAC;QACzD,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IAEL,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC;SACxC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAC7B,OAAO,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -24,19 +24,16 @@ export interface EmbeddingModelStatus {
|
|
|
24
24
|
missingFiles: string[];
|
|
25
25
|
corruptFiles: string[];
|
|
26
26
|
}
|
|
27
|
-
export type EmbeddingModelPrepareSource = "existing" | "
|
|
27
|
+
export type EmbeddingModelPrepareSource = "existing" | "download";
|
|
28
28
|
export interface EmbeddingModelPrepareResult extends EmbeddingModelStatus {
|
|
29
29
|
source: EmbeddingModelPrepareSource;
|
|
30
30
|
attemptedEndpoints: string[];
|
|
31
|
-
migratedFrom?: string;
|
|
32
31
|
}
|
|
33
32
|
export interface EmbeddingModelStoreOptions {
|
|
34
33
|
env?: NodeJS.ProcessEnv;
|
|
35
34
|
homeDir?: string;
|
|
36
35
|
cacheRoot?: string;
|
|
37
|
-
cwd?: string;
|
|
38
36
|
endpoints?: string[];
|
|
39
|
-
legacyCacheRoots?: string[];
|
|
40
37
|
allowDownload?: boolean;
|
|
41
38
|
timeoutMs?: number;
|
|
42
39
|
fetchImpl?: typeof fetch;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-model-store.d.ts","sourceRoot":"","sources":["../src/embedding-model-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedding-model-store.d.ts","sourceRoot":"","sources":["../src/embedding-model-store.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAUlF,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,qBAAqB,EAAE,SAAS,0BAA0B,EAqB7D,CAAC;AAEX,MAAM,MAAM,uBAAuB,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,CAAC;AACtE,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,CAAC;AAElE,MAAM,WAAW,wBAAyB,SAAQ,0BAA0B;IAC1E,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,uBAAuB,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,wBAAwB,EAAE,CAAC;IAClC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,2BAA2B,GAAG,UAAU,GAAG,UAAU,CAAC;AAElE,MAAM,WAAW,2BAA4B,SAAQ,oBAAoB;IACvE,MAAM,EAAE,2BAA2B,CAAC;IACpC,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,qBAAa,8BAA+B,SAAQ,KAAK;IAMrD,QAAQ,CAAC,OAAO,EAAE;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,kBAAkB,EAAE,MAAM,EAAE,CAAC;QAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB;IAVH,QAAQ,CAAC,QAAQ,iCAAiC;IAClD,QAAQ,CAAC,mBAAmB,QAAQ;gBAGlC,OAAO,EAAE,MAAM,EACN,OAAO,EAAE;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,kBAAkB,EAAE,MAAM,EAAE,CAAC;QAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB;IAMH,IAAI,YAAY,IAAI,MAAM,GAAG,SAAS,CAIrC;CACF;AAED,wBAAgB,0BAA0B,CACxC,OAAO,GAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,GAAG,KAAK,GAAG,SAAS,CAAM,GAC9E,MAAM,CAOR;AAED,wBAAgB,qBAAqB,CACnC,OAAO,GAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,GAAG,KAAK,GAAG,SAAS,CAAM,GAC9E,MAAM,CAMR;AAED,wBAAgB,8BAA8B,CAC5C,OAAO,GAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,GAAG,KAAK,CAAM,GAClE,MAAM,EAAE,CAYV;AAED,wBAAsB,qBAAqB,CACzC,OAAO,GAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,GAAG,KAAK,GAAG,SAAS,CAAM,GAC9E,OAAO,CAAC,oBAAoB,CAAC,CAyB/B;AAED,wBAAgB,qBAAqB,CACnC,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,2BAA2B,CAAC,CAYtC"}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { mkdir, open, rename, rm, stat, } from "node:fs/promises";
|
|
3
3
|
import { createReadStream } from "node:fs";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
4
|
import { dirname, join, resolve } from "node:path";
|
|
6
|
-
import {
|
|
7
|
-
import { EMBEDDING_MODEL_ID, EMBEDDING_MODEL_REVISION, } from "@devflow-tools/sdk";
|
|
5
|
+
import { EMBEDDING_MODEL_ID, EMBEDDING_MODEL_REVISION, getDevFlowStateRoot, } from "@devflow-tools/sdk";
|
|
8
6
|
export { EMBEDDING_MODEL_ID, EMBEDDING_MODEL_REVISION } from "@devflow-tools/sdk";
|
|
9
7
|
const DEFAULT_ENDPOINTS = [
|
|
10
8
|
"https://huggingface.co",
|
|
@@ -54,7 +52,7 @@ export function getEmbeddingModelCacheRoot(options = {}) {
|
|
|
54
52
|
const env = options.env ?? process.env;
|
|
55
53
|
return resolve(options.cacheRoot
|
|
56
54
|
?? env.DEVFLOW_EMBEDDING_MODEL_DIR
|
|
57
|
-
?? join(options
|
|
55
|
+
?? join(getDevFlowStateRoot(options), "models", "transformers-cache"));
|
|
58
56
|
}
|
|
59
57
|
export function getEmbeddingModelPath(options = {}) {
|
|
60
58
|
return join(getEmbeddingModelCacheRoot(options), ...EMBEDDING_MODEL_ID.split("/"), EMBEDDING_MODEL_REVISION);
|
|
@@ -117,23 +115,6 @@ async function prepareEmbeddingModelOnce(options) {
|
|
|
117
115
|
if (existing.ready) {
|
|
118
116
|
return { ...existing, source: "existing", attemptedEndpoints: [] };
|
|
119
117
|
}
|
|
120
|
-
const legacyModelPaths = resolveLegacyModelPaths(options, existing.cacheRoot);
|
|
121
|
-
for (const legacyModelPath of legacyModelPaths) {
|
|
122
|
-
if (legacyModelPath === existing.modelPath)
|
|
123
|
-
continue;
|
|
124
|
-
if (!await modelDirectoryIsValid(legacyModelPath))
|
|
125
|
-
continue;
|
|
126
|
-
await copyVerifiedModel(legacyModelPath, existing.modelPath);
|
|
127
|
-
const migrated = await inspectEmbeddingModel(options);
|
|
128
|
-
if (migrated.ready) {
|
|
129
|
-
return {
|
|
130
|
-
...migrated,
|
|
131
|
-
source: "legacy_cache",
|
|
132
|
-
attemptedEndpoints: [],
|
|
133
|
-
migratedFrom: legacyModelPath,
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
118
|
if (options.allowDownload === false) {
|
|
138
119
|
throw unavailableError(existing, [], ["No verified local embedding model was found"]);
|
|
139
120
|
}
|
|
@@ -188,40 +169,12 @@ async function inspectFileAtPath(absolutePath, manifest) {
|
|
|
188
169
|
actualSha256,
|
|
189
170
|
};
|
|
190
171
|
}
|
|
191
|
-
async function modelDirectoryIsValid(modelPath) {
|
|
192
|
-
for (const manifest of EMBEDDING_MODEL_FILES) {
|
|
193
|
-
const status = await inspectModelFile(modelPath, manifest);
|
|
194
|
-
if (status.state !== "ready")
|
|
195
|
-
return false;
|
|
196
|
-
}
|
|
197
|
-
return true;
|
|
198
|
-
}
|
|
199
172
|
async function hashFile(path) {
|
|
200
173
|
const hash = createHash("sha256");
|
|
201
174
|
for await (const chunk of createReadStream(path))
|
|
202
175
|
hash.update(chunk);
|
|
203
176
|
return hash.digest("hex");
|
|
204
177
|
}
|
|
205
|
-
async function copyVerifiedModel(source, target) {
|
|
206
|
-
for (const manifest of EMBEDDING_MODEL_FILES) {
|
|
207
|
-
await atomicCopy(join(source, manifest.path), join(target, manifest.path), manifest);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
async function atomicCopy(source, target, manifest) {
|
|
211
|
-
await mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
|
212
|
-
const temporary = `${target}.part-${process.pid}-${randomUUID()}`;
|
|
213
|
-
try {
|
|
214
|
-
await copyFile(source, temporary);
|
|
215
|
-
await chmod(temporary, 0o600);
|
|
216
|
-
const status = await inspectFileAtPath(temporary, manifest);
|
|
217
|
-
if (status.state !== "ready")
|
|
218
|
-
throw new Error(`Legacy cache file failed validation: ${manifest.path}`);
|
|
219
|
-
await rename(temporary, target);
|
|
220
|
-
}
|
|
221
|
-
finally {
|
|
222
|
-
await rm(temporary, { force: true }).catch(() => undefined);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
178
|
async function downloadModelFromEndpoint(endpoint, targetModelPath, options) {
|
|
226
179
|
const fetchImpl = options.fetchImpl ?? fetch;
|
|
227
180
|
const timeoutMs = positiveTimeout(options.timeoutMs, options.env ?? process.env);
|
|
@@ -291,27 +244,6 @@ async function atomicWrite(target, data) {
|
|
|
291
244
|
await rm(temporary, { force: true }).catch(() => undefined);
|
|
292
245
|
}
|
|
293
246
|
}
|
|
294
|
-
function resolveLegacyModelPaths(options, cacheRoot) {
|
|
295
|
-
const env = options.env ?? process.env;
|
|
296
|
-
const cwd = resolve(options.cwd ?? process.cwd());
|
|
297
|
-
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
298
|
-
const projectRoots = [cwd, env.DEVFLOW_PROJECT_ROOT, env.CLAUDE_PROJECT_DIR]
|
|
299
|
-
.filter((value) => typeof value === "string" && value.trim().length > 0)
|
|
300
|
-
.map(value => resolve(value));
|
|
301
|
-
const cacheRoots = [
|
|
302
|
-
...(options.legacyCacheRoots ?? []),
|
|
303
|
-
...splitLegacyRoots(env.DEVFLOW_EMBEDDING_LEGACY_CACHE_DIRS),
|
|
304
|
-
...projectRoots.map(root => join(root, "node_modules", "@xenova", "transformers", ".cache")),
|
|
305
|
-
resolve(moduleDir, "../../../node_modules/@xenova/transformers/.cache"),
|
|
306
|
-
resolve(moduleDir, "../../../@xenova/transformers/.cache"),
|
|
307
|
-
cacheRoot,
|
|
308
|
-
];
|
|
309
|
-
const paths = cacheRoots.flatMap(root => {
|
|
310
|
-
const modelRoot = join(resolve(root), ...EMBEDDING_MODEL_ID.split("/"));
|
|
311
|
-
return [join(modelRoot, EMBEDDING_MODEL_REVISION), modelRoot];
|
|
312
|
-
});
|
|
313
|
-
return [...new Set(paths)];
|
|
314
|
-
}
|
|
315
247
|
function modelFileUrl(endpoint, path) {
|
|
316
248
|
return `${normalizeEndpoint(endpoint)}/${EMBEDDING_MODEL_ID}/resolve/${EMBEDDING_MODEL_REVISION}/${path}`;
|
|
317
249
|
}
|
|
@@ -327,9 +259,6 @@ function normalizeEndpoint(endpoint) {
|
|
|
327
259
|
function splitEndpoints(value) {
|
|
328
260
|
return value?.split(/[;,\n]/).map(endpoint => endpoint.trim()).filter(Boolean) ?? [];
|
|
329
261
|
}
|
|
330
|
-
function splitLegacyRoots(value) {
|
|
331
|
-
return value?.split(/[\n,;]/).map(path => path.trim()).filter(Boolean) ?? [];
|
|
332
|
-
}
|
|
333
262
|
function positiveTimeout(value, env) {
|
|
334
263
|
const configured = value ?? Number(env.DEVFLOW_EMBEDDING_DOWNLOAD_TIMEOUT_MS);
|
|
335
264
|
return Number.isSafeInteger(configured) && configured > 0
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-model-store.js","sourceRoot":"","sources":["../src/embedding-model-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,KAAK,EACL,QAAQ,EACR,KAAK,EACL,IAAI,EACJ,MAAM,EACN,EAAE,EACF,IAAI,GACL,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACL,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,iBAAiB,GAAG;IACxB,wBAAwB;IACxB,uBAAuB;CACf,CAAC;AACX,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAC3C,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAgD,CAAC;AAQpF,MAAM,CAAC,MAAM,qBAAqB,GAA0C;IAC1E;QACE,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,2BAA2B;QACjC,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,kEAAkE;KAC3E;CACO,CAAC;AA4CX,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAM5C;IALF,QAAQ,GAAG,6BAA6B,CAAC;IACzC,mBAAmB,GAAG,IAAI,CAAC;IAEpC,YACE,OAAe,EACN,OAKR;QAED,KAAK,CAAC,OAAO,CAAC,CAAC;QAPN,YAAO,GAAP,OAAO,CAKf;QAGD,IAAI,CAAC,IAAI,GAAG,gCAAgC,CAAC;IAC/C,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS;YACvC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;CACF;AAED,MAAM,UAAU,0BAA0B,CACxC,UAA6E,EAAE;IAE/E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,OAAO,OAAO,CACZ,OAAO,CAAC,SAAS;WACZ,GAAG,CAAC,2BAA2B;WAC/B,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CACpF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAA6E,EAAE;IAE/E,OAAO,IAAI,CACT,0BAA0B,CAAC,OAAO,CAAC,EACnC,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,EAChC,wBAAwB,CACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,UAAiE,EAAE;IAEnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,UAAU,GAAG;QACjB,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC5B,GAAG,cAAc,CAAC,GAAG,CAAC,oBAAoB,CAAC;QAC3C,GAAG,CAAC,mBAAmB;QACvB,GAAG,CAAC,WAAW;KAChB,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,iBAAiB,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS;aACzB,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;aACxF,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAA6E,EAAE;IAE/E,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,KAAK,GAA+B,EAAE,CAAC;IAC7C,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAwB,YAAY,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,OAAO,CAAC;IACd,OAAO;QACL,KAAK,EAAE,KAAK,KAAK,OAAO;QACxB,KAAK;QACL,OAAO,EAAE,kBAAkB;QAC3B,QAAQ,EAAE,wBAAwB;QAClC,SAAS;QACT,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,WAAW,GAAG,yBAAyB,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAC;SACrE,OAAO,CAAC,GAAG,EAAE;QACZ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,WAAW,EAAE,CAAC;YACvD,mBAAmB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CAAC;IACL,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAChD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,OAAmC;IAEnC,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC9E,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;QAC/C,IAAI,eAAe,KAAK,QAAQ,CAAC,SAAS;YAAE,SAAS;QACrD,IAAI,CAAC,MAAM,qBAAqB,CAAC,eAAe,CAAC;YAAE,SAAS;QAC5D,MAAM,iBAAiB,CAAC,eAAe,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO;gBACL,GAAG,QAAQ;gBACX,MAAM,EAAE,cAAc;gBACtB,kBAAkB,EAAE,EAAE;gBACtB,YAAY,EAAE,eAAe;aAC9B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;QACpC,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,SAAS,GAAG,8BAA8B,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACvE,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxD,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;YACnE,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,gDAAgD,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;AAC3F,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAiB,EACjB,QAAoC;IAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,YAAoB,EACpB,QAAoC;IAEpC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,GAAG,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACzD,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ;YACX,YAAY;YACZ,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,QAAQ,CAAC,IAAI;SAC1B,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAClD,OAAO;QACL,GAAG,QAAQ;QACX,YAAY;QACZ,KAAK,EAAE,YAAY,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,YAAY;KACb,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,SAAiB;IACpD,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3D,IAAI,MAAM,CAAC,KAAK,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IAC7D,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,MAAc,EACd,MAAc,EACd,QAAoC;IAEpC,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,GAAG,MAAM,SAAS,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,CAAC;IAClE,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClC,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,KAAK,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACvG,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,QAAgB,EAChB,eAAuB,EACvB,OAAmC;IAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IACjF,MAAM,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC,CAAC;IACzF,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAClE,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,cAAc,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,WAAW,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,SAAuB,EACvB,GAAW,EACX,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,SAAuB,EACvB,SAAiB;IAEjB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,SAAS,CAAC,CAAC;IACrG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IACvF,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;AACvD,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAuB,EACvB,GAAW,EACX,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,IAAY;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,GAAG,MAAM,SAAS,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,CAAC;IAClE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7B,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,OAAmC,EAAE,SAAiB;IACrF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,oBAAoB,EAAE,GAAG,CAAC,kBAAkB,CAAC;SACzE,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SACxF,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG;QACjB,GAAG,CAAC,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACnC,GAAG,gBAAgB,CAAC,GAAG,CAAC,mCAAmC,CAAC;QAC5D,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;QAC5F,OAAO,CAAC,SAAS,EAAE,mDAAmD,CAAC;QACvE,OAAO,CAAC,SAAS,EAAE,sCAAsC,CAAC;QAC1D,SAAS;KACV,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,EAAE,SAAS,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,YAAY,CAAC,QAAgB,EAAE,IAAY;IAClD,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,kBAAkB,YAAY,wBAAwB,IAAI,IAAI,EAAE,CAAC;AAC5G,CAAC;AAED,SAAS,gBAAgB,CACvB,MAA4B,EAC5B,kBAA4B,EAC5B,MAAgB;IAEhB,OAAO,IAAI,8BAA8B,CACvC;QACE,8CAA8C,MAAM,CAAC,SAAS,EAAE;QAChE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KAC9C,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAC5B,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAC5D,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,OAAO,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;AACvF,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAyB;IACjD,OAAO,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;AAC/E,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,GAAsB;IACxE,MAAM,UAAU,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IAC9E,OAAO,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;QACvD,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC,2BAA2B,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;AACvF,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,IAAY;IAC/C,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,IAAI,CAAC;AACrG,CAAC"}
|
|
1
|
+
{"version":3,"file":"embedding-model-store.js","sourceRoot":"","sources":["../src/embedding-model-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,KAAK,EACL,IAAI,EACJ,MAAM,EACN,EAAE,EACF,IAAI,GACL,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,iBAAiB,GAAG;IACxB,wBAAwB;IACxB,uBAAuB;CACf,CAAC;AACX,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAC3C,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAgD,CAAC;AAQpF,MAAM,CAAC,MAAM,qBAAqB,GAA0C;IAC1E;QACE,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,kEAAkE;KAC3E;IACD;QACE,IAAI,EAAE,2BAA2B;QACjC,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,kEAAkE;KAC3E;CACO,CAAC;AAyCX,MAAM,OAAO,8BAA+B,SAAQ,KAAK;IAM5C;IALF,QAAQ,GAAG,6BAA6B,CAAC;IACzC,mBAAmB,GAAG,IAAI,CAAC;IAEpC,YACE,OAAe,EACN,OAKR;QAED,KAAK,CAAC,OAAO,CAAC,CAAC;QAPN,YAAO,GAAP,OAAO,CAKf;QAGD,IAAI,CAAC,IAAI,GAAG,gCAAgC,CAAC;IAC/C,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS;YACvC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;CACF;AAED,MAAM,UAAU,0BAA0B,CACxC,UAA6E,EAAE;IAE/E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,OAAO,OAAO,CACZ,OAAO,CAAC,SAAS;WACZ,GAAG,CAAC,2BAA2B;WAC/B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAA6E,EAAE;IAE/E,OAAO,IAAI,CACT,0BAA0B,CAAC,OAAO,CAAC,EACnC,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,EAChC,wBAAwB,CACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,UAAiE,EAAE;IAEnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,UAAU,GAAG;QACjB,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC5B,GAAG,cAAc,CAAC,GAAG,CAAC,oBAAoB,CAAC;QAC3C,GAAG,CAAC,mBAAmB;QACvB,GAAG,CAAC,WAAW;KAChB,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,iBAAiB,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS;aACzB,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;aACxF,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAA6E,EAAE;IAE/E,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,KAAK,GAA+B,EAAE,CAAC;IAC7C,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAwB,YAAY,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YACvB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,OAAO,CAAC;IACd,OAAO;QACL,KAAK,EAAE,KAAK,KAAK,OAAO;QACxB,KAAK;QACL,OAAO,EAAE,kBAAkB;QAC3B,QAAQ,EAAE,wBAAwB;QAClC,SAAS;QACT,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,WAAW,GAAG,yBAAyB,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAC;SACrE,OAAO,CAAC,GAAG,EAAE;QACZ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,WAAW,EAAE,CAAC;YACvD,mBAAmB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CAAC;IACL,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAChD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,OAAmC;IAEnC,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC;IACrE,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;QACpC,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,SAAS,GAAG,8BAA8B,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACvE,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxD,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;YACnE,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,gDAAgD,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;AAC3F,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAiB,EACjB,QAAoC;IAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,YAAoB,EACpB,QAAoC;IAEpC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,GAAG,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACzD,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ;YACX,YAAY;YACZ,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,QAAQ,CAAC,IAAI;SAC1B,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAClD,OAAO;QACL,GAAG,QAAQ;QACX,YAAY;QACZ,KAAK,EAAE,YAAY,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,YAAY;KACb,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,QAAgB,EAChB,eAAuB,EACvB,OAAmC;IAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IACjF,MAAM,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC,CAAC;IACzF,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAClE,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,IAAI,cAAc,QAAQ,CAAC,IAAI,cAAc,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,WAAW,CAAC,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,SAAuB,EACvB,GAAW,EACX,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,SAAuB,EACvB,SAAiB;IAEjB,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,SAAS,CAAC,CAAC;IACrG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IACvF,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;AACvD,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAuB,EACvB,GAAW,EACX,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,IAAY;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,GAAG,MAAM,SAAS,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,CAAC;IAClE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7B,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,QAAgB,EAAE,IAAY;IAClD,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,kBAAkB,YAAY,wBAAwB,IAAI,IAAI,EAAE,CAAC;AAC5G,CAAC;AAED,SAAS,gBAAgB,CACvB,MAA4B,EAC5B,kBAA4B,EAC5B,MAAgB;IAEhB,OAAO,IAAI,8BAA8B,CACvC;QACE,8CAA8C,MAAM,CAAC,SAAS,EAAE;QAChE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KAC9C,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAC5B,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAC5D,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,OAAO,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;AACvF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,GAAsB;IACxE,MAAM,UAAU,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IAC9E,OAAO,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;QACvD,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC,2BAA2B,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;AACvF,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,IAAY;IAC/C,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,IAAI,CAAC;AACrG,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EmbeddingProviderContract, EmbeddingWarmupOptions } from "@devflow-tools/sdk";
|
|
1
|
+
import type { EmbeddingCapabilityV1, EmbeddingProviderContract, EmbeddingPurpose, EmbeddingWarmupOptions } from "@devflow-tools/sdk";
|
|
2
2
|
export type EmbeddingStatus = "idle" | "warming" | "ready" | "failed";
|
|
3
3
|
export declare const EMBEDDING_DIMENSIONS = 384;
|
|
4
4
|
export declare class EmbeddingWarmupTimeoutError extends Error {
|
|
@@ -7,7 +7,6 @@ export declare class EmbeddingWarmupTimeoutError extends Error {
|
|
|
7
7
|
readonly retryAfterMs = 30000;
|
|
8
8
|
constructor(timeoutMs: number);
|
|
9
9
|
}
|
|
10
|
-
export declare function assertEmbeddingBatch(embeddings: Float32Array[], expectedCount: number, dimensions?: number): void;
|
|
11
10
|
type EmbeddingPipeline = (texts: string[]) => Promise<Array<{
|
|
12
11
|
data: Float32Array;
|
|
13
12
|
}>>;
|
|
@@ -18,10 +17,12 @@ export declare class EmbeddingProvider implements EmbeddingProviderContract {
|
|
|
18
17
|
private lastFailure;
|
|
19
18
|
private initializationPromise;
|
|
20
19
|
private pipeline;
|
|
20
|
+
private lastTransitionAt;
|
|
21
21
|
private errorLogPath;
|
|
22
22
|
constructor(pipelineLoader?: () => Promise<EmbeddingPipeline>);
|
|
23
23
|
isReady(): boolean;
|
|
24
24
|
getStatus(): EmbeddingStatus;
|
|
25
|
+
getCapability(purpose: EmbeddingPurpose): EmbeddingCapabilityV1;
|
|
25
26
|
getLastError(): string | null;
|
|
26
27
|
getLastFailure(): Error | null;
|
|
27
28
|
initialize(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,qBAAqB,EACrB,yBAAyB,EACzB,gBAAgB,EAChB,sBAAsB,EACvB,MAAM,oBAAoB,CAAC;AAQ5B,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;AACtE,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,qBAAa,2BAA4B,SAAQ,KAAK;IACpD,QAAQ,CAAC,QAAQ,8BAA8B;IAC/C,QAAQ,CAAC,mBAAmB,QAAQ;IACpC,QAAQ,CAAC,YAAY,SAAU;gBAEnB,SAAS,EAAE,MAAM;CAI9B;AAED,KAAK,iBAAiB,GAAG,CACvB,KAAK,EAAE,MAAM,EAAE,KACZ,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC,CAAC,CAAC;AA4L5C,qBAAa,iBAAkB,YAAW,yBAAyB;IAc/D,OAAO,CAAC,QAAQ,CAAC,cAAc;IAbjC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAAsB;IACzC,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,gBAAgB,CAAc;IACtC,OAAO,CAAC,YAAY,CAIlB;gBAGiB,cAAc,GAAE,MAAM,OAAO,CAAC,iBAAiB,CAC5C;IAGtB,OAAO,IAAI,OAAO;IAIlB,SAAS,IAAI,eAAe;IAI5B,aAAa,CAAC,OAAO,EAAE,gBAAgB,GAAG,qBAAqB;IAiC/D,YAAY,IAAI,MAAM,GAAG,IAAI;IAI7B,cAAc,IAAI,KAAK,GAAG,IAAI;IAI9B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BrB,MAAM,CAAC,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,OAAO,CAAC;IA2B9D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiB1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAkB1D,OAAO,CAAC,SAAS;YAOH,aAAa;YAUb,UAAU;CAazB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { appendFile, chmod, mkdir } from "node:fs/promises";
|
|
2
|
-
import { homedir } from "node:os";
|
|
1
|
+
import { appendFile, chmod, mkdir, readFile } from "node:fs/promises";
|
|
3
2
|
import { dirname, join } from "node:path";
|
|
4
|
-
import {
|
|
3
|
+
import { EmbeddingModelUnavailableError, prepareEmbeddingModel, } from "./embedding-model-store.js";
|
|
4
|
+
import { createEmbeddingCapability, createMiniLmEmbeddingProfile, getDevFlowStateRoot, } from "@devflow-tools/sdk";
|
|
5
|
+
import { validateEmbeddingBatch } from "@devflow-tools/provider-runtime";
|
|
5
6
|
export const EMBEDDING_DIMENSIONS = 384;
|
|
6
7
|
export class EmbeddingWarmupTimeoutError extends Error {
|
|
7
8
|
category = "embedding_warmup_timeout";
|
|
@@ -12,32 +13,6 @@ export class EmbeddingWarmupTimeoutError extends Error {
|
|
|
12
13
|
this.name = "EmbeddingWarmupTimeoutError";
|
|
13
14
|
}
|
|
14
15
|
}
|
|
15
|
-
export function assertEmbeddingBatch(embeddings, expectedCount, dimensions = EMBEDDING_DIMENSIONS) {
|
|
16
|
-
if (embeddings.length !== expectedCount) {
|
|
17
|
-
throw new Error(`Observation vector count mismatch: ${expectedCount} observations, ${embeddings.length} vectors`);
|
|
18
|
-
}
|
|
19
|
-
for (let index = 0; index < embeddings.length; index += 1) {
|
|
20
|
-
const embedding = embeddings[index];
|
|
21
|
-
if (!(embedding instanceof Float32Array) || embedding.length !== dimensions) {
|
|
22
|
-
throw new Error(`Observation vector ${index} has ${embedding.length} dimensions; expected ${dimensions}`);
|
|
23
|
-
}
|
|
24
|
-
if (embedding.some(value => !Number.isFinite(value))) {
|
|
25
|
-
throw new Error(`Observation vector ${index} contains non-finite values`);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
function splitBatchTensor(result, expectedCount) {
|
|
30
|
-
const [batchSize, embeddingSize] = result.dims ?? [];
|
|
31
|
-
if (batchSize === expectedCount
|
|
32
|
-
&& Number.isInteger(embeddingSize)
|
|
33
|
-
&& embeddingSize > 0
|
|
34
|
-
&& result.data.length === batchSize * embeddingSize) {
|
|
35
|
-
return Array.from({ length: batchSize }, (_, index) => ({
|
|
36
|
-
data: result.data.slice(index * embeddingSize, (index + 1) * embeddingSize),
|
|
37
|
-
}));
|
|
38
|
-
}
|
|
39
|
-
return [{ data: result.data }];
|
|
40
|
-
}
|
|
41
16
|
let sharedPipeline = null;
|
|
42
17
|
let sharedPipelinePromise = null;
|
|
43
18
|
let sharedModelPreparePromise = null;
|
|
@@ -55,18 +30,58 @@ function loadSharedPipeline() {
|
|
|
55
30
|
sharedPipelinePromise = (async () => {
|
|
56
31
|
sharedModelPreparePromise ??= prepareEmbeddingModel();
|
|
57
32
|
const prepared = await sharedModelPreparePromise;
|
|
58
|
-
const {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
33
|
+
const [{ Tokenizer }, { InferenceSession, Tensor }] = await Promise.all([
|
|
34
|
+
import("@huggingface/tokenizers"),
|
|
35
|
+
import("onnxruntime-web"),
|
|
36
|
+
]);
|
|
37
|
+
const [tokenizerJson, tokenizerConfigJson] = await Promise.all([
|
|
38
|
+
readFile(join(prepared.modelPath, "tokenizer.json"), "utf8"),
|
|
39
|
+
readFile(join(prepared.modelPath, "tokenizer_config.json"), "utf8"),
|
|
40
|
+
]);
|
|
41
|
+
const tokenizerConfig = JSON.parse(tokenizerConfigJson);
|
|
42
|
+
if (!Number.isSafeInteger(tokenizerConfig.model_max_length) || tokenizerConfig.model_max_length <= 0) {
|
|
43
|
+
throw new Error("Embedding tokenizer model_max_length must be a positive integer");
|
|
44
|
+
}
|
|
45
|
+
const tokenizer = new Tokenizer(JSON.parse(tokenizerJson), tokenizerConfig);
|
|
46
|
+
const padTokenId = tokenizer.token_to_id(tokenizerConfig.pad_token);
|
|
47
|
+
if (padTokenId === undefined) {
|
|
48
|
+
throw new Error(`Embedding tokenizer does not define pad token ${tokenizerConfig.pad_token}`);
|
|
49
|
+
}
|
|
50
|
+
const session = await InferenceSession.create(join(prepared.modelPath, "onnx", "model_quantized.onnx"));
|
|
65
51
|
const loaded = async (texts) => {
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
52
|
+
const encodings = texts.map(text => tokenizer.encode(text, {
|
|
53
|
+
return_token_type_ids: true,
|
|
54
|
+
}));
|
|
55
|
+
const sequenceLength = Math.max(...encodings.map(encoding => Math.min(encoding.ids.length, tokenizerConfig.model_max_length)));
|
|
56
|
+
const tensorLength = texts.length * sequenceLength;
|
|
57
|
+
const inputIds = new BigInt64Array(tensorLength);
|
|
58
|
+
const attentionMask = new BigInt64Array(tensorLength);
|
|
59
|
+
const tokenTypeIds = new BigInt64Array(tensorLength);
|
|
60
|
+
inputIds.fill(BigInt(padTokenId));
|
|
61
|
+
encodings.forEach((encoding, batchIndex) => {
|
|
62
|
+
const offset = batchIndex * sequenceLength;
|
|
63
|
+
const length = Math.min(encoding.ids.length, tokenizerConfig.model_max_length);
|
|
64
|
+
for (let tokenIndex = 0; tokenIndex < length; tokenIndex += 1) {
|
|
65
|
+
const target = offset + tokenIndex;
|
|
66
|
+
inputIds[target] = BigInt(encoding.ids[tokenIndex]);
|
|
67
|
+
attentionMask[target] = BigInt(encoding.attention_mask[tokenIndex]);
|
|
68
|
+
tokenTypeIds[target] = BigInt(encoding.token_type_ids[tokenIndex]);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
const dims = [texts.length, sequenceLength];
|
|
72
|
+
const result = await session.run({
|
|
73
|
+
input_ids: new Tensor("int64", inputIds, dims),
|
|
74
|
+
attention_mask: new Tensor("int64", attentionMask, dims),
|
|
75
|
+
token_type_ids: new Tensor("int64", tokenTypeIds, dims),
|
|
76
|
+
});
|
|
77
|
+
const output = result.last_hidden_state;
|
|
78
|
+
if (!output
|
|
79
|
+
|| !("data" in output)
|
|
80
|
+
|| output.type !== "float32"
|
|
81
|
+
|| !(output.data instanceof Float32Array)) {
|
|
82
|
+
throw new Error("Embedding model did not return float32 last_hidden_state");
|
|
83
|
+
}
|
|
84
|
+
return meanPoolAndNormalize(output.data, output.dims, attentionMask, texts.length, sequenceLength).map(data => ({ data }));
|
|
70
85
|
};
|
|
71
86
|
sharedPipeline = loaded;
|
|
72
87
|
sharedCircuit = null;
|
|
@@ -85,6 +100,44 @@ function loadSharedPipeline() {
|
|
|
85
100
|
});
|
|
86
101
|
return sharedPipelinePromise;
|
|
87
102
|
}
|
|
103
|
+
function meanPoolAndNormalize(hiddenState, dims, attentionMask, batchSize, sequenceLength) {
|
|
104
|
+
if (dims.length !== 3
|
|
105
|
+
|| dims[0] !== batchSize
|
|
106
|
+
|| dims[1] !== sequenceLength
|
|
107
|
+
|| dims[2] !== EMBEDDING_DIMENSIONS
|
|
108
|
+
|| hiddenState.length !== batchSize * sequenceLength * EMBEDDING_DIMENSIONS) {
|
|
109
|
+
throw new Error("Embedding model returned an invalid last_hidden_state shape");
|
|
110
|
+
}
|
|
111
|
+
return Array.from({ length: batchSize }, (_, batchIndex) => {
|
|
112
|
+
const embedding = new Float32Array(EMBEDDING_DIMENSIONS);
|
|
113
|
+
let tokenCount = 0;
|
|
114
|
+
for (let tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex += 1) {
|
|
115
|
+
if (attentionMask[batchIndex * sequenceLength + tokenIndex] === 0n)
|
|
116
|
+
continue;
|
|
117
|
+
tokenCount += 1;
|
|
118
|
+
const tokenOffset = (batchIndex * sequenceLength + tokenIndex) * EMBEDDING_DIMENSIONS;
|
|
119
|
+
for (let dimension = 0; dimension < EMBEDDING_DIMENSIONS; dimension += 1) {
|
|
120
|
+
embedding[dimension] += hiddenState[tokenOffset + dimension];
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (tokenCount === 0) {
|
|
124
|
+
throw new Error("Embedding tokenizer returned no attended tokens");
|
|
125
|
+
}
|
|
126
|
+
let squaredNorm = 0;
|
|
127
|
+
for (let dimension = 0; dimension < EMBEDDING_DIMENSIONS; dimension += 1) {
|
|
128
|
+
embedding[dimension] /= tokenCount;
|
|
129
|
+
squaredNorm += embedding[dimension] * embedding[dimension];
|
|
130
|
+
}
|
|
131
|
+
const norm = Math.sqrt(squaredNorm);
|
|
132
|
+
if (norm === 0) {
|
|
133
|
+
throw new Error("Embedding model returned a zero vector");
|
|
134
|
+
}
|
|
135
|
+
for (let dimension = 0; dimension < EMBEDDING_DIMENSIONS; dimension += 1) {
|
|
136
|
+
embedding[dimension] /= norm;
|
|
137
|
+
}
|
|
138
|
+
return embedding;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
88
141
|
function errorMessage(error) {
|
|
89
142
|
if (error instanceof AggregateError) {
|
|
90
143
|
return [error.message, ...error.errors.map(errorMessage)].filter(Boolean).join(": ");
|
|
@@ -121,7 +174,8 @@ export class EmbeddingProvider {
|
|
|
121
174
|
lastFailure = null;
|
|
122
175
|
initializationPromise = null;
|
|
123
176
|
pipeline = null;
|
|
124
|
-
|
|
177
|
+
lastTransitionAt = Date.now();
|
|
178
|
+
errorLogPath = join(getDevFlowStateRoot(), "errors", "embedder.log");
|
|
125
179
|
constructor(pipelineLoader = loadSharedPipeline) {
|
|
126
180
|
this.pipelineLoader = pipelineLoader;
|
|
127
181
|
}
|
|
@@ -131,6 +185,38 @@ export class EmbeddingProvider {
|
|
|
131
185
|
getStatus() {
|
|
132
186
|
return this.status;
|
|
133
187
|
}
|
|
188
|
+
getCapability(purpose) {
|
|
189
|
+
const failure = this.lastFailure;
|
|
190
|
+
const retryAfterMs = retryAfter(failure);
|
|
191
|
+
const status = this.status === "ready"
|
|
192
|
+
? "ready"
|
|
193
|
+
: this.status === "warming"
|
|
194
|
+
? "warming"
|
|
195
|
+
: this.status === "failed" && retryAfterMs !== undefined
|
|
196
|
+
? "circuit_open"
|
|
197
|
+
: this.status === "failed"
|
|
198
|
+
? "unavailable"
|
|
199
|
+
: "unavailable";
|
|
200
|
+
return createEmbeddingCapability({
|
|
201
|
+
profile: createMiniLmEmbeddingProfile(purpose),
|
|
202
|
+
status,
|
|
203
|
+
...(failure ? {
|
|
204
|
+
failure: {
|
|
205
|
+
code: status === "circuit_open"
|
|
206
|
+
? "circuit_open"
|
|
207
|
+
: failure instanceof EmbeddingWarmupTimeoutError
|
|
208
|
+
? "warmup_timeout"
|
|
209
|
+
: failure instanceof EmbeddingModelUnavailableError
|
|
210
|
+
? "model_unavailable"
|
|
211
|
+
: "provider_unavailable",
|
|
212
|
+
message: this.lastError ?? failure.message,
|
|
213
|
+
retryable: retryAfterMs !== undefined || failure instanceof EmbeddingWarmupTimeoutError,
|
|
214
|
+
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
|
215
|
+
},
|
|
216
|
+
} : {}),
|
|
217
|
+
lastTransitionAt: this.lastTransitionAt,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
134
220
|
getLastError() {
|
|
135
221
|
return this.lastError;
|
|
136
222
|
}
|
|
@@ -191,8 +277,13 @@ export class EmbeddingProvider {
|
|
|
191
277
|
await this.initialize();
|
|
192
278
|
try {
|
|
193
279
|
const results = await this.pipeline([text]);
|
|
280
|
+
const [embedding] = validateEmbeddingBatch({
|
|
281
|
+
response: results.map(result => result.data),
|
|
282
|
+
expectedCount: 1,
|
|
283
|
+
expectedDimension: EMBEDDING_DIMENSIONS,
|
|
284
|
+
});
|
|
194
285
|
this.markReady();
|
|
195
|
-
return
|
|
286
|
+
return embedding;
|
|
196
287
|
}
|
|
197
288
|
catch (error) {
|
|
198
289
|
await this.recordFailure(error);
|
|
@@ -205,8 +296,13 @@ export class EmbeddingProvider {
|
|
|
205
296
|
await this.initialize();
|
|
206
297
|
try {
|
|
207
298
|
const results = await this.pipeline(texts);
|
|
299
|
+
const embeddings = validateEmbeddingBatch({
|
|
300
|
+
response: results.map(result => result.data),
|
|
301
|
+
expectedCount: texts.length,
|
|
302
|
+
expectedDimension: EMBEDDING_DIMENSIONS,
|
|
303
|
+
});
|
|
208
304
|
this.markReady();
|
|
209
|
-
return
|
|
305
|
+
return embeddings;
|
|
210
306
|
}
|
|
211
307
|
catch (error) {
|
|
212
308
|
await this.recordFailure(error);
|
|
@@ -214,6 +310,8 @@ export class EmbeddingProvider {
|
|
|
214
310
|
}
|
|
215
311
|
}
|
|
216
312
|
markReady() {
|
|
313
|
+
if (this.status !== "ready")
|
|
314
|
+
this.lastTransitionAt = Date.now();
|
|
217
315
|
this.status = "ready";
|
|
218
316
|
this.lastError = null;
|
|
219
317
|
this.lastFailure = null;
|
|
@@ -221,6 +319,8 @@ export class EmbeddingProvider {
|
|
|
221
319
|
async recordFailure(error) {
|
|
222
320
|
const failure = normalizeError(error);
|
|
223
321
|
const message = errorMessage(failure);
|
|
322
|
+
if (this.status !== "failed")
|
|
323
|
+
this.lastTransitionAt = Date.now();
|
|
224
324
|
this.status = "failed";
|
|
225
325
|
this.lastError = message;
|
|
226
326
|
this.lastFailure = failure;
|
|
@@ -237,4 +337,12 @@ export class EmbeddingProvider {
|
|
|
237
337
|
}
|
|
238
338
|
}
|
|
239
339
|
}
|
|
340
|
+
function retryAfter(error) {
|
|
341
|
+
if (!error)
|
|
342
|
+
return undefined;
|
|
343
|
+
const candidate = error;
|
|
344
|
+
return typeof candidate.retryAfterMs === "number" && candidate.retryAfterMs >= 0
|
|
345
|
+
? candidate.retryAfterMs
|
|
346
|
+
: undefined;
|
|
347
|
+
}
|
|
240
348
|
//# sourceMappingURL=embedding-provider.js.map
|