@ai-agent-forge/plugin-memory 0.88.11 → 0.88.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/agent-forge.json +1 -1
- package/dist/inference-bin.bundle.js +116 -57
- package/dist/memory/embedding-provider.d.ts +3 -0
- package/dist/memory/embedding-provider.d.ts.map +1 -1
- package/dist/memory/embedding-provider.js +48 -17
- package/dist/memory/embedding-provider.js.map +1 -1
- package/dist/memory/embedding-reranker.d.ts +3 -0
- package/dist/memory/embedding-reranker.d.ts.map +1 -1
- package/dist/memory/embedding-reranker.js +37 -7
- package/dist/memory/embedding-reranker.js.map +1 -1
- package/dist/memory/foundation.d.ts.map +1 -1
- package/dist/memory/foundation.js +19 -0
- package/dist/memory/foundation.js.map +1 -1
- package/dist/memory/vector-channel.d.ts.map +1 -1
- package/dist/memory/vector-channel.js +96 -25
- package/dist/memory/vector-channel.js.map +1 -1
- package/dist/memory/vector-index.d.ts.map +1 -1
- package/dist/memory/vector-index.js +32 -16
- package/dist/memory/vector-index.js.map +1 -1
- package/package.json +5 -4
- package/plugin.json +1 -1
package/README.md
CHANGED
|
@@ -60,6 +60,17 @@ vector-channel enabling path — the default `light` mode (vector channel
|
|
|
60
60
|
stays disabled until `memory_enable` consent), mode `off`, and the pure
|
|
61
61
|
store/ledger paths never touch them.
|
|
62
62
|
|
|
63
|
+
本地装配失败和插件卸载会释放本实例的模型与索引。已进入的原生推理先完成再释放;
|
|
64
|
+
清理错逐资源保留原始错误并输出 stderr,不调用宿主已经撤销的插件 API;
|
|
65
|
+
某一资源或诊断输出失败不会阻断其他资源释放。
|
|
66
|
+
transformers 4.3.0 尚不提供逐调用下载取消,卸载后只阻止新推理与后续重试,
|
|
67
|
+
晚到加载会释放而不会发布通道。共享推理服务的模型由服务进程持有,会话卸载只断开客户端。
|
|
68
|
+
成功缓存不删除,非空权重和手工导入文件保持原有检查契约。
|
|
69
|
+
真实验收先经公开 `memory_enable({action:"download"})` 单独准备资源,随后用同一测试私有
|
|
70
|
+
agentDir 验证跨会话写入与召回;冷资源耗时不计入模型任务时限,也不宣称因此变快。
|
|
71
|
+
真实 fixture 同时按生产流程物化安装副本的依赖;失败时保留私有目录与 sidecar 日志。
|
|
72
|
+
详见 `docs/design/冷资源准备与取消边界.md`。
|
|
73
|
+
|
|
63
74
|
## Identity
|
|
64
75
|
|
|
65
76
|
Plugin manifest id `agent-forge.plugin.memory`; the embedded capability
|
package/agent-forge.json
CHANGED
|
@@ -2074,6 +2074,9 @@ function createTransformersEmbeddingProvider(options) {
|
|
|
2074
2074
|
const remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];
|
|
2075
2075
|
const load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);
|
|
2076
2076
|
let backendPromise = null;
|
|
2077
|
+
let disposed = false;
|
|
2078
|
+
let disposePromise;
|
|
2079
|
+
const activeCalls = /* @__PURE__ */ new Set();
|
|
2077
2080
|
const loadBackend = () => {
|
|
2078
2081
|
const existing = backendPromise;
|
|
2079
2082
|
if (existing) {
|
|
@@ -2086,13 +2089,16 @@ function createTransformersEmbeddingProvider(options) {
|
|
|
2086
2089
|
return await load(modelId, cacheDir, remoteHost);
|
|
2087
2090
|
} catch (error) {
|
|
2088
2091
|
lastError = error;
|
|
2092
|
+
if (disposed) throw error;
|
|
2089
2093
|
purgeZeroByteCacheArtifacts(join4(cacheDir, modelId));
|
|
2090
2094
|
}
|
|
2091
2095
|
try {
|
|
2092
2096
|
await new Promise((resolve2) => setTimeout(resolve2, 2e3));
|
|
2097
|
+
if (disposed) throw lastError;
|
|
2093
2098
|
return await load(modelId, cacheDir, remoteHost);
|
|
2094
2099
|
} catch (error) {
|
|
2095
2100
|
lastError = error;
|
|
2101
|
+
if (disposed) throw error;
|
|
2096
2102
|
purgeZeroByteCacheArtifacts(join4(cacheDir, modelId));
|
|
2097
2103
|
}
|
|
2098
2104
|
}
|
|
@@ -2107,28 +2113,48 @@ function createTransformersEmbeddingProvider(options) {
|
|
|
2107
2113
|
});
|
|
2108
2114
|
return attempt;
|
|
2109
2115
|
};
|
|
2116
|
+
const embed = async (texts) => {
|
|
2117
|
+
if (disposed) throw new Error("memory embedding provider is disposed");
|
|
2118
|
+
if (texts.length === 0) return [];
|
|
2119
|
+
const backend = await loadBackend();
|
|
2120
|
+
if (disposed) throw new Error("memory embedding provider is disposed");
|
|
2121
|
+
const vectors = [];
|
|
2122
|
+
for (let offset = 0; offset < texts.length; offset += batchSize) {
|
|
2123
|
+
if (disposed) throw new Error("memory embedding provider is disposed");
|
|
2124
|
+
const batchTexts = texts.slice(offset, offset + batchSize);
|
|
2125
|
+
const rows = (await backend.extract(batchTexts)).toList();
|
|
2126
|
+
for (const row of rows) {
|
|
2127
|
+
if (row.length !== dimensions) {
|
|
2128
|
+
throw new Error(
|
|
2129
|
+
`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`
|
|
2130
|
+
);
|
|
2131
|
+
}
|
|
2132
|
+
vectors.push(row);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
return vectors;
|
|
2136
|
+
};
|
|
2110
2137
|
return {
|
|
2111
2138
|
modelId,
|
|
2112
2139
|
dimensions,
|
|
2113
|
-
embed:
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
return vectors;
|
|
2140
|
+
embed: (texts) => {
|
|
2141
|
+
const call = embed(texts);
|
|
2142
|
+
activeCalls.add(call);
|
|
2143
|
+
void call.then(
|
|
2144
|
+
() => activeCalls.delete(call),
|
|
2145
|
+
() => activeCalls.delete(call)
|
|
2146
|
+
);
|
|
2147
|
+
return call;
|
|
2148
|
+
},
|
|
2149
|
+
dispose: () => {
|
|
2150
|
+
disposed = true;
|
|
2151
|
+
disposePromise ??= (async () => {
|
|
2152
|
+
const pendingBackend = backendPromise;
|
|
2153
|
+
await Promise.allSettled([...activeCalls]);
|
|
2154
|
+
const backend = await pendingBackend?.catch(() => void 0);
|
|
2155
|
+
await backend?.dispose?.();
|
|
2156
|
+
})();
|
|
2157
|
+
return disposePromise;
|
|
2132
2158
|
}
|
|
2133
2159
|
};
|
|
2134
2160
|
}
|
|
@@ -2151,6 +2177,7 @@ var init_embedding_provider = __esm({
|
|
|
2151
2177
|
transformers.env.remoteHost = remoteHost;
|
|
2152
2178
|
const extractor = await transformers.pipeline("feature-extraction", modelId, { dtype: "q8" });
|
|
2153
2179
|
return {
|
|
2180
|
+
dispose: () => extractor.dispose(),
|
|
2154
2181
|
extract: async (texts) => {
|
|
2155
2182
|
const output = await extractor(texts, { pooling: "cls", normalize: true });
|
|
2156
2183
|
return { toList: () => output.tolist() };
|
|
@@ -2169,6 +2196,9 @@ function createTransformersReranker(options) {
|
|
|
2169
2196
|
const remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST2];
|
|
2170
2197
|
const load = options.loadImpl ?? createDefaultTransformersLoadImpl2(options.moduleEntryPath);
|
|
2171
2198
|
let backendPromise = null;
|
|
2199
|
+
let disposed = false;
|
|
2200
|
+
let disposePromise;
|
|
2201
|
+
const activeCalls = /* @__PURE__ */ new Set();
|
|
2172
2202
|
const loadBackend = () => {
|
|
2173
2203
|
const existing = backendPromise;
|
|
2174
2204
|
if (existing) {
|
|
@@ -2181,13 +2211,16 @@ function createTransformersReranker(options) {
|
|
|
2181
2211
|
return await load(modelId, cacheDir, remoteHost);
|
|
2182
2212
|
} catch (error) {
|
|
2183
2213
|
lastError = error;
|
|
2214
|
+
if (disposed) throw error;
|
|
2184
2215
|
purgeZeroByteCacheArtifacts(join5(cacheDir, modelId));
|
|
2185
2216
|
}
|
|
2186
2217
|
try {
|
|
2187
2218
|
await new Promise((resolve2) => setTimeout(resolve2, 2e3));
|
|
2219
|
+
if (disposed) throw lastError;
|
|
2188
2220
|
return await load(modelId, cacheDir, remoteHost);
|
|
2189
2221
|
} catch (error) {
|
|
2190
2222
|
lastError = error;
|
|
2223
|
+
if (disposed) throw error;
|
|
2191
2224
|
purgeZeroByteCacheArtifacts(join5(cacheDir, modelId));
|
|
2192
2225
|
}
|
|
2193
2226
|
}
|
|
@@ -2202,14 +2235,33 @@ function createTransformersReranker(options) {
|
|
|
2202
2235
|
});
|
|
2203
2236
|
return attempt;
|
|
2204
2237
|
};
|
|
2238
|
+
const rerank = async (query, docs) => {
|
|
2239
|
+
if (disposed) throw new Error("memory reranker is disposed");
|
|
2240
|
+
if (docs.length === 0) return [];
|
|
2241
|
+
const backend = await loadBackend();
|
|
2242
|
+
if (disposed) throw new Error("memory reranker is disposed");
|
|
2243
|
+
return backend.score(query, [...docs]);
|
|
2244
|
+
};
|
|
2205
2245
|
return {
|
|
2206
2246
|
modelId,
|
|
2207
|
-
rerank:
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2247
|
+
rerank: (query, docs) => {
|
|
2248
|
+
const call = rerank(query, docs);
|
|
2249
|
+
activeCalls.add(call);
|
|
2250
|
+
void call.then(
|
|
2251
|
+
() => activeCalls.delete(call),
|
|
2252
|
+
() => activeCalls.delete(call)
|
|
2253
|
+
);
|
|
2254
|
+
return call;
|
|
2255
|
+
},
|
|
2256
|
+
dispose: () => {
|
|
2257
|
+
disposed = true;
|
|
2258
|
+
disposePromise ??= (async () => {
|
|
2259
|
+
const pendingBackend = backendPromise;
|
|
2260
|
+
await Promise.allSettled([...activeCalls]);
|
|
2261
|
+
const backend = await pendingBackend?.catch(() => void 0);
|
|
2262
|
+
await backend?.dispose?.();
|
|
2263
|
+
})();
|
|
2264
|
+
return disposePromise;
|
|
2213
2265
|
}
|
|
2214
2266
|
};
|
|
2215
2267
|
}
|
|
@@ -2233,6 +2285,9 @@ var init_embedding_reranker = __esm({
|
|
|
2233
2285
|
dtype: "q8"
|
|
2234
2286
|
});
|
|
2235
2287
|
return {
|
|
2288
|
+
dispose: async () => {
|
|
2289
|
+
await model.dispose();
|
|
2290
|
+
},
|
|
2236
2291
|
score: async (query, docs) => {
|
|
2237
2292
|
const inputs = tokenizer(
|
|
2238
2293
|
docs.map(() => query),
|
|
@@ -2258,6 +2313,19 @@ async function createMemoryVectorIndex(options) {
|
|
|
2258
2313
|
);
|
|
2259
2314
|
const db = await lancedb.connect(options.dbPath);
|
|
2260
2315
|
let table = null;
|
|
2316
|
+
let closed = false;
|
|
2317
|
+
let closePromise;
|
|
2318
|
+
const activeCalls = /* @__PURE__ */ new Set();
|
|
2319
|
+
const withOpenIndex = (operation) => {
|
|
2320
|
+
if (closed) return Promise.reject(new Error("memory vector index is closed"));
|
|
2321
|
+
const call = operation();
|
|
2322
|
+
activeCalls.add(call);
|
|
2323
|
+
void call.then(
|
|
2324
|
+
() => activeCalls.delete(call),
|
|
2325
|
+
() => activeCalls.delete(call)
|
|
2326
|
+
);
|
|
2327
|
+
return call;
|
|
2328
|
+
};
|
|
2261
2329
|
const ensureTable = async () => {
|
|
2262
2330
|
if (table) {
|
|
2263
2331
|
return table;
|
|
@@ -2277,7 +2345,7 @@ async function createMemoryVectorIndex(options) {
|
|
|
2277
2345
|
};
|
|
2278
2346
|
return {
|
|
2279
2347
|
replicaId: "memory-vector-index",
|
|
2280
|
-
upsert: async (
|
|
2348
|
+
upsert: (entries) => withOpenIndex(async () => {
|
|
2281
2349
|
if (entries.length === 0) {
|
|
2282
2350
|
return;
|
|
2283
2351
|
}
|
|
@@ -2302,8 +2370,8 @@ async function createMemoryVectorIndex(options) {
|
|
|
2302
2370
|
return;
|
|
2303
2371
|
}
|
|
2304
2372
|
await current.mergeInsert("memoryId").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(rows);
|
|
2305
|
-
},
|
|
2306
|
-
remove: async (
|
|
2373
|
+
}),
|
|
2374
|
+
remove: (memoryIds) => withOpenIndex(async () => {
|
|
2307
2375
|
if (memoryIds.length === 0) {
|
|
2308
2376
|
return;
|
|
2309
2377
|
}
|
|
@@ -2313,8 +2381,8 @@ async function createMemoryVectorIndex(options) {
|
|
|
2313
2381
|
}
|
|
2314
2382
|
const predicate = `memoryId IN (${memoryIds.map((id) => `'${escapeSqlLiteral(id)}'`).join(",")})`;
|
|
2315
2383
|
await current.delete(predicate);
|
|
2316
|
-
},
|
|
2317
|
-
queryKnn:
|
|
2384
|
+
}),
|
|
2385
|
+
queryKnn: (vector, query) => withOpenIndex(async () => {
|
|
2318
2386
|
if (vector.length !== dimensions) {
|
|
2319
2387
|
throw new Error(`query vector dimension mismatch: got ${vector.length}, index declares ${dimensions}`);
|
|
2320
2388
|
}
|
|
@@ -2324,40 +2392,45 @@ async function createMemoryVectorIndex(options) {
|
|
|
2324
2392
|
}
|
|
2325
2393
|
const rows = await current.search(Array.from(vector)).where(`owner = '${escapeSqlLiteral(query.owner)}'`).limit(query.limit).toArray();
|
|
2326
2394
|
return rows.map((row) => ({ memoryId: row.memoryId, score: 1 / (1 + row._distance) }));
|
|
2327
|
-
},
|
|
2328
|
-
queryFts:
|
|
2395
|
+
}),
|
|
2396
|
+
queryFts: (queryText, query) => withOpenIndex(async () => {
|
|
2329
2397
|
const current = await ensureTable();
|
|
2330
2398
|
if (!current) {
|
|
2331
2399
|
return [];
|
|
2332
2400
|
}
|
|
2333
2401
|
const rows = await current.search(queryText).where(`owner = '${escapeSqlLiteral(query.owner)}'`).limit(query.limit).toArray();
|
|
2334
2402
|
return rows.map((row, rank) => ({ memoryId: row.memoryId, score: 1 / (RRF_K + rank + 1) }));
|
|
2335
|
-
},
|
|
2336
|
-
listMemoryIds: async () => {
|
|
2403
|
+
}),
|
|
2404
|
+
listMemoryIds: () => withOpenIndex(async () => {
|
|
2337
2405
|
const current = await ensureTable();
|
|
2338
2406
|
if (!current) {
|
|
2339
2407
|
return /* @__PURE__ */ new Set();
|
|
2340
2408
|
}
|
|
2341
2409
|
const rows = await current.query().select(["memoryId"]).toArray();
|
|
2342
2410
|
return new Set(rows.map((row) => row.memoryId));
|
|
2343
|
-
},
|
|
2344
|
-
count: async () => {
|
|
2411
|
+
}),
|
|
2412
|
+
count: () => withOpenIndex(async () => {
|
|
2345
2413
|
const current = await ensureTable();
|
|
2346
2414
|
if (!current) {
|
|
2347
2415
|
return 0;
|
|
2348
2416
|
}
|
|
2349
2417
|
return current.countRows();
|
|
2350
|
-
},
|
|
2351
|
-
getStoredModelId: async () => {
|
|
2418
|
+
}),
|
|
2419
|
+
getStoredModelId: () => withOpenIndex(async () => {
|
|
2352
2420
|
const current = await ensureTable();
|
|
2353
2421
|
if (!current) {
|
|
2354
2422
|
return void 0;
|
|
2355
2423
|
}
|
|
2356
2424
|
const rows = await current.query().select(["modelId"]).limit(1).toArray();
|
|
2357
2425
|
return rows[0]?.modelId;
|
|
2358
|
-
},
|
|
2359
|
-
close:
|
|
2360
|
-
|
|
2426
|
+
}),
|
|
2427
|
+
close: () => {
|
|
2428
|
+
closed = true;
|
|
2429
|
+
closePromise ??= (async () => {
|
|
2430
|
+
await Promise.allSettled([...activeCalls]);
|
|
2431
|
+
db.close();
|
|
2432
|
+
})();
|
|
2433
|
+
return closePromise;
|
|
2361
2434
|
}
|
|
2362
2435
|
};
|
|
2363
2436
|
}
|
|
@@ -10835,7 +10908,7 @@ var FrameDecoder = class {
|
|
|
10835
10908
|
};
|
|
10836
10909
|
|
|
10837
10910
|
// ../../packages/protocol/src/schemas.ts
|
|
10838
|
-
var PROTOCOL_VERSION =
|
|
10911
|
+
var PROTOCOL_VERSION = 2;
|
|
10839
10912
|
var IdSchema = build_default.String({ minLength: 1 });
|
|
10840
10913
|
var TimestampSchema = build_default.Integer({ minimum: 0 });
|
|
10841
10914
|
var Base64Schema = build_default.String({ pattern: "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" });
|
|
@@ -10874,12 +10947,6 @@ var ModelRefSchema = StrictObject({
|
|
|
10874
10947
|
provider: IdSchema,
|
|
10875
10948
|
id: IdSchema
|
|
10876
10949
|
});
|
|
10877
|
-
var ModelCostSchema = StrictObject({
|
|
10878
|
-
input: build_default.Number({ minimum: 0 }),
|
|
10879
|
-
output: build_default.Number({ minimum: 0 }),
|
|
10880
|
-
cacheRead: build_default.Number({ minimum: 0 }),
|
|
10881
|
-
cacheWrite: build_default.Number({ minimum: 0 })
|
|
10882
|
-
});
|
|
10883
10950
|
var ModelMetadataSchema = StrictObject({
|
|
10884
10951
|
provider: IdSchema,
|
|
10885
10952
|
id: IdSchema,
|
|
@@ -10889,7 +10956,6 @@ var ModelMetadataSchema = StrictObject({
|
|
|
10889
10956
|
input: build_default.Array(build_default.Union([build_default.Literal("text"), build_default.Literal("image")])),
|
|
10890
10957
|
contextWindow: build_default.Integer({ minimum: 1 }),
|
|
10891
10958
|
maxTokens: build_default.Integer({ minimum: 1 }),
|
|
10892
|
-
cost: ModelCostSchema,
|
|
10893
10959
|
supportedThinkingLevels: build_default.Array(ThinkingLevelSchema, { minItems: 1 }),
|
|
10894
10960
|
authenticated: build_default.Boolean()
|
|
10895
10961
|
});
|
|
@@ -10922,14 +10988,7 @@ var UsageSchema = StrictObject({
|
|
|
10922
10988
|
cacheRead: build_default.Integer({ minimum: 0 }),
|
|
10923
10989
|
cacheWrite: build_default.Integer({ minimum: 0 }),
|
|
10924
10990
|
reasoning: build_default.Optional(build_default.Integer({ minimum: 0 })),
|
|
10925
|
-
totalTokens: build_default.Integer({ minimum: 0 })
|
|
10926
|
-
cost: StrictObject({
|
|
10927
|
-
input: build_default.Number({ minimum: 0 }),
|
|
10928
|
-
output: build_default.Number({ minimum: 0 }),
|
|
10929
|
-
cacheRead: build_default.Number({ minimum: 0 }),
|
|
10930
|
-
cacheWrite: build_default.Number({ minimum: 0 }),
|
|
10931
|
-
total: build_default.Number({ minimum: 0 })
|
|
10932
|
-
})
|
|
10991
|
+
totalTokens: build_default.Integer({ minimum: 0 })
|
|
10933
10992
|
});
|
|
10934
10993
|
var UserTranscriptItemSchema = StrictObject({
|
|
10935
10994
|
id: IdSchema,
|
|
@@ -22,6 +22,8 @@ export interface MemoryEmbeddingProviderV1 {
|
|
|
22
22
|
* responsibility (低档 bge-small-zh 系查询侧指令由调用方负责添加).
|
|
23
23
|
*/
|
|
24
24
|
embed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;
|
|
25
|
+
/** 仅释放本实例拥有的模型;等待已进入的推理完成,不删除缓存文件。 */
|
|
26
|
+
dispose?(): Promise<void>;
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
29
|
* Minimal structural seam around a transformers.js feature-extraction
|
|
@@ -32,6 +34,7 @@ export interface TransformersEmbeddingBackendV1 {
|
|
|
32
34
|
extract: (texts: string[]) => Promise<{
|
|
33
35
|
toList: () => number[][];
|
|
34
36
|
}>;
|
|
37
|
+
dispose?(): Promise<void>;
|
|
35
38
|
}
|
|
36
39
|
export type TransformersEmbeddingLoadImpl = (modelId: string, cacheDir: string, remoteHost: string) => Promise<TransformersEmbeddingBackendV1>;
|
|
37
40
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../../src/memory/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH,MAAM,WAAW,yBAAyB;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;CACzE;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA8B;IAC9C,gFAAgF;IAChF,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,MAAM,EAAE,EAAE,CAAA;KAAE,CAAC,CAAC;CACpE;AAED,MAAM,MAAM,6BAA6B,GAAG,CAC3C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,KACd,OAAO,CAAC,8BAA8B,CAAC,CAAC;AA2C7C;;;;;GAKG;AACH,wBAAgB,wCAAwC,CAAC,OAAO,CAAC,EAAE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC7B,GAAG,yBAAyB,CAO5B;AAmBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mCAAmC,CAAC,OAAO,EAAE;IAC5D,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,6BAA6B,CAAC;IAClD,iGAAiG;IACjG,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,yBAAyB,CA0E5B","sourcesContent":["/**\n * Memory embedding providers for hybrid retrieval (混合检索 · 向量侧).\n *\n * POC-verified (2026-09, 100k corpus): high tier (Xenova/bge-m3, 1024d) and\n * low tier (Xenova/bge-small-zh-v1.5, 512d) inference is deterministic\n * bit-exact across runs; high-tier drift queries reach re@10 100% after\n * reranking, low tier 83%. Tier presets are caller configuration — this module\n * only exposes a modelId-parameterized interface (宪法: 不绑死单一 provider).\n *\n * Runtime dep @huggingface/transformers is lazy-loaded via `await import` at\n * first embed (惯例参考 packages/agent-forge/src/utils/photon.ts); type\n * positions use type-only imports. The deterministic mock provider keeps tests\n * offline (门禁: deterministic mock AI; 真实模型不进 CI).\n */\n\n/**\n * Batch embedding provider. Implementations must be deterministic for\n * identical input and must not silently degrade: any failure propagates.\n */\nimport { join } from \"node:path\";\nimport { importHostOrBareModule } from \"./host-module-import.ts\";\nimport { annotateUpstreamListingPoison, purgeZeroByteCacheArtifacts } from \"./model-cache-hygiene.ts\";\n\nexport interface MemoryEmbeddingProviderV1 {\n\treadonly modelId: string;\n\treadonly dimensions: number;\n\t/**\n\t * Batch embed. Vectors are L2-normalized float arrays. Deterministic for\n\t * identical input. Plain text mapping — no instruction prefix is added\n\t * here; query-side instructions for bge-small-zh family are the caller's\n\t * responsibility (低档 bge-small-zh 系查询侧指令由调用方负责添加).\n\t */\n\tembed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;\n}\n\n/**\n * Minimal structural seam around a transformers.js feature-extraction\n * pipeline, so tests can inject a fake without touching the real loader.\n */\nexport interface TransformersEmbeddingBackendV1 {\n\t/** Extract features for a batch of texts; one vector per input, input order. */\n\textract: (texts: string[]) => Promise<{ toList: () => number[][] }>;\n}\n\nexport type TransformersEmbeddingLoadImpl = (\n\tmodelId: string,\n\tcacheDir: string,\n\tremoteHost: string,\n) => Promise<TransformersEmbeddingBackendV1>;\n\nconst DEFAULT_TRANSFORMERS_MODEL_ID = \"Xenova/bge-m3\";\nconst DEFAULT_TRANSFORMERS_DIMENSIONS = 1024;\nconst DEFAULT_TRANSFORMERS_BATCH_SIZE = 16;\nconst DEFAULT_REMOTE_HOST = \"https://huggingface.co\";\n\nfunction fnv1a32(input: string): number {\n\tlet hash = 0x811c9dc5;\n\tfor (let offset = 0; offset < input.length; offset += 1) {\n\t\thash ^= input.charCodeAt(offset);\n\t\thash = Math.imul(hash, 0x01000193);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction embedDeterministicMock(text: string, dimensions: number): number[] {\n\tconst values = new Array<number>(dimensions).fill(0);\n\tconst tokens = text.toLowerCase().split(/[^\\p{L}\\p{N}]+/u);\n\tfor (const token of tokens) {\n\t\tif (token.length === 0) {\n\t\t\tcontinue;\n\t\t}\n\t\t// 多轮混淆累加: 每个 token 经 FNV-1a 后再派生 3 个维度沉点,\n\t\t// 放大不同文本间的向量差异; 全程无随机、无时钟。\n\t\tlet mixed = fnv1a32(token);\n\t\tfor (let round = 0; round < 3; round += 1) {\n\t\t\tmixed = Math.imul(mixed ^ 0x9e3779b9, 0x85ebca6b) >>> 0;\n\t\t\tvalues[mixed % dimensions] += 1;\n\t\t}\n\t}\n\tlet sumSquares = 0;\n\tfor (const value of values) {\n\t\tsumSquares += value * value;\n\t}\n\tif (sumSquares === 0) {\n\t\t// 空文本/全标点: 确定性零向量; L2 范数为 0, 不归一化 (避免除零 NaN)。\n\t\treturn values;\n\t}\n\tconst norm = Math.sqrt(sumSquares);\n\treturn values.map((value) => value / norm);\n}\n\n/**\n * Deterministic mock embedding provider for offline tests: tokenizes by\n * `[^\\p{L}\\p{N}]+`, hashes each token with FNV-1a plus multi-round mixing into\n * dimension indices, then L2-normalizes. Same text → same vector, no random,\n * no clock. POC 已用真实模型验证 bit-exact; mock 只承担确定性测试门禁。\n */\nexport function createDeterministicMockEmbeddingProvider(options?: {\n\treadonly dimensions?: number;\n}): MemoryEmbeddingProviderV1 {\n\tconst dimensions = options?.dimensions ?? 64;\n\treturn {\n\t\tmodelId: \"mock-embedding@1\",\n\t\tdimensions,\n\t\tembed: async (texts) => texts.map((text) => embedDeterministicMock(text, dimensions)),\n\t};\n}\n\nconst createDefaultTransformersLoadImpl = (moduleEntryPath?: string): TransformersEmbeddingLoadImpl => {\n\treturn async (modelId, cacheDir, remoteHost) => {\n\t\tconst transformers = await importHostOrBareModule<typeof import(\"@huggingface/transformers\")>(\n\t\t\tmoduleEntryPath ?? \"@huggingface/transformers\",\n\t\t);\n\t\ttransformers.env.cacheDir = cacheDir;\n\t\ttransformers.env.remoteHost = remoteHost;\n\t\tconst extractor = await transformers.pipeline(\"feature-extraction\", modelId, { dtype: \"q8\" });\n\t\treturn {\n\t\t\textract: async (texts) => {\n\t\t\t\tconst output = await extractor(texts, { pooling: \"cls\", normalize: true });\n\t\t\t\treturn { toList: () => output.tolist() as number[][] };\n\t\t\t},\n\t\t};\n\t};\n};\n\n/**\n * Transformers.js-backed embedding provider (POC 档位: 高档默认\n * Xenova/bge-m3/1024d; 低档由调用方传 modelId \"Xenova/bge-small-zh-v1.5\" +\n * dimensions 512)。\n *\n * 镜像序列: 首次 embed 时按 `remoteHosts` 顺序逐个尝试加载 (调用方通常传\n * [\"https://huggingface.co\", \"https://hf-mirror.com\"]); 某 host 加载抛错则\n * 尝试下一个, 全部失败时抛最后一个错误。host 确定后模型句柄只加载一次\n * (并发 embed 共享同一 in-flight promise); 加载失败不缓存, 下次 embed 重新\n * 走完整序列, 失败原因原样保留, 不静默降级。\n *\n * `loadImpl` 是测试 seam: 注入后绕过真实加载, 签名\n * (modelId, cacheDir, remoteHost) => backend。\n */\nexport function createTransformersEmbeddingProvider(options: {\n\treadonly modelId?: string;\n\treadonly dimensions?: number;\n\treadonly cacheDir: string;\n\treadonly remoteHosts?: readonly string[];\n\treadonly batchSize?: number;\n\treadonly loadImpl?: TransformersEmbeddingLoadImpl;\n\t/** Host-resolved transformers entry path (install-store visibility); undefined = bare import. */\n\treadonly moduleEntryPath?: string;\n}): MemoryEmbeddingProviderV1 {\n\tconst modelId = options.modelId ?? DEFAULT_TRANSFORMERS_MODEL_ID;\n\tconst dimensions = options.dimensions ?? DEFAULT_TRANSFORMERS_DIMENSIONS;\n\tconst batchSize = options.batchSize ?? DEFAULT_TRANSFORMERS_BATCH_SIZE;\n\tconst cacheDir = options.cacheDir;\n\tconst remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];\n\tconst load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);\n\n\tlet backendPromise: Promise<TransformersEmbeddingBackendV1> | null = null;\n\tconst loadBackend = (): Promise<TransformersEmbeddingBackendV1> => {\n\t\tconst existing = backendPromise;\n\t\tif (existing) {\n\t\t\treturn existing;\n\t\t}\n\t\tconst attempt = (async () => {\n\t\t\tlet lastError: unknown;\n\t\t\tfor (const remoteHost of remoteHosts) {\n\t\t\t\ttry {\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\t// 镜像轮换前清 0 字节残骸: 失败下载留下的空文件会被 transformers\n\t\t\t\t\t// 当作缓存命中, 毒化下一个 host 的重试 (model-cache-hygiene)。\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t\t// 同 host 一次短退避重试 (2026-10-06 live E2E): 覆盖 rejection 型瞬断\n\t\t\t\t// (fetch 网络错/5xx)。对\"仓库 API 清单缺文件\"型不生效——该结果被\n\t\t\t\t// transformers 模块级 memoize (见 annotateUpstreamListingPoison),\n\t\t\t\t// 进程内换 host 也拿同一空清单。\n\t\t\t\ttry {\n\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, 2_000));\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (lastError === undefined) throw new Error(`no remote hosts configured for model ${modelId}`);\n\t\t\tthrow annotateUpstreamListingPoison(lastError);\n\t\t})();\n\t\tbackendPromise = attempt;\n\t\tattempt.catch(() => {\n\t\t\t// 失败不缓存: 下次 embed 重新走镜像序列 (显式失败, 不静默降级)。\n\t\t\tif (backendPromise === attempt) {\n\t\t\t\tbackendPromise = null;\n\t\t\t}\n\t\t});\n\t\treturn attempt;\n\t};\n\n\treturn {\n\t\tmodelId,\n\t\tdimensions,\n\t\tembed: async (texts) => {\n\t\t\tif (texts.length === 0) {\n\t\t\t\treturn [];\n\t\t\t}\n\t\t\tconst backend = await loadBackend();\n\t\t\tconst vectors: number[][] = [];\n\t\t\tfor (let offset = 0; offset < texts.length; offset += batchSize) {\n\t\t\t\tconst batchTexts = texts.slice(offset, offset + batchSize);\n\t\t\t\tconst rows = (await backend.extract(batchTexts)).toList();\n\t\t\t\tfor (const row of rows) {\n\t\t\t\t\tif (row.length !== dimensions) {\n\t\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t\t`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tvectors.push(row);\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn vectors;\n\t\t},\n\t};\n}\n"]}
|
|
1
|
+
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../../src/memory/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH,MAAM,WAAW,yBAAyB;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IACzE,oGAAsC;IACtC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,8BAA8B;IAC9C,gFAAgF;IAChF,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,MAAM,EAAE,EAAE,CAAA;KAAE,CAAC,CAAC;IACpE,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,MAAM,6BAA6B,GAAG,CAC3C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,KACd,OAAO,CAAC,8BAA8B,CAAC,CAAC;AA2C7C;;;;;GAKG;AACH,wBAAgB,wCAAwC,CAAC,OAAO,CAAC,EAAE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC7B,GAAG,yBAAyB,CAO5B;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mCAAmC,CAAC,OAAO,EAAE;IAC5D,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,6BAA6B,CAAC;IAClD,iGAAiG;IACjG,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,yBAAyB,CAoG5B","sourcesContent":["/**\n * Memory embedding providers for hybrid retrieval (混合检索 · 向量侧).\n *\n * POC-verified (2026-09, 100k corpus): high tier (Xenova/bge-m3, 1024d) and\n * low tier (Xenova/bge-small-zh-v1.5, 512d) inference is deterministic\n * bit-exact across runs; high-tier drift queries reach re@10 100% after\n * reranking, low tier 83%. Tier presets are caller configuration — this module\n * only exposes a modelId-parameterized interface (宪法: 不绑死单一 provider).\n *\n * Runtime dep @huggingface/transformers is lazy-loaded via `await import` at\n * first embed (惯例参考 packages/agent-forge/src/utils/photon.ts); type\n * positions use type-only imports. The deterministic mock provider keeps tests\n * offline (门禁: deterministic mock AI; 真实模型不进 CI).\n */\n\n/**\n * Batch embedding provider. Implementations must be deterministic for\n * identical input and must not silently degrade: any failure propagates.\n */\nimport { join } from \"node:path\";\nimport { importHostOrBareModule } from \"./host-module-import.ts\";\nimport { annotateUpstreamListingPoison, purgeZeroByteCacheArtifacts } from \"./model-cache-hygiene.ts\";\n\nexport interface MemoryEmbeddingProviderV1 {\n\treadonly modelId: string;\n\treadonly dimensions: number;\n\t/**\n\t * Batch embed. Vectors are L2-normalized float arrays. Deterministic for\n\t * identical input. Plain text mapping — no instruction prefix is added\n\t * here; query-side instructions for bge-small-zh family are the caller's\n\t * responsibility (低档 bge-small-zh 系查询侧指令由调用方负责添加).\n\t */\n\tembed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;\n\t/** 仅释放本实例拥有的模型;等待已进入的推理完成,不删除缓存文件。 */\n\tdispose?(): Promise<void>;\n}\n\n/**\n * Minimal structural seam around a transformers.js feature-extraction\n * pipeline, so tests can inject a fake without touching the real loader.\n */\nexport interface TransformersEmbeddingBackendV1 {\n\t/** Extract features for a batch of texts; one vector per input, input order. */\n\textract: (texts: string[]) => Promise<{ toList: () => number[][] }>;\n\tdispose?(): Promise<void>;\n}\n\nexport type TransformersEmbeddingLoadImpl = (\n\tmodelId: string,\n\tcacheDir: string,\n\tremoteHost: string,\n) => Promise<TransformersEmbeddingBackendV1>;\n\nconst DEFAULT_TRANSFORMERS_MODEL_ID = \"Xenova/bge-m3\";\nconst DEFAULT_TRANSFORMERS_DIMENSIONS = 1024;\nconst DEFAULT_TRANSFORMERS_BATCH_SIZE = 16;\nconst DEFAULT_REMOTE_HOST = \"https://huggingface.co\";\n\nfunction fnv1a32(input: string): number {\n\tlet hash = 0x811c9dc5;\n\tfor (let offset = 0; offset < input.length; offset += 1) {\n\t\thash ^= input.charCodeAt(offset);\n\t\thash = Math.imul(hash, 0x01000193);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction embedDeterministicMock(text: string, dimensions: number): number[] {\n\tconst values = new Array<number>(dimensions).fill(0);\n\tconst tokens = text.toLowerCase().split(/[^\\p{L}\\p{N}]+/u);\n\tfor (const token of tokens) {\n\t\tif (token.length === 0) {\n\t\t\tcontinue;\n\t\t}\n\t\t// 多轮混淆累加: 每个 token 经 FNV-1a 后再派生 3 个维度沉点,\n\t\t// 放大不同文本间的向量差异; 全程无随机、无时钟。\n\t\tlet mixed = fnv1a32(token);\n\t\tfor (let round = 0; round < 3; round += 1) {\n\t\t\tmixed = Math.imul(mixed ^ 0x9e3779b9, 0x85ebca6b) >>> 0;\n\t\t\tvalues[mixed % dimensions] += 1;\n\t\t}\n\t}\n\tlet sumSquares = 0;\n\tfor (const value of values) {\n\t\tsumSquares += value * value;\n\t}\n\tif (sumSquares === 0) {\n\t\t// 空文本/全标点: 确定性零向量; L2 范数为 0, 不归一化 (避免除零 NaN)。\n\t\treturn values;\n\t}\n\tconst norm = Math.sqrt(sumSquares);\n\treturn values.map((value) => value / norm);\n}\n\n/**\n * Deterministic mock embedding provider for offline tests: tokenizes by\n * `[^\\p{L}\\p{N}]+`, hashes each token with FNV-1a plus multi-round mixing into\n * dimension indices, then L2-normalizes. Same text → same vector, no random,\n * no clock. POC 已用真实模型验证 bit-exact; mock 只承担确定性测试门禁。\n */\nexport function createDeterministicMockEmbeddingProvider(options?: {\n\treadonly dimensions?: number;\n}): MemoryEmbeddingProviderV1 {\n\tconst dimensions = options?.dimensions ?? 64;\n\treturn {\n\t\tmodelId: \"mock-embedding@1\",\n\t\tdimensions,\n\t\tembed: async (texts) => texts.map((text) => embedDeterministicMock(text, dimensions)),\n\t};\n}\n\nconst createDefaultTransformersLoadImpl = (moduleEntryPath?: string): TransformersEmbeddingLoadImpl => {\n\treturn async (modelId, cacheDir, remoteHost) => {\n\t\tconst transformers = await importHostOrBareModule<typeof import(\"@huggingface/transformers\")>(\n\t\t\tmoduleEntryPath ?? \"@huggingface/transformers\",\n\t\t);\n\t\ttransformers.env.cacheDir = cacheDir;\n\t\ttransformers.env.remoteHost = remoteHost;\n\t\tconst extractor = await transformers.pipeline(\"feature-extraction\", modelId, { dtype: \"q8\" });\n\t\treturn {\n\t\t\tdispose: () => extractor.dispose(),\n\t\t\textract: async (texts) => {\n\t\t\t\tconst output = await extractor(texts, { pooling: \"cls\", normalize: true });\n\t\t\t\treturn { toList: () => output.tolist() as number[][] };\n\t\t\t},\n\t\t};\n\t};\n};\n\n/**\n * Transformers.js-backed embedding provider (POC 档位: 高档默认\n * Xenova/bge-m3/1024d; 低档由调用方传 modelId \"Xenova/bge-small-zh-v1.5\" +\n * dimensions 512)。\n *\n * 镜像序列: 首次 embed 时按 `remoteHosts` 顺序逐个尝试加载 (调用方通常传\n * [\"https://huggingface.co\", \"https://hf-mirror.com\"]); 某 host 加载抛错则\n * 尝试下一个, 全部失败时抛最后一个错误。host 确定后模型句柄只加载一次\n * (并发 embed 共享同一 in-flight promise); 加载失败不缓存, 下次 embed 重新\n * 走完整序列, 失败原因原样保留, 不静默降级。\n *\n * `loadImpl` 是测试 seam: 注入后绕过真实加载, 签名\n * (modelId, cacheDir, remoteHost) => backend。\n */\nexport function createTransformersEmbeddingProvider(options: {\n\treadonly modelId?: string;\n\treadonly dimensions?: number;\n\treadonly cacheDir: string;\n\treadonly remoteHosts?: readonly string[];\n\treadonly batchSize?: number;\n\treadonly loadImpl?: TransformersEmbeddingLoadImpl;\n\t/** Host-resolved transformers entry path (install-store visibility); undefined = bare import. */\n\treadonly moduleEntryPath?: string;\n}): MemoryEmbeddingProviderV1 {\n\tconst modelId = options.modelId ?? DEFAULT_TRANSFORMERS_MODEL_ID;\n\tconst dimensions = options.dimensions ?? DEFAULT_TRANSFORMERS_DIMENSIONS;\n\tconst batchSize = options.batchSize ?? DEFAULT_TRANSFORMERS_BATCH_SIZE;\n\tconst cacheDir = options.cacheDir;\n\tconst remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];\n\tconst load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);\n\n\tlet backendPromise: Promise<TransformersEmbeddingBackendV1> | null = null;\n\tlet disposed = false;\n\tlet disposePromise: Promise<void> | undefined;\n\tconst activeCalls = new Set<Promise<unknown>>();\n\tconst loadBackend = (): Promise<TransformersEmbeddingBackendV1> => {\n\t\tconst existing = backendPromise;\n\t\tif (existing) {\n\t\t\treturn existing;\n\t\t}\n\t\tconst attempt = (async () => {\n\t\t\tlet lastError: unknown;\n\t\t\tfor (const remoteHost of remoteHosts) {\n\t\t\t\ttry {\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tif (disposed) throw error;\n\t\t\t\t\t// 镜像轮换前清 0 字节残骸: 失败下载留下的空文件会被 transformers\n\t\t\t\t\t// 当作缓存命中, 毒化下一个 host 的重试 (model-cache-hygiene)。\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t\t// 同 host 一次短退避重试 (2026-10-06 live E2E): 覆盖 rejection 型瞬断\n\t\t\t\t// (fetch 网络错/5xx)。对\"仓库 API 清单缺文件\"型不生效——该结果被\n\t\t\t\t// transformers 模块级 memoize (见 annotateUpstreamListingPoison),\n\t\t\t\t// 进程内换 host 也拿同一空清单。\n\t\t\t\ttry {\n\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, 2_000));\n\t\t\t\t\tif (disposed) throw lastError;\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tif (disposed) throw error;\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (lastError === undefined) throw new Error(`no remote hosts configured for model ${modelId}`);\n\t\t\tthrow annotateUpstreamListingPoison(lastError);\n\t\t})();\n\t\tbackendPromise = attempt;\n\t\tattempt.catch(() => {\n\t\t\t// 失败不缓存: 下次 embed 重新走镜像序列 (显式失败, 不静默降级)。\n\t\t\tif (backendPromise === attempt) {\n\t\t\t\tbackendPromise = null;\n\t\t\t}\n\t\t});\n\t\treturn attempt;\n\t};\n\n\tconst embed = async (texts: readonly string[]): Promise<readonly (readonly number[])[]> => {\n\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\tif (texts.length === 0) return [];\n\t\tconst backend = await loadBackend();\n\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\tconst vectors: number[][] = [];\n\t\tfor (let offset = 0; offset < texts.length; offset += batchSize) {\n\t\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\t\tconst batchTexts = texts.slice(offset, offset + batchSize);\n\t\t\tconst rows = (await backend.extract(batchTexts)).toList();\n\t\t\tfor (const row of rows) {\n\t\t\t\tif (row.length !== dimensions) {\n\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tvectors.push(row);\n\t\t\t}\n\t\t}\n\t\treturn vectors;\n\t};\n\treturn {\n\t\tmodelId,\n\t\tdimensions,\n\t\tembed: (texts) => {\n\t\t\tconst call = embed(texts);\n\t\t\tactiveCalls.add(call);\n\t\t\tvoid call.then(\n\t\t\t\t() => activeCalls.delete(call),\n\t\t\t\t() => activeCalls.delete(call),\n\t\t\t);\n\t\t\treturn call;\n\t\t},\n\t\tdispose: () => {\n\t\t\tdisposed = true;\n\t\t\tdisposePromise ??= (async () => {\n\t\t\t\tconst pendingBackend = backendPromise;\n\t\t\t\tawait Promise.allSettled([...activeCalls]);\n\t\t\t\tconst backend = await pendingBackend?.catch(() => undefined);\n\t\t\t\tawait backend?.dispose?.();\n\t\t\t})();\n\t\t\treturn disposePromise;\n\t\t},\n\t};\n}\n"]}
|
|
@@ -78,6 +78,7 @@ const createDefaultTransformersLoadImpl = (moduleEntryPath) => {
|
|
|
78
78
|
transformers.env.remoteHost = remoteHost;
|
|
79
79
|
const extractor = await transformers.pipeline("feature-extraction", modelId, { dtype: "q8" });
|
|
80
80
|
return {
|
|
81
|
+
dispose: () => extractor.dispose(),
|
|
81
82
|
extract: async (texts) => {
|
|
82
83
|
const output = await extractor(texts, { pooling: "cls", normalize: true });
|
|
83
84
|
return { toList: () => output.tolist() };
|
|
@@ -107,6 +108,9 @@ export function createTransformersEmbeddingProvider(options) {
|
|
|
107
108
|
const remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];
|
|
108
109
|
const load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);
|
|
109
110
|
let backendPromise = null;
|
|
111
|
+
let disposed = false;
|
|
112
|
+
let disposePromise;
|
|
113
|
+
const activeCalls = new Set();
|
|
110
114
|
const loadBackend = () => {
|
|
111
115
|
const existing = backendPromise;
|
|
112
116
|
if (existing) {
|
|
@@ -120,6 +124,8 @@ export function createTransformersEmbeddingProvider(options) {
|
|
|
120
124
|
}
|
|
121
125
|
catch (error) {
|
|
122
126
|
lastError = error;
|
|
127
|
+
if (disposed)
|
|
128
|
+
throw error;
|
|
123
129
|
// 镜像轮换前清 0 字节残骸: 失败下载留下的空文件会被 transformers
|
|
124
130
|
// 当作缓存命中, 毒化下一个 host 的重试 (model-cache-hygiene)。
|
|
125
131
|
purgeZeroByteCacheArtifacts(join(cacheDir, modelId));
|
|
@@ -130,10 +136,14 @@ export function createTransformersEmbeddingProvider(options) {
|
|
|
130
136
|
// 进程内换 host 也拿同一空清单。
|
|
131
137
|
try {
|
|
132
138
|
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
|
139
|
+
if (disposed)
|
|
140
|
+
throw lastError;
|
|
133
141
|
return await load(modelId, cacheDir, remoteHost);
|
|
134
142
|
}
|
|
135
143
|
catch (error) {
|
|
136
144
|
lastError = error;
|
|
145
|
+
if (disposed)
|
|
146
|
+
throw error;
|
|
137
147
|
purgeZeroByteCacheArtifacts(join(cacheDir, modelId));
|
|
138
148
|
}
|
|
139
149
|
}
|
|
@@ -150,26 +160,47 @@ export function createTransformersEmbeddingProvider(options) {
|
|
|
150
160
|
});
|
|
151
161
|
return attempt;
|
|
152
162
|
};
|
|
163
|
+
const embed = async (texts) => {
|
|
164
|
+
if (disposed)
|
|
165
|
+
throw new Error("memory embedding provider is disposed");
|
|
166
|
+
if (texts.length === 0)
|
|
167
|
+
return [];
|
|
168
|
+
const backend = await loadBackend();
|
|
169
|
+
if (disposed)
|
|
170
|
+
throw new Error("memory embedding provider is disposed");
|
|
171
|
+
const vectors = [];
|
|
172
|
+
for (let offset = 0; offset < texts.length; offset += batchSize) {
|
|
173
|
+
if (disposed)
|
|
174
|
+
throw new Error("memory embedding provider is disposed");
|
|
175
|
+
const batchTexts = texts.slice(offset, offset + batchSize);
|
|
176
|
+
const rows = (await backend.extract(batchTexts)).toList();
|
|
177
|
+
for (const row of rows) {
|
|
178
|
+
if (row.length !== dimensions) {
|
|
179
|
+
throw new Error(`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`);
|
|
180
|
+
}
|
|
181
|
+
vectors.push(row);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return vectors;
|
|
185
|
+
};
|
|
153
186
|
return {
|
|
154
187
|
modelId,
|
|
155
188
|
dimensions,
|
|
156
|
-
embed:
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
}
|
|
172
|
-
return vectors;
|
|
189
|
+
embed: (texts) => {
|
|
190
|
+
const call = embed(texts);
|
|
191
|
+
activeCalls.add(call);
|
|
192
|
+
void call.then(() => activeCalls.delete(call), () => activeCalls.delete(call));
|
|
193
|
+
return call;
|
|
194
|
+
},
|
|
195
|
+
dispose: () => {
|
|
196
|
+
disposed = true;
|
|
197
|
+
disposePromise ??= (async () => {
|
|
198
|
+
const pendingBackend = backendPromise;
|
|
199
|
+
await Promise.allSettled([...activeCalls]);
|
|
200
|
+
const backend = await pendingBackend?.catch(() => undefined);
|
|
201
|
+
await backend?.dispose?.();
|
|
202
|
+
})();
|
|
203
|
+
return disposePromise;
|
|
173
204
|
},
|
|
174
205
|
};
|
|
175
206
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../../src/memory/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH;;;GAGG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,6BAA6B,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AA6BtG,MAAM,6BAA6B,GAAG,eAAe,CAAC;AACtD,MAAM,+BAA+B,GAAG,IAAI,CAAC;AAC7C,MAAM,+BAA+B,GAAG,EAAE,CAAC;AAC3C,MAAM,mBAAmB,GAAG,wBAAwB,CAAC;AAErD,SAAS,OAAO,CAAC,KAAa,EAAU;IACvC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AAAA,CAClB;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,UAAkB,EAAY;IAC3E,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,SAAS;QACV,CAAC;QACD,8EAA0C;QAC1C,uEAA2B;QAC3B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC3C,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,UAAU,IAAI,KAAK,GAAG,KAAK,CAAC;IAC7B,CAAC;IACD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACtB,8FAA8C;QAC9C,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AAAA,CAC3C;AAED;;;;;GAKG;AACH,MAAM,UAAU,wCAAwC,CAAC,OAExD,EAA6B;IAC7B,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,EAAE,CAAC;IAC7C,OAAO;QACN,OAAO,EAAE,kBAAkB;QAC3B,UAAU;QACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;KACrF,CAAC;AAAA,CACF;AAED,MAAM,iCAAiC,GAAG,CAAC,eAAwB,EAAiC,EAAE,CAAC;IACtG,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAChD,eAAe,IAAI,2BAA2B,CAC9C,CAAC;QACF,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACrC,YAAY,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;QACzC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,OAAO;YACN,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3E,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAgB,EAAE,CAAC;YAAA,CACvD;SACD,CAAC;IAAA,CACF,CAAC;AAAA,CACF,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mCAAmC,CAAC,OASnD,EAA6B;IAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,6BAA6B,CAAC;IACjE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,+BAA+B,CAAC;IACzE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,+BAA+B,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,IAAI,iCAAiC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5F,IAAI,cAAc,GAAmD,IAAI,CAAC;IAC1E,MAAM,WAAW,GAAG,GAA4C,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,cAAc,CAAC;QAChC,IAAI,QAAQ,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC;QACjB,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,IAAI,SAAkB,CAAC;YACvB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACJ,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,SAAS,GAAG,KAAK,CAAC;oBAClB,uFAA2C;oBAC3C,8EAAgD;oBAChD,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;gBACD,mFAAyD;gBACzD,wFAA4C;gBAC5C,sEAA8D;gBAC9D,6CAAqB;gBACrB,IAAI,CAAC;oBACJ,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC3D,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,SAAS,GAAG,KAAK,CAAC;oBAClB,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;YACF,CAAC;YACD,IAAI,SAAS,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,OAAO,EAAE,CAAC,CAAC;YAChG,MAAM,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAAA,CAC/C,CAAC,EAAE,CAAC;QACL,cAAc,GAAG,OAAO,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACnB,yFAAyC;YACzC,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;gBAChC,cAAc,GAAG,IAAI,CAAC;YACvB,CAAC;QAAA,CACD,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IAAA,CACf,CAAC;IAEF,OAAO;QACN,OAAO;QACP,UAAU;QACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAC;YACX,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;YACpC,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBACjE,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;gBAC3D,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC1D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACxB,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC/B,MAAM,IAAI,KAAK,CACd,uCAAuC,OAAO,aAAa,GAAG,CAAC,MAAM,4BAA4B,UAAU,EAAE,CAC7G,CAAC;oBACH,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACF,CAAC;YACD,OAAO,OAAO,CAAC;QAAA,CACf;KACD,CAAC;AAAA,CACF","sourcesContent":["/**\n * Memory embedding providers for hybrid retrieval (混合检索 · 向量侧).\n *\n * POC-verified (2026-09, 100k corpus): high tier (Xenova/bge-m3, 1024d) and\n * low tier (Xenova/bge-small-zh-v1.5, 512d) inference is deterministic\n * bit-exact across runs; high-tier drift queries reach re@10 100% after\n * reranking, low tier 83%. Tier presets are caller configuration — this module\n * only exposes a modelId-parameterized interface (宪法: 不绑死单一 provider).\n *\n * Runtime dep @huggingface/transformers is lazy-loaded via `await import` at\n * first embed (惯例参考 packages/agent-forge/src/utils/photon.ts); type\n * positions use type-only imports. The deterministic mock provider keeps tests\n * offline (门禁: deterministic mock AI; 真实模型不进 CI).\n */\n\n/**\n * Batch embedding provider. Implementations must be deterministic for\n * identical input and must not silently degrade: any failure propagates.\n */\nimport { join } from \"node:path\";\nimport { importHostOrBareModule } from \"./host-module-import.ts\";\nimport { annotateUpstreamListingPoison, purgeZeroByteCacheArtifacts } from \"./model-cache-hygiene.ts\";\n\nexport interface MemoryEmbeddingProviderV1 {\n\treadonly modelId: string;\n\treadonly dimensions: number;\n\t/**\n\t * Batch embed. Vectors are L2-normalized float arrays. Deterministic for\n\t * identical input. Plain text mapping — no instruction prefix is added\n\t * here; query-side instructions for bge-small-zh family are the caller's\n\t * responsibility (低档 bge-small-zh 系查询侧指令由调用方负责添加).\n\t */\n\tembed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;\n}\n\n/**\n * Minimal structural seam around a transformers.js feature-extraction\n * pipeline, so tests can inject a fake without touching the real loader.\n */\nexport interface TransformersEmbeddingBackendV1 {\n\t/** Extract features for a batch of texts; one vector per input, input order. */\n\textract: (texts: string[]) => Promise<{ toList: () => number[][] }>;\n}\n\nexport type TransformersEmbeddingLoadImpl = (\n\tmodelId: string,\n\tcacheDir: string,\n\tremoteHost: string,\n) => Promise<TransformersEmbeddingBackendV1>;\n\nconst DEFAULT_TRANSFORMERS_MODEL_ID = \"Xenova/bge-m3\";\nconst DEFAULT_TRANSFORMERS_DIMENSIONS = 1024;\nconst DEFAULT_TRANSFORMERS_BATCH_SIZE = 16;\nconst DEFAULT_REMOTE_HOST = \"https://huggingface.co\";\n\nfunction fnv1a32(input: string): number {\n\tlet hash = 0x811c9dc5;\n\tfor (let offset = 0; offset < input.length; offset += 1) {\n\t\thash ^= input.charCodeAt(offset);\n\t\thash = Math.imul(hash, 0x01000193);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction embedDeterministicMock(text: string, dimensions: number): number[] {\n\tconst values = new Array<number>(dimensions).fill(0);\n\tconst tokens = text.toLowerCase().split(/[^\\p{L}\\p{N}]+/u);\n\tfor (const token of tokens) {\n\t\tif (token.length === 0) {\n\t\t\tcontinue;\n\t\t}\n\t\t// 多轮混淆累加: 每个 token 经 FNV-1a 后再派生 3 个维度沉点,\n\t\t// 放大不同文本间的向量差异; 全程无随机、无时钟。\n\t\tlet mixed = fnv1a32(token);\n\t\tfor (let round = 0; round < 3; round += 1) {\n\t\t\tmixed = Math.imul(mixed ^ 0x9e3779b9, 0x85ebca6b) >>> 0;\n\t\t\tvalues[mixed % dimensions] += 1;\n\t\t}\n\t}\n\tlet sumSquares = 0;\n\tfor (const value of values) {\n\t\tsumSquares += value * value;\n\t}\n\tif (sumSquares === 0) {\n\t\t// 空文本/全标点: 确定性零向量; L2 范数为 0, 不归一化 (避免除零 NaN)。\n\t\treturn values;\n\t}\n\tconst norm = Math.sqrt(sumSquares);\n\treturn values.map((value) => value / norm);\n}\n\n/**\n * Deterministic mock embedding provider for offline tests: tokenizes by\n * `[^\\p{L}\\p{N}]+`, hashes each token with FNV-1a plus multi-round mixing into\n * dimension indices, then L2-normalizes. Same text → same vector, no random,\n * no clock. POC 已用真实模型验证 bit-exact; mock 只承担确定性测试门禁。\n */\nexport function createDeterministicMockEmbeddingProvider(options?: {\n\treadonly dimensions?: number;\n}): MemoryEmbeddingProviderV1 {\n\tconst dimensions = options?.dimensions ?? 64;\n\treturn {\n\t\tmodelId: \"mock-embedding@1\",\n\t\tdimensions,\n\t\tembed: async (texts) => texts.map((text) => embedDeterministicMock(text, dimensions)),\n\t};\n}\n\nconst createDefaultTransformersLoadImpl = (moduleEntryPath?: string): TransformersEmbeddingLoadImpl => {\n\treturn async (modelId, cacheDir, remoteHost) => {\n\t\tconst transformers = await importHostOrBareModule<typeof import(\"@huggingface/transformers\")>(\n\t\t\tmoduleEntryPath ?? \"@huggingface/transformers\",\n\t\t);\n\t\ttransformers.env.cacheDir = cacheDir;\n\t\ttransformers.env.remoteHost = remoteHost;\n\t\tconst extractor = await transformers.pipeline(\"feature-extraction\", modelId, { dtype: \"q8\" });\n\t\treturn {\n\t\t\textract: async (texts) => {\n\t\t\t\tconst output = await extractor(texts, { pooling: \"cls\", normalize: true });\n\t\t\t\treturn { toList: () => output.tolist() as number[][] };\n\t\t\t},\n\t\t};\n\t};\n};\n\n/**\n * Transformers.js-backed embedding provider (POC 档位: 高档默认\n * Xenova/bge-m3/1024d; 低档由调用方传 modelId \"Xenova/bge-small-zh-v1.5\" +\n * dimensions 512)。\n *\n * 镜像序列: 首次 embed 时按 `remoteHosts` 顺序逐个尝试加载 (调用方通常传\n * [\"https://huggingface.co\", \"https://hf-mirror.com\"]); 某 host 加载抛错则\n * 尝试下一个, 全部失败时抛最后一个错误。host 确定后模型句柄只加载一次\n * (并发 embed 共享同一 in-flight promise); 加载失败不缓存, 下次 embed 重新\n * 走完整序列, 失败原因原样保留, 不静默降级。\n *\n * `loadImpl` 是测试 seam: 注入后绕过真实加载, 签名\n * (modelId, cacheDir, remoteHost) => backend。\n */\nexport function createTransformersEmbeddingProvider(options: {\n\treadonly modelId?: string;\n\treadonly dimensions?: number;\n\treadonly cacheDir: string;\n\treadonly remoteHosts?: readonly string[];\n\treadonly batchSize?: number;\n\treadonly loadImpl?: TransformersEmbeddingLoadImpl;\n\t/** Host-resolved transformers entry path (install-store visibility); undefined = bare import. */\n\treadonly moduleEntryPath?: string;\n}): MemoryEmbeddingProviderV1 {\n\tconst modelId = options.modelId ?? DEFAULT_TRANSFORMERS_MODEL_ID;\n\tconst dimensions = options.dimensions ?? DEFAULT_TRANSFORMERS_DIMENSIONS;\n\tconst batchSize = options.batchSize ?? DEFAULT_TRANSFORMERS_BATCH_SIZE;\n\tconst cacheDir = options.cacheDir;\n\tconst remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];\n\tconst load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);\n\n\tlet backendPromise: Promise<TransformersEmbeddingBackendV1> | null = null;\n\tconst loadBackend = (): Promise<TransformersEmbeddingBackendV1> => {\n\t\tconst existing = backendPromise;\n\t\tif (existing) {\n\t\t\treturn existing;\n\t\t}\n\t\tconst attempt = (async () => {\n\t\t\tlet lastError: unknown;\n\t\t\tfor (const remoteHost of remoteHosts) {\n\t\t\t\ttry {\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\t// 镜像轮换前清 0 字节残骸: 失败下载留下的空文件会被 transformers\n\t\t\t\t\t// 当作缓存命中, 毒化下一个 host 的重试 (model-cache-hygiene)。\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t\t// 同 host 一次短退避重试 (2026-10-06 live E2E): 覆盖 rejection 型瞬断\n\t\t\t\t// (fetch 网络错/5xx)。对\"仓库 API 清单缺文件\"型不生效——该结果被\n\t\t\t\t// transformers 模块级 memoize (见 annotateUpstreamListingPoison),\n\t\t\t\t// 进程内换 host 也拿同一空清单。\n\t\t\t\ttry {\n\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, 2_000));\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (lastError === undefined) throw new Error(`no remote hosts configured for model ${modelId}`);\n\t\t\tthrow annotateUpstreamListingPoison(lastError);\n\t\t})();\n\t\tbackendPromise = attempt;\n\t\tattempt.catch(() => {\n\t\t\t// 失败不缓存: 下次 embed 重新走镜像序列 (显式失败, 不静默降级)。\n\t\t\tif (backendPromise === attempt) {\n\t\t\t\tbackendPromise = null;\n\t\t\t}\n\t\t});\n\t\treturn attempt;\n\t};\n\n\treturn {\n\t\tmodelId,\n\t\tdimensions,\n\t\tembed: async (texts) => {\n\t\t\tif (texts.length === 0) {\n\t\t\t\treturn [];\n\t\t\t}\n\t\t\tconst backend = await loadBackend();\n\t\t\tconst vectors: number[][] = [];\n\t\t\tfor (let offset = 0; offset < texts.length; offset += batchSize) {\n\t\t\t\tconst batchTexts = texts.slice(offset, offset + batchSize);\n\t\t\t\tconst rows = (await backend.extract(batchTexts)).toList();\n\t\t\t\tfor (const row of rows) {\n\t\t\t\t\tif (row.length !== dimensions) {\n\t\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t\t`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tvectors.push(row);\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn vectors;\n\t\t},\n\t};\n}\n"]}
|
|
1
|
+
{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../../src/memory/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH;;;GAGG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,6BAA6B,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AAgCtG,MAAM,6BAA6B,GAAG,eAAe,CAAC;AACtD,MAAM,+BAA+B,GAAG,IAAI,CAAC;AAC7C,MAAM,+BAA+B,GAAG,EAAE,CAAC;AAC3C,MAAM,mBAAmB,GAAG,wBAAwB,CAAC;AAErD,SAAS,OAAO,CAAC,KAAa,EAAU;IACvC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AAAA,CAClB;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,UAAkB,EAAY;IAC3E,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,SAAS;QACV,CAAC;QACD,8EAA0C;QAC1C,uEAA2B;QAC3B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC3C,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,UAAU,IAAI,KAAK,GAAG,KAAK,CAAC;IAC7B,CAAC;IACD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACtB,8FAA8C;QAC9C,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AAAA,CAC3C;AAED;;;;;GAKG;AACH,MAAM,UAAU,wCAAwC,CAAC,OAExD,EAA6B;IAC7B,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,EAAE,CAAC;IAC7C,OAAO;QACN,OAAO,EAAE,kBAAkB;QAC3B,UAAU;QACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;KACrF,CAAC;AAAA,CACF;AAED,MAAM,iCAAiC,GAAG,CAAC,eAAwB,EAAiC,EAAE,CAAC;IACtG,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAChD,eAAe,IAAI,2BAA2B,CAC9C,CAAC;QACF,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACrC,YAAY,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;QACzC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,OAAO;YACN,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE;YAClC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3E,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAgB,EAAE,CAAC;YAAA,CACvD;SACD,CAAC;IAAA,CACF,CAAC;AAAA,CACF,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mCAAmC,CAAC,OASnD,EAA6B;IAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,6BAA6B,CAAC;IACjE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,+BAA+B,CAAC;IACzE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,+BAA+B,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,IAAI,iCAAiC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5F,IAAI,cAAc,GAAmD,IAAI,CAAC;IAC1E,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,cAAyC,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAoB,CAAC;IAChD,MAAM,WAAW,GAAG,GAA4C,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,cAAc,CAAC;QAChC,IAAI,QAAQ,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC;QACjB,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,IAAI,SAAkB,CAAC;YACvB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACJ,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,SAAS,GAAG,KAAK,CAAC;oBAClB,IAAI,QAAQ;wBAAE,MAAM,KAAK,CAAC;oBAC1B,uFAA2C;oBAC3C,8EAAgD;oBAChD,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;gBACD,mFAAyD;gBACzD,wFAA4C;gBAC5C,sEAA8D;gBAC9D,6CAAqB;gBACrB,IAAI,CAAC;oBACJ,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC3D,IAAI,QAAQ;wBAAE,MAAM,SAAS,CAAC;oBAC9B,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,SAAS,GAAG,KAAK,CAAC;oBAClB,IAAI,QAAQ;wBAAE,MAAM,KAAK,CAAC;oBAC1B,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;YACF,CAAC;YACD,IAAI,SAAS,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,OAAO,EAAE,CAAC,CAAC;YAChG,MAAM,6BAA6B,CAAC,SAAS,CAAC,CAAC;QAAA,CAC/C,CAAC,EAAE,CAAC;QACL,cAAc,GAAG,OAAO,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACnB,yFAAyC;YACzC,IAAI,cAAc,KAAK,OAAO,EAAE,CAAC;gBAChC,cAAc,GAAG,IAAI,CAAC;YACvB,CAAC;QAAA,CACD,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IAAA,CACf,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,EAAE,KAAwB,EAA2C,EAAE,CAAC;QAC1F,IAAI,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACvE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,IAAI,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACvE,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YACjE,IAAI,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACvE,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;YAC3D,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC1D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CACd,uCAAuC,OAAO,aAAa,GAAG,CAAC,MAAM,4BAA4B,UAAU,EAAE,CAC7G,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACF,CAAC;QACD,OAAO,OAAO,CAAC;IAAA,CACf,CAAC;IACF,OAAO;QACN,OAAO;QACP,UAAU;QACV,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACtB,KAAK,IAAI,CAAC,IAAI,CACb,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAC9B,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAC9B,CAAC;YACF,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC;YAChB,cAAc,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC/B,MAAM,cAAc,GAAG,cAAc,CAAC;gBACtC,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;gBAC3C,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC7D,MAAM,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAAA,CAC3B,CAAC,EAAE,CAAC;YACL,OAAO,cAAc,CAAC;QAAA,CACtB;KACD,CAAC;AAAA,CACF","sourcesContent":["/**\n * Memory embedding providers for hybrid retrieval (混合检索 · 向量侧).\n *\n * POC-verified (2026-09, 100k corpus): high tier (Xenova/bge-m3, 1024d) and\n * low tier (Xenova/bge-small-zh-v1.5, 512d) inference is deterministic\n * bit-exact across runs; high-tier drift queries reach re@10 100% after\n * reranking, low tier 83%. Tier presets are caller configuration — this module\n * only exposes a modelId-parameterized interface (宪法: 不绑死单一 provider).\n *\n * Runtime dep @huggingface/transformers is lazy-loaded via `await import` at\n * first embed (惯例参考 packages/agent-forge/src/utils/photon.ts); type\n * positions use type-only imports. The deterministic mock provider keeps tests\n * offline (门禁: deterministic mock AI; 真实模型不进 CI).\n */\n\n/**\n * Batch embedding provider. Implementations must be deterministic for\n * identical input and must not silently degrade: any failure propagates.\n */\nimport { join } from \"node:path\";\nimport { importHostOrBareModule } from \"./host-module-import.ts\";\nimport { annotateUpstreamListingPoison, purgeZeroByteCacheArtifacts } from \"./model-cache-hygiene.ts\";\n\nexport interface MemoryEmbeddingProviderV1 {\n\treadonly modelId: string;\n\treadonly dimensions: number;\n\t/**\n\t * Batch embed. Vectors are L2-normalized float arrays. Deterministic for\n\t * identical input. Plain text mapping — no instruction prefix is added\n\t * here; query-side instructions for bge-small-zh family are the caller's\n\t * responsibility (低档 bge-small-zh 系查询侧指令由调用方负责添加).\n\t */\n\tembed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;\n\t/** 仅释放本实例拥有的模型;等待已进入的推理完成,不删除缓存文件。 */\n\tdispose?(): Promise<void>;\n}\n\n/**\n * Minimal structural seam around a transformers.js feature-extraction\n * pipeline, so tests can inject a fake without touching the real loader.\n */\nexport interface TransformersEmbeddingBackendV1 {\n\t/** Extract features for a batch of texts; one vector per input, input order. */\n\textract: (texts: string[]) => Promise<{ toList: () => number[][] }>;\n\tdispose?(): Promise<void>;\n}\n\nexport type TransformersEmbeddingLoadImpl = (\n\tmodelId: string,\n\tcacheDir: string,\n\tremoteHost: string,\n) => Promise<TransformersEmbeddingBackendV1>;\n\nconst DEFAULT_TRANSFORMERS_MODEL_ID = \"Xenova/bge-m3\";\nconst DEFAULT_TRANSFORMERS_DIMENSIONS = 1024;\nconst DEFAULT_TRANSFORMERS_BATCH_SIZE = 16;\nconst DEFAULT_REMOTE_HOST = \"https://huggingface.co\";\n\nfunction fnv1a32(input: string): number {\n\tlet hash = 0x811c9dc5;\n\tfor (let offset = 0; offset < input.length; offset += 1) {\n\t\thash ^= input.charCodeAt(offset);\n\t\thash = Math.imul(hash, 0x01000193);\n\t}\n\treturn hash >>> 0;\n}\n\nfunction embedDeterministicMock(text: string, dimensions: number): number[] {\n\tconst values = new Array<number>(dimensions).fill(0);\n\tconst tokens = text.toLowerCase().split(/[^\\p{L}\\p{N}]+/u);\n\tfor (const token of tokens) {\n\t\tif (token.length === 0) {\n\t\t\tcontinue;\n\t\t}\n\t\t// 多轮混淆累加: 每个 token 经 FNV-1a 后再派生 3 个维度沉点,\n\t\t// 放大不同文本间的向量差异; 全程无随机、无时钟。\n\t\tlet mixed = fnv1a32(token);\n\t\tfor (let round = 0; round < 3; round += 1) {\n\t\t\tmixed = Math.imul(mixed ^ 0x9e3779b9, 0x85ebca6b) >>> 0;\n\t\t\tvalues[mixed % dimensions] += 1;\n\t\t}\n\t}\n\tlet sumSquares = 0;\n\tfor (const value of values) {\n\t\tsumSquares += value * value;\n\t}\n\tif (sumSquares === 0) {\n\t\t// 空文本/全标点: 确定性零向量; L2 范数为 0, 不归一化 (避免除零 NaN)。\n\t\treturn values;\n\t}\n\tconst norm = Math.sqrt(sumSquares);\n\treturn values.map((value) => value / norm);\n}\n\n/**\n * Deterministic mock embedding provider for offline tests: tokenizes by\n * `[^\\p{L}\\p{N}]+`, hashes each token with FNV-1a plus multi-round mixing into\n * dimension indices, then L2-normalizes. Same text → same vector, no random,\n * no clock. POC 已用真实模型验证 bit-exact; mock 只承担确定性测试门禁。\n */\nexport function createDeterministicMockEmbeddingProvider(options?: {\n\treadonly dimensions?: number;\n}): MemoryEmbeddingProviderV1 {\n\tconst dimensions = options?.dimensions ?? 64;\n\treturn {\n\t\tmodelId: \"mock-embedding@1\",\n\t\tdimensions,\n\t\tembed: async (texts) => texts.map((text) => embedDeterministicMock(text, dimensions)),\n\t};\n}\n\nconst createDefaultTransformersLoadImpl = (moduleEntryPath?: string): TransformersEmbeddingLoadImpl => {\n\treturn async (modelId, cacheDir, remoteHost) => {\n\t\tconst transformers = await importHostOrBareModule<typeof import(\"@huggingface/transformers\")>(\n\t\t\tmoduleEntryPath ?? \"@huggingface/transformers\",\n\t\t);\n\t\ttransformers.env.cacheDir = cacheDir;\n\t\ttransformers.env.remoteHost = remoteHost;\n\t\tconst extractor = await transformers.pipeline(\"feature-extraction\", modelId, { dtype: \"q8\" });\n\t\treturn {\n\t\t\tdispose: () => extractor.dispose(),\n\t\t\textract: async (texts) => {\n\t\t\t\tconst output = await extractor(texts, { pooling: \"cls\", normalize: true });\n\t\t\t\treturn { toList: () => output.tolist() as number[][] };\n\t\t\t},\n\t\t};\n\t};\n};\n\n/**\n * Transformers.js-backed embedding provider (POC 档位: 高档默认\n * Xenova/bge-m3/1024d; 低档由调用方传 modelId \"Xenova/bge-small-zh-v1.5\" +\n * dimensions 512)。\n *\n * 镜像序列: 首次 embed 时按 `remoteHosts` 顺序逐个尝试加载 (调用方通常传\n * [\"https://huggingface.co\", \"https://hf-mirror.com\"]); 某 host 加载抛错则\n * 尝试下一个, 全部失败时抛最后一个错误。host 确定后模型句柄只加载一次\n * (并发 embed 共享同一 in-flight promise); 加载失败不缓存, 下次 embed 重新\n * 走完整序列, 失败原因原样保留, 不静默降级。\n *\n * `loadImpl` 是测试 seam: 注入后绕过真实加载, 签名\n * (modelId, cacheDir, remoteHost) => backend。\n */\nexport function createTransformersEmbeddingProvider(options: {\n\treadonly modelId?: string;\n\treadonly dimensions?: number;\n\treadonly cacheDir: string;\n\treadonly remoteHosts?: readonly string[];\n\treadonly batchSize?: number;\n\treadonly loadImpl?: TransformersEmbeddingLoadImpl;\n\t/** Host-resolved transformers entry path (install-store visibility); undefined = bare import. */\n\treadonly moduleEntryPath?: string;\n}): MemoryEmbeddingProviderV1 {\n\tconst modelId = options.modelId ?? DEFAULT_TRANSFORMERS_MODEL_ID;\n\tconst dimensions = options.dimensions ?? DEFAULT_TRANSFORMERS_DIMENSIONS;\n\tconst batchSize = options.batchSize ?? DEFAULT_TRANSFORMERS_BATCH_SIZE;\n\tconst cacheDir = options.cacheDir;\n\tconst remoteHosts = options.remoteHosts ?? [DEFAULT_REMOTE_HOST];\n\tconst load = options.loadImpl ?? createDefaultTransformersLoadImpl(options.moduleEntryPath);\n\n\tlet backendPromise: Promise<TransformersEmbeddingBackendV1> | null = null;\n\tlet disposed = false;\n\tlet disposePromise: Promise<void> | undefined;\n\tconst activeCalls = new Set<Promise<unknown>>();\n\tconst loadBackend = (): Promise<TransformersEmbeddingBackendV1> => {\n\t\tconst existing = backendPromise;\n\t\tif (existing) {\n\t\t\treturn existing;\n\t\t}\n\t\tconst attempt = (async () => {\n\t\t\tlet lastError: unknown;\n\t\t\tfor (const remoteHost of remoteHosts) {\n\t\t\t\ttry {\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tif (disposed) throw error;\n\t\t\t\t\t// 镜像轮换前清 0 字节残骸: 失败下载留下的空文件会被 transformers\n\t\t\t\t\t// 当作缓存命中, 毒化下一个 host 的重试 (model-cache-hygiene)。\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t\t// 同 host 一次短退避重试 (2026-10-06 live E2E): 覆盖 rejection 型瞬断\n\t\t\t\t// (fetch 网络错/5xx)。对\"仓库 API 清单缺文件\"型不生效——该结果被\n\t\t\t\t// transformers 模块级 memoize (见 annotateUpstreamListingPoison),\n\t\t\t\t// 进程内换 host 也拿同一空清单。\n\t\t\t\ttry {\n\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, 2_000));\n\t\t\t\t\tif (disposed) throw lastError;\n\t\t\t\t\treturn await load(modelId, cacheDir, remoteHost);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tif (disposed) throw error;\n\t\t\t\t\tpurgeZeroByteCacheArtifacts(join(cacheDir, modelId));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (lastError === undefined) throw new Error(`no remote hosts configured for model ${modelId}`);\n\t\t\tthrow annotateUpstreamListingPoison(lastError);\n\t\t})();\n\t\tbackendPromise = attempt;\n\t\tattempt.catch(() => {\n\t\t\t// 失败不缓存: 下次 embed 重新走镜像序列 (显式失败, 不静默降级)。\n\t\t\tif (backendPromise === attempt) {\n\t\t\t\tbackendPromise = null;\n\t\t\t}\n\t\t});\n\t\treturn attempt;\n\t};\n\n\tconst embed = async (texts: readonly string[]): Promise<readonly (readonly number[])[]> => {\n\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\tif (texts.length === 0) return [];\n\t\tconst backend = await loadBackend();\n\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\tconst vectors: number[][] = [];\n\t\tfor (let offset = 0; offset < texts.length; offset += batchSize) {\n\t\t\tif (disposed) throw new Error(\"memory embedding provider is disposed\");\n\t\t\tconst batchTexts = texts.slice(offset, offset + batchSize);\n\t\t\tconst rows = (await backend.extract(batchTexts)).toList();\n\t\t\tfor (const row of rows) {\n\t\t\t\tif (row.length !== dimensions) {\n\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t`embedding dimension mismatch: model ${modelId} returned ${row.length} dims, provider declares ${dimensions}`,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tvectors.push(row);\n\t\t\t}\n\t\t}\n\t\treturn vectors;\n\t};\n\treturn {\n\t\tmodelId,\n\t\tdimensions,\n\t\tembed: (texts) => {\n\t\t\tconst call = embed(texts);\n\t\t\tactiveCalls.add(call);\n\t\t\tvoid call.then(\n\t\t\t\t() => activeCalls.delete(call),\n\t\t\t\t() => activeCalls.delete(call),\n\t\t\t);\n\t\t\treturn call;\n\t\t},\n\t\tdispose: () => {\n\t\t\tdisposed = true;\n\t\t\tdisposePromise ??= (async () => {\n\t\t\t\tconst pendingBackend = backendPromise;\n\t\t\t\tawait Promise.allSettled([...activeCalls]);\n\t\t\t\tconst backend = await pendingBackend?.catch(() => undefined);\n\t\t\t\tawait backend?.dispose?.();\n\t\t\t})();\n\t\t\treturn disposePromise;\n\t\t},\n\t};\n}\n"]}
|
|
@@ -22,6 +22,8 @@ export interface MemoryRerankerV1 {
|
|
|
22
22
|
readonly modelId: string;
|
|
23
23
|
/** Score each doc against the query; higher = more relevant (raw logits, order-preserving). */
|
|
24
24
|
rerank(query: string, docs: readonly string[]): Promise<readonly number[]>;
|
|
25
|
+
/** 释放本实例模型;在途原生推理完成后关闭,不删除缓存。 */
|
|
26
|
+
dispose?(): Promise<void>;
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
29
|
* Minimal structural seam around a transformers.js sequence-classification
|
|
@@ -30,6 +32,7 @@ export interface MemoryRerankerV1 {
|
|
|
30
32
|
export interface TransformersRerankerBackendV1 {
|
|
31
33
|
/** Score docs against the query; result[i] corresponds to docs[i]. */
|
|
32
34
|
score: (query: string, docs: string[]) => Promise<number[]>;
|
|
35
|
+
dispose?(): Promise<void>;
|
|
33
36
|
}
|
|
34
37
|
export type TransformersRerankerLoadImpl = (modelId: string, cacheDir: string, remoteHost: string) => Promise<TransformersRerankerBackendV1>;
|
|
35
38
|
/**
|