@kenz1117/dsh-engram 0.9.6 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +15 -3
- package/README.md +15 -3
- package/lib/client.js +2 -2
- package/lib/client.js.map +1 -1
- package/lib/index.js +371 -92
- package/package.json +4 -3
- package/scripts/backfill-embeddings.mjs +298 -40
package/lib/index.js
CHANGED
|
@@ -524,6 +524,13 @@ const zh$1 = {
|
|
|
524
524
|
"store.migrationAlreadyMigrated": "[dsh-engram] 旧库已有迁移记录({names}):{pointer};先前迁移的库可能含有当前目录的记忆。{manual}",
|
|
525
525
|
"store.mergeTargetMissing": "write-disposition: 并入目标 {id} 不存在",
|
|
526
526
|
"store.embedderDownloadFailed": "嵌入模型下载/加载失败;检索已降级为纯关键词模式,可检查网络后重试",
|
|
527
|
+
"store.embedderDimensionMismatch": "嵌入向量维度不一致:库里是 {expected} 维,嵌入器给出 {found} 维。比较与写入直接拒绝,而不是逐维截断——请把模型改回去,或用新模型重建库中向量",
|
|
528
|
+
"store.embedderRemoteUnreachable": "嵌入端点不可达({baseUrl},超时 {timeoutMs} 毫秒)",
|
|
529
|
+
"store.embedderRemoteHttp": "嵌入端点返回 HTTP {status}{detail}",
|
|
530
|
+
"store.embedderRemoteBody": "嵌入端点的响应不是可解析的 JSON",
|
|
531
|
+
"store.embedderRemoteCount": "嵌入端点返回 {received} 个向量,而本次请求为 {expected} 个",
|
|
532
|
+
"store.embedderRemoteCountNa": "非数组",
|
|
533
|
+
"store.embedderRemoteEmpty": "嵌入端点返回了空向量",
|
|
527
534
|
"store.budgetDropped": "(另有 {count} 条未展示:缩小查询范围或降低 limit 后重试)"
|
|
528
535
|
};
|
|
529
536
|
/** Английский набор — образец полноты. */
|
|
@@ -557,6 +564,13 @@ const en$1 = {
|
|
|
557
564
|
"store.migrationAlreadyMigrated": "[dsh-engram] the legacy database already has a migration record ({names}): {pointer}; the database migrated earlier may hold memories of the current directory. {manual}",
|
|
558
565
|
"store.mergeTargetMissing": "write-disposition: merge target {id} does not exist",
|
|
559
566
|
"store.embedderDownloadFailed": "embedding model download/load failed; retrieval has degraded to keyword-only mode — check the network and retry",
|
|
567
|
+
"store.embedderDimensionMismatch": "embedding dimension mismatch: the store holds {expected}-dimensional vectors, the embedder returned {found}. Comparison and writes are refused instead of being silently truncated — switch the model back, or re-embed the store with the new one",
|
|
568
|
+
"store.embedderRemoteUnreachable": "embedding endpoint unreachable ({baseUrl}, timeout {timeoutMs} ms)",
|
|
569
|
+
"store.embedderRemoteHttp": "embedding endpoint returned HTTP {status}{detail}",
|
|
570
|
+
"store.embedderRemoteBody": "embedding endpoint returned a body that is not parseable JSON",
|
|
571
|
+
"store.embedderRemoteCount": "embedding endpoint returned {received} vectors instead of the requested {expected}",
|
|
572
|
+
"store.embedderRemoteCountNa": "not an array",
|
|
573
|
+
"store.embedderRemoteEmpty": "embedding endpoint returned an empty vector",
|
|
560
574
|
"store.budgetDropped": "({count} more not shown: narrow the query or lower the limit and retry)"
|
|
561
575
|
};
|
|
562
576
|
/** Русский набор. */
|
|
@@ -590,6 +604,13 @@ const ru$1 = {
|
|
|
590
604
|
"store.migrationAlreadyMigrated": "[dsh-engram] У старой базы уже есть запись о переносе ({names}): {pointer}; в базе, перенесённой ранее, могут быть воспоминания текущего каталога. {manual}",
|
|
591
605
|
"store.mergeTargetMissing": "write-disposition: цель слияния {id} не существует",
|
|
592
606
|
"store.embedderDownloadFailed": "Не удалось скачать/загрузить модель эмбеддингов; поиск переведён в режим только по ключевым словам — проверьте сеть и повторите",
|
|
607
|
+
"store.embedderDimensionMismatch": "размерность векторов не совпадает: в базе {expected} измерений, эмбеддер вернул {found}. Сравнение и запись отвергаются, а не усекаются молча — верните прежнюю модель или пересчитайте векторы новой",
|
|
608
|
+
"store.embedderRemoteUnreachable": "эндпоинт эмбеддингов недоступен ({baseUrl}, таймаут {timeoutMs} мс)",
|
|
609
|
+
"store.embedderRemoteHttp": "эндпоинт эмбеддингов вернул HTTP {status}{detail}",
|
|
610
|
+
"store.embedderRemoteBody": "эндпоинт эмбеддингов вернул тело, которое не удалось разобрать как JSON",
|
|
611
|
+
"store.embedderRemoteCount": "эндпоинт эмбеддингов вернул {received} векторов вместо запрошенных {expected}",
|
|
612
|
+
"store.embedderRemoteCountNa": "не массив",
|
|
613
|
+
"store.embedderRemoteEmpty": "эндпоинт эмбеддингов вернул пустой вектор",
|
|
593
614
|
"store.budgetDropped": "(ещё {count} не показано: сузьте запрос или уменьшите limit и повторите)"
|
|
594
615
|
};
|
|
595
616
|
//#endregion
|
|
@@ -1677,6 +1698,99 @@ function message(key, params) {
|
|
|
1677
1698
|
return messageIn(current, key, params);
|
|
1678
1699
|
}
|
|
1679
1700
|
//#endregion
|
|
1701
|
+
//#region src/types.ts
|
|
1702
|
+
/** 从任意字符串铸造品牌化 id(存储层入库前调用)。 */
|
|
1703
|
+
function asMemoryId(raw) {
|
|
1704
|
+
return raw;
|
|
1705
|
+
}
|
|
1706
|
+
/** 从任意字符串铸造品牌化实体 id(存储层入库前调用)。 */
|
|
1707
|
+
function asEntityId(raw) {
|
|
1708
|
+
return raw;
|
|
1709
|
+
}
|
|
1710
|
+
/** 从任意字符串铸造品牌化事实 id(存储层入库前调用)。 */
|
|
1711
|
+
function asFactId(raw) {
|
|
1712
|
+
return raw;
|
|
1713
|
+
}
|
|
1714
|
+
/** 从任意字符串铸造品牌化信念 id(存储层入库前调用)。 */
|
|
1715
|
+
function asObservationId(raw) {
|
|
1716
|
+
return raw;
|
|
1717
|
+
}
|
|
1718
|
+
/** 归一化比较键:trim + 合并连续空白 + 小写(消解第一版只做精确归一匹配,不做相似合并)。 */
|
|
1719
|
+
function normalizeEntityName(raw) {
|
|
1720
|
+
return raw.trim().replace(/\s+/g, " ").toLowerCase();
|
|
1721
|
+
}
|
|
1722
|
+
/** episode 情景时间线的时间邻近扩展缺省窗口:锚点 createdAt ± 60 分钟。 */
|
|
1723
|
+
const EPISODE_PROXIMITY_MS_DEFAULT = 36e5;
|
|
1724
|
+
/** dsh-engram 统一错误:加载/使用期的可诊断失败都抛此类型。 */
|
|
1725
|
+
var EngramError = class extends Error {
|
|
1726
|
+
/** 机器可读原因码,如 EMBEDDER_DOWNLOAD_FAILED / SCHEMA_INCOMPATIBLE。 */
|
|
1727
|
+
code;
|
|
1728
|
+
constructor(code, message, options) {
|
|
1729
|
+
super(message, options);
|
|
1730
|
+
this.name = "EngramError";
|
|
1731
|
+
this.code = code;
|
|
1732
|
+
}
|
|
1733
|
+
};
|
|
1734
|
+
//#endregion
|
|
1735
|
+
//#region src/embedder/local.ts
|
|
1736
|
+
/**
|
|
1737
|
+
* transformers.js 本地嵌入实现:量化权重(q8),离线推理。模型首次使用需联网
|
|
1738
|
+
* 下载到 cacheDir;之后完全离线,数据不出机。
|
|
1739
|
+
*
|
|
1740
|
+
* 默认模型仍是 Xenova/bge-small-zh-v1.5(中文 bge 小模型,512 维),但模型选择是
|
|
1741
|
+
* 部署属性而不是代码属性——处理俄语文本要用多语言模型,因此模型标识与池化方式
|
|
1742
|
+
* 都来自配置。
|
|
1743
|
+
* @module @kenz1117/dsh-engram/embedder/local
|
|
1744
|
+
*/
|
|
1745
|
+
/** 模型标识(bge 中文小模型,512 维输出);保留为默认值以兼容旧配置。 */
|
|
1746
|
+
const ENGRAM_MODEL_ID = "Xenova/bge-small-zh-v1.5";
|
|
1747
|
+
/** 进程内单例:重复调用 createLocalEmbedder 共享同一模型实例,避免重复加载权重。 */
|
|
1748
|
+
let cached;
|
|
1749
|
+
/**
|
|
1750
|
+
* 创建本地嵌入器。
|
|
1751
|
+
* @param cacheDir - 模型缓存目录;不存在会自动创建(0o700)。
|
|
1752
|
+
* @param remoteHost - 模型下载端点;默认 huggingface.co,网络受限环境可配镜像(如 https://hf-mirror.com)。
|
|
1753
|
+
* @param modelId - 模型标识;默认 ENGRAM_MODEL_ID。更换模型会让进程内单例失效并重新加载。
|
|
1754
|
+
* @param pooling - 池化方式;默认 mean。
|
|
1755
|
+
* @returns 就绪的 EngramEmbedder。
|
|
1756
|
+
* @throws EngramError(code=EMBEDDER_DOWNLOAD_FAILED) 模型下载/加载失败(调用方据此降级)。
|
|
1757
|
+
*/
|
|
1758
|
+
async function createLocalEmbedder(cacheDir, remoteHost, modelId = ENGRAM_MODEL_ID, pooling = "mean") {
|
|
1759
|
+
if (cached?.dir === cacheDir && cached.model === modelId && cached.pooling === pooling) return cached.embedder;
|
|
1760
|
+
await mkdir(cacheDir, {
|
|
1761
|
+
recursive: true,
|
|
1762
|
+
mode: 448
|
|
1763
|
+
});
|
|
1764
|
+
try {
|
|
1765
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
1766
|
+
env.cacheDir = cacheDir;
|
|
1767
|
+
if (remoteHost !== void 0 && remoteHost !== "") env.remoteHost = remoteHost;
|
|
1768
|
+
const extractor = await pipeline("feature-extraction", modelId, { dtype: "q8" });
|
|
1769
|
+
const embedder = {
|
|
1770
|
+
model: modelId,
|
|
1771
|
+
async embed(texts) {
|
|
1772
|
+
if (texts.length === 0) return [];
|
|
1773
|
+
return (await extractor([...texts], {
|
|
1774
|
+
pooling,
|
|
1775
|
+
normalize: true
|
|
1776
|
+
})).tolist().map((list) => new Float32Array(list));
|
|
1777
|
+
},
|
|
1778
|
+
close: async () => {
|
|
1779
|
+
if (cached?.embedder === embedder) cached = void 0;
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
cached = {
|
|
1783
|
+
dir: cacheDir,
|
|
1784
|
+
model: modelId,
|
|
1785
|
+
pooling,
|
|
1786
|
+
embedder
|
|
1787
|
+
};
|
|
1788
|
+
return embedder;
|
|
1789
|
+
} catch (error) {
|
|
1790
|
+
throw new EngramError("EMBEDDER_DOWNLOAD_FAILED", message("store.embedderDownloadFailed"), { cause: error });
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
//#endregion
|
|
1680
1794
|
//#region src/jev/stale.ts
|
|
1681
1795
|
/**
|
|
1682
1796
|
* Поля Jev прежнего формата: ключ и адрес когда-то хранились у самого плагина
|
|
@@ -1760,6 +1874,7 @@ const CONFIG_KEYS = /* @__PURE__ */ new Set([
|
|
|
1760
1874
|
"historyBackfillIncludeSeeded",
|
|
1761
1875
|
"historyBackfillIncludeNoCwd",
|
|
1762
1876
|
"jev",
|
|
1877
|
+
"embedder",
|
|
1763
1878
|
"language",
|
|
1764
1879
|
"redactWords"
|
|
1765
1880
|
]);
|
|
@@ -1773,6 +1888,18 @@ const REFLECT_MODES = /* @__PURE__ */ new Set([
|
|
|
1773
1888
|
"suggest",
|
|
1774
1889
|
"auto"
|
|
1775
1890
|
]);
|
|
1891
|
+
/** `embedder` 块的键:未知键按错误处理,而不是静默忽略。 */
|
|
1892
|
+
const EMBEDDER_KEYS = /* @__PURE__ */ new Set([
|
|
1893
|
+
"kind",
|
|
1894
|
+
"localModel",
|
|
1895
|
+
"localPooling",
|
|
1896
|
+
"model",
|
|
1897
|
+
"baseUrl",
|
|
1898
|
+
"apiKey",
|
|
1899
|
+
"apiKeyEnv",
|
|
1900
|
+
"batchSize",
|
|
1901
|
+
"timeoutMs"
|
|
1902
|
+
]);
|
|
1776
1903
|
/** Schemastery 校验面(cordis.yml 读取时校验)。 */
|
|
1777
1904
|
const Config = z.object({
|
|
1778
1905
|
dbDir: z.string(),
|
|
@@ -1813,6 +1940,17 @@ const Config = z.object({
|
|
|
1813
1940
|
deferAcceptBelow: z.number().min(0).max(.5),
|
|
1814
1941
|
contradictMinProbability: z.number().min(.5).max(1)
|
|
1815
1942
|
}),
|
|
1943
|
+
embedder: z.object({
|
|
1944
|
+
kind: z.string(),
|
|
1945
|
+
localModel: z.string(),
|
|
1946
|
+
localPooling: z.string(),
|
|
1947
|
+
model: z.string(),
|
|
1948
|
+
baseUrl: z.string(),
|
|
1949
|
+
apiKey: z.string(),
|
|
1950
|
+
apiKeyEnv: z.string(),
|
|
1951
|
+
batchSize: z.number().step(1).min(1).max(256),
|
|
1952
|
+
timeoutMs: z.number().step(1).min(1e3).max(6e5)
|
|
1953
|
+
}),
|
|
1816
1954
|
language: z.string(),
|
|
1817
1955
|
redactWords: z.array(z.string())
|
|
1818
1956
|
});
|
|
@@ -1867,6 +2005,26 @@ function resolveConfig(config = {}) {
|
|
|
1867
2005
|
if (jev.contradictMinProbability !== void 0 && (jev.contradictMinProbability < .5 || jev.contradictMinProbability > 1)) throw new Error("dsh-engram: jev.contradictMinProbability must be in [0.5, 1]");
|
|
1868
2006
|
if ((jev.deferAcceptBelow ?? .15) >= (jev.deferMergeAbove ?? .85)) throw new Error("dsh-engram: jev.deferAcceptBelow must be less than jev.deferMergeAbove");
|
|
1869
2007
|
}
|
|
2008
|
+
const embedderConfig = config.embedder;
|
|
2009
|
+
const embedderKind = embedderConfig?.kind ?? "local";
|
|
2010
|
+
if (embedderConfig !== void 0) {
|
|
2011
|
+
for (const key of Object.keys(embedderConfig)) if (!EMBEDDER_KEYS.has(key)) throw new Error(`dsh-engram: unknown embedder key "${key}"`);
|
|
2012
|
+
if (embedderConfig.kind !== void 0 && embedderConfig.kind !== "local" && embedderConfig.kind !== "remote") throw new Error("dsh-engram: embedder.kind must be one of local|remote");
|
|
2013
|
+
if (embedderConfig.localPooling !== void 0 && embedderConfig.localPooling !== "mean" && embedderConfig.localPooling !== "cls") throw new Error("dsh-engram: embedder.localPooling must be one of mean|cls");
|
|
2014
|
+
if (embedderConfig.batchSize !== void 0 && (!Number.isInteger(embedderConfig.batchSize) || embedderConfig.batchSize < 1 || embedderConfig.batchSize > 256)) throw new Error("dsh-engram: embedder.batchSize must be an integer in [1, 256]");
|
|
2015
|
+
if (embedderConfig.timeoutMs !== void 0 && (!Number.isInteger(embedderConfig.timeoutMs) || embedderConfig.timeoutMs < 1e3 || embedderConfig.timeoutMs > 6e5)) throw new Error("dsh-engram: embedder.timeoutMs must be an integer in [1000, 600000]");
|
|
2016
|
+
const baseUrl = embedderConfig.baseUrl;
|
|
2017
|
+
if (baseUrl !== void 0 && baseUrl !== "") try {
|
|
2018
|
+
const parsed = new URL(baseUrl);
|
|
2019
|
+
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") throw new Error("unsupported protocol");
|
|
2020
|
+
} catch {
|
|
2021
|
+
throw new Error("dsh-engram: embedder.baseUrl must be an http(s) URL");
|
|
2022
|
+
}
|
|
2023
|
+
if (embedderKind === "remote") {
|
|
2024
|
+
if (embedderConfig.model === void 0 || embedderConfig.model === "") throw new Error("dsh-engram: embedder.kind=remote requires embedder.model");
|
|
2025
|
+
if (baseUrl === void 0 || baseUrl === "") throw new Error("dsh-engram: embedder.kind=remote requires embedder.baseUrl");
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
1870
2028
|
const dbDir = config.dbDir ?? join(homedir(), ".dsh", "engram");
|
|
1871
2029
|
return {
|
|
1872
2030
|
dbDir,
|
|
@@ -1912,6 +2070,19 @@ function resolveConfig(config = {}) {
|
|
|
1912
2070
|
deferMergeAbove: config.jev?.deferMergeAbove ?? .85,
|
|
1913
2071
|
deferAcceptBelow: config.jev?.deferAcceptBelow ?? .15,
|
|
1914
2072
|
contradictMinProbability: config.jev?.contradictMinProbability ?? .8
|
|
2073
|
+
},
|
|
2074
|
+
embedder: {
|
|
2075
|
+
kind: embedderKind,
|
|
2076
|
+
localModel: embedderConfig?.localModel ?? "Xenova/bge-small-zh-v1.5",
|
|
2077
|
+
localPooling: embedderConfig?.localPooling ?? "mean",
|
|
2078
|
+
remote: embedderKind === "remote" && embedderConfig?.model !== void 0 && embedderConfig.baseUrl !== void 0 ? {
|
|
2079
|
+
model: embedderConfig.model,
|
|
2080
|
+
baseUrl: embedderConfig.baseUrl,
|
|
2081
|
+
apiKey: embedderConfig.apiKey,
|
|
2082
|
+
apiKeyEnv: embedderConfig.apiKeyEnv,
|
|
2083
|
+
batchSize: embedderConfig.batchSize ?? 32,
|
|
2084
|
+
timeoutMs: embedderConfig.timeoutMs ?? 3e4
|
|
2085
|
+
} : void 0
|
|
1915
2086
|
}
|
|
1916
2087
|
};
|
|
1917
2088
|
}
|
|
@@ -2112,89 +2283,146 @@ function resolveRoomInput(input) {
|
|
|
2112
2283
|
}
|
|
2113
2284
|
}
|
|
2114
2285
|
//#endregion
|
|
2115
|
-
//#region src/
|
|
2116
|
-
/** 从任意字符串铸造品牌化 id(存储层入库前调用)。 */
|
|
2117
|
-
function asMemoryId(raw) {
|
|
2118
|
-
return raw;
|
|
2119
|
-
}
|
|
2120
|
-
/** 从任意字符串铸造品牌化实体 id(存储层入库前调用)。 */
|
|
2121
|
-
function asEntityId(raw) {
|
|
2122
|
-
return raw;
|
|
2123
|
-
}
|
|
2124
|
-
/** 从任意字符串铸造品牌化事实 id(存储层入库前调用)。 */
|
|
2125
|
-
function asFactId(raw) {
|
|
2126
|
-
return raw;
|
|
2127
|
-
}
|
|
2128
|
-
/** 从任意字符串铸造品牌化信念 id(存储层入库前调用)。 */
|
|
2129
|
-
function asObservationId(raw) {
|
|
2130
|
-
return raw;
|
|
2131
|
-
}
|
|
2132
|
-
/** 归一化比较键:trim + 合并连续空白 + 小写(消解第一版只做精确归一匹配,不做相似合并)。 */
|
|
2133
|
-
function normalizeEntityName(raw) {
|
|
2134
|
-
return raw.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2135
|
-
}
|
|
2136
|
-
/** episode 情景时间线的时间邻近扩展缺省窗口:锚点 createdAt ± 60 分钟。 */
|
|
2137
|
-
const EPISODE_PROXIMITY_MS_DEFAULT = 36e5;
|
|
2138
|
-
/** dsh-engram 统一错误:加载/使用期的可诊断失败都抛此类型。 */
|
|
2139
|
-
var EngramError = class extends Error {
|
|
2140
|
-
/** 机器可读原因码,如 EMBEDDER_DOWNLOAD_FAILED / SCHEMA_INCOMPATIBLE。 */
|
|
2141
|
-
code;
|
|
2142
|
-
constructor(code, message, options) {
|
|
2143
|
-
super(message, options);
|
|
2144
|
-
this.name = "EngramError";
|
|
2145
|
-
this.code = code;
|
|
2146
|
-
}
|
|
2147
|
-
};
|
|
2148
|
-
//#endregion
|
|
2149
|
-
//#region src/embedder/local.ts
|
|
2286
|
+
//#region src/embedder/remote.ts
|
|
2150
2287
|
/**
|
|
2151
|
-
*
|
|
2152
|
-
*
|
|
2153
|
-
*
|
|
2288
|
+
* OpenAI 兼容端点嵌入器:`POST {baseUrl}/embeddings`。
|
|
2289
|
+
*
|
|
2290
|
+
* 端点既可以是公开的嵌入服务,也可以是局域网内的推理服务器(LM Studio、Ollama、vLLM)。
|
|
2291
|
+
* 对插件而言区别只在 baseUrl 的取值:「本地/远程」的边界是部署决定,不是代码决定,
|
|
2292
|
+
* 因此实现只有一份。
|
|
2293
|
+
*
|
|
2294
|
+
* 密钥只从配置或环境变量读取:不进入日志、不进入错误文本、不进入诊断字段 model。
|
|
2295
|
+
* @module @kenz1117/dsh-engram/embedder/remote
|
|
2154
2296
|
*/
|
|
2155
|
-
/**
|
|
2156
|
-
const
|
|
2157
|
-
/**
|
|
2158
|
-
|
|
2297
|
+
/** 单次请求的文本条数上限:过大的批通常被服务端整体拒绝。 */
|
|
2298
|
+
const DEFAULT_BATCH_SIZE = 32;
|
|
2299
|
+
/** 单次请求的超时毫秒数。 */
|
|
2300
|
+
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
2159
2301
|
/**
|
|
2160
|
-
*
|
|
2161
|
-
*
|
|
2162
|
-
|
|
2302
|
+
* 加载期试探请求的正文:只用来验证端点可达且模型存在,不参与任何检索。
|
|
2303
|
+
* 用 ASCII 常量而不是 message():这段文字发往外部服务,不是给人看的界面文本。
|
|
2304
|
+
*/
|
|
2305
|
+
const PROBE_TEXT = "dsh-engram: embedder probe";
|
|
2306
|
+
/**
|
|
2307
|
+
* 把向量归一到单位长度。
|
|
2308
|
+
*
|
|
2309
|
+
* 接口承诺归一化向量,这同时消除了余弦与点积的差别:范数非零时两者相等。
|
|
2310
|
+
* 零向量原样返回——调用方把它当作「没有信号」。
|
|
2311
|
+
*/
|
|
2312
|
+
function toUnitVector(values) {
|
|
2313
|
+
const out = new Float32Array(values.length);
|
|
2314
|
+
let sumSquares = 0;
|
|
2315
|
+
for (let i = 0; i < values.length; i++) sumSquares += values[i] * values[i];
|
|
2316
|
+
if (sumSquares === 0) return out;
|
|
2317
|
+
const scale = 1 / Math.sqrt(sumSquares);
|
|
2318
|
+
for (let i = 0; i < values.length; i++) out[i] = values[i] * scale;
|
|
2319
|
+
return out;
|
|
2320
|
+
}
|
|
2321
|
+
/**
|
|
2322
|
+
* 创建远程嵌入器。
|
|
2323
|
+
*
|
|
2324
|
+
* 创建时先发一次试探请求:端点不可达、密钥错误、模型缺失都在加载期暴露,
|
|
2325
|
+
* 调用方据此把检索降级为纯关键词,而不是等到第一次写入才失败。
|
|
2326
|
+
*
|
|
2327
|
+
* @param options - 端点参数。
|
|
2163
2328
|
* @returns 就绪的 EngramEmbedder。
|
|
2164
|
-
* @throws EngramError(code=
|
|
2329
|
+
* @throws EngramError(code=EMBEDDER_REMOTE_FAILED) 试探请求失败。
|
|
2165
2330
|
*/
|
|
2166
|
-
async function
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
const
|
|
2177
|
-
const
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
pooling: "mean",
|
|
2183
|
-
normalize: true
|
|
2184
|
-
})).tolist().map((list) => new Float32Array(list));
|
|
2185
|
-
},
|
|
2186
|
-
close: async () => {
|
|
2187
|
-
if (cached?.embedder === embedder) cached = void 0;
|
|
2188
|
-
}
|
|
2189
|
-
};
|
|
2190
|
-
cached = {
|
|
2191
|
-
dir: cacheDir,
|
|
2192
|
-
embedder
|
|
2331
|
+
async function createRemoteEmbedder(options) {
|
|
2332
|
+
const endpoint = `${options.baseUrl.replace(/\/+$/, "")}/embeddings`;
|
|
2333
|
+
const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
2334
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
2335
|
+
const headers = { "Content-Type": "application/json" };
|
|
2336
|
+
/**
|
|
2337
|
+
* 请求头在每次请求时组装,而不是在创建时确定:变量名由配置给出,值却可能
|
|
2338
|
+
* 在运行中变化(轮换密钥、注入到测试环境),换密钥不该要求重启插件。
|
|
2339
|
+
*/
|
|
2340
|
+
const requestHeaders = () => {
|
|
2341
|
+
const fromEnv = options.apiKeyEnv === void 0 || options.apiKeyEnv === "" ? void 0 : process.env[options.apiKeyEnv];
|
|
2342
|
+
const apiKey = fromEnv !== void 0 && fromEnv !== "" ? fromEnv : options.apiKey;
|
|
2343
|
+
if (apiKey === void 0 || apiKey === "") return headers;
|
|
2344
|
+
return {
|
|
2345
|
+
...headers,
|
|
2346
|
+
Authorization: `Bearer ${apiKey}`
|
|
2193
2347
|
};
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2348
|
+
};
|
|
2349
|
+
/** 单次请求;服务端给了 index 就按 index 还原顺序。 */
|
|
2350
|
+
const request = async (texts) => {
|
|
2351
|
+
let response;
|
|
2352
|
+
try {
|
|
2353
|
+
response = await fetch(endpoint, {
|
|
2354
|
+
method: "POST",
|
|
2355
|
+
headers: requestHeaders(),
|
|
2356
|
+
body: JSON.stringify({
|
|
2357
|
+
model: options.model,
|
|
2358
|
+
input: [...texts],
|
|
2359
|
+
encoding_format: "float"
|
|
2360
|
+
}),
|
|
2361
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
2362
|
+
});
|
|
2363
|
+
} catch (error) {
|
|
2364
|
+
throw new EngramError("EMBEDDER_REMOTE_FAILED", message("store.embedderRemoteUnreachable", {
|
|
2365
|
+
baseUrl: options.baseUrl,
|
|
2366
|
+
timeoutMs
|
|
2367
|
+
}), { cause: error });
|
|
2368
|
+
}
|
|
2369
|
+
if (!response.ok) {
|
|
2370
|
+
const detail = await response.text().catch(() => "");
|
|
2371
|
+
throw new EngramError("EMBEDDER_REMOTE_FAILED", message("store.embedderRemoteHttp", {
|
|
2372
|
+
status: response.status,
|
|
2373
|
+
detail: detail === "" ? "" : `: ${detail.slice(0, 200)}`
|
|
2374
|
+
}));
|
|
2375
|
+
}
|
|
2376
|
+
let payload;
|
|
2377
|
+
try {
|
|
2378
|
+
payload = await response.json();
|
|
2379
|
+
} catch (error) {
|
|
2380
|
+
throw new EngramError("EMBEDDER_REMOTE_FAILED", message("store.embedderRemoteBody"), { cause: error });
|
|
2381
|
+
}
|
|
2382
|
+
const data = payload?.data;
|
|
2383
|
+
if (!Array.isArray(data) || data.length !== texts.length) throw new EngramError("EMBEDDER_REMOTE_FAILED", message("store.embedderRemoteCount", {
|
|
2384
|
+
received: Array.isArray(data) ? data.length : message("store.embedderRemoteCountNa"),
|
|
2385
|
+
expected: texts.length
|
|
2386
|
+
}));
|
|
2387
|
+
return (data.every((item) => typeof item.index === "number") ? [...data].sort((left, right) => (left.index ?? 0) - (right.index ?? 0)) : data).map((item) => {
|
|
2388
|
+
const values = item.embedding;
|
|
2389
|
+
if (!Array.isArray(values) || values.length === 0) throw new EngramError("EMBEDDER_REMOTE_FAILED", message("store.embedderRemoteEmpty"));
|
|
2390
|
+
return toUnitVector(values);
|
|
2391
|
+
});
|
|
2392
|
+
};
|
|
2393
|
+
const embed = async (texts) => {
|
|
2394
|
+
if (texts.length === 0) return [];
|
|
2395
|
+
const vectors = [];
|
|
2396
|
+
for (let start = 0; start < texts.length; start += batchSize) vectors.push(...await request(texts.slice(start, start + batchSize)));
|
|
2397
|
+
return vectors;
|
|
2398
|
+
};
|
|
2399
|
+
await embed([PROBE_TEXT]);
|
|
2400
|
+
return {
|
|
2401
|
+
model: options.model,
|
|
2402
|
+
embed,
|
|
2403
|
+
close: async () => {}
|
|
2404
|
+
};
|
|
2405
|
+
}
|
|
2406
|
+
//#endregion
|
|
2407
|
+
//#region src/embedder/select.ts
|
|
2408
|
+
/**
|
|
2409
|
+
* 按配置创建嵌入器。
|
|
2410
|
+
* @param config - 插件解析后的配置。
|
|
2411
|
+
* @returns 就绪的 EngramEmbedder。
|
|
2412
|
+
* @throws EngramError 本地模型加载失败,或远程端点没有响应。
|
|
2413
|
+
*/
|
|
2414
|
+
function createEmbedder(config) {
|
|
2415
|
+
const embedder = config.embedder;
|
|
2416
|
+
const remote = embedder.remote;
|
|
2417
|
+
if (remote !== void 0) return createRemoteEmbedder({
|
|
2418
|
+
model: remote.model,
|
|
2419
|
+
baseUrl: remote.baseUrl,
|
|
2420
|
+
apiKey: remote.apiKey,
|
|
2421
|
+
apiKeyEnv: remote.apiKeyEnv,
|
|
2422
|
+
batchSize: remote.batchSize,
|
|
2423
|
+
timeoutMs: remote.timeoutMs
|
|
2424
|
+
});
|
|
2425
|
+
return createLocalEmbedder(config.modelCacheDir, config.hfEndpoint, embedder.localModel, embedder.localPooling);
|
|
2198
2426
|
}
|
|
2199
2427
|
//#endregion
|
|
2200
2428
|
//#region src/llm/client.ts
|
|
@@ -3594,27 +3822,27 @@ async function runHistoryBackfill(deps, defaults, overrides, onProgress, signal)
|
|
|
3594
3822
|
* 向量相似度共享实现:store 检索与巩固聚类共用一份,替代历史上的两处复制。
|
|
3595
3823
|
* @module @kenz1117/dsh-engram/vec
|
|
3596
3824
|
*/
|
|
3597
|
-
/** 维度不匹配告警只发一次(模块级标记);上游单一嵌入模型下维度恒等,永不触发。 */
|
|
3598
|
-
let warnedDimensionMismatch = false;
|
|
3599
3825
|
/**
|
|
3600
3826
|
* 余弦相似度;任一向量零范数时返回 0。
|
|
3601
|
-
*
|
|
3602
|
-
*
|
|
3603
|
-
*
|
|
3827
|
+
*
|
|
3828
|
+
* 维度不同的两个向量不比较,而是抛错。逐维截断会得到一个「看起来像相似度」的数,
|
|
3829
|
+
* 而嵌入器现在是可配置的部署属性:换模型之后存量向量与新向量维度不同是常规情形,
|
|
3830
|
+
* 不是意外。与其静默降级、让人按错误的相似度做合并决定,不如在第一次比较时就说清
|
|
3831
|
+
* 两边的维度——错误原样交给上层,人就有机会改回模型,或用新模型重建向量。
|
|
3604
3832
|
* @param a - 向量 A。
|
|
3605
3833
|
* @param b - 向量 B。
|
|
3606
3834
|
* @returns 余弦相似度,范围约 [-1, 1];零范数输入返回 0。
|
|
3835
|
+
* @throws EngramError(code=EMBEDDER_DIMENSION_MISMATCH) 两向量维度不同。
|
|
3607
3836
|
*/
|
|
3608
3837
|
function cosine(a, b) {
|
|
3609
|
-
if (a.length !== b.length
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
}
|
|
3838
|
+
if (a.length !== b.length) throw new EngramError("EMBEDDER_DIMENSION_MISMATCH", message("store.embedderDimensionMismatch", {
|
|
3839
|
+
expected: b.length,
|
|
3840
|
+
found: a.length
|
|
3841
|
+
}));
|
|
3613
3842
|
let dot = 0;
|
|
3614
3843
|
let na = 0;
|
|
3615
3844
|
let nb = 0;
|
|
3616
|
-
|
|
3617
|
-
for (let i = 0; i < len; i++) {
|
|
3845
|
+
for (let i = 0; i < a.length; i++) {
|
|
3618
3846
|
dot += a[i] * b[i];
|
|
3619
3847
|
na += a[i] * a[i];
|
|
3620
3848
|
nb += b[i] * b[i];
|
|
@@ -3622,6 +3850,17 @@ function cosine(a, b) {
|
|
|
3622
3850
|
if (na === 0 || nb === 0) return 0;
|
|
3623
3851
|
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
3624
3852
|
}
|
|
3853
|
+
/**
|
|
3854
|
+
* 是不是「维度不一致」这类错误。
|
|
3855
|
+
*
|
|
3856
|
+
* 这类错误在比较向量时产生,要由调用方区分:能承受的调用方(检索)降级继续,
|
|
3857
|
+
* 不能承受的(写入)照原样抛出。区分的依据是代码而不是文本——文本会随语言变。
|
|
3858
|
+
* @param error - 接住的值。
|
|
3859
|
+
* @returns true,若是带 EMBEDDER_DIMENSION_MISMATCH 代码的 EngramError。
|
|
3860
|
+
*/
|
|
3861
|
+
function isDimensionMismatch(error) {
|
|
3862
|
+
return error instanceof EngramError && error.code === "EMBEDDER_DIMENSION_MISMATCH";
|
|
3863
|
+
}
|
|
3625
3864
|
//#endregion
|
|
3626
3865
|
//#region src/consolidation/run.ts
|
|
3627
3866
|
/** 单条整理动作(写入 op_log 的统一标记)。 */
|
|
@@ -7372,6 +7611,34 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
7372
7611
|
});
|
|
7373
7612
|
}
|
|
7374
7613
|
}
|
|
7614
|
+
/**
|
|
7615
|
+
* 库中已有向量的维度(按浮点数个数计,不按字节)。
|
|
7616
|
+
*
|
|
7617
|
+
* 打开时读一次,且两张带向量的表一起读:记忆与信念彼此比较,任何一边对不上
|
|
7618
|
+
* 都会让语义检索失真。库里还没有向量时维度未知——由第一次写入定下来,在那之前
|
|
7619
|
+
* 没有可拒绝的东西。
|
|
7620
|
+
*/
|
|
7621
|
+
const storedDimRow = db.prepare(`SELECT LENGTH(embedding) AS bytes FROM nodes WHERE embedding IS NOT NULL
|
|
7622
|
+
UNION ALL SELECT LENGTH(embedding) AS bytes FROM observations WHERE embedding IS NOT NULL LIMIT 1`).get();
|
|
7623
|
+
let storedDim = storedDimRow === void 0 ? void 0 : storedDimRow.bytes / Float32Array.BYTES_PER_ELEMENT;
|
|
7624
|
+
/** 降级为关键词检索的告警只打一次:同一个进程里重复刷同一句没有信息量。 */
|
|
7625
|
+
let warnedDegradedRead = false;
|
|
7626
|
+
/**
|
|
7627
|
+
* 放行向量入库,或拒绝并说出两边的维度。
|
|
7628
|
+
*
|
|
7629
|
+
* 严格而不是「能比多少算多少」:截断后的相似度仍是个像样的数字,合并决策却会
|
|
7630
|
+
* 据此做出。空库由第一次写入定维;此后必须一致,换模型就要重建向量。
|
|
7631
|
+
* @throws EngramError(code=EMBEDDER_DIMENSION_MISMATCH) 维度与库中不符。
|
|
7632
|
+
*/
|
|
7633
|
+
const guardDimension = (vector) => {
|
|
7634
|
+
const length = vector instanceof Float32Array ? vector.length : vector.byteLength / Float32Array.BYTES_PER_ELEMENT;
|
|
7635
|
+
const expected = storedDim ?? length;
|
|
7636
|
+
if (length !== expected || !Number.isInteger(length)) throw new EngramError("EMBEDDER_DIMENSION_MISMATCH", message("store.embedderDimensionMismatch", {
|
|
7637
|
+
expected,
|
|
7638
|
+
found: length
|
|
7639
|
+
}));
|
|
7640
|
+
storedDim = length;
|
|
7641
|
+
};
|
|
7375
7642
|
const sqlGet = db.prepare("SELECT * FROM nodes WHERE id = ?");
|
|
7376
7643
|
const sqlInsert = db.prepare(`INSERT INTO nodes
|
|
7377
7644
|
(id, scope, kind, content, importance, confidence, status, created_at, last_accessed_at, access_count,
|
|
@@ -7515,6 +7782,7 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
7515
7782
|
* 事务由调用方持有(withTransaction)。
|
|
7516
7783
|
*/
|
|
7517
7784
|
const insertRecord = (id, input, content, importance, confidence, at, sourceSessionId, embedding, imagery, op) => {
|
|
7785
|
+
if (embedding !== null) guardDimension(embedding);
|
|
7518
7786
|
const stored = embedding === null ? null : embedding instanceof Float32Array ? vecToBlob(embedding) : embedding;
|
|
7519
7787
|
const initialReview = input.initialReviewAt ?? null;
|
|
7520
7788
|
sqlInsert.run(id, input.scope, input.kind, content, importance, confidence, at, at, sourceSessionId, input.sourceRound ?? null, input.sourceSeq ?? null, stored, imageryToJson(imagery), input.slot?.room ?? null, input.slot?.index ?? null, input.imageryScore ?? null, initialReview, initialReview === null ? null : 2.5, initialReview === null ? null : 0);
|
|
@@ -7668,8 +7936,7 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
7668
7936
|
});
|
|
7669
7937
|
});
|
|
7670
7938
|
let degraded = true;
|
|
7671
|
-
if (queryVector !== void 0) {
|
|
7672
|
-
degraded = false;
|
|
7939
|
+
if (queryVector !== void 0) try {
|
|
7673
7940
|
vectorPool(query.scopes, query.rooms).map((row) => ({
|
|
7674
7941
|
row,
|
|
7675
7942
|
sim: cosine(queryVector, blobToVec(row.embedding))
|
|
@@ -7685,6 +7952,16 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
7685
7952
|
via: "both"
|
|
7686
7953
|
});
|
|
7687
7954
|
});
|
|
7955
|
+
degraded = false;
|
|
7956
|
+
} catch (error) {
|
|
7957
|
+
if (!isDimensionMismatch(error)) throw error;
|
|
7958
|
+
if (!warnedDegradedRead) {
|
|
7959
|
+
warnedDegradedRead = true;
|
|
7960
|
+
console.warn(message("store.embedderDimensionMismatch", {
|
|
7961
|
+
expected: storedDim ?? queryVector.length,
|
|
7962
|
+
found: queryVector.length
|
|
7963
|
+
}));
|
|
7964
|
+
}
|
|
7688
7965
|
}
|
|
7689
7966
|
if (rankBoost.recencyWeight !== 0 || rankBoost.proofWeight !== 0) {
|
|
7690
7967
|
const now = Date.now();
|
|
@@ -8449,6 +8726,7 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
8449
8726
|
const now = Date.now();
|
|
8450
8727
|
const sourceIds = [...new Set(input.sourceIds.map(String))].map(asMemoryId);
|
|
8451
8728
|
if (sourceIds.length === 0) throw new EngramError("BAD_INPUT", message("store.observationEvidenceRequired"));
|
|
8729
|
+
if (input.embedding !== void 0) guardDimension(input.embedding);
|
|
8452
8730
|
const stored = input.embedding === void 0 ? null : vecToBlob(input.embedding);
|
|
8453
8731
|
withTransaction(() => {
|
|
8454
8732
|
sqlObservationInsert.run(id, input.scope, input.belief, JSON.stringify(sourceIds.map(String)), sourceIds.length, stored, now, now, now);
|
|
@@ -8470,6 +8748,7 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
|
|
|
8470
8748
|
if (existing === void 0) return void 0;
|
|
8471
8749
|
const merged = [.../* @__PURE__ */ new Set([...observationRowToRecord(existing).sourceIds.map(String), ...addSourceIds.map(String)])].map(asMemoryId);
|
|
8472
8750
|
const now = Date.now();
|
|
8751
|
+
if (embedding !== void 0) guardDimension(embedding);
|
|
8473
8752
|
const stored = embedding === void 0 ? null : vecToBlob(embedding);
|
|
8474
8753
|
withTransaction(() => {
|
|
8475
8754
|
sqlObservationUpdate.run(belief, "active", JSON.stringify(merged.map(String)), merged.length, stored, now, now, String(id));
|
|
@@ -11834,7 +12113,7 @@ function estimateTokens(text) {
|
|
|
11834
12113
|
}
|
|
11835
12114
|
//#endregion
|
|
11836
12115
|
//#region package.json
|
|
11837
|
-
var version = "0.9.
|
|
12116
|
+
var version = "0.9.7";
|
|
11838
12117
|
//#endregion
|
|
11839
12118
|
//#region src/index.ts
|
|
11840
12119
|
/**
|
|
@@ -12283,7 +12562,7 @@ function apply(ctx, config = {}) {
|
|
|
12283
12562
|
* 面板「项目」scope、工具读写与自动摄取的 project 落点因此同一口径。
|
|
12284
12563
|
*/
|
|
12285
12564
|
const resolveProjectStore = (cwd) => cwd === void 0 ? openStore("project") : openStoreForProjectCwd(cwd);
|
|
12286
|
-
const embedder =
|
|
12565
|
+
const embedder = createEmbedder(resolved).catch((error) => {
|
|
12287
12566
|
console.warn(message("core.warnEmbedderUnavailable"), error);
|
|
12288
12567
|
});
|
|
12289
12568
|
const logIngestRequest = (data) => {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kenz1117/dsh-engram",
|
|
3
3
|
"description": "Cross-session long-term memory for DeepSeek Harness (memory palace · AGI Architecture Exploration on dsh-market): dual-scope SQLite memory graph, hybrid retrieval, provenance audit, and a knowledge flywheel.",
|
|
4
|
-
"version": "0.9.
|
|
4
|
+
"version": "0.9.7",
|
|
5
5
|
"icon": "icon.svg",
|
|
6
6
|
"logo": "icon-256.svg",
|
|
7
7
|
"publishConfig": {
|
|
@@ -110,8 +110,9 @@
|
|
|
110
110
|
"@types/react-dom": "^18.3.1",
|
|
111
111
|
"lightningcss": "^1.32.0",
|
|
112
112
|
"tsdown": "^0.23.1",
|
|
113
|
-
"typescript": "^
|
|
114
|
-
"
|
|
113
|
+
"typescript": "^7.0.2",
|
|
114
|
+
"vite": "^8.0.0",
|
|
115
|
+
"vitest": "^5.0.4"
|
|
115
116
|
},
|
|
116
117
|
"packageManager": "pnpm@10.15.0"
|
|
117
118
|
}
|