@co-engram/core 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dreaming/decay.d.ts.map +1 -1
- package/dist/dreaming/decay.js +1 -1
- package/dist/dreaming/decay.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +29 -4
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/zh.d.ts +24 -5
- package/dist/i18n/zh.d.ts.map +1 -1
- package/dist/i18n/zh.js +29 -2
- package/dist/i18n/zh.js.map +1 -1
- package/dist/importance/vector.d.ts +4 -4
- package/dist/importance/vector.d.ts.map +1 -1
- package/dist/importance/vector.js +7 -12
- package/dist/importance/vector.js.map +1 -1
- package/dist/lifecycle/freshness.d.ts +25 -6
- package/dist/lifecycle/freshness.d.ts.map +1 -1
- package/dist/lifecycle/freshness.js +37 -21
- package/dist/lifecycle/freshness.js.map +1 -1
- package/dist/merge-driver.cjs +1 -1
- package/dist/observability/proposal-engine.d.ts +2 -1
- package/dist/observability/proposal-engine.d.ts.map +1 -1
- package/dist/observability/proposal-engine.js +6 -3
- package/dist/observability/proposal-engine.js.map +1 -1
- package/dist/prompt-builder/builder.d.ts +3 -0
- package/dist/prompt-builder/builder.d.ts.map +1 -1
- package/dist/prompt-builder/builder.js +45 -3
- package/dist/prompt-builder/builder.js.map +1 -1
- package/dist/prompt-builder/index.d.ts +1 -0
- package/dist/prompt-builder/index.d.ts.map +1 -1
- package/dist/prompt-builder/index.js +1 -0
- package/dist/prompt-builder/index.js.map +1 -1
- package/dist/prompt-builder/path-overview.d.ts +35 -0
- package/dist/prompt-builder/path-overview.d.ts.map +1 -0
- package/dist/prompt-builder/path-overview.js +44 -0
- package/dist/prompt-builder/path-overview.js.map +1 -0
- package/dist/prompt-builder/types.d.ts +11 -0
- package/dist/prompt-builder/types.d.ts.map +1 -1
- package/dist/retrieval/index.d.ts +2 -0
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +2 -0
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/scoring.d.ts +7 -4
- package/dist/retrieval/scoring.d.ts.map +1 -1
- package/dist/retrieval/scoring.js +9 -16
- package/dist/retrieval/scoring.js.map +1 -1
- package/dist/retrieval/search-engine.d.ts +58 -0
- package/dist/retrieval/search-engine.d.ts.map +1 -0
- package/dist/retrieval/search-engine.js +53 -0
- package/dist/retrieval/search-engine.js.map +1 -0
- package/dist/retrieval/sqlite-orchestrator.d.ts +68 -0
- package/dist/retrieval/sqlite-orchestrator.d.ts.map +1 -0
- package/dist/retrieval/sqlite-orchestrator.js +148 -0
- package/dist/retrieval/sqlite-orchestrator.js.map +1 -0
- package/dist/storage/bootstrap.d.ts +31 -0
- package/dist/storage/bootstrap.d.ts.map +1 -0
- package/dist/storage/bootstrap.js +89 -0
- package/dist/storage/bootstrap.js.map +1 -0
- package/dist/storage/index-db-cursor.d.ts +27 -0
- package/dist/storage/index-db-cursor.d.ts.map +1 -0
- package/dist/storage/index-db-cursor.js +59 -0
- package/dist/storage/index-db-cursor.js.map +1 -0
- package/dist/storage/index-db.d.ts +112 -0
- package/dist/storage/index-db.d.ts.map +1 -0
- package/dist/storage/index-db.js +183 -0
- package/dist/storage/index-db.js.map +1 -0
- package/dist/storage/index.d.ts +3 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/repository.d.ts +30 -2
- package/dist/storage/repository.d.ts.map +1 -1
- package/dist/storage/repository.js +73 -2
- package/dist/storage/repository.js.map +1 -1
- package/dist/tools/audit-query-tool.d.ts +2 -2
- package/dist/tools/engram-tools.d.ts.map +1 -1
- package/dist/tools/engram-tools.js +2 -2
- package/dist/tools/engram-tools.js.map +1 -1
- package/dist/tools/proposal-tools.d.ts.map +1 -1
- package/dist/tools/proposal-tools.js +1 -0
- package/dist/tools/proposal-tools.js.map +1 -1
- package/dist/tools/schemas.d.ts +78 -70
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +6 -0
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/tool.d.ts +1 -1
- package/dist/tools/tool.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Language } from "../i18n/index.js";
|
|
2
|
+
import { type SearchEngine, type SearchEngineType } from "../retrieval/search-engine.js";
|
|
3
|
+
import { EngramRepository } from "./repository.js";
|
|
4
|
+
import { IndexDb } from "./index-db.js";
|
|
5
|
+
export interface BootstrapOptions {
|
|
6
|
+
readonly dataRoot: string;
|
|
7
|
+
readonly language?: Language;
|
|
8
|
+
/** 注入 env(测试用);默认 process.env */
|
|
9
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
10
|
+
}
|
|
11
|
+
export interface BootstrapResult {
|
|
12
|
+
readonly repository: EngramRepository;
|
|
13
|
+
readonly searchEngine: SearchEngine;
|
|
14
|
+
readonly engineType: SearchEngineType;
|
|
15
|
+
/** SQLite 模式下的 indexDb 引用;memory 模式为 undefined。host 持有以在
|
|
16
|
+
* 关闭时显式 close()(进程退出时 OS 自动回收 fd,但测试 / 显式资源管理
|
|
17
|
+
* 需要确定性释放)。 */
|
|
18
|
+
readonly indexDb?: IndexDb;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 装配 EngramRepository + SearchEngine。
|
|
22
|
+
*
|
|
23
|
+
* Engine 选择:env.CO_ENGRAM_SEARCH_ENGINE=sqlite → SQLite FTS5 模式;
|
|
24
|
+
* 未设置 / 其他值 → 默认 memory(in-memory Intl.Segmenter)。
|
|
25
|
+
*
|
|
26
|
+
* Cold start 行为(sqlite 模式):db 为空时,从 repository.listEngrams()
|
|
27
|
+
* 枚举全量 engram,readEngram() 读详情,转 EngramIndexEntry[] 后
|
|
28
|
+
* rebuildFromEntries() 一次性灌入。已有数据的 db 不重建。
|
|
29
|
+
*/
|
|
30
|
+
export declare function bootstrapRepositoryAndSearch(opts: BootstrapOptions): BootstrapResult;
|
|
31
|
+
//# sourceMappingURL=bootstrap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/storage/bootstrap.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAyB,MAAM,eAAe,CAAC;AAE/D,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAC7B,iCAAiC;IACjC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC;;oBAEgB;IAChB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,gBAAgB,GACrB,eAAe,CAkDjB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// packages/core/src/storage/bootstrap.ts
|
|
2
|
+
//
|
|
3
|
+
// 装配 EngramRepository + SearchEngine —— 把 host adapter(claude-code-mcp /
|
|
4
|
+
// openclaw-plugin)共用的初始化逻辑抽到 core,避免双宿主行为漂移。
|
|
5
|
+
//
|
|
6
|
+
// 默认行为(memory 模式)完全等同此前各 host 内联的 new SearchOrchestrator()
|
|
7
|
+
// + rebuildSearchIndex。sqlite 模式额外做:
|
|
8
|
+
// 1. 打开 .co-engram/index.db(WAL,首次自动建 schema)
|
|
9
|
+
// 2. 把 indexDb 注入 EngramRepository(开启 write-through)
|
|
10
|
+
// 3. Cold start:db 为空时从 engrams/*.md 全量重建索引
|
|
11
|
+
// 4. 用 SqliteSearchEngineAdapter 包装 SearchEngine
|
|
12
|
+
//
|
|
13
|
+
// @module @co-engram/core/storage
|
|
14
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { createSearchEngine, resolveSearchEngineType, } from "../retrieval/search-engine.js";
|
|
17
|
+
import { EngramRepository } from "./repository.js";
|
|
18
|
+
import { IndexDb } from "./index-db.js";
|
|
19
|
+
/**
|
|
20
|
+
* 装配 EngramRepository + SearchEngine。
|
|
21
|
+
*
|
|
22
|
+
* Engine 选择:env.CO_ENGRAM_SEARCH_ENGINE=sqlite → SQLite FTS5 模式;
|
|
23
|
+
* 未设置 / 其他值 → 默认 memory(in-memory Intl.Segmenter)。
|
|
24
|
+
*
|
|
25
|
+
* Cold start 行为(sqlite 模式):db 为空时,从 repository.listEngrams()
|
|
26
|
+
* 枚举全量 engram,readEngram() 读详情,转 EngramIndexEntry[] 后
|
|
27
|
+
* rebuildFromEntries() 一次性灌入。已有数据的 db 不重建。
|
|
28
|
+
*/
|
|
29
|
+
export function bootstrapRepositoryAndSearch(opts) {
|
|
30
|
+
const engineType = resolveSearchEngineType(opts.env);
|
|
31
|
+
if (engineType === "sqlite") {
|
|
32
|
+
const dbDir = join(opts.dataRoot, ".co-engram");
|
|
33
|
+
if (!existsSync(dbDir))
|
|
34
|
+
mkdirSync(dbDir, { recursive: true });
|
|
35
|
+
const dbPath = join(dbDir, "index.db");
|
|
36
|
+
const indexDb = new IndexDb({ dbPath });
|
|
37
|
+
indexDb.open();
|
|
38
|
+
// 先构造 repository with indexDb,write-through 在后续所有写入路径生效
|
|
39
|
+
const repository = new EngramRepository({
|
|
40
|
+
rootPath: opts.dataRoot,
|
|
41
|
+
...(opts.language ? { language: opts.language } : {}),
|
|
42
|
+
}, indexDb);
|
|
43
|
+
// Cold start:db 空时全量重建
|
|
44
|
+
const count = indexDb.prepare("SELECT count(*) as n FROM engrams").get();
|
|
45
|
+
if (count.n === 0) {
|
|
46
|
+
const entries = [];
|
|
47
|
+
// listEngrams 返回 EngramCatalogEntry[],取 .id 作为 readEngram 入参
|
|
48
|
+
for (const catalog of repository.listEngrams()) {
|
|
49
|
+
try {
|
|
50
|
+
const engram = repository.readEngram(catalog.id);
|
|
51
|
+
if (!engram)
|
|
52
|
+
continue;
|
|
53
|
+
entries.push(engramToIndexEntry(engram));
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// 跳过损坏 engram;doctor 自愈路径会单独处理
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
indexDb.rebuildFromEntries(entries);
|
|
60
|
+
}
|
|
61
|
+
const searchEngine = createSearchEngine({ type: "sqlite", indexDb });
|
|
62
|
+
return { repository, searchEngine, engineType, indexDb };
|
|
63
|
+
}
|
|
64
|
+
// memory 模式:行为与原 host 内联代码等价
|
|
65
|
+
const repository = new EngramRepository({
|
|
66
|
+
rootPath: opts.dataRoot,
|
|
67
|
+
...(opts.language ? { language: opts.language } : {}),
|
|
68
|
+
});
|
|
69
|
+
const searchEngine = createSearchEngine({ type: "memory" });
|
|
70
|
+
return { repository, searchEngine, engineType };
|
|
71
|
+
}
|
|
72
|
+
/** Engram → EngramIndexEntry 投影(与 repository.syncEngramToIndex 同义) */
|
|
73
|
+
function engramToIndexEntry(e) {
|
|
74
|
+
return {
|
|
75
|
+
id: e.id,
|
|
76
|
+
title: e.title,
|
|
77
|
+
kind: e.kind,
|
|
78
|
+
importance: e.importance,
|
|
79
|
+
confidence: e.confidence,
|
|
80
|
+
updatedAt: Date.parse(e.updatedAt),
|
|
81
|
+
contentSize: e.contentSize,
|
|
82
|
+
visibility: e.visibility,
|
|
83
|
+
status: e.status,
|
|
84
|
+
domainTags: [...e.domainTags],
|
|
85
|
+
summary: e.summary,
|
|
86
|
+
contentTokens: e.content,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=bootstrap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/storage/bootstrap.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,yEAAyE;AACzE,6CAA6C;AAC7C,EAAE;AACF,2DAA2D;AAC3D,qCAAqC;AACrC,gDAAgD;AAChD,uDAAuD;AACvD,8CAA8C;AAC9C,mDAAmD;AACnD,EAAE;AACF,kCAAkC;AAClC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EACL,kBAAkB,EAClB,uBAAuB,GAGxB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAyB,MAAM,eAAe,CAAC;AAmB/D;;;;;;;;;GASG;AACH,MAAM,UAAU,4BAA4B,CAC1C,IAAsB;IAEtB,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErD,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,SAAS,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAEvC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,EAAE,CAAC;QAEf,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,gBAAgB,CACrC;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,EACD,OAAO,CACR,CAAC;QAEF,uBAAuB;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,EAErE,CAAC;QACF,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAClB,MAAM,OAAO,GAAuB,EAAE,CAAC;YACvC,6DAA6D;YAC7D,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACjD,IAAI,CAAC,MAAM;wBAAE,SAAS;oBACtB,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBAAC,MAAM,CAAC;oBACP,+BAA+B;gBACjC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,YAAY,GAAG,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IAC3D,CAAC;IAED,6BAA6B;IAC7B,MAAM,UAAU,GAAG,IAAI,gBAAgB,CAAC;QACtC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC5D,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AAClD,CAAC;AAED,sEAAsE;AACtE,SAAS,kBAAkB,CAAC,CAAS;IACnC,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAClC,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;QAC7B,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,aAAa,EAAE,CAAC,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 稳定排序键:importance DESC, updatedAt DESC, id ASC
|
|
3
|
+
*
|
|
4
|
+
* cursor 是 opaque base64url token,encode 当前页最后一条的 sort key,
|
|
5
|
+
* 让下一页查询能"接着"这个位置往后取(避免 offset 漂移导致的数据跳过/重复)。
|
|
6
|
+
*
|
|
7
|
+
* 选择 base64url 而非 base64:cursor 经常出现在 URL query string,
|
|
8
|
+
* base64url 无需 URL encode(+ / = 等特殊字符)。
|
|
9
|
+
*/
|
|
10
|
+
export interface SortKey {
|
|
11
|
+
readonly importance: number;
|
|
12
|
+
readonly updatedAt: number;
|
|
13
|
+
readonly id: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function encodeCursor(key: SortKey): string;
|
|
16
|
+
export declare function decodeCursor(cursor: string): SortKey;
|
|
17
|
+
/**
|
|
18
|
+
* 比较两个 sort key,返回 -1 / 0 / 1。
|
|
19
|
+
* 返回 -1 表示 a 排在 b 前面;1 表示 a 排在 b 后面;0 表示相等。
|
|
20
|
+
*
|
|
21
|
+
* 顺序:importance 大→小;同分 updatedAt 新→旧;再同分 id 字典序升序(稳定)。
|
|
22
|
+
*
|
|
23
|
+
* 用于 cursor 分页:取出 cursor 之后,过滤掉所有 compareSortKey(key, cursor) <= 0 的项
|
|
24
|
+
* (cursor 之前或等于的项跳过),从下一个开始取。
|
|
25
|
+
*/
|
|
26
|
+
export declare function compareSortKey(a: SortKey, b: SortKey): number;
|
|
27
|
+
//# sourceMappingURL=index-db-cursor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-db-cursor.d.ts","sourceRoot":"","sources":["../../src/storage/index-db-cursor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAIjD;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAwBpD;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,GAAG,MAAM,CAK7D"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 稳定排序键:importance DESC, updatedAt DESC, id ASC
|
|
3
|
+
*
|
|
4
|
+
* cursor 是 opaque base64url token,encode 当前页最后一条的 sort key,
|
|
5
|
+
* 让下一页查询能"接着"这个位置往后取(避免 offset 漂移导致的数据跳过/重复)。
|
|
6
|
+
*
|
|
7
|
+
* 选择 base64url 而非 base64:cursor 经常出现在 URL query string,
|
|
8
|
+
* base64url 无需 URL encode(+ / = 等特殊字符)。
|
|
9
|
+
*/
|
|
10
|
+
export function encodeCursor(key) {
|
|
11
|
+
// 用数组而非 object:更紧凑,且避免 key 顺序歧义
|
|
12
|
+
const json = JSON.stringify([key.importance, key.updatedAt, key.id]);
|
|
13
|
+
return Buffer.from(json, "utf8").toString("base64url");
|
|
14
|
+
}
|
|
15
|
+
export function decodeCursor(cursor) {
|
|
16
|
+
if (!cursor)
|
|
17
|
+
throw new Error("invalid cursor: empty");
|
|
18
|
+
let json;
|
|
19
|
+
try {
|
|
20
|
+
json = Buffer.from(cursor, "base64url").toString("utf8");
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
throw new Error("invalid cursor: base64 decode failed");
|
|
24
|
+
}
|
|
25
|
+
let arr;
|
|
26
|
+
try {
|
|
27
|
+
arr = JSON.parse(json);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
throw new Error("invalid cursor: JSON parse failed");
|
|
31
|
+
}
|
|
32
|
+
if (!Array.isArray(arr) ||
|
|
33
|
+
arr.length !== 3 ||
|
|
34
|
+
typeof arr[0] !== "number" ||
|
|
35
|
+
typeof arr[1] !== "number" ||
|
|
36
|
+
typeof arr[2] !== "string") {
|
|
37
|
+
throw new Error("invalid cursor: shape mismatch");
|
|
38
|
+
}
|
|
39
|
+
return { importance: arr[0], updatedAt: arr[1], id: arr[2] };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* 比较两个 sort key,返回 -1 / 0 / 1。
|
|
43
|
+
* 返回 -1 表示 a 排在 b 前面;1 表示 a 排在 b 后面;0 表示相等。
|
|
44
|
+
*
|
|
45
|
+
* 顺序:importance 大→小;同分 updatedAt 新→旧;再同分 id 字典序升序(稳定)。
|
|
46
|
+
*
|
|
47
|
+
* 用于 cursor 分页:取出 cursor 之后,过滤掉所有 compareSortKey(key, cursor) <= 0 的项
|
|
48
|
+
* (cursor 之前或等于的项跳过),从下一个开始取。
|
|
49
|
+
*/
|
|
50
|
+
export function compareSortKey(a, b) {
|
|
51
|
+
if (a.importance !== b.importance)
|
|
52
|
+
return a.importance > b.importance ? -1 : 1;
|
|
53
|
+
if (a.updatedAt !== b.updatedAt)
|
|
54
|
+
return a.updatedAt > b.updatedAt ? -1 : 1;
|
|
55
|
+
if (a.id !== b.id)
|
|
56
|
+
return a.id < b.id ? -1 : 1;
|
|
57
|
+
return 0;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=index-db-cursor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-db-cursor.js","sourceRoot":"","sources":["../../src/storage/index-db-cursor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,MAAM,UAAU,YAAY,CAAC,GAAY;IACxC,gCAAgC;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IAC1C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACtD,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACJ,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACtD,CAAC;IACD,IACC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QACnB,GAAG,CAAC,MAAM,KAAK,CAAC;QAChB,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ;QAC1B,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ;QAC1B,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,EACzB,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,CAAU,EAAE,CAAU;IACpD,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;QAAE,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACV,CAAC"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
declare const DatabaseSync: typeof import("node:sqlite").DatabaseSync;
|
|
2
|
+
type SqliteDb = InstanceType<typeof DatabaseSync>;
|
|
3
|
+
type Statement = ReturnType<SqliteDb["prepare"]>;
|
|
4
|
+
export interface IndexDbOptions {
|
|
5
|
+
readonly dbPath: string;
|
|
6
|
+
/** true = 删除现有 db 文件后重建(plan Task 1.3 暂未实现 reset 逻辑,占位字段) */
|
|
7
|
+
readonly reset?: boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* engram 在索引层的一份精简投影(只含搜索/排序/FTS 需要的字段)。
|
|
11
|
+
*
|
|
12
|
+
* domainTags 是多值字段,在 SQLite 端拆 engram_domains 表;其余标量字段直接落 engrams 主表。
|
|
13
|
+
* contentTokens 是已切分的纯文本(可能含空格分隔的 token),用于 FTS5 trigram 倒排。
|
|
14
|
+
*/
|
|
15
|
+
export interface EngramIndexEntry {
|
|
16
|
+
readonly id: string;
|
|
17
|
+
readonly title: string;
|
|
18
|
+
readonly kind: string;
|
|
19
|
+
readonly importance: number;
|
|
20
|
+
readonly confidence: number;
|
|
21
|
+
/** epoch ms */
|
|
22
|
+
readonly updatedAt: number;
|
|
23
|
+
readonly contentSize: number;
|
|
24
|
+
readonly visibility: string;
|
|
25
|
+
readonly status: string;
|
|
26
|
+
readonly domainTags: readonly string[];
|
|
27
|
+
readonly summary: string;
|
|
28
|
+
readonly contentTokens: string;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* 突触(synapse)索引条目。外键 from_id / to_id 引用 engrams 主表,
|
|
32
|
+
* 删除任一端 engram 时由 ON DELETE CASCADE 自动清空。
|
|
33
|
+
*/
|
|
34
|
+
export interface SynapseIndexEntry {
|
|
35
|
+
readonly id: string;
|
|
36
|
+
readonly fromId: string;
|
|
37
|
+
readonly toId: string;
|
|
38
|
+
readonly kind: string;
|
|
39
|
+
readonly weight: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* SQLite 索引库封装:打开 / schema 初始化 / 显式事务。
|
|
43
|
+
*
|
|
44
|
+
* 设计要点:
|
|
45
|
+
* - WAL 模式:多读单写,显著降低并发读写冲突,适合 engram write-through 场景。
|
|
46
|
+
* - 显式 BEGIN/COMMIT/ROLLBACK:node:sqlite 默认隐式事务,显式包裹保证多语句原子性。
|
|
47
|
+
* - schema 通过外部 .sql 文件管理,便于审计和迁移工具比对。
|
|
48
|
+
*/
|
|
49
|
+
export declare class IndexDb {
|
|
50
|
+
private db;
|
|
51
|
+
private readonly opts;
|
|
52
|
+
constructor(opts: IndexDbOptions);
|
|
53
|
+
open(): void;
|
|
54
|
+
close(): void;
|
|
55
|
+
prepare(sql: string): Statement;
|
|
56
|
+
exec(sql: string): void;
|
|
57
|
+
/**
|
|
58
|
+
* 在事务中执行回调,失败 rollback。
|
|
59
|
+
* 注意:node:sqlite 的 DatabaseSync 默认每条语句隐式事务,
|
|
60
|
+
* 但我们用 BEGIN/COMMIT/ROLLBACK 显式包裹,确保多语句原子性。
|
|
61
|
+
*/
|
|
62
|
+
transaction<T>(fn: () => T): T;
|
|
63
|
+
private requireOpen;
|
|
64
|
+
/**
|
|
65
|
+
* UPSERT 一个 engram 索引条目:主表 + domains(全量替换)+ FTS(delete+insert)。
|
|
66
|
+
*
|
|
67
|
+
* 实现要点:
|
|
68
|
+
* - 整体在事务内,任一步失败 rollback,保证主表/domains/FTS 三地一致。
|
|
69
|
+
* - domains 用 DELETE+INSERT 而非 ON CONFLICT:语义上是"全量替换 tag 集合",
|
|
70
|
+
* 调用方传入的新集合就是真理源,旧 tag 不在该集合内就该被清掉。
|
|
71
|
+
* - FTS5 不支持 ON CONFLICT,只能 delete + insert;否则会有重复倒排项。
|
|
72
|
+
*/
|
|
73
|
+
upsertEngram(entry: EngramIndexEntry): void;
|
|
74
|
+
/**
|
|
75
|
+
* 内部 UPSERT(无事务包裹)。调用方必须已在外层 transaction 内。
|
|
76
|
+
*
|
|
77
|
+
* 单条写入走 upsertEngram;批量 cold start rebuild 走 rebuildFromEntries,
|
|
78
|
+
* 后者在单个 transaction 里循环调用本方法,避免每条都 BEGIN/COMMIT。
|
|
79
|
+
*/
|
|
80
|
+
private upsertEngramUnsafe;
|
|
81
|
+
/**
|
|
82
|
+
* Cold start 重建:用 entries 全量替换 SQLite 索引内容。
|
|
83
|
+
*
|
|
84
|
+
* 用例:host 启动时检测到 .co-engram/index.db 缺失或损坏 → 扫描所有
|
|
85
|
+
* engrams/*.md → 调本方法一次性灌入。
|
|
86
|
+
*
|
|
87
|
+
* 实现:
|
|
88
|
+
* - 单个 transaction 内:DELETE engrams(CASCADE 清 domains + synapses)
|
|
89
|
+
* + DELETE engram_fts(FTS 不参与外键,手动清)+ 批量 upsertEngramUnsafe。
|
|
90
|
+
* - 中途任一步失败 → rollback,SQLite 保持原状(数据可能旧但一致)。
|
|
91
|
+
* - 调用方决定是否清空 synapses:本方法不重建 synapses(它们由 synapse
|
|
92
|
+
* create 路径单独维护),CASCADE 会把它们一并清掉,rebuild 期间 synapse
|
|
93
|
+
* 数据丢失是可接受的(可以从 markdown frontmatter 重新解析)。
|
|
94
|
+
*/
|
|
95
|
+
rebuildFromEntries(entries: readonly EngramIndexEntry[]): void;
|
|
96
|
+
/**
|
|
97
|
+
* UPSERT 一条 synapse。无 FTS,单语句即可,不需要事务包裹。
|
|
98
|
+
*/
|
|
99
|
+
upsertSynapse(s: SynapseIndexEntry): void;
|
|
100
|
+
/**
|
|
101
|
+
* 删除一个 engram 及其所有派生数据。
|
|
102
|
+
*
|
|
103
|
+
* - engrams 主表:显式 DELETE。
|
|
104
|
+
* - engram_domains / synapses(from/to):外键 ON DELETE CASCADE 自动清。
|
|
105
|
+
* - engram_fts:FTS5 表不参与外键,必须显式 DELETE。
|
|
106
|
+
*
|
|
107
|
+
* 包在事务里:即便 FTS 删除失败,主表删除也会 rollback,保持一致。
|
|
108
|
+
*/
|
|
109
|
+
deleteEngram(engramId: string): void;
|
|
110
|
+
}
|
|
111
|
+
export {};
|
|
112
|
+
//# sourceMappingURL=index-db.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-db.d.ts","sourceRoot":"","sources":["../../src/storage/index-db.ts"],"names":[],"mappings":"AAiBA,QAAA,MAAM,YAAY,2CAA4B,CAAC;AAI/C,KAAK,QAAQ,GAAG,YAAY,CAAC,OAAO,YAAY,CAAC,CAAC;AAClD,KAAK,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AAKjD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,eAAe;IACf,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,EAAE,CAAyB;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;gBAE1B,IAAI,EAAE,cAAc;IAIhC,IAAI,IAAI,IAAI;IAaZ,KAAK,IAAI,IAAI;IAMb,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS;IAK/B,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAKvB;;;;OAIG;IACH,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;IAiB9B,OAAO,CAAC,WAAW;IAMnB;;;;;;;;OAQG;IACH,YAAY,CAAC,KAAK,EAAE,gBAAgB,GAAG,IAAI;IAI3C;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,GAAG,IAAI;IAU9D;;OAEG;IACH,aAAa,CAAC,CAAC,EAAE,iBAAiB,GAAG,IAAI;IAYzC;;;;;;;;OAQG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;CAMrC"}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
// packages/core/src/storage/index-db.ts
|
|
2
|
+
// 用 createRequire 绕过 Vite 静态分析:`import { DatabaseSync } from "node:sqlite"`
|
|
3
|
+
// 在 vitest 2.x 下会被 Vite resolver 拦截,报 `Failed to load url sqlite`
|
|
4
|
+
// (stripping `node:` prefix)。createRequire 在运行时调用,Vite 无法静态解析,
|
|
5
|
+
// 直接走 Node builtin resolver —— 同 test/storage/node-sqlite-smoke.test.ts 的 workaround。
|
|
6
|
+
//
|
|
7
|
+
// 字符串拼接 `"node:" + "sqlite"` 进一步防止 Vite 把整个 require 调用静态化为
|
|
8
|
+
// bare import 解析。type import 在编译时被擦除,不被 resolver 拦截,可以安全使用。
|
|
9
|
+
import { createRequire } from "node:module";
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
import { dirname, join } from "node:path";
|
|
12
|
+
import { fileURLToPath } from "node:url";
|
|
13
|
+
const require = createRequire(import.meta.url);
|
|
14
|
+
const SQLITE_MODULE = "node:" + "sqlite";
|
|
15
|
+
const sqliteModule = require(SQLITE_MODULE);
|
|
16
|
+
const DatabaseSync = sqliteModule.DatabaseSync;
|
|
17
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
18
|
+
const SCHEMA_PATH = join(__dirname, "index-db-schema.sql");
|
|
19
|
+
/**
|
|
20
|
+
* SQLite 索引库封装:打开 / schema 初始化 / 显式事务。
|
|
21
|
+
*
|
|
22
|
+
* 设计要点:
|
|
23
|
+
* - WAL 模式:多读单写,显著降低并发读写冲突,适合 engram write-through 场景。
|
|
24
|
+
* - 显式 BEGIN/COMMIT/ROLLBACK:node:sqlite 默认隐式事务,显式包裹保证多语句原子性。
|
|
25
|
+
* - schema 通过外部 .sql 文件管理,便于审计和迁移工具比对。
|
|
26
|
+
*/
|
|
27
|
+
export class IndexDb {
|
|
28
|
+
db = null;
|
|
29
|
+
opts;
|
|
30
|
+
constructor(opts) {
|
|
31
|
+
this.opts = opts;
|
|
32
|
+
}
|
|
33
|
+
open() {
|
|
34
|
+
if (this.db)
|
|
35
|
+
return;
|
|
36
|
+
this.db = new DatabaseSync(this.opts.dbPath);
|
|
37
|
+
// WAL 模式 + 外键 + 性能调优
|
|
38
|
+
this.db.exec("PRAGMA journal_mode = WAL");
|
|
39
|
+
this.db.exec("PRAGMA synchronous = NORMAL");
|
|
40
|
+
this.db.exec("PRAGMA foreign_keys = ON");
|
|
41
|
+
this.db.exec("PRAGMA busy_timeout = 5000");
|
|
42
|
+
// 初始化 schema
|
|
43
|
+
const schema = readFileSync(SCHEMA_PATH, "utf8");
|
|
44
|
+
this.db.exec(schema);
|
|
45
|
+
}
|
|
46
|
+
close() {
|
|
47
|
+
if (!this.db)
|
|
48
|
+
return;
|
|
49
|
+
this.db.close();
|
|
50
|
+
this.db = null;
|
|
51
|
+
}
|
|
52
|
+
prepare(sql) {
|
|
53
|
+
this.requireOpen();
|
|
54
|
+
return this.db.prepare(sql);
|
|
55
|
+
}
|
|
56
|
+
exec(sql) {
|
|
57
|
+
this.requireOpen();
|
|
58
|
+
this.db.exec(sql);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* 在事务中执行回调,失败 rollback。
|
|
62
|
+
* 注意:node:sqlite 的 DatabaseSync 默认每条语句隐式事务,
|
|
63
|
+
* 但我们用 BEGIN/COMMIT/ROLLBACK 显式包裹,确保多语句原子性。
|
|
64
|
+
*/
|
|
65
|
+
transaction(fn) {
|
|
66
|
+
this.requireOpen();
|
|
67
|
+
this.db.exec("BEGIN");
|
|
68
|
+
try {
|
|
69
|
+
const result = fn();
|
|
70
|
+
this.db.exec("COMMIT");
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
try {
|
|
75
|
+
this.db.exec("ROLLBACK");
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// ignore rollback failure
|
|
79
|
+
}
|
|
80
|
+
throw err;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
requireOpen() {
|
|
84
|
+
if (!this.db) {
|
|
85
|
+
throw new Error("IndexDb not opened. Call open() first.");
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* UPSERT 一个 engram 索引条目:主表 + domains(全量替换)+ FTS(delete+insert)。
|
|
90
|
+
*
|
|
91
|
+
* 实现要点:
|
|
92
|
+
* - 整体在事务内,任一步失败 rollback,保证主表/domains/FTS 三地一致。
|
|
93
|
+
* - domains 用 DELETE+INSERT 而非 ON CONFLICT:语义上是"全量替换 tag 集合",
|
|
94
|
+
* 调用方传入的新集合就是真理源,旧 tag 不在该集合内就该被清掉。
|
|
95
|
+
* - FTS5 不支持 ON CONFLICT,只能 delete + insert;否则会有重复倒排项。
|
|
96
|
+
*/
|
|
97
|
+
upsertEngram(entry) {
|
|
98
|
+
this.transaction(() => this.upsertEngramUnsafe(entry));
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* 内部 UPSERT(无事务包裹)。调用方必须已在外层 transaction 内。
|
|
102
|
+
*
|
|
103
|
+
* 单条写入走 upsertEngram;批量 cold start rebuild 走 rebuildFromEntries,
|
|
104
|
+
* 后者在单个 transaction 里循环调用本方法,避免每条都 BEGIN/COMMIT。
|
|
105
|
+
*/
|
|
106
|
+
upsertEngramUnsafe(entry) {
|
|
107
|
+
this.prepare(`
|
|
108
|
+
INSERT INTO engrams (id, title, kind, importance, confidence, updated_at, content_size, visibility, status)
|
|
109
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
110
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
111
|
+
title = excluded.title,
|
|
112
|
+
kind = excluded.kind,
|
|
113
|
+
importance = excluded.importance,
|
|
114
|
+
confidence = excluded.confidence,
|
|
115
|
+
updated_at = excluded.updated_at,
|
|
116
|
+
content_size = excluded.content_size,
|
|
117
|
+
visibility = excluded.visibility,
|
|
118
|
+
status = excluded.status
|
|
119
|
+
`).run(entry.id, entry.title, entry.kind, entry.importance, entry.confidence, entry.updatedAt, entry.contentSize, entry.visibility, entry.status);
|
|
120
|
+
// domains 全量替换
|
|
121
|
+
this.prepare("DELETE FROM engram_domains WHERE engram_id = ?").run(entry.id);
|
|
122
|
+
const insertDomain = this.prepare("INSERT OR IGNORE INTO engram_domains (engram_id, domain) VALUES (?, ?)");
|
|
123
|
+
for (const d of entry.domainTags) {
|
|
124
|
+
insertDomain.run(entry.id, d);
|
|
125
|
+
}
|
|
126
|
+
// FTS5 不支持 ON CONFLICT,delete + insert
|
|
127
|
+
this.prepare("DELETE FROM engram_fts WHERE id = ?").run(entry.id);
|
|
128
|
+
this.prepare("INSERT INTO engram_fts (id, title, summary, content_tokens) VALUES (?, ?, ?, ?)").run(entry.id, entry.title, entry.summary, entry.contentTokens);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Cold start 重建:用 entries 全量替换 SQLite 索引内容。
|
|
132
|
+
*
|
|
133
|
+
* 用例:host 启动时检测到 .co-engram/index.db 缺失或损坏 → 扫描所有
|
|
134
|
+
* engrams/*.md → 调本方法一次性灌入。
|
|
135
|
+
*
|
|
136
|
+
* 实现:
|
|
137
|
+
* - 单个 transaction 内:DELETE engrams(CASCADE 清 domains + synapses)
|
|
138
|
+
* + DELETE engram_fts(FTS 不参与外键,手动清)+ 批量 upsertEngramUnsafe。
|
|
139
|
+
* - 中途任一步失败 → rollback,SQLite 保持原状(数据可能旧但一致)。
|
|
140
|
+
* - 调用方决定是否清空 synapses:本方法不重建 synapses(它们由 synapse
|
|
141
|
+
* create 路径单独维护),CASCADE 会把它们一并清掉,rebuild 期间 synapse
|
|
142
|
+
* 数据丢失是可接受的(可以从 markdown frontmatter 重新解析)。
|
|
143
|
+
*/
|
|
144
|
+
rebuildFromEntries(entries) {
|
|
145
|
+
this.transaction(() => {
|
|
146
|
+
this.exec("DELETE FROM engrams");
|
|
147
|
+
this.exec("DELETE FROM engram_fts");
|
|
148
|
+
for (const e of entries) {
|
|
149
|
+
this.upsertEngramUnsafe(e);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* UPSERT 一条 synapse。无 FTS,单语句即可,不需要事务包裹。
|
|
155
|
+
*/
|
|
156
|
+
upsertSynapse(s) {
|
|
157
|
+
this.prepare(`
|
|
158
|
+
INSERT INTO synapses (id, from_id, to_id, kind, weight)
|
|
159
|
+
VALUES (?, ?, ?, ?, ?)
|
|
160
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
161
|
+
from_id = excluded.from_id,
|
|
162
|
+
to_id = excluded.to_id,
|
|
163
|
+
kind = excluded.kind,
|
|
164
|
+
weight = excluded.weight
|
|
165
|
+
`).run(s.id, s.fromId, s.toId, s.kind, s.weight);
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* 删除一个 engram 及其所有派生数据。
|
|
169
|
+
*
|
|
170
|
+
* - engrams 主表:显式 DELETE。
|
|
171
|
+
* - engram_domains / synapses(from/to):外键 ON DELETE CASCADE 自动清。
|
|
172
|
+
* - engram_fts:FTS5 表不参与外键,必须显式 DELETE。
|
|
173
|
+
*
|
|
174
|
+
* 包在事务里:即便 FTS 删除失败,主表删除也会 rollback,保持一致。
|
|
175
|
+
*/
|
|
176
|
+
deleteEngram(engramId) {
|
|
177
|
+
this.transaction(() => {
|
|
178
|
+
this.prepare("DELETE FROM engrams WHERE id = ?").run(engramId);
|
|
179
|
+
this.prepare("DELETE FROM engram_fts WHERE id = ?").run(engramId);
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
//# sourceMappingURL=index-db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-db.js","sourceRoot":"","sources":["../../src/storage/index-db.ts"],"names":[],"mappings":"AAAA,wCAAwC;AAExC,4EAA4E;AAC5E,kEAAkE;AAClE,+DAA+D;AAC/D,sFAAsF;AACtF,EAAE;AACF,2DAA2D;AAC3D,4DAA4D;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,CAAC;AACzC,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,CAAiC,CAAC;AAC5E,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;AAO/C,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;AA0C3D;;;;;;;GAOG;AACH,MAAM,OAAO,OAAO;IACV,EAAE,GAAoB,IAAI,CAAC;IAClB,IAAI,CAAiB;IAEtC,YAAY,IAAoB;QAC9B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,EAAE;YAAE,OAAO;QACpB,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,qBAAqB;QACrB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC3C,aAAa;QACb,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QACrB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,EAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,CAAC,GAAW;QACd,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAI,EAAW;QACxB,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC;YACpB,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC;gBACH,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,YAAY,CAAC,KAAuB;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,KAAuB;QAChD,IAAI,CAAC,OAAO,CAAC;;;;;;;;;;;;KAYZ,CAAC,CAAC,GAAG,CACJ,KAAK,CAAC,EAAE,EACR,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,MAAM,CACb,CAAC;QACF,eAAe;QACf,IAAI,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAC/B,wEAAwE,CACzE,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACjC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QACD,uCAAuC;QACvC,IAAI,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,CACV,iFAAiF,CAClF,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,OAAoC;QACrD,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;YACpB,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACpC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,CAAoB;QAChC,IAAI,CAAC,OAAO,CAAC;;;;;;;;KAQZ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAED;;;;;;;;OAQG;IACH,YAAY,CAAC,QAAgB;QAC3B,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;YACpB,IAAI,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/D,IAAI,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/dist/storage/index.d.ts
CHANGED
|
@@ -13,4 +13,7 @@ export * from "./engram-index.js";
|
|
|
13
13
|
export * from "./repository.js";
|
|
14
14
|
export * from "./infra-doctor.js";
|
|
15
15
|
export * from "./index-cleanup.js";
|
|
16
|
+
export { IndexDb } from "./index-db.js";
|
|
17
|
+
export type { SynapseIndexEntry } from "./index-db.js";
|
|
18
|
+
export * from "./bootstrap.js";
|
|
16
19
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AAInC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,cAAc,gBAAgB,CAAC"}
|
package/dist/storage/index.js
CHANGED
|
@@ -13,4 +13,9 @@ export * from "./engram-index.js";
|
|
|
13
13
|
export * from "./repository.js";
|
|
14
14
|
export * from "./infra-doctor.js";
|
|
15
15
|
export * from "./index-cleanup.js";
|
|
16
|
+
// index-db.js 显式 re-export:EngramIndexEntry 与 types/repository-types 的同名,
|
|
17
|
+
// 不能 export *;只 re-export 公共符号,需要 EngramIndexEntry 类型时直接从
|
|
18
|
+
// ./index-db.js import(避免命名冲突 + 保持 SQLite 索引层类型在受控范围)。
|
|
19
|
+
export { IndexDb } from "./index-db.js";
|
|
20
|
+
export * from "./bootstrap.js";
|
|
16
21
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,0EAA0E;AAC1E,0DAA0D;AAC1D,uDAAuD;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,cAAc,gBAAgB,CAAC"}
|
|
@@ -17,6 +17,7 @@ import type { EngramIndexEntry, DoctorReport, PathTreeNode } from "../types/repo
|
|
|
17
17
|
import { type Language } from "../i18n/index.js";
|
|
18
18
|
import { type EngramFile } from "./engram-store.js";
|
|
19
19
|
import { type EngramIndexMap } from "./engram-index.js";
|
|
20
|
+
import { IndexDb } from "./index-db.js";
|
|
20
21
|
/** Repository 配置 */
|
|
21
22
|
export interface RepositoryConfig {
|
|
22
23
|
/** 仓库根目录(~/team-memory/) */
|
|
@@ -61,7 +62,6 @@ export type ExternalMarkdownHook = (params: ExternalMarkdownHookParams) => void;
|
|
|
61
62
|
* 所有读取方法接受 stable id (ULID)。从 path 读 engram 用 readEngramByPath。
|
|
62
63
|
*/
|
|
63
64
|
export declare class EngramRepository {
|
|
64
|
-
private readonly config;
|
|
65
65
|
private indexCache;
|
|
66
66
|
/**
|
|
67
67
|
* 当前 indexCache 对应的 engram-index.json 磁盘 mtime。
|
|
@@ -107,8 +107,36 @@ export declare class EngramRepository {
|
|
|
107
107
|
* 未设置时 → watcher 发现新 .md 仅记录 orphan,不自动接受(noop)。
|
|
108
108
|
*/
|
|
109
109
|
private externalMarkdownHook;
|
|
110
|
+
private readonly config;
|
|
110
111
|
private readonly language;
|
|
111
|
-
|
|
112
|
+
/**
|
|
113
|
+
* 可选 SQLite 索引层(用于 FTS 召回 / 排序)。
|
|
114
|
+
*
|
|
115
|
+
* 由 host adapter(claude-code-mcp / openclaw-plugin)在装配阶段注入。
|
|
116
|
+
* 未注入时(向后兼容)所有写入路径行为不变;注入后,createEngram /
|
|
117
|
+
* updateEngram / deleteEngram / mutateFrontmatter 在文件落盘成功后会
|
|
118
|
+
* 透明地把 engram 投影 upsert / delete 到 SQLite。
|
|
119
|
+
*
|
|
120
|
+
* 写失败由调用方决定是否致命:本 repository 默认 fail-silent(SQLite 是
|
|
121
|
+
* 派生数据,文件源真理仍然有效;doctor 自愈 + cold start rebuild 最终会
|
|
122
|
+
* 修复 SQLite 与文件的不一致)。
|
|
123
|
+
*/
|
|
124
|
+
private readonly indexDb?;
|
|
125
|
+
constructor(config: RepositoryConfig, indexDb?: IndexDb);
|
|
126
|
+
/**
|
|
127
|
+
* 把 EngramFrontmatter + content 投影成 EngramIndexEntry,同步到 SQLite。
|
|
128
|
+
*
|
|
129
|
+
* 字段映射决策:
|
|
130
|
+
* - contentTokens = content 全文。FTS5 trigram 主要用于召回,词频统计
|
|
131
|
+
* 不影响排序;SQLite page cache 自管索引大小。content 通常 < 2KB,
|
|
132
|
+
* 直接全量灌入。
|
|
133
|
+
* - updatedAt 由 ISO string 转 epoch ms(Date.parse),与 IndexDb 的
|
|
134
|
+
* INTEGER 列对齐。
|
|
135
|
+
* - status / visibility / kind 等 union 类型直接 stringify 落 VARCHAR。
|
|
136
|
+
*
|
|
137
|
+
* Fail-silent:SQLite 是派生层,任何写失败都不阻塞文件源真理。
|
|
138
|
+
*/
|
|
139
|
+
private syncEngramToIndex;
|
|
112
140
|
/** 当前写入语言(读取时自动兼容任意语言格式) */
|
|
113
141
|
get currentLanguage(): Language;
|
|
114
142
|
get rootPath(): string;
|