@avantf/dsh-mem 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/lib/client.js +548 -1195
- package/lib/dsh-build.json +1 -0
- package/lib/engine/graph-recall/in-memory.d.ts +93 -0
- package/lib/engine/graph-recall/index.d.ts +22 -0
- package/lib/engine/graph-recall/traverse.d.ts +119 -0
- package/lib/engine/graph-recall/types.d.ts +188 -0
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/config/loader.d.ts +2 -3
- package/lib/engine/mem/config/paths.d.ts +2 -13
- package/lib/engine/mem/db/dao/contradictions.d.ts +4 -22
- package/lib/engine/mem/db/dao/entities.d.ts +80 -7
- package/lib/engine/mem/db/dao/fact_sources.d.ts +3 -13
- package/lib/engine/mem/db/dao/facts.d.ts +311 -56
- package/lib/engine/mem/db/dao/triples.d.ts +1 -26
- package/lib/engine/mem/db/graph.d.ts +249 -0
- package/lib/engine/mem/db/journal_notes.d.ts +35 -0
- package/lib/engine/mem/db/migrations.d.ts +9 -0
- package/lib/engine/mem/db/port.d.ts +22 -3
- package/lib/engine/mem/db/schema.d.ts +119 -0
- package/lib/engine/mem/db/tokenizer.d.ts +9 -0
- package/lib/engine/mem/dbdrivers/fake/driver.d.ts +75 -0
- package/lib/engine/mem/dbdrivers/fake/index.d.ts +7 -0
- package/lib/engine/mem/dbdrivers/fake/surface.d.ts +34 -0
- package/lib/engine/mem/dbdrivers/fake/transport.d.ts +61 -0
- package/lib/engine/mem/dbdrivers/sqlite/connection.d.ts +14 -0
- package/lib/engine/mem/dbdrivers/sqlite/contradictions.d.ts +35 -0
- package/lib/engine/mem/dbdrivers/sqlite/dao/index.d.ts +16 -0
- package/lib/engine/mem/dbdrivers/sqlite/driver.d.ts +54 -0
- package/lib/engine/mem/dbdrivers/sqlite/facts.d.ts +117 -0
- package/lib/engine/mem/dbdrivers/sqlite/index.d.ts +6 -0
- package/lib/engine/mem/dbdrivers/sqlite/migrations.d.ts +15 -0
- package/lib/engine/mem/dbdrivers/sqlite/relations.d.ts +62 -0
- package/lib/engine/mem/dbdrivers/sqlite/search.d.ts +56 -0
- package/lib/engine/mem/dbdrivers/sqlite/sources.d.ts +37 -0
- package/lib/engine/mem/defaults.d.ts +3 -4
- package/lib/engine/mem/dispatch.d.ts +7 -32
- package/lib/engine/mem/entities/extract.d.ts +44 -47
- package/lib/engine/mem/eval/derived_runner.d.ts +6 -1
- package/lib/engine/mem/eval/relation_gold.d.ts +251 -0
- package/lib/engine/mem/eval/write_metrics.d.ts +1 -1
- package/lib/engine/mem/facts_io.d.ts +50 -10
- package/lib/engine/mem/index.d.ts +6 -2
- package/lib/engine/mem/interfaces/capabilities.d.ts +30 -0
- package/lib/engine/mem/interfaces/common.d.ts +11 -0
- package/lib/engine/mem/interfaces/errors.d.ts +65 -0
- package/lib/engine/mem/interfaces/index.d.ts +18 -0
- package/lib/engine/mem/interfaces/migrations.d.ts +21 -0
- package/lib/engine/mem/interfaces/repositories.d.ts +471 -0
- package/lib/engine/mem/interfaces/search_index.d.ts +113 -0
- package/lib/engine/mem/interfaces/storage_backend.d.ts +84 -0
- package/lib/engine/mem/interfaces/storage_driver.d.ts +43 -0
- package/lib/engine/mem/interfaces/version.d.ts +34 -0
- package/lib/engine/mem/lifecycle/contradiction.d.ts +2 -2
- package/lib/engine/mem/modelBootstrap.d.ts +1 -1
- package/lib/engine/mem/relations/confidence.d.ts +39 -0
- package/lib/engine/mem/relations/extract.d.ts +128 -0
- package/lib/engine/mem/relations/index.d.ts +32 -0
- package/lib/engine/mem/relations/name-quality.d.ts +78 -0
- package/lib/engine/mem/relations/normalize.d.ts +275 -0
- package/lib/engine/mem/relations/patterns.d.ts +67 -0
- package/lib/engine/mem/relations/statement.d.ts +88 -0
- package/lib/engine/mem/relations/types.d.ts +79 -0
- package/lib/engine/mem/runtime.d.ts +61 -88
- package/lib/engine/mem/storage/create.d.ts +42 -0
- package/lib/engine/mem/storage/errors.d.ts +65 -0
- package/lib/engine/mem/storage/index.d.ts +15 -0
- package/lib/engine/mem/storage/ports.d.ts +472 -0
- package/lib/engine/mem/storage/registry.d.ts +37 -0
- package/lib/engine/mem/storage/sqlite.d.ts +28 -0
- package/lib/engine/mem/store/common.d.ts +1 -1
- package/lib/engine/mem/store/entity_leg.d.ts +59 -2
- package/lib/engine/mem/store/entity_sweep.d.ts +4 -3
- package/lib/engine/mem/store/explicit_relations.d.ts +52 -0
- package/lib/engine/mem/store/facts_restore.d.ts +10 -2
- package/lib/engine/mem/store/floors.d.ts +4 -1
- package/lib/engine/mem/store/graph_reader.d.ts +146 -0
- package/lib/engine/mem/store/hybrid.d.ts +50 -13
- package/lib/engine/mem/store/kinds.d.ts +47 -0
- package/lib/engine/mem/store/legs.d.ts +24 -29
- package/lib/engine/mem/store/lexical.d.ts +36 -45
- package/lib/engine/mem/store/memory.d.ts +471 -83
- package/lib/engine/mem/store/recall_policy.d.ts +49 -0
- package/lib/engine/mem/store/relation_backfill.d.ts +210 -0
- package/lib/engine/mem/store/relation_planner.d.ts +191 -0
- package/lib/engine/mem/store/secret_guard.d.ts +0 -1
- package/lib/engine/mem/store/self_query.d.ts +4 -2
- package/lib/engine/mem/store/time_filter.d.ts +121 -0
- package/lib/engine/mem/store/titles.d.ts +52 -0
- package/lib/engine/mem/store/vector_repair.d.ts +44 -54
- package/lib/engine/mem-contract/config.d.ts +44 -38
- package/lib/engine/mem-contract/text.d.ts +1 -1
- package/lib/engine/mem-contract/tools.d.ts +145 -132
- package/lib/engine/mem-contract/types.d.ts +364 -288
- package/lib/index.js +12048 -11337
- package/lib/interface-version.json +1 -1
- package/lib/types/client/index.d.ts +2 -2
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/client/paging.d.ts +3 -12
- package/lib/types/client/paging.d.ts.map +1 -1
- package/lib/types/client/summarize.d.ts.map +1 -1
- package/lib/types/envinit-bootstrap.d.ts +1 -1
- package/lib/types/envinit.d.ts +2 -6
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/hints.d.ts +28 -24
- package/lib/types/hints.d.ts.map +1 -1
- package/lib/types/index.d.ts +21 -40
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/inject.d.ts +3 -2
- package/lib/types/inject.d.ts.map +1 -1
- package/lib/types/interface_gate.d.ts +1 -1
- package/lib/types/prompt.d.ts +17 -41
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/provision.d.ts +3 -0
- package/lib/types/provision.d.ts.map +1 -1
- package/lib/types/remote.d.ts +14 -7
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/tool_schema.d.ts +4 -0
- package/lib/types/tool_schema.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +5 -7
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +15 -14
- package/lib/engine/mem/config/domains.d.ts +0 -20
- package/lib/engine/mem/db/dao/chunks.d.ts +0 -231
- package/lib/engine/mem/db/dao/documents.d.ts +0 -51
- package/lib/engine/mem/db/hash.d.ts +0 -2
- package/lib/engine/mem/db/knowledge.d.ts +0 -17
- package/lib/engine/mem/eval/index.d.ts +0 -4
- package/lib/engine/mem/git.d.ts +0 -73
- package/lib/engine/mem/router.d.ts +0 -51
- package/lib/engine/mem/store/doc_files.d.ts +0 -125
- package/lib/engine/mem/store/document_text.d.ts +0 -47
- package/lib/engine/mem/store/ingest_guard.d.ts +0 -25
- package/lib/engine/mem/store/knowledge.d.ts +0 -598
- package/lib/engine/mem/store/source_picker.d.ts +0 -43
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
- package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
- package/lib/types/client/domains.d.ts +0 -53
- package/lib/types/client/domains.d.ts.map +0 -1
- package/lib/types/client/ingest.d.ts +0 -24
- package/lib/types/client/ingest.d.ts.map +0 -1
- package/lib/types/open.d.ts +0 -77
- package/lib/types/open.d.ts.map +0 -1
- package/lib/types/reconcile.d.ts +0 -64
- package/lib/types/reconcile.d.ts.map +0 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avantf/dsh-mem",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "avantf-mem DSH native Cordis plugin: host service + tools + RPC + client
|
|
3
|
+
"version": "0.8.0",
|
|
4
|
+
"description": "avantf-mem DSH native Cordis plugin: host service + memory tools + RPC + client 记忆 tab",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"types": "lib/types/index.d.ts",
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
},
|
|
43
43
|
"peerDependencies": {
|
|
44
44
|
"@deepseek-ai/cordis": "^4.0.0",
|
|
45
|
+
"@deepseek-ai/dsh-llm": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
|
|
45
46
|
"@deepseek-ai/dsh-system-prompt": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
|
|
46
47
|
"@deepseek-ai/dsh-tools": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
|
|
47
48
|
"@deepseek-ai/dsh-typert-protocol": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
|
|
@@ -52,6 +53,9 @@
|
|
|
52
53
|
"@deepseek-ai/cordis": {
|
|
53
54
|
"optional": true
|
|
54
55
|
},
|
|
56
|
+
"@deepseek-ai/dsh-llm": {
|
|
57
|
+
"optional": true
|
|
58
|
+
},
|
|
55
59
|
"@deepseek-ai/dsh-system-prompt": {
|
|
56
60
|
"optional": true
|
|
57
61
|
},
|
|
@@ -67,8 +71,6 @@
|
|
|
67
71
|
},
|
|
68
72
|
"dependencies": {
|
|
69
73
|
"@avantf/dsh-plugin-base": ">=0.3.0 <1.0.0",
|
|
70
|
-
"exceljs": "^4.4.0",
|
|
71
|
-
"unpdf": "^1.8.1",
|
|
72
74
|
"yaml": "^2.5.0"
|
|
73
75
|
},
|
|
74
76
|
"optionalDependencies": {
|
|
@@ -78,10 +80,9 @@
|
|
|
78
80
|
},
|
|
79
81
|
"devDependencies": {
|
|
80
82
|
"@avantf/dsh-plugin-base": ">=0.3.0 <1.0.0",
|
|
81
|
-
"@avantf/mem": "0.
|
|
82
|
-
"@avantf/mem-contract": "0.
|
|
83
|
-
"@avantf/mem-
|
|
84
|
-
"@avantf/mem-provision": "0.7.0",
|
|
83
|
+
"@avantf/mem": "0.8.0",
|
|
84
|
+
"@avantf/mem-contract": "0.8.0",
|
|
85
|
+
"@avantf/mem-provision": "0.8.0",
|
|
85
86
|
"@types/node": "^22.0.0",
|
|
86
87
|
"@types/react": "^18.3.0",
|
|
87
88
|
"lightningcss": "^1.32.0",
|
|
@@ -98,16 +99,16 @@
|
|
|
98
99
|
},
|
|
99
100
|
"repository": {
|
|
100
101
|
"type": "git",
|
|
101
|
-
"url": "git@gitee.com:ffeng86/dsh-
|
|
102
|
-
"directory": "
|
|
102
|
+
"url": "git@gitee.com:ffeng86/dsh-mem.git",
|
|
103
|
+
"directory": "packages/plugin"
|
|
103
104
|
},
|
|
104
105
|
"author": "qunqi.ff",
|
|
105
106
|
"scripts": {
|
|
106
|
-
"build": "node ../../scripts/link-dsh.mjs && node ../../scripts/link-envinit.mjs && tsc -p tsconfig.json && node ../../scripts/copy-envinit-bootstrap.mjs && node ../../scripts/carry-engine-types.mjs && tsdown",
|
|
107
|
+
"build": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-dsh.mjs && node ../../scripts/link-envinit.mjs && tsc -p tsconfig.json && node ../../scripts/copy-envinit-bootstrap.mjs && node ../../scripts/carry-engine-types.mjs && tsdown",
|
|
107
108
|
"bundle": "tsdown",
|
|
108
109
|
"watch": "tsdown --watch",
|
|
109
|
-
"typecheck": "node ../../scripts/link-dsh.mjs && tsc -p tsconfig.test.json",
|
|
110
|
-
"test": "vitest run",
|
|
111
|
-
"test:dsh": "vitest run --config vitest.dsh.config.ts"
|
|
110
|
+
"typecheck": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-dsh.mjs && tsc -p tsconfig.test.json",
|
|
111
|
+
"test": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-envinit.mjs && vitest run",
|
|
112
|
+
"test:dsh": "node ../../../build/scripts/link-base.mjs && vitest run --config vitest.dsh.config.ts"
|
|
112
113
|
}
|
|
113
114
|
}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Append one 知识域 to the STORE config's top-level `domains` allowlist
|
|
3
|
-
* (`~/.avantf/configs/knowledge.yaml`, layer ③ of `loader.ts`) and return the new list.
|
|
4
|
-
*
|
|
5
|
-
* This is the write half of the UI-only `kbAddDomain` path, so the user adds a domain once in the
|
|
6
|
-
* 知识 tab instead of hand-editing YAML. It writes to the STORE config rather than the common one:
|
|
7
|
-
* `knowledge.domains` is resolved from layer ③ (the file the loader parses with
|
|
8
|
-
* `KnowledgeConfigSchema`), and a second source of truth would make "where do I change it?" a
|
|
9
|
-
* question again.
|
|
10
|
-
*
|
|
11
|
-
* `parseDocument` + `toString` (never `parse`/`stringify`): the comments in that file are the
|
|
12
|
-
* documentation for the keys around them, and a round-trip that drops them quietly turns a
|
|
13
|
-
* documented config into a bare list. When `domains` is already a sequence the node is mutated IN
|
|
14
|
-
* PLACE (`seq.add`), which keeps the comments attached to the key and its items; a missing key is
|
|
15
|
-
* created.
|
|
16
|
-
*
|
|
17
|
-
* The write is temp file + atomic rename, the same shape as the managed document copy
|
|
18
|
-
* (`doc_files.ts`): a crash mid-write must not leave a truncated config the next boot cannot parse.
|
|
19
|
-
*/
|
|
20
|
-
export declare function addKnowledgeDomain(configPath: string, domain: string): string[];
|
|
@@ -1,231 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `doc_chunks` + `chunk_entities` + the `doc_chunks_fts` index — the knowledge read model.
|
|
3
|
-
*
|
|
4
|
-
* Every id list here is batched (`batches()`): candidate sets are derived from the CORPUS
|
|
5
|
-
* (entity overlap, semantic top-k), not from the request, so a plain `IN (…)` would trip
|
|
6
|
-
* SQLite's bind-parameter cap on a large store. That failure is silent on the search path
|
|
7
|
-
* (it just drops a leg), which is exactly what batching prevents.
|
|
8
|
-
*/
|
|
9
|
-
import type { DocumentChunk } from '../../../mem-contract/index.js';
|
|
10
|
-
import type { Db } from '../port.js';
|
|
11
|
-
/** One chunk to persist, in document order (`idx` is the position, not the array index). */
|
|
12
|
-
interface ChunkInsert {
|
|
13
|
-
idx: number;
|
|
14
|
-
text: string;
|
|
15
|
-
headingsPath: string;
|
|
16
|
-
sourceRef: string;
|
|
17
|
-
charStart: number;
|
|
18
|
-
charEnd: number;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* A chunk plus the state of everything DERIVED from it (DESIGN §20).
|
|
22
|
-
*
|
|
23
|
-
* `content_hash` says which text the derivations were built from, `embedding_model` which
|
|
24
|
-
* vector space they live in, `entities_version` which extraction rules produced the entity
|
|
25
|
-
* rows. `reindex` decides per row what actually needs redoing from these; without them it has
|
|
26
|
-
* to assume every row is stale and pay for the whole corpus.
|
|
27
|
-
*/
|
|
28
|
-
export interface ChunkStateRow {
|
|
29
|
-
chunk_id: number;
|
|
30
|
-
text: string;
|
|
31
|
-
content_hash: string | null;
|
|
32
|
-
embedding_model: string | null;
|
|
33
|
-
entities_version: number | null;
|
|
34
|
-
/** `1` when a non-NULL vector blob is stored (SQLite has no boolean). */
|
|
35
|
-
has_vector: number;
|
|
36
|
-
}
|
|
37
|
-
/** A persisted chunk with the entity names extracted at ingest time. */
|
|
38
|
-
interface ChunkEntityRow {
|
|
39
|
-
chunk_id: number;
|
|
40
|
-
names: readonly string[];
|
|
41
|
-
}
|
|
42
|
-
/** A search hit row: chunk body plus the owning document's identity. */
|
|
43
|
-
export interface ChunkHitRow {
|
|
44
|
-
chunk_id: number;
|
|
45
|
-
text: string;
|
|
46
|
-
source_ref: string;
|
|
47
|
-
domain: string;
|
|
48
|
-
source: string;
|
|
49
|
-
created_at: string | null;
|
|
50
|
-
updated_at: string | null;
|
|
51
|
-
}
|
|
52
|
-
export declare class ChunksDao {
|
|
53
|
-
private readonly db;
|
|
54
|
-
constructor(db: Db);
|
|
55
|
-
idsForDoc(docId: number): number[];
|
|
56
|
-
/** Re-ingest replaces the chunk set wholesale (entities/FTS follow via cascade/triggers). */
|
|
57
|
-
deleteForDoc(docId: number): void;
|
|
58
|
-
/** Insert the doc's chunks in one statement per chunk; returns the new ids in order. */
|
|
59
|
-
insertMany(docId: number, chunks: readonly ChunkInsert[]): number[];
|
|
60
|
-
/** Chunk bodies by id (entity extraction + vector encoding after ingest). */
|
|
61
|
-
texts(ids: readonly number[]): {
|
|
62
|
-
chunk_id: number;
|
|
63
|
-
text: string;
|
|
64
|
-
}[];
|
|
65
|
-
/** Chunk rows for the catalog detail view, in document order (`DocumentDetail.chunks`). */
|
|
66
|
-
rowsForDoc(docId: number): DocumentChunk[];
|
|
67
|
-
/** Persist a chunk vector together with the vector space it belongs to. */
|
|
68
|
-
setVector(chunkId: number, bytes: Buffer, embeddingModel: string): void;
|
|
69
|
-
/**
|
|
70
|
-
* The doc's chunks WITH their stored vectors — the reuse index for a re-ingest.
|
|
71
|
-
*
|
|
72
|
-
* A re-ingest deletes and re-inserts every chunk (new ids), so without this the only way to keep a
|
|
73
|
-
* vector is to re-encode the same text; `vectorReusable` already knows how to tell "unchanged", but
|
|
74
|
-
* only `reindex` could use it. Read BEFORE the replace, inside its transaction.
|
|
75
|
-
*/
|
|
76
|
-
vectorStateForDoc(docId: number): (ChunkStateRow & {
|
|
77
|
-
vec: Buffer | null;
|
|
78
|
-
})[];
|
|
79
|
-
/**
|
|
80
|
-
* Write a batch of vectors in ONE transaction.
|
|
81
|
-
*
|
|
82
|
-
* `setVector` is autocommit: each chunk costs a commit (measured 14.2 µs/row against 2.2 µs in a
|
|
83
|
-
* transaction, and on a WAL database every commit appends its dirty pages). Callers encode in
|
|
84
|
-
* batches anyway, so this is the same loop with the boundary moved out — and the transaction
|
|
85
|
-
* NEVER spans an `await` (the caller encodes first, then calls this).
|
|
86
|
-
*/
|
|
87
|
-
setVectors(rows: readonly {
|
|
88
|
-
chunk_id: number;
|
|
89
|
-
bytes: Buffer;
|
|
90
|
-
embeddingModel: string;
|
|
91
|
-
}[]): void;
|
|
92
|
-
/** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
|
|
93
|
-
setEntitiesVersion(ids: readonly number[], version: number): void;
|
|
94
|
-
/** Every persisted chunk vector, for rebuilding the in-memory index at open. */
|
|
95
|
-
/**
|
|
96
|
-
* Every persisted chunk vector, with the space it was written in.
|
|
97
|
-
*
|
|
98
|
-
* `embedding_model` is selected too (not just the bytes) so a reload can DETECT a model swap: a
|
|
99
|
-
* dim-valid vector from another space is still ranked, and without this the two spaces would mix
|
|
100
|
-
* in one index silently — the memory side had the check, the knowledge side did not.
|
|
101
|
-
*/
|
|
102
|
-
vectorRows(): {
|
|
103
|
-
id: number;
|
|
104
|
-
vec: Buffer;
|
|
105
|
-
embedding_model: string | null;
|
|
106
|
-
}[];
|
|
107
|
-
/**
|
|
108
|
-
* Vector-space health of the WHOLE chunk corpus, by kind, WITHOUT decoding a single blob.
|
|
109
|
-
*
|
|
110
|
-
* The same two counts the open-time warning reports (`stale` = wrong width, `space_stale` = same
|
|
111
|
-
* width but another recorded space — `embedding_model IS NOT` is null-safe, so a row with no
|
|
112
|
-
* recorded space counts as foreign exactly like the JS comparison in `reloadIndex`) plus `missing`
|
|
113
|
-
* (no vector at all), which is the third reason the semantic leg cannot use a chunk.
|
|
114
|
-
*
|
|
115
|
-
* This is what lets the shared repair flow read "what is stale" cheaply per slice instead of
|
|
116
|
-
* re-scanning and hashing the corpus (`corpusState` reads every text; a batched migration must
|
|
117
|
-
* not).
|
|
118
|
-
*
|
|
119
|
-
* The JS rule itself is `store/vector_space.ts` (`classifyVectorSpace` / `countVectorSpace`), and
|
|
120
|
-
* this projection is bound to it by the cross assertions in
|
|
121
|
-
* `test/knowledge_vector_migration.spec.ts` (the startup warning's count, the health count and the
|
|
122
|
-
* migration's `remaining` all have to be the same number). NOTHING here changes when the JS rule
|
|
123
|
-
* changes: the mirror is checked, not shared.
|
|
124
|
-
*/
|
|
125
|
-
vectorSpaceCounts(space: string, expectedBytes: number): {
|
|
126
|
-
stale: number;
|
|
127
|
-
space_stale: number;
|
|
128
|
-
missing: number;
|
|
129
|
-
};
|
|
130
|
-
/** Chunks with no persisted vector — the repair flow's `missing` term in `remaining`. */
|
|
131
|
-
countMissingVectors(): number;
|
|
132
|
-
/**
|
|
133
|
-
* Chunks the semantic leg cannot use (no vector / wrong width / another space), bounded and
|
|
134
|
-
* ordered by id — the repair flow's encode source.
|
|
135
|
-
*
|
|
136
|
-
* The content-hash check `vectorReusable` adds is deliberately NOT here: a chunk whose vector is
|
|
137
|
-
* in the current space is usable even if its text was edited behind the store's back, and that
|
|
138
|
-
* (rarer) case belongs to `kb_reindex`, which re-derives every leg. Repairing the SPACE must not
|
|
139
|
-
* pay a jieba/hash pass over the corpus to find rows the space change did not touch.
|
|
140
|
-
*/
|
|
141
|
-
staleVectorRows(space: string, expectedBytes: number, limit: number): {
|
|
142
|
-
chunk_id: number;
|
|
143
|
-
text: string;
|
|
144
|
-
}[];
|
|
145
|
-
count(): number;
|
|
146
|
-
/**
|
|
147
|
-
* Replace the entity rows of the given chunks in one transaction.
|
|
148
|
-
*
|
|
149
|
-
* `replace` is REQUIRED, not defaulted: `true` clears the old rows first (the entity sweep, where
|
|
150
|
-
* the extractor may have changed), `false` is the ingest path, where the chunk ids are freshly
|
|
151
|
-
* inserted and an absent old row set. A default would let one caller silently inherit the other's
|
|
152
|
-
* semantics — an extra `DELETE` per chunk on ingest, or an append-on-top-of-stale sweep.
|
|
153
|
-
*/
|
|
154
|
-
replaceEntities(rows: readonly ChunkEntityRow[], replace: boolean): void;
|
|
155
|
-
/** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
|
|
156
|
-
entityBags(ids: readonly number[]): Map<number, string[]>;
|
|
157
|
-
/** Chunk body + owning-document identity for the fused candidate ids. */
|
|
158
|
-
hits(ids: readonly number[]): ChunkHitRow[];
|
|
159
|
-
/** `(chunk_id, domain, source)` for the semantic top-k, to apply the domain/source filter. */
|
|
160
|
-
meta(ids: readonly number[]): {
|
|
161
|
-
chunk_id: number;
|
|
162
|
-
domain: string;
|
|
163
|
-
source: string;
|
|
164
|
-
}[];
|
|
165
|
-
/**
|
|
166
|
-
* Corpus rows for the rebuild legs, optionally scoped to one domain, together with the state
|
|
167
|
-
* of their derivations. This is the `reindex` source: the caller compares `content_hash` /
|
|
168
|
-
* `embedding_model` / `entities_version` against what it is about to produce and only pays
|
|
169
|
-
* for the rows that are actually out of date (DESIGN §20).
|
|
170
|
-
*/
|
|
171
|
-
corpusState(domain?: string): ChunkStateRow[];
|
|
172
|
-
/**
|
|
173
|
-
* Chunk rows whose `chunk_entities` were produced by OLDER extraction rules — the entity sweep's
|
|
174
|
-
* selection, bounded and ordered by id.
|
|
175
|
-
*
|
|
176
|
-
* `COALESCE(entities_version, 0) < :version`, not `entities_version != :version`: the column is
|
|
177
|
-
* NULLABLE on purpose (a row written before the derived-state columns existed must look stale
|
|
178
|
-
* exactly once and be adopted by the first rebuild — see `ChunkStateRow`), and a bare `<` would
|
|
179
|
-
* evaluate to NULL for those rows and let them hide from the sweep forever.
|
|
180
|
-
*
|
|
181
|
-
* The predicate is index-served in id order (`doc_chunks` is keyed by `chunk_id`), so the LIMIT
|
|
182
|
-
* bounds what is READ, not only what is rebuilt — the same property `FactsDao.staleEntityRows`
|
|
183
|
-
* documents for the memory side.
|
|
184
|
-
*/
|
|
185
|
-
chunkEntityRows(version: number, limit: number, domain?: string): {
|
|
186
|
-
chunk_id: number;
|
|
187
|
-
text: string;
|
|
188
|
-
}[];
|
|
189
|
-
/** How many chunks still carry entity rows from older rules — the sweep's `deferred` source. */
|
|
190
|
-
countStaleEntityChunks(version: number, domain?: string): number;
|
|
191
|
-
/** Rebuild the FTS index from the content table (write-path maintenance). */
|
|
192
|
-
rebuildFts(): void;
|
|
193
|
-
/** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
|
|
194
|
-
/**
|
|
195
|
-
* Entity-overlap candidates, highest-score first and CAPPED (see the memory store's sibling;
|
|
196
|
-
* the score/order tail itself lives in `./shared.js#entityCandidateTail`).
|
|
197
|
-
*
|
|
198
|
-
* `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
|
|
199
|
-
* `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
|
|
200
|
-
* caller is expected to enforce the real `cap` on the union AFTER scoring it (it re-computes the
|
|
201
|
-
* true Jaccard in JS): a union of per-batch tops is not a top-N, and a leg that returns more than
|
|
202
|
-
* the cap makes `scores.size === legCap` — the "was this leg trimmed" signal — report the opposite
|
|
203
|
-
* of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
|
|
204
|
-
*/
|
|
205
|
-
candidatesByEntityNames(names: readonly string[], domain: string | undefined, source: string | undefined, limit: number, queryWidth: number, widthCap: number): number[];
|
|
206
|
-
/**
|
|
207
|
-
* How many chunks carry each name — the knowledge side of the entity leg's document frequency.
|
|
208
|
-
*
|
|
209
|
-
* Mirrors `EntitiesDao.activeDocFrequency`: only the caller's query names are looked up. There is
|
|
210
|
-
* no status column to filter on (see `candidatesByEntityNames`), so "document frequency" is over
|
|
211
|
-
* all chunks, which is the population the leg ranks.
|
|
212
|
-
*/
|
|
213
|
-
docFrequency(names: readonly string[]): Map<string, number>;
|
|
214
|
-
/** FTS leg (`bm25` rank; negated by the caller so higher = better). */
|
|
215
|
-
ftsSearch(ftsQuery: string, domain?: string, source?: string, limit?: number): {
|
|
216
|
-
id: number;
|
|
217
|
-
rank: number;
|
|
218
|
-
}[];
|
|
219
|
-
/**
|
|
220
|
-
* The knowledge store's SHORT-QUERY fallback leg — the sibling of `FactsDao.ftsSubstringSearch`
|
|
221
|
-
* (see it for the score/tie/verify notes; the predicate itself is shared through
|
|
222
|
-
* `db/tokenizer.ts#likeSubstring`). It reads `doc_chunks` directly rather than the external-content
|
|
223
|
-
* FTS table: no index can serve a two-character pattern anyway, and the `documents` join is where
|
|
224
|
-
* the domain/source filter lives.
|
|
225
|
-
*/
|
|
226
|
-
ftsSubstringSearch(terms: readonly string[], domain?: string, source?: string, limit?: number): {
|
|
227
|
-
id: number;
|
|
228
|
-
rank: number;
|
|
229
|
-
}[];
|
|
230
|
-
}
|
|
231
|
-
export {};
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `documents` — the knowledge corpus identity table (`domain → source → title`).
|
|
3
|
-
*
|
|
4
|
-
* Identity is the `(domain, source, title)` unique key, so `upsert` is `INSERT OR IGNORE`
|
|
5
|
-
* followed by a lookup: a re-ingest of the same document must land on the SAME `doc_id`
|
|
6
|
-
* (chunks are replaced under it), never create a second row.
|
|
7
|
-
*/
|
|
8
|
-
import type { Db } from '../port.js';
|
|
9
|
-
import type { DocumentRecord, DocumentSummary } from '../../../mem-contract/index.js';
|
|
10
|
-
/**
|
|
11
|
-
* The stored document rows. The PAYLOAD shapes live in the contract (`kb list`/`kb detail` return
|
|
12
|
-
* them verbatim, and the settings page imports them), so these are aliases rather than a second
|
|
13
|
-
* declaration — a column added here without the contract would be a type error, not a silent
|
|
14
|
-
* extra field on the wire.
|
|
15
|
-
*/
|
|
16
|
-
type DocumentRow = DocumentSummary;
|
|
17
|
-
type DocumentDetailRow = DocumentRecord;
|
|
18
|
-
export declare class DocumentsDao {
|
|
19
|
-
private readonly db;
|
|
20
|
-
constructor(db: Db);
|
|
21
|
-
/** Idempotent upsert by the unique key; returns the existing or newly created id. */
|
|
22
|
-
upsert(domain: string, source: string, title: string, sourceUri?: string): number;
|
|
23
|
-
/**
|
|
24
|
-
* One row by the unique identity `(domain, source, title)`, or `null` when no such document
|
|
25
|
-
* exists. Walks the `UNIQUE(domain, source, title)` index — the add-only guard's existence test.
|
|
26
|
-
*/
|
|
27
|
-
find(domain: string, source: string, title: string): DocumentRow | null;
|
|
28
|
-
/**
|
|
29
|
-
* Bump `updated_at` and record the URI that was actually read. `COALESCE` keeps the
|
|
30
|
-
* previous URI when this ingest did not come from one (plain text has none).
|
|
31
|
-
*/
|
|
32
|
-
touch(docId: number, sourceUri?: string): void;
|
|
33
|
-
/**
|
|
34
|
-
* Catalog page: newest first, optionally scoped to one domain and/or source.
|
|
35
|
-
*
|
|
36
|
-
* `limit`/`offset` page it for the UI (which renders one page at a time); omitting `limit` keeps
|
|
37
|
-
* the historical "everything" behavior the CLI and MCP surfaces expect.
|
|
38
|
-
*/
|
|
39
|
-
list(domain?: string, source?: string, limit?: number, offset?: number): DocumentRow[];
|
|
40
|
-
get(docId: number): DocumentDetailRow | null;
|
|
41
|
-
/**
|
|
42
|
-
* The distinct domains the library already holds, sorted. This is the "already in the library"
|
|
43
|
-
* half of the domain allowlist: a name in use must stay usable (and selectable) even when the
|
|
44
|
-
* configured allowlist would not have accepted it as a NEW domain.
|
|
45
|
-
*/
|
|
46
|
-
domains(): string[];
|
|
47
|
-
/** `true` when a document was actually deleted (cascades chunks/entities/FTS). */
|
|
48
|
-
remove(docId: number): boolean;
|
|
49
|
-
count(): number;
|
|
50
|
-
}
|
|
51
|
-
export {};
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { type OpenedStore, type StoreSchema } from './store.js';
|
|
2
|
-
/** Knowledge DB schema: documents under `domain → source`, split into chunks. */
|
|
3
|
-
export declare const KNOWLEDGE_DDL: string[];
|
|
4
|
-
/**
|
|
5
|
-
* The external-content FTS5 table, created separately so the tokenizer can be
|
|
6
|
-
* chosen at open time (same DESIGN §16 mitigation as the memory store).
|
|
7
|
-
*/
|
|
8
|
-
export declare function chunksFtsTableDdl(tokenizer: string): string;
|
|
9
|
-
/**
|
|
10
|
-
* Ordered schema steps. Step 1 is the base schema and MUST stay idempotent: a database
|
|
11
|
-
* created before this mechanism existed reports `user_version = 0` while already holding it.
|
|
12
|
-
*/
|
|
13
|
-
export declare const KNOWLEDGE_SCHEMA: StoreSchema;
|
|
14
|
-
/** Open (or create) the knowledge DB, bringing its schema up to the current version. */
|
|
15
|
-
export declare function openKnowledgeDb(path: string): OpenedStore['db'];
|
|
16
|
-
/** Open and report which migrations ran (tests, and any explicit upgrade path). */
|
|
17
|
-
export declare function openKnowledgeStore(path: string): OpenedStore;
|
package/lib/engine/mem/git.d.ts
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
import type { AvantfLogger } from '../mem-contract/index.js';
|
|
2
|
-
/**
|
|
3
|
-
* A reusable git wrapper for the stores' on-disk artifacts.
|
|
4
|
-
*
|
|
5
|
-
* It is deliberately root-agnostic: the knowledge corpus (`knowledge.docs.dir`, the default root)
|
|
6
|
-
* is one use, and "share a whole store" is another — point `root` at `~/.avantf/knowledge` or
|
|
7
|
-
* `~/.avantf` and pass an `ignore` list so the databases stay out of the history. The caller owns
|
|
8
|
-
* WHAT is versioned; this owns HOW, and nothing else in the codebase needs to know git exists.
|
|
9
|
-
*
|
|
10
|
-
* THREE PROPERTIES ARE LOAD-BEARING, and they are why this is not just `execFileSync` at each
|
|
11
|
-
* call site:
|
|
12
|
-
*
|
|
13
|
-
* - **It never throws.** Every caller writes here AFTER its real mission is durable (the row is
|
|
14
|
-
* committed, the document is indexed). A missing `git`, a locked index, or an unwritable repo
|
|
15
|
-
* must not turn a successful operation into a failed one — so failures are warned and swallowed,
|
|
16
|
-
* the same rule `IngestResult.file_error` follows for the managed file itself.
|
|
17
|
-
* - **It is always recognisable as automatic, and never invents an identity.** Every commit
|
|
18
|
-
* carries the `Automatic: avantf-mem` trailer. When the machine has an identity the commit is
|
|
19
|
-
* authored with it (the repo is the user's, and they may commit in it by hand — attributing
|
|
20
|
-
* their machine's mission to a bot by default would be the wrong kind of surprise); when it has
|
|
21
|
-
* NONE, the repo gets a LOCAL bot identity rather than failing or writing to the user's GLOBAL
|
|
22
|
-
* config, which is what lets the commit happen at all.
|
|
23
|
-
* - **It never commits an empty change.** `git status --porcelain` gates the commit, because
|
|
24
|
-
* `git add` + `git commit` with nothing staged fails, and a store write that changed nothing
|
|
25
|
-
* (a re-ingest of identical text) is common.
|
|
26
|
-
*/
|
|
27
|
-
type GitMode = 'auto' | 'off';
|
|
28
|
-
interface GitRepoOptions {
|
|
29
|
-
/** The directory this repo tracks. `init`/`commit` no-op when it does not exist. */
|
|
30
|
-
root: string;
|
|
31
|
-
/** `auto` (default) = create the repo on first commit; `off` = never invoke git at all. */
|
|
32
|
-
mode?: GitMode;
|
|
33
|
-
/**
|
|
34
|
-
* Patterns written to `.gitignore` when the repo is CREATED. An existing `.gitignore` is left
|
|
35
|
-
* alone. Only matters when `root` is broader than a text directory — e.g. pointing at a store
|
|
36
|
-
* directory brings the SQLite files with it, and those must never enter the history.
|
|
37
|
-
*/
|
|
38
|
-
ignore?: readonly string[];
|
|
39
|
-
/** Committer used when the machine has no identity. Defaults to the avantf bot. */
|
|
40
|
-
identity?: {
|
|
41
|
-
name: string;
|
|
42
|
-
email: string;
|
|
43
|
-
};
|
|
44
|
-
logger?: AvantfLogger;
|
|
45
|
-
timeoutMs?: number;
|
|
46
|
-
}
|
|
47
|
-
export declare class GitRepo {
|
|
48
|
-
readonly root: string;
|
|
49
|
-
private readonly mode;
|
|
50
|
-
private readonly ignore;
|
|
51
|
-
private readonly identity;
|
|
52
|
-
private readonly logger?;
|
|
53
|
-
private readonly timeoutMs;
|
|
54
|
-
constructor(options: GitRepoOptions);
|
|
55
|
-
/** Whether git is in play at all (a disabled repo does nothing, silently and by design). */
|
|
56
|
-
get enabled(): boolean;
|
|
57
|
-
/**
|
|
58
|
-
* Create the repository if it is not one yet (writing `.gitignore` when it does). Idempotent and
|
|
59
|
-
* never throws; returns whether a repository exists afterwards.
|
|
60
|
-
*/
|
|
61
|
-
init(): boolean;
|
|
62
|
-
/** Stage and commit whatever changed under `root`. No-op when nothing did. Never throws. */
|
|
63
|
-
commit(message: string): void;
|
|
64
|
-
/** Commit subjects, newest first. `[]` when git is off, absent, or the repo has no commits. */
|
|
65
|
-
history(limit?: number): string[];
|
|
66
|
-
/** Current branch, or `null` when unavailable (never throws — this is diagnostics only). */
|
|
67
|
-
branch(): string | null;
|
|
68
|
-
private ensureIdentity;
|
|
69
|
-
private writeIgnore;
|
|
70
|
-
private warn;
|
|
71
|
-
private git;
|
|
72
|
-
}
|
|
73
|
-
export {};
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { type RecallHit, type RecallResult, type RetrievalFloorDrops } from '../mem-contract/index.js';
|
|
2
|
-
interface CrossQueryOptions {
|
|
3
|
-
limit?: number;
|
|
4
|
-
kind?: 'all' | 'fact' | 'doc_chunk';
|
|
5
|
-
domain?: string;
|
|
6
|
-
source?: string;
|
|
7
|
-
/**
|
|
8
|
-
* Token budget for the merged result; `0` = unlimited, omitted = the caller already bounded
|
|
9
|
-
* the legs (the runtime lifts the per-store budgets for the pool and bounds the OUTCOME here,
|
|
10
|
-
* because fusion can reorder which entries deserve the text).
|
|
11
|
-
*/
|
|
12
|
-
maxTokens?: number;
|
|
13
|
-
/**
|
|
14
|
-
* The knowledge leg's floor report. `KnowledgeStore.search` answers with hits only, so the
|
|
15
|
-
* runtime hands its captured `HybridResult` drops here; the memory side already arrives on
|
|
16
|
-
* `memory.dropped_by_floor`. Without this, the merged result would report only half the floors.
|
|
17
|
-
*/
|
|
18
|
-
kbDroppedByFloor?: RetrievalFloorDrops;
|
|
19
|
-
}
|
|
20
|
-
/** The memory search result the router derives degradation/weights from. */
|
|
21
|
-
type CrossQueryMemory = Pick<RecallResult, 'hits' | 'degraded' | 'weights' | 'floors' | 'dropped_by_floor'>;
|
|
22
|
-
/**
|
|
23
|
-
* Joint min-max normalization over the MERGED pool — the rule the cross-store score
|
|
24
|
-
* contract is defined by (DESIGN §7).
|
|
25
|
-
*
|
|
26
|
-
* Degenerate case (every candidate carries the same score — including a pool of one):
|
|
27
|
-
* the raw score is KEPT. A collapsed range encodes no ranking information, so rescaling
|
|
28
|
-
* it would invent a "perfect match" out of an absolute fused score that may say
|
|
29
|
-
* otherwise. `minMaxNormalize` maps the same case to 1 instead, which is right there
|
|
30
|
-
* because it only ever feeds a relative weighted sum inside ONE fusion path; the two
|
|
31
|
-
* differ deliberately rather than by oversight.
|
|
32
|
-
*
|
|
33
|
-
* Returns COPIES: the inputs are the stores' own result objects, and callers still
|
|
34
|
-
* read their original scores.
|
|
35
|
-
*/
|
|
36
|
-
export declare function normalizeMergedScores<T extends {
|
|
37
|
-
score: number;
|
|
38
|
-
}>(pool: T[]): T[];
|
|
39
|
-
/**
|
|
40
|
-
* Cross-retrieval router: merge memory + knowledge hits, joint min-max normalize
|
|
41
|
-
* the final fused scores over the *combined* pool (so the two stores' scores are
|
|
42
|
-
* comparable), then rank by score.
|
|
43
|
-
*
|
|
44
|
-
* Degradation and weights are inherited from the memory search result rather
|
|
45
|
-
* than hardcoded: both stores share the same semantic adapter, so
|
|
46
|
-
* `memory.degraded`/`memory.weights` already describe whether the semantic leg
|
|
47
|
-
* ran. Reporting a fixed `degraded: true` here made every `kb_query` look
|
|
48
|
-
* degraded even when embeddings were live.
|
|
49
|
-
*/
|
|
50
|
-
export declare function crossQuery(memory: CrossQueryMemory, kb: RecallHit[], opts?: CrossQueryOptions): RecallResult;
|
|
51
|
-
export {};
|
|
@@ -1,125 +0,0 @@
|
|
|
1
|
-
/** The frontmatter a managed file carries; `content_hash` is the body AS INGESTED. */
|
|
2
|
-
interface DocFileMeta {
|
|
3
|
-
doc_id: number;
|
|
4
|
-
domain: string;
|
|
5
|
-
source: string;
|
|
6
|
-
title: string;
|
|
7
|
-
source_uri: string | null;
|
|
8
|
-
ingested_at: string;
|
|
9
|
-
content_hash: string;
|
|
10
|
-
/**
|
|
11
|
-
* The `@avantf/mem-convert` converter that produced the body, when the source was converted
|
|
12
|
-
* (absent for text and PDF, and for every file written before this field existed — readers must
|
|
13
|
-
* treat it as optional, which is why it is not `null`).
|
|
14
|
-
*/
|
|
15
|
-
converter?: string;
|
|
16
|
-
}
|
|
17
|
-
/** One managed file as the file system sees it now. */
|
|
18
|
-
interface DocFileState {
|
|
19
|
-
path: string;
|
|
20
|
-
stale: boolean;
|
|
21
|
-
missing: boolean;
|
|
22
|
-
}
|
|
23
|
-
/** A parsed managed file: frontmatter (possibly partial — a hand-written file may omit it) + body. */
|
|
24
|
-
interface ParsedDocFile {
|
|
25
|
-
meta: Partial<DocFileMeta>;
|
|
26
|
-
body: string;
|
|
27
|
-
}
|
|
28
|
-
/** One path segment, made safe and human-readable; `fallback` covers names that sanitize away. */
|
|
29
|
-
export declare function sanitizeSegment(raw: string, fallback: string): string;
|
|
30
|
-
/**
|
|
31
|
-
* A stable title for a body that arrived as TEXT (a paste), where there is no file name to borrow.
|
|
32
|
-
*
|
|
33
|
-
* Pasted text has no `source_uri` to name it after, and `source` defaults to the same value for
|
|
34
|
-
* every paste — so without this, every untitled paste into one domain collapsed onto one identity
|
|
35
|
-
* and the second ingest silently replaced the first. The rule: the first Markdown ATX heading,
|
|
36
|
-
* else the first non-empty line with its block markers stripped.
|
|
37
|
-
*
|
|
38
|
-
* Fenced code is stepped over in BOTH halves: `#include` is not a heading, and a fence opener is
|
|
39
|
-
* not a title. `MAX_SEGMENT` is the same cap the file name uses, so a long line cannot produce a
|
|
40
|
-
* title the managed path has to truncate differently. Empty input falls back to `untitled`, and the
|
|
41
|
-
* result depends only on the text (same body → same title), which is what keeps a re-paste
|
|
42
|
-
* landing on the SAME identity instead of quietly creating a sibling every time.
|
|
43
|
-
*/
|
|
44
|
-
export declare function deriveDocTitle(text: string): string;
|
|
45
|
-
/** sha256 of a document body — the staleness key. */
|
|
46
|
-
export declare function bodyHash(body: string): string;
|
|
47
|
-
/**
|
|
48
|
-
* Render one managed file: frontmatter (JSON-encoded values, so any title/URI survives) + body.
|
|
49
|
-
*
|
|
50
|
-
* The body is written VERBATIM — no trailing newline is added. `content_hash` records the ingested
|
|
51
|
-
* text, and `state` compares it against the file's body, so anything this function normalizes
|
|
52
|
-
* would make every document read as edited the moment it was written.
|
|
53
|
-
*
|
|
54
|
-
* An `undefined` value is OMITTED rather than rendered: JSON.stringify(undefined) is the bare token
|
|
55
|
-
* `undefined`, which the tolerant parser would then hand back as the string "undefined". Optional
|
|
56
|
-
* keys (today: `converter`) therefore simply do not appear for the documents that lack them.
|
|
57
|
-
*/
|
|
58
|
-
export declare function renderDocFile(meta: DocFileMeta, body: string): string;
|
|
59
|
-
/**
|
|
60
|
-
* Split one managed file into frontmatter and body.
|
|
61
|
-
*
|
|
62
|
-
* Deliberately tolerant: a file without frontmatter (or with a hand-mangled one) is ALL body —
|
|
63
|
-
* the user's content is never thrown away because a header key is missing, and the caller decides
|
|
64
|
-
* what an unknown `doc_id` means.
|
|
65
|
-
*/
|
|
66
|
-
export declare function parseDocFile(text: string): ParsedDocFile;
|
|
67
|
-
/**
|
|
68
|
-
* The managed document directory.
|
|
69
|
-
*
|
|
70
|
-
* Paths are `<root>/<domain>/<source>/<title>.md`, sanitized per segment. Collisions resolve by
|
|
71
|
-
* asking the EXISTING file who it belongs to: a file whose frontmatter names another `doc_id` is
|
|
72
|
-
* left alone and this document takes the `<title>~<doc_id>.md` variant. Without that check two
|
|
73
|
-
* documents that sanitize to the same name would take turns clobbering each other.
|
|
74
|
-
*/
|
|
75
|
-
export declare class DocFiles {
|
|
76
|
-
private readonly root;
|
|
77
|
-
constructor(root: string);
|
|
78
|
-
/** The directory holding one document's file (created on write, not here). */
|
|
79
|
-
dirFor(domain: string, source: string): string;
|
|
80
|
-
/**
|
|
81
|
-
* The file NAME for a title, minus the directory: a title that is already a Markdown file name
|
|
82
|
-
* (`kb ingest --uri notes.md` names the document after the file) must not become `notes.md.md`.
|
|
83
|
-
*/
|
|
84
|
-
fileName(title: string, suffix?: string): string;
|
|
85
|
-
/** The base path for a title (before collision handling). */
|
|
86
|
-
basePath(domain: string, source: string, title: string): string;
|
|
87
|
-
/**
|
|
88
|
-
* The file this document owns: the base path, or its `~<doc_id>` variant when the base path is
|
|
89
|
-
* taken by a DIFFERENT document. Purely derived — no state is written to answer this.
|
|
90
|
-
*/
|
|
91
|
-
pathFor(docId: number, domain: string, source: string, title: string): string;
|
|
92
|
-
/** Write the managed file for one document (atomic: temp file + rename). Returns its path. */
|
|
93
|
-
write(meta: DocFileMeta, body: string): string;
|
|
94
|
-
/** Parse one managed file, or `null` when it does not exist. */
|
|
95
|
-
read(path: string): ParsedDocFile | null;
|
|
96
|
-
/**
|
|
97
|
-
* A cheap per-file stamp — `mtimeMs:size`, no reads — or `null` when the file is absent.
|
|
98
|
-
*
|
|
99
|
-
* This is a TRIGGER, never a judgement: same-second edits, coarse filesystem timestamps and
|
|
100
|
-
* `cp -p` can all leave it unchanged, so a matching stamp means "probably unchanged" and only
|
|
101
|
-
* `state()` (which hashes the body) decides. The point is that the common case costs one `stat`
|
|
102
|
-
* per file instead of reading and hashing every file in the corpus.
|
|
103
|
-
*/
|
|
104
|
-
stamp(path: string): string | null;
|
|
105
|
-
/** Every `.md` path under the root. A directory walk WITHOUT reads — no frontmatter, no hashing. */
|
|
106
|
-
listPaths(): string[];
|
|
107
|
-
/** Is this document's file there, and does it still match what was ingested? */
|
|
108
|
-
state(docId: number, domain: string, source: string, title: string): DocFileState;
|
|
109
|
-
/** Delete this document's file. `false` when there was nothing to delete. */
|
|
110
|
-
remove(docId: number, domain: string, source: string, title: string): boolean;
|
|
111
|
-
/**
|
|
112
|
-
* Every `.md` under the root, with the `doc_id` its frontmatter names (when it names one).
|
|
113
|
-
* Used to report orphans — files left behind by a document that no longer exists.
|
|
114
|
-
*
|
|
115
|
-
* ASYNC and batched: the walk itself is an async recursive `readdir` (a `readdirSync` gathers
|
|
116
|
-
* every entry in one ~41 ms call at 10k files), and the per-file parse is a `read` — at 10k
|
|
117
|
-
* documents that is a second full read of the corpus on top of `sync`'s staleness pass, and it
|
|
118
|
-
* used to be one uninterrupted synchronous block (performance review §7.5 / P6).
|
|
119
|
-
*/
|
|
120
|
-
scan(): Promise<{
|
|
121
|
-
path: string;
|
|
122
|
-
doc_id?: number;
|
|
123
|
-
}[]>;
|
|
124
|
-
}
|
|
125
|
-
export {};
|