@avantf/dsh-mem 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +4 -0
  2. package/lib/client.js +548 -1195
  3. package/lib/dsh-build.json +1 -0
  4. package/lib/engine/graph-recall/in-memory.d.ts +93 -0
  5. package/lib/engine/graph-recall/index.d.ts +22 -0
  6. package/lib/engine/graph-recall/traverse.d.ts +119 -0
  7. package/lib/engine/graph-recall/types.d.ts +188 -0
  8. package/lib/engine/mem/config/config_files.d.ts +1 -1
  9. package/lib/engine/mem/config/loader.d.ts +2 -3
  10. package/lib/engine/mem/config/paths.d.ts +2 -13
  11. package/lib/engine/mem/db/dao/contradictions.d.ts +4 -22
  12. package/lib/engine/mem/db/dao/entities.d.ts +80 -7
  13. package/lib/engine/mem/db/dao/fact_sources.d.ts +3 -13
  14. package/lib/engine/mem/db/dao/facts.d.ts +311 -56
  15. package/lib/engine/mem/db/dao/triples.d.ts +1 -26
  16. package/lib/engine/mem/db/graph.d.ts +249 -0
  17. package/lib/engine/mem/db/journal_notes.d.ts +35 -0
  18. package/lib/engine/mem/db/migrations.d.ts +9 -0
  19. package/lib/engine/mem/db/port.d.ts +22 -3
  20. package/lib/engine/mem/db/schema.d.ts +119 -0
  21. package/lib/engine/mem/db/tokenizer.d.ts +9 -0
  22. package/lib/engine/mem/dbdrivers/fake/driver.d.ts +75 -0
  23. package/lib/engine/mem/dbdrivers/fake/index.d.ts +7 -0
  24. package/lib/engine/mem/dbdrivers/fake/surface.d.ts +34 -0
  25. package/lib/engine/mem/dbdrivers/fake/transport.d.ts +61 -0
  26. package/lib/engine/mem/dbdrivers/sqlite/connection.d.ts +14 -0
  27. package/lib/engine/mem/dbdrivers/sqlite/contradictions.d.ts +35 -0
  28. package/lib/engine/mem/dbdrivers/sqlite/dao/index.d.ts +16 -0
  29. package/lib/engine/mem/dbdrivers/sqlite/driver.d.ts +54 -0
  30. package/lib/engine/mem/dbdrivers/sqlite/facts.d.ts +117 -0
  31. package/lib/engine/mem/dbdrivers/sqlite/index.d.ts +6 -0
  32. package/lib/engine/mem/dbdrivers/sqlite/migrations.d.ts +15 -0
  33. package/lib/engine/mem/dbdrivers/sqlite/relations.d.ts +62 -0
  34. package/lib/engine/mem/dbdrivers/sqlite/search.d.ts +56 -0
  35. package/lib/engine/mem/dbdrivers/sqlite/sources.d.ts +37 -0
  36. package/lib/engine/mem/defaults.d.ts +3 -4
  37. package/lib/engine/mem/dispatch.d.ts +7 -32
  38. package/lib/engine/mem/entities/extract.d.ts +44 -47
  39. package/lib/engine/mem/eval/derived_runner.d.ts +6 -1
  40. package/lib/engine/mem/eval/relation_gold.d.ts +251 -0
  41. package/lib/engine/mem/eval/write_metrics.d.ts +1 -1
  42. package/lib/engine/mem/facts_io.d.ts +50 -10
  43. package/lib/engine/mem/index.d.ts +6 -2
  44. package/lib/engine/mem/interfaces/capabilities.d.ts +30 -0
  45. package/lib/engine/mem/interfaces/common.d.ts +11 -0
  46. package/lib/engine/mem/interfaces/errors.d.ts +65 -0
  47. package/lib/engine/mem/interfaces/index.d.ts +18 -0
  48. package/lib/engine/mem/interfaces/migrations.d.ts +21 -0
  49. package/lib/engine/mem/interfaces/repositories.d.ts +471 -0
  50. package/lib/engine/mem/interfaces/search_index.d.ts +113 -0
  51. package/lib/engine/mem/interfaces/storage_backend.d.ts +84 -0
  52. package/lib/engine/mem/interfaces/storage_driver.d.ts +43 -0
  53. package/lib/engine/mem/interfaces/version.d.ts +34 -0
  54. package/lib/engine/mem/lifecycle/contradiction.d.ts +2 -2
  55. package/lib/engine/mem/modelBootstrap.d.ts +1 -1
  56. package/lib/engine/mem/relations/confidence.d.ts +39 -0
  57. package/lib/engine/mem/relations/extract.d.ts +128 -0
  58. package/lib/engine/mem/relations/index.d.ts +32 -0
  59. package/lib/engine/mem/relations/name-quality.d.ts +78 -0
  60. package/lib/engine/mem/relations/normalize.d.ts +275 -0
  61. package/lib/engine/mem/relations/patterns.d.ts +67 -0
  62. package/lib/engine/mem/relations/statement.d.ts +88 -0
  63. package/lib/engine/mem/relations/types.d.ts +79 -0
  64. package/lib/engine/mem/runtime.d.ts +61 -88
  65. package/lib/engine/mem/storage/create.d.ts +42 -0
  66. package/lib/engine/mem/storage/errors.d.ts +65 -0
  67. package/lib/engine/mem/storage/index.d.ts +15 -0
  68. package/lib/engine/mem/storage/ports.d.ts +472 -0
  69. package/lib/engine/mem/storage/registry.d.ts +37 -0
  70. package/lib/engine/mem/storage/sqlite.d.ts +28 -0
  71. package/lib/engine/mem/store/common.d.ts +1 -1
  72. package/lib/engine/mem/store/entity_leg.d.ts +59 -2
  73. package/lib/engine/mem/store/entity_sweep.d.ts +4 -3
  74. package/lib/engine/mem/store/explicit_relations.d.ts +52 -0
  75. package/lib/engine/mem/store/facts_restore.d.ts +10 -2
  76. package/lib/engine/mem/store/floors.d.ts +4 -1
  77. package/lib/engine/mem/store/graph_reader.d.ts +146 -0
  78. package/lib/engine/mem/store/hybrid.d.ts +50 -13
  79. package/lib/engine/mem/store/kinds.d.ts +47 -0
  80. package/lib/engine/mem/store/legs.d.ts +24 -29
  81. package/lib/engine/mem/store/lexical.d.ts +36 -45
  82. package/lib/engine/mem/store/memory.d.ts +471 -83
  83. package/lib/engine/mem/store/recall_policy.d.ts +49 -0
  84. package/lib/engine/mem/store/relation_backfill.d.ts +210 -0
  85. package/lib/engine/mem/store/relation_planner.d.ts +191 -0
  86. package/lib/engine/mem/store/secret_guard.d.ts +0 -1
  87. package/lib/engine/mem/store/self_query.d.ts +4 -2
  88. package/lib/engine/mem/store/time_filter.d.ts +121 -0
  89. package/lib/engine/mem/store/titles.d.ts +52 -0
  90. package/lib/engine/mem/store/vector_repair.d.ts +44 -54
  91. package/lib/engine/mem-contract/config.d.ts +44 -38
  92. package/lib/engine/mem-contract/text.d.ts +1 -1
  93. package/lib/engine/mem-contract/tools.d.ts +145 -132
  94. package/lib/engine/mem-contract/types.d.ts +364 -288
  95. package/lib/index.js +12048 -11337
  96. package/lib/interface-version.json +1 -1
  97. package/lib/types/client/index.d.ts +2 -2
  98. package/lib/types/client/index.d.ts.map +1 -1
  99. package/lib/types/client/paging.d.ts +3 -12
  100. package/lib/types/client/paging.d.ts.map +1 -1
  101. package/lib/types/client/summarize.d.ts.map +1 -1
  102. package/lib/types/envinit-bootstrap.d.ts +1 -1
  103. package/lib/types/envinit.d.ts +2 -6
  104. package/lib/types/envinit.d.ts.map +1 -1
  105. package/lib/types/hints.d.ts +28 -24
  106. package/lib/types/hints.d.ts.map +1 -1
  107. package/lib/types/index.d.ts +21 -40
  108. package/lib/types/index.d.ts.map +1 -1
  109. package/lib/types/inject.d.ts +3 -2
  110. package/lib/types/inject.d.ts.map +1 -1
  111. package/lib/types/interface_gate.d.ts +1 -1
  112. package/lib/types/prompt.d.ts +17 -41
  113. package/lib/types/prompt.d.ts.map +1 -1
  114. package/lib/types/provision.d.ts +3 -0
  115. package/lib/types/provision.d.ts.map +1 -1
  116. package/lib/types/remote.d.ts +14 -7
  117. package/lib/types/remote.d.ts.map +1 -1
  118. package/lib/types/tool_schema.d.ts +4 -0
  119. package/lib/types/tool_schema.d.ts.map +1 -1
  120. package/lib/types/vectorMigration.d.ts +5 -7
  121. package/lib/types/vectorMigration.d.ts.map +1 -1
  122. package/package.json +15 -14
  123. package/lib/engine/mem/config/domains.d.ts +0 -20
  124. package/lib/engine/mem/db/dao/chunks.d.ts +0 -231
  125. package/lib/engine/mem/db/dao/documents.d.ts +0 -51
  126. package/lib/engine/mem/db/hash.d.ts +0 -2
  127. package/lib/engine/mem/db/knowledge.d.ts +0 -17
  128. package/lib/engine/mem/eval/index.d.ts +0 -4
  129. package/lib/engine/mem/git.d.ts +0 -73
  130. package/lib/engine/mem/router.d.ts +0 -51
  131. package/lib/engine/mem/store/doc_files.d.ts +0 -125
  132. package/lib/engine/mem/store/document_text.d.ts +0 -47
  133. package/lib/engine/mem/store/ingest_guard.d.ts +0 -25
  134. package/lib/engine/mem/store/knowledge.d.ts +0 -598
  135. package/lib/engine/mem/store/source_picker.d.ts +0 -43
  136. package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
  137. package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
  138. package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
  139. package/lib/types/client/domains.d.ts +0 -53
  140. package/lib/types/client/domains.d.ts.map +0 -1
  141. package/lib/types/client/ingest.d.ts +0 -24
  142. package/lib/types/client/ingest.d.ts.map +0 -1
  143. package/lib/types/open.d.ts +0 -77
  144. package/lib/types/open.d.ts.map +0 -1
  145. package/lib/types/reconcile.d.ts +0 -64
  146. package/lib/types/reconcile.d.ts.map +0 -1
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@avantf/dsh-mem",
3
- "version": "0.7.0",
4
- "description": "avantf-mem DSH native Cordis plugin: host service + tools + RPC + client tabs 记忆/知识",
3
+ "version": "0.8.0",
4
+ "description": "avantf-mem DSH native Cordis plugin: host service + memory tools + RPC + client 记忆 tab",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",
7
7
  "types": "lib/types/index.d.ts",
@@ -42,6 +42,7 @@
42
42
  },
43
43
  "peerDependencies": {
44
44
  "@deepseek-ai/cordis": "^4.0.0",
45
+ "@deepseek-ai/dsh-llm": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
45
46
  "@deepseek-ai/dsh-system-prompt": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
46
47
  "@deepseek-ai/dsh-tools": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
47
48
  "@deepseek-ai/dsh-typert-protocol": "^0.1.5-rc.2 || ^0.1.7-rc.2 || ^0.2.0-rc.2",
@@ -52,6 +53,9 @@
52
53
  "@deepseek-ai/cordis": {
53
54
  "optional": true
54
55
  },
56
+ "@deepseek-ai/dsh-llm": {
57
+ "optional": true
58
+ },
55
59
  "@deepseek-ai/dsh-system-prompt": {
56
60
  "optional": true
57
61
  },
@@ -67,8 +71,6 @@
67
71
  },
68
72
  "dependencies": {
69
73
  "@avantf/dsh-plugin-base": ">=0.3.0 <1.0.0",
70
- "exceljs": "^4.4.0",
71
- "unpdf": "^1.8.1",
72
74
  "yaml": "^2.5.0"
73
75
  },
74
76
  "optionalDependencies": {
@@ -78,10 +80,9 @@
78
80
  },
79
81
  "devDependencies": {
80
82
  "@avantf/dsh-plugin-base": ">=0.3.0 <1.0.0",
81
- "@avantf/mem": "0.7.0",
82
- "@avantf/mem-contract": "0.7.0",
83
- "@avantf/mem-convert": "0.7.0",
84
- "@avantf/mem-provision": "0.7.0",
83
+ "@avantf/mem": "0.8.0",
84
+ "@avantf/mem-contract": "0.8.0",
85
+ "@avantf/mem-provision": "0.8.0",
85
86
  "@types/node": "^22.0.0",
86
87
  "@types/react": "^18.3.0",
87
88
  "lightningcss": "^1.32.0",
@@ -98,16 +99,16 @@
98
99
  },
99
100
  "repository": {
100
101
  "type": "git",
101
- "url": "git@gitee.com:ffeng86/dsh-plugins.git",
102
- "directory": "mem/packages/plugin"
102
+ "url": "git@gitee.com:ffeng86/dsh-mem.git",
103
+ "directory": "packages/plugin"
103
104
  },
104
105
  "author": "qunqi.ff",
105
106
  "scripts": {
106
- "build": "node ../../scripts/link-dsh.mjs && node ../../scripts/link-envinit.mjs && tsc -p tsconfig.json && node ../../scripts/copy-envinit-bootstrap.mjs && node ../../scripts/carry-engine-types.mjs && tsdown",
107
+ "build": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-dsh.mjs && node ../../scripts/link-envinit.mjs && tsc -p tsconfig.json && node ../../scripts/copy-envinit-bootstrap.mjs && node ../../scripts/carry-engine-types.mjs && tsdown",
107
108
  "bundle": "tsdown",
108
109
  "watch": "tsdown --watch",
109
- "typecheck": "node ../../scripts/link-dsh.mjs && tsc -p tsconfig.test.json",
110
- "test": "vitest run",
111
- "test:dsh": "vitest run --config vitest.dsh.config.ts"
110
+ "typecheck": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-dsh.mjs && tsc -p tsconfig.test.json",
111
+ "test": "node ../../../build/scripts/link-base.mjs && node ../../scripts/link-envinit.mjs && vitest run",
112
+ "test:dsh": "node ../../../build/scripts/link-base.mjs && vitest run --config vitest.dsh.config.ts"
112
113
  }
113
114
  }
@@ -1,20 +0,0 @@
1
- /**
2
- * Append one 知识域 to the STORE config's top-level `domains` allowlist
3
- * (`~/.avantf/configs/knowledge.yaml`, layer ③ of `loader.ts`) and return the new list.
4
- *
5
- * This is the write half of the UI-only `kbAddDomain` path, so the user adds a domain once in the
6
- * 知识 tab instead of hand-editing YAML. It writes to the STORE config rather than the common one:
7
- * `knowledge.domains` is resolved from layer ③ (the file the loader parses with
8
- * `KnowledgeConfigSchema`), and a second source of truth would make "where do I change it?" a
9
- * question again.
10
- *
11
- * `parseDocument` + `toString` (never `parse`/`stringify`): the comments in that file are the
12
- * documentation for the keys around them, and a round-trip that drops them quietly turns a
13
- * documented config into a bare list. When `domains` is already a sequence the node is mutated IN
14
- * PLACE (`seq.add`), which keeps the comments attached to the key and its items; a missing key is
15
- * created.
16
- *
17
- * The write is temp file + atomic rename, the same shape as the managed document copy
18
- * (`doc_files.ts`): a crash mid-write must not leave a truncated config the next boot cannot parse.
19
- */
20
- export declare function addKnowledgeDomain(configPath: string, domain: string): string[];
@@ -1,231 +0,0 @@
1
- /**
2
- * `doc_chunks` + `chunk_entities` + the `doc_chunks_fts` index — the knowledge read model.
3
- *
4
- * Every id list here is batched (`batches()`): candidate sets are derived from the CORPUS
5
- * (entity overlap, semantic top-k), not from the request, so a plain `IN (…)` would trip
6
- * SQLite's bind-parameter cap on a large store. That failure is silent on the search path
7
- * (it just drops a leg), which is exactly what batching prevents.
8
- */
9
- import type { DocumentChunk } from '../../../mem-contract/index.js';
10
- import type { Db } from '../port.js';
11
- /** One chunk to persist, in document order (`idx` is the position, not the array index). */
12
- interface ChunkInsert {
13
- idx: number;
14
- text: string;
15
- headingsPath: string;
16
- sourceRef: string;
17
- charStart: number;
18
- charEnd: number;
19
- }
20
- /**
21
- * A chunk plus the state of everything DERIVED from it (DESIGN §20).
22
- *
23
- * `content_hash` says which text the derivations were built from, `embedding_model` which
24
- * vector space they live in, `entities_version` which extraction rules produced the entity
25
- * rows. `reindex` decides per row what actually needs redoing from these; without them it has
26
- * to assume every row is stale and pay for the whole corpus.
27
- */
28
- export interface ChunkStateRow {
29
- chunk_id: number;
30
- text: string;
31
- content_hash: string | null;
32
- embedding_model: string | null;
33
- entities_version: number | null;
34
- /** `1` when a non-NULL vector blob is stored (SQLite has no boolean). */
35
- has_vector: number;
36
- }
37
- /** A persisted chunk with the entity names extracted at ingest time. */
38
- interface ChunkEntityRow {
39
- chunk_id: number;
40
- names: readonly string[];
41
- }
42
- /** A search hit row: chunk body plus the owning document's identity. */
43
- export interface ChunkHitRow {
44
- chunk_id: number;
45
- text: string;
46
- source_ref: string;
47
- domain: string;
48
- source: string;
49
- created_at: string | null;
50
- updated_at: string | null;
51
- }
52
- export declare class ChunksDao {
53
- private readonly db;
54
- constructor(db: Db);
55
- idsForDoc(docId: number): number[];
56
- /** Re-ingest replaces the chunk set wholesale (entities/FTS follow via cascade/triggers). */
57
- deleteForDoc(docId: number): void;
58
- /** Insert the doc's chunks in one statement per chunk; returns the new ids in order. */
59
- insertMany(docId: number, chunks: readonly ChunkInsert[]): number[];
60
- /** Chunk bodies by id (entity extraction + vector encoding after ingest). */
61
- texts(ids: readonly number[]): {
62
- chunk_id: number;
63
- text: string;
64
- }[];
65
- /** Chunk rows for the catalog detail view, in document order (`DocumentDetail.chunks`). */
66
- rowsForDoc(docId: number): DocumentChunk[];
67
- /** Persist a chunk vector together with the vector space it belongs to. */
68
- setVector(chunkId: number, bytes: Buffer, embeddingModel: string): void;
69
- /**
70
- * The doc's chunks WITH their stored vectors — the reuse index for a re-ingest.
71
- *
72
- * A re-ingest deletes and re-inserts every chunk (new ids), so without this the only way to keep a
73
- * vector is to re-encode the same text; `vectorReusable` already knows how to tell "unchanged", but
74
- * only `reindex` could use it. Read BEFORE the replace, inside its transaction.
75
- */
76
- vectorStateForDoc(docId: number): (ChunkStateRow & {
77
- vec: Buffer | null;
78
- })[];
79
- /**
80
- * Write a batch of vectors in ONE transaction.
81
- *
82
- * `setVector` is autocommit: each chunk costs a commit (measured 14.2 µs/row against 2.2 µs in a
83
- * transaction, and on a WAL database every commit appends its dirty pages). Callers encode in
84
- * batches anyway, so this is the same loop with the boundary moved out — and the transaction
85
- * NEVER spans an `await` (the caller encodes first, then calls this).
86
- */
87
- setVectors(rows: readonly {
88
- chunk_id: number;
89
- bytes: Buffer;
90
- embeddingModel: string;
91
- }[]): void;
92
- /** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
93
- setEntitiesVersion(ids: readonly number[], version: number): void;
94
- /** Every persisted chunk vector, for rebuilding the in-memory index at open. */
95
- /**
96
- * Every persisted chunk vector, with the space it was written in.
97
- *
98
- * `embedding_model` is selected too (not just the bytes) so a reload can DETECT a model swap: a
99
- * dim-valid vector from another space is still ranked, and without this the two spaces would mix
100
- * in one index silently — the memory side had the check, the knowledge side did not.
101
- */
102
- vectorRows(): {
103
- id: number;
104
- vec: Buffer;
105
- embedding_model: string | null;
106
- }[];
107
- /**
108
- * Vector-space health of the WHOLE chunk corpus, by kind, WITHOUT decoding a single blob.
109
- *
110
- * The same two counts the open-time warning reports (`stale` = wrong width, `space_stale` = same
111
- * width but another recorded space — `embedding_model IS NOT` is null-safe, so a row with no
112
- * recorded space counts as foreign exactly like the JS comparison in `reloadIndex`) plus `missing`
113
- * (no vector at all), which is the third reason the semantic leg cannot use a chunk.
114
- *
115
- * This is what lets the shared repair flow read "what is stale" cheaply per slice instead of
116
- * re-scanning and hashing the corpus (`corpusState` reads every text; a batched migration must
117
- * not).
118
- *
119
- * The JS rule itself is `store/vector_space.ts` (`classifyVectorSpace` / `countVectorSpace`), and
120
- * this projection is bound to it by the cross assertions in
121
- * `test/knowledge_vector_migration.spec.ts` (the startup warning's count, the health count and the
122
- * migration's `remaining` all have to be the same number). NOTHING here changes when the JS rule
123
- * changes: the mirror is checked, not shared.
124
- */
125
- vectorSpaceCounts(space: string, expectedBytes: number): {
126
- stale: number;
127
- space_stale: number;
128
- missing: number;
129
- };
130
- /** Chunks with no persisted vector — the repair flow's `missing` term in `remaining`. */
131
- countMissingVectors(): number;
132
- /**
133
- * Chunks the semantic leg cannot use (no vector / wrong width / another space), bounded and
134
- * ordered by id — the repair flow's encode source.
135
- *
136
- * The content-hash check `vectorReusable` adds is deliberately NOT here: a chunk whose vector is
137
- * in the current space is usable even if its text was edited behind the store's back, and that
138
- * (rarer) case belongs to `kb_reindex`, which re-derives every leg. Repairing the SPACE must not
139
- * pay a jieba/hash pass over the corpus to find rows the space change did not touch.
140
- */
141
- staleVectorRows(space: string, expectedBytes: number, limit: number): {
142
- chunk_id: number;
143
- text: string;
144
- }[];
145
- count(): number;
146
- /**
147
- * Replace the entity rows of the given chunks in one transaction.
148
- *
149
- * `replace` is REQUIRED, not defaulted: `true` clears the old rows first (the entity sweep, where
150
- * the extractor may have changed), `false` is the ingest path, where the chunk ids are freshly
151
- * inserted and an absent old row set. A default would let one caller silently inherit the other's
152
- * semantics — an extra `DELETE` per chunk on ingest, or an append-on-top-of-stale sweep.
153
- */
154
- replaceEntities(rows: readonly ChunkEntityRow[], replace: boolean): void;
155
- /** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
156
- entityBags(ids: readonly number[]): Map<number, string[]>;
157
- /** Chunk body + owning-document identity for the fused candidate ids. */
158
- hits(ids: readonly number[]): ChunkHitRow[];
159
- /** `(chunk_id, domain, source)` for the semantic top-k, to apply the domain/source filter. */
160
- meta(ids: readonly number[]): {
161
- chunk_id: number;
162
- domain: string;
163
- source: string;
164
- }[];
165
- /**
166
- * Corpus rows for the rebuild legs, optionally scoped to one domain, together with the state
167
- * of their derivations. This is the `reindex` source: the caller compares `content_hash` /
168
- * `embedding_model` / `entities_version` against what it is about to produce and only pays
169
- * for the rows that are actually out of date (DESIGN §20).
170
- */
171
- corpusState(domain?: string): ChunkStateRow[];
172
- /**
173
- * Chunk rows whose `chunk_entities` were produced by OLDER extraction rules — the entity sweep's
174
- * selection, bounded and ordered by id.
175
- *
176
- * `COALESCE(entities_version, 0) < :version`, not `entities_version != :version`: the column is
177
- * NULLABLE on purpose (a row written before the derived-state columns existed must look stale
178
- * exactly once and be adopted by the first rebuild — see `ChunkStateRow`), and a bare `<` would
179
- * evaluate to NULL for those rows and let them hide from the sweep forever.
180
- *
181
- * The predicate is index-served in id order (`doc_chunks` is keyed by `chunk_id`), so the LIMIT
182
- * bounds what is READ, not only what is rebuilt — the same property `FactsDao.staleEntityRows`
183
- * documents for the memory side.
184
- */
185
- chunkEntityRows(version: number, limit: number, domain?: string): {
186
- chunk_id: number;
187
- text: string;
188
- }[];
189
- /** How many chunks still carry entity rows from older rules — the sweep's `deferred` source. */
190
- countStaleEntityChunks(version: number, domain?: string): number;
191
- /** Rebuild the FTS index from the content table (write-path maintenance). */
192
- rebuildFts(): void;
193
- /** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
194
- /**
195
- * Entity-overlap candidates, highest-score first and CAPPED (see the memory store's sibling;
196
- * the score/order tail itself lives in `./shared.js#entityCandidateTail`).
197
- *
198
- * `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
199
- * `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
200
- * caller is expected to enforce the real `cap` on the union AFTER scoring it (it re-computes the
201
- * true Jaccard in JS): a union of per-batch tops is not a top-N, and a leg that returns more than
202
- * the cap makes `scores.size === legCap` — the "was this leg trimmed" signal — report the opposite
203
- * of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
204
- */
205
- candidatesByEntityNames(names: readonly string[], domain: string | undefined, source: string | undefined, limit: number, queryWidth: number, widthCap: number): number[];
206
- /**
207
- * How many chunks carry each name — the knowledge side of the entity leg's document frequency.
208
- *
209
- * Mirrors `EntitiesDao.activeDocFrequency`: only the caller's query names are looked up. There is
210
- * no status column to filter on (see `candidatesByEntityNames`), so "document frequency" is over
211
- * all chunks, which is the population the leg ranks.
212
- */
213
- docFrequency(names: readonly string[]): Map<string, number>;
214
- /** FTS leg (`bm25` rank; negated by the caller so higher = better). */
215
- ftsSearch(ftsQuery: string, domain?: string, source?: string, limit?: number): {
216
- id: number;
217
- rank: number;
218
- }[];
219
- /**
220
- * The knowledge store's SHORT-QUERY fallback leg — the sibling of `FactsDao.ftsSubstringSearch`
221
- * (see it for the score/tie/verify notes; the predicate itself is shared through
222
- * `db/tokenizer.ts#likeSubstring`). It reads `doc_chunks` directly rather than the external-content
223
- * FTS table: no index can serve a two-character pattern anyway, and the `documents` join is where
224
- * the domain/source filter lives.
225
- */
226
- ftsSubstringSearch(terms: readonly string[], domain?: string, source?: string, limit?: number): {
227
- id: number;
228
- rank: number;
229
- }[];
230
- }
231
- export {};
@@ -1,51 +0,0 @@
1
- /**
2
- * `documents` — the knowledge corpus identity table (`domain → source → title`).
3
- *
4
- * Identity is the `(domain, source, title)` unique key, so `upsert` is `INSERT OR IGNORE`
5
- * followed by a lookup: a re-ingest of the same document must land on the SAME `doc_id`
6
- * (chunks are replaced under it), never create a second row.
7
- */
8
- import type { Db } from '../port.js';
9
- import type { DocumentRecord, DocumentSummary } from '../../../mem-contract/index.js';
10
- /**
11
- * The stored document rows. The PAYLOAD shapes live in the contract (`kb list`/`kb detail` return
12
- * them verbatim, and the settings page imports them), so these are aliases rather than a second
13
- * declaration — a column added here without the contract would be a type error, not a silent
14
- * extra field on the wire.
15
- */
16
- type DocumentRow = DocumentSummary;
17
- type DocumentDetailRow = DocumentRecord;
18
- export declare class DocumentsDao {
19
- private readonly db;
20
- constructor(db: Db);
21
- /** Idempotent upsert by the unique key; returns the existing or newly created id. */
22
- upsert(domain: string, source: string, title: string, sourceUri?: string): number;
23
- /**
24
- * One row by the unique identity `(domain, source, title)`, or `null` when no such document
25
- * exists. Walks the `UNIQUE(domain, source, title)` index — the add-only guard's existence test.
26
- */
27
- find(domain: string, source: string, title: string): DocumentRow | null;
28
- /**
29
- * Bump `updated_at` and record the URI that was actually read. `COALESCE` keeps the
30
- * previous URI when this ingest did not come from one (plain text has none).
31
- */
32
- touch(docId: number, sourceUri?: string): void;
33
- /**
34
- * Catalog page: newest first, optionally scoped to one domain and/or source.
35
- *
36
- * `limit`/`offset` page it for the UI (which renders one page at a time); omitting `limit` keeps
37
- * the historical "everything" behavior the CLI and MCP surfaces expect.
38
- */
39
- list(domain?: string, source?: string, limit?: number, offset?: number): DocumentRow[];
40
- get(docId: number): DocumentDetailRow | null;
41
- /**
42
- * The distinct domains the library already holds, sorted. This is the "already in the library"
43
- * half of the domain allowlist: a name in use must stay usable (and selectable) even when the
44
- * configured allowlist would not have accepted it as a NEW domain.
45
- */
46
- domains(): string[];
47
- /** `true` when a document was actually deleted (cascades chunks/entities/FTS). */
48
- remove(docId: number): boolean;
49
- count(): number;
50
- }
51
- export {};
@@ -1,2 +0,0 @@
1
- /** Stable content digest (sha1: change detection, not security). */
2
- export declare function contentHash(text: string): string;
@@ -1,17 +0,0 @@
1
- import { type OpenedStore, type StoreSchema } from './store.js';
2
- /** Knowledge DB schema: documents under `domain → source`, split into chunks. */
3
- export declare const KNOWLEDGE_DDL: string[];
4
- /**
5
- * The external-content FTS5 table, created separately so the tokenizer can be
6
- * chosen at open time (same DESIGN §16 mitigation as the memory store).
7
- */
8
- export declare function chunksFtsTableDdl(tokenizer: string): string;
9
- /**
10
- * Ordered schema steps. Step 1 is the base schema and MUST stay idempotent: a database
11
- * created before this mechanism existed reports `user_version = 0` while already holding it.
12
- */
13
- export declare const KNOWLEDGE_SCHEMA: StoreSchema;
14
- /** Open (or create) the knowledge DB, bringing its schema up to the current version. */
15
- export declare function openKnowledgeDb(path: string): OpenedStore['db'];
16
- /** Open and report which migrations ran (tests, and any explicit upgrade path). */
17
- export declare function openKnowledgeStore(path: string): OpenedStore;
@@ -1,4 +0,0 @@
1
- export * from './metrics.js';
2
- export * from './loader.js';
3
- export * from './runner.js';
4
- export * from './write_metrics.js';
@@ -1,73 +0,0 @@
1
- import type { AvantfLogger } from '../mem-contract/index.js';
2
- /**
3
- * A reusable git wrapper for the stores' on-disk artifacts.
4
- *
5
- * It is deliberately root-agnostic: the knowledge corpus (`knowledge.docs.dir`, the default root)
6
- * is one use, and "share a whole store" is another — point `root` at `~/.avantf/knowledge` or
7
- * `~/.avantf` and pass an `ignore` list so the databases stay out of the history. The caller owns
8
- * WHAT is versioned; this owns HOW, and nothing else in the codebase needs to know git exists.
9
- *
10
- * THREE PROPERTIES ARE LOAD-BEARING, and they are why this is not just `execFileSync` at each
11
- * call site:
12
- *
13
- * - **It never throws.** Every caller writes here AFTER its real mission is durable (the row is
14
- * committed, the document is indexed). A missing `git`, a locked index, or an unwritable repo
15
- * must not turn a successful operation into a failed one — so failures are warned and swallowed,
16
- * the same rule `IngestResult.file_error` follows for the managed file itself.
17
- * - **It is always recognisable as automatic, and never invents an identity.** Every commit
18
- * carries the `Automatic: avantf-mem` trailer. When the machine has an identity the commit is
19
- * authored with it (the repo is the user's, and they may commit in it by hand — attributing
20
- * their machine's mission to a bot by default would be the wrong kind of surprise); when it has
21
- * NONE, the repo gets a LOCAL bot identity rather than failing or writing to the user's GLOBAL
22
- * config, which is what lets the commit happen at all.
23
- * - **It never commits an empty change.** `git status --porcelain` gates the commit, because
24
- * `git add` + `git commit` with nothing staged fails, and a store write that changed nothing
25
- * (a re-ingest of identical text) is common.
26
- */
27
- type GitMode = 'auto' | 'off';
28
- interface GitRepoOptions {
29
- /** The directory this repo tracks. `init`/`commit` no-op when it does not exist. */
30
- root: string;
31
- /** `auto` (default) = create the repo on first commit; `off` = never invoke git at all. */
32
- mode?: GitMode;
33
- /**
34
- * Patterns written to `.gitignore` when the repo is CREATED. An existing `.gitignore` is left
35
- * alone. Only matters when `root` is broader than a text directory — e.g. pointing at a store
36
- * directory brings the SQLite files with it, and those must never enter the history.
37
- */
38
- ignore?: readonly string[];
39
- /** Committer used when the machine has no identity. Defaults to the avantf bot. */
40
- identity?: {
41
- name: string;
42
- email: string;
43
- };
44
- logger?: AvantfLogger;
45
- timeoutMs?: number;
46
- }
47
- export declare class GitRepo {
48
- readonly root: string;
49
- private readonly mode;
50
- private readonly ignore;
51
- private readonly identity;
52
- private readonly logger?;
53
- private readonly timeoutMs;
54
- constructor(options: GitRepoOptions);
55
- /** Whether git is in play at all (a disabled repo does nothing, silently and by design). */
56
- get enabled(): boolean;
57
- /**
58
- * Create the repository if it is not one yet (writing `.gitignore` when it does). Idempotent and
59
- * never throws; returns whether a repository exists afterwards.
60
- */
61
- init(): boolean;
62
- /** Stage and commit whatever changed under `root`. No-op when nothing did. Never throws. */
63
- commit(message: string): void;
64
- /** Commit subjects, newest first. `[]` when git is off, absent, or the repo has no commits. */
65
- history(limit?: number): string[];
66
- /** Current branch, or `null` when unavailable (never throws — this is diagnostics only). */
67
- branch(): string | null;
68
- private ensureIdentity;
69
- private writeIgnore;
70
- private warn;
71
- private git;
72
- }
73
- export {};
@@ -1,51 +0,0 @@
1
- import { type RecallHit, type RecallResult, type RetrievalFloorDrops } from '../mem-contract/index.js';
2
- interface CrossQueryOptions {
3
- limit?: number;
4
- kind?: 'all' | 'fact' | 'doc_chunk';
5
- domain?: string;
6
- source?: string;
7
- /**
8
- * Token budget for the merged result; `0` = unlimited, omitted = the caller already bounded
9
- * the legs (the runtime lifts the per-store budgets for the pool and bounds the OUTCOME here,
10
- * because fusion can reorder which entries deserve the text).
11
- */
12
- maxTokens?: number;
13
- /**
14
- * The knowledge leg's floor report. `KnowledgeStore.search` answers with hits only, so the
15
- * runtime hands its captured `HybridResult` drops here; the memory side already arrives on
16
- * `memory.dropped_by_floor`. Without this, the merged result would report only half the floors.
17
- */
18
- kbDroppedByFloor?: RetrievalFloorDrops;
19
- }
20
- /** The memory search result the router derives degradation/weights from. */
21
- type CrossQueryMemory = Pick<RecallResult, 'hits' | 'degraded' | 'weights' | 'floors' | 'dropped_by_floor'>;
22
- /**
23
- * Joint min-max normalization over the MERGED pool — the rule the cross-store score
24
- * contract is defined by (DESIGN §7).
25
- *
26
- * Degenerate case (every candidate carries the same score — including a pool of one):
27
- * the raw score is KEPT. A collapsed range encodes no ranking information, so rescaling
28
- * it would invent a "perfect match" out of an absolute fused score that may say
29
- * otherwise. `minMaxNormalize` maps the same case to 1 instead, which is right there
30
- * because it only ever feeds a relative weighted sum inside ONE fusion path; the two
31
- * differ deliberately rather than by oversight.
32
- *
33
- * Returns COPIES: the inputs are the stores' own result objects, and callers still
34
- * read their original scores.
35
- */
36
- export declare function normalizeMergedScores<T extends {
37
- score: number;
38
- }>(pool: T[]): T[];
39
- /**
40
- * Cross-retrieval router: merge memory + knowledge hits, joint min-max normalize
41
- * the final fused scores over the *combined* pool (so the two stores' scores are
42
- * comparable), then rank by score.
43
- *
44
- * Degradation and weights are inherited from the memory search result rather
45
- * than hardcoded: both stores share the same semantic adapter, so
46
- * `memory.degraded`/`memory.weights` already describe whether the semantic leg
47
- * ran. Reporting a fixed `degraded: true` here made every `kb_query` look
48
- * degraded even when embeddings were live.
49
- */
50
- export declare function crossQuery(memory: CrossQueryMemory, kb: RecallHit[], opts?: CrossQueryOptions): RecallResult;
51
- export {};
@@ -1,125 +0,0 @@
1
- /** The frontmatter a managed file carries; `content_hash` is the body AS INGESTED. */
2
- interface DocFileMeta {
3
- doc_id: number;
4
- domain: string;
5
- source: string;
6
- title: string;
7
- source_uri: string | null;
8
- ingested_at: string;
9
- content_hash: string;
10
- /**
11
- * The `@avantf/mem-convert` converter that produced the body, when the source was converted
12
- * (absent for text and PDF, and for every file written before this field existed — readers must
13
- * treat it as optional, which is why it is not `null`).
14
- */
15
- converter?: string;
16
- }
17
- /** One managed file as the file system sees it now. */
18
- interface DocFileState {
19
- path: string;
20
- stale: boolean;
21
- missing: boolean;
22
- }
23
- /** A parsed managed file: frontmatter (possibly partial — a hand-written file may omit it) + body. */
24
- interface ParsedDocFile {
25
- meta: Partial<DocFileMeta>;
26
- body: string;
27
- }
28
- /** One path segment, made safe and human-readable; `fallback` covers names that sanitize away. */
29
- export declare function sanitizeSegment(raw: string, fallback: string): string;
30
- /**
31
- * A stable title for a body that arrived as TEXT (a paste), where there is no file name to borrow.
32
- *
33
- * Pasted text has no `source_uri` to name it after, and `source` defaults to the same value for
34
- * every paste — so without this, every untitled paste into one domain collapsed onto one identity
35
- * and the second ingest silently replaced the first. The rule: the first Markdown ATX heading,
36
- * else the first non-empty line with its block markers stripped.
37
- *
38
- * Fenced code is stepped over in BOTH halves: `#include` is not a heading, and a fence opener is
39
- * not a title. `MAX_SEGMENT` is the same cap the file name uses, so a long line cannot produce a
40
- * title the managed path has to truncate differently. Empty input falls back to `untitled`, and the
41
- * result depends only on the text (same body → same title), which is what keeps a re-paste
42
- * landing on the SAME identity instead of quietly creating a sibling every time.
43
- */
44
- export declare function deriveDocTitle(text: string): string;
45
- /** sha256 of a document body — the staleness key. */
46
- export declare function bodyHash(body: string): string;
47
- /**
48
- * Render one managed file: frontmatter (JSON-encoded values, so any title/URI survives) + body.
49
- *
50
- * The body is written VERBATIM — no trailing newline is added. `content_hash` records the ingested
51
- * text, and `state` compares it against the file's body, so anything this function normalizes
52
- * would make every document read as edited the moment it was written.
53
- *
54
- * An `undefined` value is OMITTED rather than rendered: JSON.stringify(undefined) is the bare token
55
- * `undefined`, which the tolerant parser would then hand back as the string "undefined". Optional
56
- * keys (today: `converter`) therefore simply do not appear for the documents that lack them.
57
- */
58
- export declare function renderDocFile(meta: DocFileMeta, body: string): string;
59
- /**
60
- * Split one managed file into frontmatter and body.
61
- *
62
- * Deliberately tolerant: a file without frontmatter (or with a hand-mangled one) is ALL body —
63
- * the user's content is never thrown away because a header key is missing, and the caller decides
64
- * what an unknown `doc_id` means.
65
- */
66
- export declare function parseDocFile(text: string): ParsedDocFile;
67
- /**
68
- * The managed document directory.
69
- *
70
- * Paths are `<root>/<domain>/<source>/<title>.md`, sanitized per segment. Collisions resolve by
71
- * asking the EXISTING file who it belongs to: a file whose frontmatter names another `doc_id` is
72
- * left alone and this document takes the `<title>~<doc_id>.md` variant. Without that check two
73
- * documents that sanitize to the same name would take turns clobbering each other.
74
- */
75
- export declare class DocFiles {
76
- private readonly root;
77
- constructor(root: string);
78
- /** The directory holding one document's file (created on write, not here). */
79
- dirFor(domain: string, source: string): string;
80
- /**
81
- * The file NAME for a title, minus the directory: a title that is already a Markdown file name
82
- * (`kb ingest --uri notes.md` names the document after the file) must not become `notes.md.md`.
83
- */
84
- fileName(title: string, suffix?: string): string;
85
- /** The base path for a title (before collision handling). */
86
- basePath(domain: string, source: string, title: string): string;
87
- /**
88
- * The file this document owns: the base path, or its `~<doc_id>` variant when the base path is
89
- * taken by a DIFFERENT document. Purely derived — no state is written to answer this.
90
- */
91
- pathFor(docId: number, domain: string, source: string, title: string): string;
92
- /** Write the managed file for one document (atomic: temp file + rename). Returns its path. */
93
- write(meta: DocFileMeta, body: string): string;
94
- /** Parse one managed file, or `null` when it does not exist. */
95
- read(path: string): ParsedDocFile | null;
96
- /**
97
- * A cheap per-file stamp — `mtimeMs:size`, no reads — or `null` when the file is absent.
98
- *
99
- * This is a TRIGGER, never a judgement: same-second edits, coarse filesystem timestamps and
100
- * `cp -p` can all leave it unchanged, so a matching stamp means "probably unchanged" and only
101
- * `state()` (which hashes the body) decides. The point is that the common case costs one `stat`
102
- * per file instead of reading and hashing every file in the corpus.
103
- */
104
- stamp(path: string): string | null;
105
- /** Every `.md` path under the root. A directory walk WITHOUT reads — no frontmatter, no hashing. */
106
- listPaths(): string[];
107
- /** Is this document's file there, and does it still match what was ingested? */
108
- state(docId: number, domain: string, source: string, title: string): DocFileState;
109
- /** Delete this document's file. `false` when there was nothing to delete. */
110
- remove(docId: number, domain: string, source: string, title: string): boolean;
111
- /**
112
- * Every `.md` under the root, with the `doc_id` its frontmatter names (when it names one).
113
- * Used to report orphans — files left behind by a document that no longer exists.
114
- *
115
- * ASYNC and batched: the walk itself is an async recursive `readdir` (a `readdirSync` gathers
116
- * every entry in one ~41 ms call at 10k files), and the per-file parse is a `read` — at 10k
117
- * documents that is a second full read of the corpus on top of `sync`'s staleness pass, and it
118
- * used to be one uninterrupted synchronous block (performance review §7.5 / P6).
119
- */
120
- scan(): Promise<{
121
- path: string;
122
- doc_id?: number;
123
- }[]>;
124
- }
125
- export {};