@gmickel/gno 1.45.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +46 -0
  3. package/assets/skill/SKILL.md +7 -6
  4. package/assets/skill/cli-reference.md +14 -6
  5. package/assets/skill/mcp-reference.md +4 -1
  6. package/assets/spa-production.json.gz +0 -0
  7. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
  8. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
  9. package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
  10. package/browser-extension/dist/manifest.json +1 -1
  11. package/browser-extension/dist/preview.html +1 -1
  12. package/browser-extension/dist/service-worker.js +32 -33
  13. package/bunfig.toml +2 -0
  14. package/package.json +40 -26
  15. package/spec/cli.md +30 -11
  16. package/spec/db/schema.sql +146 -1
  17. package/spec/mcp.md +26 -0
  18. package/src/app/context-runtime-types.ts +3 -0
  19. package/src/app/context-runtime.ts +2 -0
  20. package/src/cli/commands/ask.ts +6 -1
  21. package/src/cli/commands/daemon.ts +21 -8
  22. package/src/cli/commands/embed.ts +77 -41
  23. package/src/cli/commands/mcp/install.ts +20 -0
  24. package/src/cli/commands/mcp/paths.ts +25 -0
  25. package/src/cli/commands/mcp/status.ts +6 -0
  26. package/src/cli/detach.ts +3 -2
  27. package/src/cli/program.ts +6 -0
  28. package/src/config/types.ts +3 -3
  29. package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
  30. package/src/converters/adapters/officeparser/adapter.ts +1 -2
  31. package/src/converters/versions.ts +6 -8
  32. package/src/core/context-evidence.ts +8 -4
  33. package/src/core/job-manager.ts +95 -13
  34. package/src/core/network-boundary-inventory.ts +10 -0
  35. package/src/core/shutdown-budget.ts +45 -0
  36. package/src/embed/backlog.ts +107 -4
  37. package/src/embed/batch.ts +42 -2
  38. package/src/embed/fingerprint.ts +16 -0
  39. package/src/embed/retry.ts +113 -5
  40. package/src/embed/variant-backlog.ts +105 -0
  41. package/src/embed/variant-plan.ts +62 -0
  42. package/src/embed/variant-retry.ts +113 -0
  43. package/src/ingestion/graph-reconciliation.ts +327 -0
  44. package/src/ingestion/sync.ts +9 -272
  45. package/src/llm/http-inference.ts +6 -0
  46. package/src/llm/httpEmbedding.ts +37 -6
  47. package/src/llm/httpGeneration.ts +18 -3
  48. package/src/llm/httpRerank.ts +23 -5
  49. package/src/llm/inference-cancellation.ts +168 -0
  50. package/src/llm/inference-scope.ts +202 -0
  51. package/src/llm/lazy-ports.ts +115 -0
  52. package/src/llm/native-worker/client.ts +541 -0
  53. package/src/llm/native-worker/dispatcher.ts +228 -0
  54. package/src/llm/native-worker/embedding-identity.ts +33 -0
  55. package/src/llm/native-worker/entry.ts +173 -0
  56. package/src/llm/native-worker/errors.ts +32 -0
  57. package/src/llm/native-worker/evaluation.ts +16 -0
  58. package/src/llm/native-worker/owned-exit.ts +108 -0
  59. package/src/llm/native-worker/owner.ts +141 -0
  60. package/src/llm/native-worker/ports.ts +317 -0
  61. package/src/llm/native-worker/protocol.ts +442 -0
  62. package/src/llm/native-worker/runtime-config.ts +92 -0
  63. package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
  64. package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
  65. package/src/llm/nodeLlamaCpp/generation.ts +34 -5
  66. package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
  67. package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
  68. package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
  69. package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
  70. package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
  71. package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
  72. package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
  73. package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
  74. package/src/llm/types.ts +35 -5
  75. package/src/mcp/context.ts +27 -0
  76. package/src/mcp/http-transport.ts +12 -10
  77. package/src/mcp/server.ts +3 -0
  78. package/src/mcp/tool-profile.ts +30 -8
  79. package/src/mcp/tools/context.ts +8 -11
  80. package/src/mcp/tools/embed.ts +1 -1
  81. package/src/mcp/tools/index-cmd.ts +1 -1
  82. package/src/mcp/tools/index.ts +10 -8
  83. package/src/mcp/tools/query.ts +14 -30
  84. package/src/mcp/tools/vsearch.ts +1 -1
  85. package/src/pipeline/answer.ts +23 -3
  86. package/src/pipeline/claim-verifier.ts +6 -0
  87. package/src/pipeline/expansion.ts +43 -40
  88. package/src/pipeline/explain.ts +6 -2
  89. package/src/pipeline/filters.ts +63 -0
  90. package/src/pipeline/fusion.ts +29 -9
  91. package/src/pipeline/graph-retrieval.ts +29 -9
  92. package/src/pipeline/hybrid.ts +198 -55
  93. package/src/pipeline/hydration.ts +161 -0
  94. package/src/pipeline/owner-fusion.ts +87 -0
  95. package/src/pipeline/rerank.ts +35 -11
  96. package/src/pipeline/search.ts +13 -2
  97. package/src/pipeline/types.ts +5 -3
  98. package/src/pipeline/vsearch.ts +87 -7
  99. package/src/sdk/client.ts +47 -3
  100. package/src/sdk/embed.ts +63 -39
  101. package/src/serve/background-runtime.ts +1 -1
  102. package/src/serve/context.ts +41 -56
  103. package/src/serve/embed-scheduler.ts +58 -35
  104. package/src/serve/public/components/IndexingProgress.tsx +46 -60
  105. package/src/serve/public/globals.built.css +1 -1
  106. package/src/serve/public/lib/shiki-language-ids.ts +14 -0
  107. package/src/serve/resident-admission.ts +36 -36
  108. package/src/serve/resident-background-work.ts +20 -2
  109. package/src/serve/resident-request.ts +11 -5
  110. package/src/serve/resident-runtime.ts +97 -61
  111. package/src/serve/resident-shutdown.ts +153 -0
  112. package/src/serve/routes/api.ts +3 -1
  113. package/src/serve/server.ts +47 -26
  114. package/src/store/migrations/028-vector-variants.ts +54 -0
  115. package/src/store/migrations/029-graph-reference-state.ts +77 -0
  116. package/src/store/migrations/index.ts +4 -0
  117. package/src/store/sqlite/adapter.ts +251 -183
  118. package/src/store/sqlite/eligibility.ts +174 -0
  119. package/src/store/sqlite/graph-edge-application.ts +66 -0
  120. package/src/store/sqlite/graph-reference-state.ts +194 -0
  121. package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
  122. package/src/store/types.ts +80 -12
  123. package/src/store/vector/eligibility.ts +36 -0
  124. package/src/store/vector/freshness.ts +33 -6
  125. package/src/store/vector/lazy.ts +81 -0
  126. package/src/store/vector/sqlite-vec.ts +106 -54
  127. package/src/store/vector/stats.ts +14 -3
  128. package/src/store/vector/types.ts +35 -2
  129. package/src/store/vector/variant-search.ts +192 -0
  130. package/src/store/vector/variants.ts +451 -0
  131. package/vendor/converters/markitdown-ts/LICENSE +21 -0
  132. package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
  133. package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
  134. package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
  135. package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
  136. package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
  137. package/vendor/converters/markitdown-ts/package.json +77 -0
  138. package/vendor/converters/officeparser/LICENSE +21 -0
  139. package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
  140. package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
  141. package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
  142. package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
  143. package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
  144. package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
  145. package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
  146. package/vendor/converters/officeparser/dist/cli.js +381 -0
  147. package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
  148. package/vendor/converters/officeparser/dist/defaults.js +218 -0
  149. package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
  150. package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
  151. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
  152. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
  153. package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
  154. package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
  155. package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
  156. package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
  157. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
  158. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
  159. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
  160. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
  161. package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
  162. package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
  163. package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
  164. package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
  165. package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
  166. package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
  167. package/vendor/converters/officeparser/dist/index.d.ts +60 -0
  168. package/vendor/converters/officeparser/dist/index.js +72 -0
  169. package/vendor/converters/officeparser/dist/index.mjs +18 -0
  170. package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
  171. package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
  172. package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
  173. package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
  174. package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
  175. package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
  176. package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
  177. package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
  178. package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
  179. package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
  180. package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
  181. package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
  182. package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
  183. package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
  184. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
  185. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
  186. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
  187. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
  188. package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
  189. package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
  190. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
  191. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
  192. package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
  193. package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
  194. package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
  195. package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
  196. package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
  197. package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
  198. package/vendor/converters/officeparser/dist/types.js +107 -0
  199. package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
  200. package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
  201. package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
  202. package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
  203. package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
  204. package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
  205. package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
  206. package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
  207. package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
  208. package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
  209. package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
  210. package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
  211. package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
  212. package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
  213. package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
  214. package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
  215. package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
  216. package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
  217. package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
  218. package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
  219. package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
  220. package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
  221. package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
  222. package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
  223. package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
  224. package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
  225. package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
  226. package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
  227. package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
  228. package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
  229. package/vendor/converters/officeparser/package.json +147 -0
  230. package/vendor/converters/upstream-manifest.json +124 -0
  231. package/vendor/dependency-fixes/README.md +77 -0
  232. package/vendor/dependency-fixes/vendor-converters.py +83 -0
  233. package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip +0 -0
  234. package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip.sha256 +0 -1
  235. package/browser-extension/dist/chunk-627emwpj.js +0 -75
  236. /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
@@ -0,0 +1,174 @@
1
+ import type { Database } from "bun:sqlite";
2
+
3
+ import type { DocumentEligibilityOptions } from "../types";
4
+
5
+ import { matchesExcludedText } from "../../pipeline/exclude";
6
+
7
+ /** Unbounded owner selection; compose with FTS rowids or chunk mirror ownership
8
+ * before ranking/LIMIT. EXISTS keeps multiple tags/scopes from duplicating owners.
9
+ * Retains store filter semantics; lexical runtime language remains reserved. */
10
+ export function buildEligibleDocumentQuery(
11
+ options: DocumentEligibilityOptions,
12
+ db?: Database
13
+ ): { sql: string; params: (string | number)[] } {
14
+ // Build tag filter conditions using EXISTS subqueries
15
+ const conditions: string[] = ["d.active = 1"];
16
+ const params: (string | number)[] = [];
17
+ if (options.chunkLanguage) {
18
+ conditions.push(
19
+ "EXISTS (SELECT 1 FROM content_chunks lc WHERE lc.mirror_hash = d.mirror_hash AND lc.language = ?)"
20
+ );
21
+ params.push(options.chunkLanguage);
22
+ }
23
+
24
+ // tagsAny: document has at least one of these tags
25
+ if (options.tagsAny && options.tagsAny.length > 0) {
26
+ const placeholders = options.tagsAny.map(() => "?").join(",");
27
+ conditions.push(
28
+ `EXISTS (SELECT 1 FROM doc_tags dt WHERE dt.document_id = d.id AND dt.tag IN (${placeholders}))`
29
+ );
30
+ params.push(...options.tagsAny);
31
+ }
32
+
33
+ // tagsAll: document has all of these tags
34
+ if (options.tagsAll && options.tagsAll.length > 0) {
35
+ for (const tag of options.tagsAll) {
36
+ conditions.push(
37
+ "EXISTS (SELECT 1 FROM doc_tags dt WHERE dt.document_id = d.id AND dt.tag = ?)"
38
+ );
39
+ params.push(tag);
40
+ }
41
+ }
42
+
43
+ if (options.since) {
44
+ conditions.push("d.source_mtime >= ?");
45
+ params.push(options.since);
46
+ }
47
+ if (options.until) {
48
+ conditions.push("d.source_mtime <= ?");
49
+ params.push(options.until);
50
+ }
51
+ if (
52
+ !options.semanticMetadata &&
53
+ options.categories &&
54
+ options.categories.length > 0
55
+ ) {
56
+ const placeholders = options.categories.map(() => "?").join(",");
57
+ conditions.push(
58
+ `(d.content_type IN (${placeholders}) OR EXISTS (SELECT 1 FROM json_each(COALESCE(d.categories, '[]')) jc WHERE jc.value IN (${placeholders})))`
59
+ );
60
+ params.push(...options.categories, ...options.categories);
61
+ }
62
+ if (!options.semanticMetadata && options.author) {
63
+ conditions.push("LOWER(COALESCE(d.author, '')) LIKE ?");
64
+ params.push(`%${options.author.toLowerCase()}%`);
65
+ }
66
+
67
+ if (options.allowedMirrorHashes !== undefined) {
68
+ conditions.push("d.mirror_hash IN (SELECT value FROM json_each(?))");
69
+ params.push(JSON.stringify(options.allowedMirrorHashes));
70
+ }
71
+ if (options.collection) {
72
+ conditions.push("d.collection = ?");
73
+ params.push(options.collection);
74
+ }
75
+ if (options.relPathPrefix !== undefined) {
76
+ conditions.push(
77
+ "(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(?) + 1) = ? || '/')"
78
+ );
79
+ params.push(
80
+ options.relPathPrefix,
81
+ options.relPathPrefix,
82
+ options.relPathPrefix
83
+ );
84
+ }
85
+ if (options.memoryScopesAny?.length) {
86
+ const placeholders = options.memoryScopesAny.map(() => "?").join(",");
87
+ conditions.push(
88
+ `EXISTS (SELECT 1 FROM doc_memory_scopes ms WHERE ms.document_id = d.id AND ms.scope IN (${placeholders}))`
89
+ );
90
+ params.push(...options.memoryScopesAny);
91
+ }
92
+ if (options.excludeSuperseded) {
93
+ conditions.push(
94
+ "NOT EXISTS (SELECT 1 FROM doc_edges se JOIN documents sd ON sd.id = se.src_doc_id AND sd.active = 1 WHERE se.dst_doc_id = d.id AND se.edge_type = 'supersedes')"
95
+ );
96
+ }
97
+ if (
98
+ options.exclude?.length ||
99
+ (options.semanticMetadata && (options.author || options.categories?.length))
100
+ ) {
101
+ if (!db)
102
+ throw new Error("Exclusion eligibility requires document metadata");
103
+ // One bulk read, preserving JavaScript Unicode case folding and exclusions
104
+ // across every chunk, including chunks outside a later language selection.
105
+ const rows = db
106
+ .query<
107
+ {
108
+ id: number;
109
+ title: string | null;
110
+ path: string;
111
+ text: string | null;
112
+ author: string | null;
113
+ content_type: string | null;
114
+ categories: string | null;
115
+ },
116
+ (string | number)[]
117
+ >(`
118
+ SELECT d.id, d.title, d.author, d.content_type, d.categories, COALESCE(d.record_source_path, d.rel_path) AS path, cc.text
119
+ FROM documents d LEFT JOIN content_chunks cc ON cc.mirror_hash = d.mirror_hash
120
+ WHERE ${conditions.join(" AND ")}
121
+ `)
122
+ .iterate(...params);
123
+ const denied = new Set<number>();
124
+ for (const row of rows) {
125
+ const categories: unknown =
126
+ options.semanticMetadata || options.excludeMetadata
127
+ ? JSON.parse(row.categories ?? "[]")
128
+ : [];
129
+ if (
130
+ !Array.isArray(categories) ||
131
+ !categories.every((value): value is string => typeof value === "string")
132
+ ) {
133
+ denied.add(row.id);
134
+ continue;
135
+ }
136
+ if (options.semanticMetadata) {
137
+ const wanted = options.categories?.map((value) => value.toLowerCase());
138
+ if (
139
+ (options.author &&
140
+ !(row.author ?? "")
141
+ .toLowerCase()
142
+ .includes(options.author.toLowerCase())) ||
143
+ (wanted?.length &&
144
+ ![row.content_type ?? "", ...categories].some((value) =>
145
+ wanted.includes(value.toLowerCase())
146
+ ))
147
+ ) {
148
+ denied.add(row.id);
149
+ continue;
150
+ }
151
+ }
152
+ if (
153
+ matchesExcludedText(
154
+ [
155
+ row.title ?? "",
156
+ row.path,
157
+ row.text ?? "",
158
+ ...(options.excludeMetadata
159
+ ? [row.author ?? "", row.content_type ?? "", ...categories]
160
+ : []),
161
+ ],
162
+ options.exclude ?? []
163
+ )
164
+ )
165
+ denied.add(row.id);
166
+ }
167
+ conditions.push("d.id NOT IN (SELECT value FROM json_each(?))");
168
+ params.push(JSON.stringify([...denied]));
169
+ }
170
+ return {
171
+ sql: `SELECT d.id, d.mirror_hash FROM documents d WHERE ${conditions.join(" AND ")}`,
172
+ params,
173
+ };
174
+ }
@@ -0,0 +1,66 @@
1
+ import type { Database } from "bun:sqlite";
2
+
3
+ import type { DocEdgeConfidence, DocEdgeSource } from "../types";
4
+
5
+ export interface DesiredGraphEdge {
6
+ sourceId: number;
7
+ targetId: number;
8
+ edgeType: string;
9
+ confidence: DocEdgeConfidence;
10
+ source: DocEdgeSource;
11
+ }
12
+
13
+ /** Apply an exact scoped set without replacing retained edge identities/timestamps.
14
+ * Synchronous staging and application run in one transaction, including nested
15
+ * projection transactions. The temporary table is connection-local scratch only. */
16
+ export function applyGraphEdges(
17
+ db: Database,
18
+ edges: DesiredGraphEdge[],
19
+ sources: DocEdgeSource[],
20
+ sourceIds?: number[]
21
+ ): number {
22
+ return db.transaction(() => {
23
+ db.exec(`CREATE TEMP TABLE IF NOT EXISTS graph_edge_application (
24
+ src_doc_id INTEGER NOT NULL, dst_doc_id INTEGER NOT NULL,
25
+ edge_type TEXT NOT NULL, confidence TEXT NOT NULL, source TEXT NOT NULL,
26
+ PRIMARY KEY(src_doc_id, dst_doc_id, edge_type, source)
27
+ )`);
28
+ db.run("DELETE FROM graph_edge_application");
29
+ const stage = db.query(`INSERT INTO graph_edge_application
30
+ (src_doc_id, dst_doc_id, edge_type, confidence, source) VALUES (?, ?, ?, ?, ?)
31
+ ON CONFLICT(src_doc_id, dst_doc_id, edge_type, source) DO UPDATE SET confidence=excluded.confidence`);
32
+ for (const edge of edges)
33
+ stage.run(
34
+ edge.sourceId,
35
+ edge.targetId,
36
+ edge.edgeType,
37
+ edge.confidence,
38
+ edge.source
39
+ );
40
+ const same =
41
+ "s.src_doc_id=e.src_doc_id AND s.dst_doc_id=e.dst_doc_id AND s.edge_type=e.edge_type AND s.source=e.source";
42
+ db.run(
43
+ `DELETE FROM doc_edges AS e
44
+ WHERE e.source IN (SELECT value FROM json_each(?))
45
+ ${sourceIds ? "AND e.src_doc_id IN (SELECT value FROM json_each(?))" : ""}
46
+ AND NOT EXISTS (SELECT 1 FROM graph_edge_application s WHERE ${same})`,
47
+ sourceIds
48
+ ? [JSON.stringify(sources), JSON.stringify(sourceIds)]
49
+ : [JSON.stringify(sources)]
50
+ );
51
+ db.run(`UPDATE doc_edges AS e SET confidence=(
52
+ SELECT s.confidence FROM graph_edge_application s WHERE ${same})
53
+ WHERE e.src_doc_id IN (SELECT src_doc_id FROM graph_edge_application)
54
+ AND EXISTS (SELECT 1 FROM graph_edge_application s WHERE ${same} AND s.confidence IS NOT e.confidence)`);
55
+ db.run(`INSERT INTO doc_edges(src_doc_id, dst_doc_id, edge_type, confidence, source)
56
+ SELECT s.src_doc_id, s.dst_doc_id, s.edge_type, s.confidence, s.source
57
+ FROM graph_edge_application s
58
+ WHERE NOT EXISTS (SELECT 1 FROM doc_edges e WHERE ${same})
59
+ ORDER BY s.rowid`);
60
+ // Bun run().changes includes trigger side effects; SQLite changes() does not.
61
+ return (
62
+ db.query<{ count: number }, []>("SELECT changes() AS count").get()
63
+ ?.count ?? 0
64
+ );
65
+ })();
66
+ }
@@ -0,0 +1,194 @@
1
+ import type { Database } from "bun:sqlite";
2
+
3
+ import type {
4
+ GraphProjectionState,
5
+ GraphReferenceDocument,
6
+ GraphReferenceInventory,
7
+ GraphReferenceStore,
8
+ } from "../types";
9
+
10
+ import { buildWikiMatchExpression } from "../../core/graph-resolver";
11
+ import { normalizeWikiName } from "../../core/links";
12
+
13
+ const SNAPSHOT_COLUMNS = `document_id AS documentId, collection, rel_path AS relPath,
14
+ docid, uri, title, mirror_hash AS mirrorHash, source_hash AS sourceHash,
15
+ content_type AS contentType`;
16
+
17
+ /** Raw references preserve resolver precedence; resolution belongs to ingestion.
18
+ * Keep old snapshots through deletion/rename until the affected closure is consumed.
19
+ * No second parsed-link inventory: use doc_links alongside these frontmatter rows. */
20
+ export function createGraphReferenceStore(db: Database): GraphReferenceStore {
21
+ function state(
22
+ version: number,
23
+ configFingerprint: string
24
+ ): GraphProjectionState {
25
+ const row = db
26
+ .query<
27
+ {
28
+ epoch: number;
29
+ version: number | null;
30
+ configFingerprint: string | null;
31
+ dirty: number;
32
+ inProgress: number;
33
+ },
34
+ []
35
+ >(`SELECT epoch, version, config_fingerprint AS configFingerprint, dirty, in_progress AS inProgress
36
+ FROM graph_projection_state WHERE id = 1`)
37
+ .get();
38
+ if (!row)
39
+ throw new Error("Missing graph projection state; rebuild required");
40
+ return {
41
+ ...row,
42
+ dirty: row.dirty !== 0,
43
+ inProgress: row.inProgress !== 0,
44
+ complete:
45
+ row.dirty === 0 &&
46
+ row.inProgress === 0 &&
47
+ row.version === version &&
48
+ row.configFingerprint === configFingerprint,
49
+ };
50
+ }
51
+ return {
52
+ state,
53
+ begin(version, configFingerprint) {
54
+ if (!Number.isSafeInteger(version) || version < 1 || !configFingerprint)
55
+ throw new Error("Invalid graph projection identity");
56
+ return db.transaction(() => {
57
+ const previous = state(version, configFingerprint);
58
+ if (
59
+ previous.version !== version ||
60
+ previous.configFingerprint !== configFingerprint
61
+ )
62
+ db.run("DELETE FROM graph_reference_documents");
63
+ db.run(
64
+ `UPDATE graph_projection_state SET epoch = epoch + 1, dirty = 1, in_progress = 1, version = ?, config_fingerprint = ? WHERE id = 1`,
65
+ [version, configFingerprint]
66
+ );
67
+ return state(version, configFingerprint).epoch;
68
+ })();
69
+ },
70
+ readInventory() {
71
+ const docs = db
72
+ .query<GraphReferenceDocument, []>(
73
+ `SELECT ${SNAPSHOT_COLUMNS} FROM graph_reference_documents ORDER BY document_id`
74
+ )
75
+ .all();
76
+ const refs = db
77
+ .query<{ sourceId: number; edgeType: string; target: string }, []>(
78
+ `SELECT source_doc_id AS sourceId, edge_type AS edgeType, target FROM graph_frontmatter_references ORDER BY source_doc_id, ordinal`
79
+ )
80
+ .all();
81
+ const inventories = new Map<number, GraphReferenceInventory>(
82
+ docs.map((document) => [
83
+ document.documentId,
84
+ { document, references: [] },
85
+ ])
86
+ );
87
+ for (const ref of refs)
88
+ inventories
89
+ .get(ref.sourceId)
90
+ ?.references.push({ edgeType: ref.edgeType, target: ref.target });
91
+ return [...inventories.values()];
92
+ },
93
+ incomingLinkSources(identities) {
94
+ if (identities.length === 0) return [];
95
+ const targets = identities.map((d) => ({
96
+ collection: d.collection,
97
+ rel_path: d.relPath,
98
+ title: d.title,
99
+ wiki_title: normalizeWikiName(
100
+ d.title ?? d.relPath.split("/").pop() ?? d.relPath
101
+ ),
102
+ wiki_rel: normalizeWikiName(d.relPath),
103
+ wiki_stem: normalizeWikiName(d.relPath.replace(/\.[^/.]+$/, "")),
104
+ }));
105
+ return db
106
+ .query<{ sourceId: number }, [string]>(`
107
+ WITH targets AS (
108
+ SELECT json_extract(value, '$.collection') AS collection,
109
+ json_extract(value, '$.rel_path') AS rel_path,
110
+ json_extract(value, '$.title') AS title,
111
+ json_extract(value, '$.wiki_title') AS wiki_title,
112
+ json_extract(value, '$.wiki_rel') AS wiki_rel,
113
+ json_extract(value, '$.wiki_stem') AS wiki_stem
114
+ FROM json_each(?)
115
+ )
116
+ SELECT DISTINCT dl.source_doc_id AS sourceId
117
+ FROM doc_links dl JOIN documents src ON src.id = dl.source_doc_id AND src.active = 1
118
+ JOIN targets t ON (
119
+ (t.collection = COALESCE(dl.target_collection, src.collection) AND (
120
+ (dl.link_type = 'markdown' AND dl.target_ref_norm = t.rel_path) OR
121
+ (dl.link_type = 'wiki' AND ${buildWikiMatchExpression("t", "dl.target_ref_norm")})
122
+ )) OR
123
+ (dl.link_type = 'wiki' AND dl.target_ref_norm IN (t.wiki_title, t.wiki_rel, t.wiki_stem))
124
+ OR (dl.link_type = 'markdown' AND t.collection = src.collection AND dl.target_ref_norm = t.rel_path)
125
+ )
126
+ ORDER BY dl.source_doc_id
127
+ `)
128
+ .all(JSON.stringify(targets))
129
+ .map((row) => row.sourceId);
130
+ },
131
+ writeInventory({ document: d, references }) {
132
+ db.transaction(() => {
133
+ db.run("UPDATE graph_projection_state SET dirty = 1 WHERE id = 1");
134
+ db.run(
135
+ `INSERT INTO graph_reference_documents
136
+ (document_id, collection, rel_path, docid, uri, title, mirror_hash, source_hash, content_type)
137
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
138
+ ON CONFLICT(document_id) DO UPDATE SET collection=excluded.collection, rel_path=excluded.rel_path,
139
+ docid=excluded.docid, uri=excluded.uri, title=excluded.title, mirror_hash=excluded.mirror_hash,
140
+ source_hash=excluded.source_hash, content_type=excluded.content_type`,
141
+ [
142
+ d.documentId,
143
+ d.collection,
144
+ d.relPath,
145
+ d.docid,
146
+ d.uri,
147
+ d.title,
148
+ d.mirrorHash,
149
+ d.sourceHash,
150
+ d.contentType,
151
+ ]
152
+ );
153
+ db.run(
154
+ "DELETE FROM graph_frontmatter_references WHERE source_doc_id = ?",
155
+ [d.documentId]
156
+ );
157
+ for (const [ordinal, ref] of references.entries()) {
158
+ db.run(
159
+ "INSERT INTO graph_frontmatter_references(source_doc_id, ordinal, edge_type, target) VALUES (?, ?, ?, ?)",
160
+ [d.documentId, ordinal, ref.edgeType, ref.target]
161
+ );
162
+ }
163
+ })();
164
+ },
165
+ complete(expectedEpoch) {
166
+ db.transaction(() => {
167
+ const row = db
168
+ .query<{ epoch: number; version: number | null }, []>(
169
+ "SELECT epoch, version FROM graph_projection_state WHERE id = 1"
170
+ )
171
+ .get();
172
+ if (!row?.version || row.epoch !== expectedEpoch)
173
+ throw new Error("Graph inputs changed during projection");
174
+ const missing = db
175
+ .query<{ id: number }, []>(`SELECT d.id FROM documents d
176
+ LEFT JOIN graph_reference_documents r ON r.document_id = d.id
177
+ WHERE d.active = 1 AND (r.document_id IS NULL
178
+ OR r.collection IS NOT d.collection OR r.rel_path IS NOT d.rel_path
179
+ OR r.docid IS NOT d.docid OR r.uri IS NOT d.uri OR r.title IS NOT d.title
180
+ OR r.mirror_hash IS NOT d.mirror_hash OR r.source_hash IS NOT d.source_hash
181
+ OR r.content_type IS NOT d.content_type) LIMIT 1`)
182
+ .get();
183
+ if (missing)
184
+ throw new Error("Graph reference inventory is incomplete or stale");
185
+ db.run(
186
+ "DELETE FROM graph_reference_documents WHERE document_id NOT IN (SELECT id FROM documents WHERE active = 1)"
187
+ );
188
+ db.run(
189
+ "UPDATE graph_projection_state SET dirty = 0, in_progress = 0 WHERE id = 1"
190
+ );
191
+ })();
192
+ },
193
+ };
194
+ }
@@ -0,0 +1,79 @@
1
+ /** Conservative input invalidation for providers without verified variants. */
2
+ import type { Database } from "bun:sqlite";
3
+
4
+ import { formatDocForEmbedding } from "../../pipeline/contextual";
5
+
6
+ type TitleRow = { title: string | null };
7
+ export type LegacyTitleSnapshot = Map<string, (string | null)[]>;
8
+
9
+ function activeTitle(db: Database, mirror: string): TitleRow | null {
10
+ return db
11
+ .query<TitleRow, [string]>(`
12
+ SELECT title FROM documents WHERE mirror_hash = ? AND active = 1
13
+ ORDER BY id LIMIT 1
14
+ `)
15
+ .get(mirror);
16
+ }
17
+
18
+ /** Must be captured within the same transaction as the document mutation. */
19
+ export function snapshotLegacyTitles(
20
+ db: Database,
21
+ mirrors: (string | null | undefined)[]
22
+ ): LegacyTitleSnapshot {
23
+ const snapshot: LegacyTitleSnapshot = new Map();
24
+ for (const mirror of new Set(mirrors)) {
25
+ if (!mirror) continue;
26
+ const active = activeTitle(db, mirror);
27
+ // All inactive titles must agree before a restored source can reuse an
28
+ // unproven legacy row. There is no persisted legacy input provenance.
29
+ const titles = active
30
+ ? [active.title]
31
+ : db
32
+ .query<TitleRow, [string]>(
33
+ "SELECT DISTINCT title FROM documents WHERE mirror_hash = ?"
34
+ )
35
+ .all(mirror)
36
+ .map((row) => row.title);
37
+ snapshot.set(mirror, titles);
38
+ }
39
+ return snapshot;
40
+ }
41
+
42
+ /** Only legacy vectors are invalidated; canonical chunks and variants survive. */
43
+ export function reconcileLegacyTitles(
44
+ db: Database,
45
+ before: LegacyTitleSnapshot
46
+ ): void {
47
+ for (const [mirror, titles] of before) {
48
+ const next = activeTitle(db, mirror);
49
+ // Retain vectors when the final owner disappears, for identical restore.
50
+ if (!next || (titles.length === 1 && titles[0] === next.title)) continue;
51
+ const rows = db
52
+ .query<{ seq: number; model: string; text: string }, [string]>(`
53
+ SELECT v.seq, v.model, c.text FROM content_vectors v
54
+ JOIN content_chunks c ON c.mirror_hash = v.mirror_hash AND c.seq = v.seq
55
+ WHERE v.mirror_hash = ?
56
+ `)
57
+ .all(mirror);
58
+ for (const row of rows) {
59
+ const input = formatDocForEmbedding(
60
+ row.text,
61
+ next.title ?? undefined,
62
+ row.model
63
+ );
64
+ if (
65
+ titles.length &&
66
+ titles.every(
67
+ (title) =>
68
+ formatDocForEmbedding(row.text, title ?? undefined, row.model) ===
69
+ input
70
+ )
71
+ )
72
+ continue;
73
+ db.run(
74
+ "DELETE FROM content_vectors WHERE mirror_hash = ? AND seq = ? AND model = ?",
75
+ [mirror, row.seq, row.model]
76
+ );
77
+ }
78
+ }
79
+ }
@@ -615,22 +615,65 @@ export interface IngestErrorInput {
615
615
  // Search Types
616
616
  // ─────────────────────────────────────────────────────────────────────────────
617
617
 
618
- /** Options for FTS search */
619
- export interface FtsSearchOptions {
620
- /** Max results to return */
621
- limit?: number;
618
+ /** Existing SQL owner filters shared by lexical and chunk candidate selection. */
619
+ export interface GraphReferenceDocument {
620
+ documentId: number;
621
+ collection: string;
622
+ relPath: string;
623
+ docid: string;
624
+ uri: string;
625
+ title: string | null;
626
+ mirrorHash: string | null;
627
+ sourceHash: string;
628
+ contentType: string | null;
629
+ }
630
+
631
+ export interface GraphFrontmatterReference {
632
+ edgeType: string;
633
+ target: string;
634
+ }
635
+
636
+ export interface GraphReferenceInventory {
637
+ document: GraphReferenceDocument;
638
+ references: GraphFrontmatterReference[];
639
+ }
640
+
641
+ export interface GraphProjectionState {
642
+ epoch: number;
643
+ version: number | null;
644
+ configFingerprint: string | null;
645
+ dirty: boolean;
646
+ /** Interrupted projection requires full recovery; ordinary input dirtiness does not. */
647
+ inProgress: boolean;
648
+ complete: boolean;
649
+ }
650
+
651
+ /** Synchronous internal operations compose with the adapter transaction. */
652
+ export interface GraphReferenceStore {
653
+ state(version: number, configFingerprint: string): GraphProjectionState;
654
+ begin(version: number, configFingerprint: string): number;
655
+ readInventory(): GraphReferenceInventory[];
656
+ /** Active parsed-link sources matching any old/new target candidate. */
657
+ incomingLinkSources(identities: GraphReferenceDocument[]): number[];
658
+ writeInventory(inventory: GraphReferenceInventory): void;
659
+ complete(expectedEpoch: number): void;
660
+ }
661
+
662
+ export interface DocumentEligibilityOptions {
663
+ /** Internal owner eligibility; public lexical language remains reserved. */
664
+ chunkLanguage?: string;
665
+ /** Include author, content type and categories in whole-owner exclusion. */
666
+ excludeMetadata?: boolean;
667
+ /** Internal vector/hybrid JavaScript metadata matching semantics. */
668
+ semanticMetadata?: boolean;
669
+ /** Internal caller allowlist; undefined is unrestricted, empty denies all. */
670
+ allowedMirrorHashes?: string[];
671
+ /** Whole-document title/path/chunk exclusions, applied before the budget. */
672
+ exclude?: string[];
622
673
  /** Filter by collection */
623
674
  collection?: string;
624
675
  /** Internal exact relative-path boundary applied before ranking and LIMIT. */
625
676
  relPathPrefix?: string;
626
- /**
627
- * Language hint (reserved for future use).
628
- * Note: FTS5 snowball tokenizer is language-aware at index time,
629
- * so runtime language filtering is not currently implemented.
630
- */
631
- language?: string;
632
- /** Include snippet with highlights */
633
- snippet?: boolean;
634
677
  /** Filter to docs with ANY of these tags */
635
678
  tagsAny?: string[];
636
679
  /** Filter to docs with ALL of these tags */
@@ -654,6 +697,19 @@ export interface FtsSearchOptions {
654
697
  * `supersedes` pointing at them). Applied inside the candidate subquery.
655
698
  */
656
699
  excludeSuperseded?: boolean;
700
+ }
701
+
702
+ export interface FtsSearchOptions extends DocumentEligibilityOptions {
703
+ /** Max eligible ranked results to return (filters run before this budget) */
704
+ limit?: number;
705
+ /**
706
+ * Language hint (reserved for future use).
707
+ * Note: FTS5 snowball tokenizer is language-aware at index time,
708
+ * so runtime language filtering is not currently implemented.
709
+ */
710
+ language?: string;
711
+ /** Include snippet with highlights */
712
+ snippet?: boolean;
657
713
  /** Match documents containing ANY positive term instead of ALL of them. */
658
714
  anyTerm?: boolean;
659
715
  }
@@ -1998,6 +2054,15 @@ export interface StorePort {
1998
2054
  mirrorHashes: string[]
1999
2055
  ): Promise<StoreResult<Map<string, ChunkRow[]>>>;
2000
2056
 
2057
+ /**
2058
+ * Optional exact (mirrorHash, seq) batch hydration. Same mapping as
2059
+ * getChunksBatch, but only requested sequences; missing pairs are omitted.
2060
+ * Callers fall back to whole-hash batching when this capability is absent.
2061
+ */
2062
+ getChunksBySequenceBatch?(
2063
+ keys: { mirrorHash: string; seq: number }[]
2064
+ ): Promise<StoreResult<Map<string, ChunkRow[]>>>;
2065
+
2001
2066
  // ─────────────────────────────────────────────────────────────────────────
2002
2067
  // FTS Search
2003
2068
  // ─────────────────────────────────────────────────────────────────────────
@@ -2112,6 +2177,9 @@ export interface StorePort {
2112
2177
  */
2113
2178
  getLinksForDoc(documentId: number): Promise<StoreResult<DocLinkRow[]>>;
2114
2179
 
2180
+ /** Missing capability or stale state requires full graph reconciliation. */
2181
+ graphReferenceStore?(): GraphReferenceStore;
2182
+
2115
2183
  /**
2116
2184
  * Get backlinks pointing to a document.
2117
2185
  * Uses target_ref_norm for matching (wiki=normalized title with path fallbacks, markdown=rel_path).
@@ -0,0 +1,36 @@
1
+ import type { Database } from "bun:sqlite";
2
+
3
+ import type { VectorSearchOptions } from "./types";
4
+
5
+ import { buildEligibleDocumentQuery } from "../sqlite/eligibility";
6
+
7
+ /** Correlated canonical chunk domain; owner predicates stay in the shared builder. */
8
+ export function buildEligibleVectorQuery(
9
+ db: Database,
10
+ options: VectorSearchOptions
11
+ ): { sql: string; params: (string | number)[] } {
12
+ const eligibility = options.eligibility ?? {};
13
+ const owners = buildEligibleDocumentQuery(
14
+ { ...eligibility, semanticMetadata: true },
15
+ db
16
+ );
17
+ const conditions = [
18
+ "cc.mirror_hash = v.mirror_hash",
19
+ "cc.seq = v.seq",
20
+ `cc.mirror_hash IN (SELECT mirror_hash FROM (${owners.sql}))`,
21
+ ];
22
+ const params = [...owners.params];
23
+ if (eligibility.language) {
24
+ conditions.push("cc.language = ?");
25
+ params.push(eligibility.language);
26
+ }
27
+ // Caller scope intersects owner scope; an explicitly empty array denies all.
28
+ if (options.allowedMirrorHashes !== undefined) {
29
+ conditions.push("cc.mirror_hash IN (SELECT value FROM json_each(?))");
30
+ params.push(JSON.stringify(options.allowedMirrorHashes));
31
+ }
32
+ return {
33
+ sql: `SELECT 1 FROM content_chunks cc WHERE ${conditions.join(" AND ")}`,
34
+ params,
35
+ };
36
+ }