@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
3
|
+
import type { DocumentEligibilityOptions } from "../types";
|
|
4
|
+
|
|
5
|
+
import { matchesExcludedText } from "../../pipeline/exclude";
|
|
6
|
+
|
|
7
|
+
/** Unbounded owner selection; compose with FTS rowids or chunk mirror ownership
|
|
8
|
+
* before ranking/LIMIT. EXISTS keeps multiple tags/scopes from duplicating owners.
|
|
9
|
+
* Retains store filter semantics; lexical runtime language remains reserved. */
|
|
10
|
+
export function buildEligibleDocumentQuery(
|
|
11
|
+
options: DocumentEligibilityOptions,
|
|
12
|
+
db?: Database
|
|
13
|
+
): { sql: string; params: (string | number)[] } {
|
|
14
|
+
// Build tag filter conditions using EXISTS subqueries
|
|
15
|
+
const conditions: string[] = ["d.active = 1"];
|
|
16
|
+
const params: (string | number)[] = [];
|
|
17
|
+
if (options.chunkLanguage) {
|
|
18
|
+
conditions.push(
|
|
19
|
+
"EXISTS (SELECT 1 FROM content_chunks lc WHERE lc.mirror_hash = d.mirror_hash AND lc.language = ?)"
|
|
20
|
+
);
|
|
21
|
+
params.push(options.chunkLanguage);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// tagsAny: document has at least one of these tags
|
|
25
|
+
if (options.tagsAny && options.tagsAny.length > 0) {
|
|
26
|
+
const placeholders = options.tagsAny.map(() => "?").join(",");
|
|
27
|
+
conditions.push(
|
|
28
|
+
`EXISTS (SELECT 1 FROM doc_tags dt WHERE dt.document_id = d.id AND dt.tag IN (${placeholders}))`
|
|
29
|
+
);
|
|
30
|
+
params.push(...options.tagsAny);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// tagsAll: document has all of these tags
|
|
34
|
+
if (options.tagsAll && options.tagsAll.length > 0) {
|
|
35
|
+
for (const tag of options.tagsAll) {
|
|
36
|
+
conditions.push(
|
|
37
|
+
"EXISTS (SELECT 1 FROM doc_tags dt WHERE dt.document_id = d.id AND dt.tag = ?)"
|
|
38
|
+
);
|
|
39
|
+
params.push(tag);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (options.since) {
|
|
44
|
+
conditions.push("d.source_mtime >= ?");
|
|
45
|
+
params.push(options.since);
|
|
46
|
+
}
|
|
47
|
+
if (options.until) {
|
|
48
|
+
conditions.push("d.source_mtime <= ?");
|
|
49
|
+
params.push(options.until);
|
|
50
|
+
}
|
|
51
|
+
if (
|
|
52
|
+
!options.semanticMetadata &&
|
|
53
|
+
options.categories &&
|
|
54
|
+
options.categories.length > 0
|
|
55
|
+
) {
|
|
56
|
+
const placeholders = options.categories.map(() => "?").join(",");
|
|
57
|
+
conditions.push(
|
|
58
|
+
`(d.content_type IN (${placeholders}) OR EXISTS (SELECT 1 FROM json_each(COALESCE(d.categories, '[]')) jc WHERE jc.value IN (${placeholders})))`
|
|
59
|
+
);
|
|
60
|
+
params.push(...options.categories, ...options.categories);
|
|
61
|
+
}
|
|
62
|
+
if (!options.semanticMetadata && options.author) {
|
|
63
|
+
conditions.push("LOWER(COALESCE(d.author, '')) LIKE ?");
|
|
64
|
+
params.push(`%${options.author.toLowerCase()}%`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (options.allowedMirrorHashes !== undefined) {
|
|
68
|
+
conditions.push("d.mirror_hash IN (SELECT value FROM json_each(?))");
|
|
69
|
+
params.push(JSON.stringify(options.allowedMirrorHashes));
|
|
70
|
+
}
|
|
71
|
+
if (options.collection) {
|
|
72
|
+
conditions.push("d.collection = ?");
|
|
73
|
+
params.push(options.collection);
|
|
74
|
+
}
|
|
75
|
+
if (options.relPathPrefix !== undefined) {
|
|
76
|
+
conditions.push(
|
|
77
|
+
"(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(?) + 1) = ? || '/')"
|
|
78
|
+
);
|
|
79
|
+
params.push(
|
|
80
|
+
options.relPathPrefix,
|
|
81
|
+
options.relPathPrefix,
|
|
82
|
+
options.relPathPrefix
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
if (options.memoryScopesAny?.length) {
|
|
86
|
+
const placeholders = options.memoryScopesAny.map(() => "?").join(",");
|
|
87
|
+
conditions.push(
|
|
88
|
+
`EXISTS (SELECT 1 FROM doc_memory_scopes ms WHERE ms.document_id = d.id AND ms.scope IN (${placeholders}))`
|
|
89
|
+
);
|
|
90
|
+
params.push(...options.memoryScopesAny);
|
|
91
|
+
}
|
|
92
|
+
if (options.excludeSuperseded) {
|
|
93
|
+
conditions.push(
|
|
94
|
+
"NOT EXISTS (SELECT 1 FROM doc_edges se JOIN documents sd ON sd.id = se.src_doc_id AND sd.active = 1 WHERE se.dst_doc_id = d.id AND se.edge_type = 'supersedes')"
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
if (
|
|
98
|
+
options.exclude?.length ||
|
|
99
|
+
(options.semanticMetadata && (options.author || options.categories?.length))
|
|
100
|
+
) {
|
|
101
|
+
if (!db)
|
|
102
|
+
throw new Error("Exclusion eligibility requires document metadata");
|
|
103
|
+
// One bulk read, preserving JavaScript Unicode case folding and exclusions
|
|
104
|
+
// across every chunk, including chunks outside a later language selection.
|
|
105
|
+
const rows = db
|
|
106
|
+
.query<
|
|
107
|
+
{
|
|
108
|
+
id: number;
|
|
109
|
+
title: string | null;
|
|
110
|
+
path: string;
|
|
111
|
+
text: string | null;
|
|
112
|
+
author: string | null;
|
|
113
|
+
content_type: string | null;
|
|
114
|
+
categories: string | null;
|
|
115
|
+
},
|
|
116
|
+
(string | number)[]
|
|
117
|
+
>(`
|
|
118
|
+
SELECT d.id, d.title, d.author, d.content_type, d.categories, COALESCE(d.record_source_path, d.rel_path) AS path, cc.text
|
|
119
|
+
FROM documents d LEFT JOIN content_chunks cc ON cc.mirror_hash = d.mirror_hash
|
|
120
|
+
WHERE ${conditions.join(" AND ")}
|
|
121
|
+
`)
|
|
122
|
+
.iterate(...params);
|
|
123
|
+
const denied = new Set<number>();
|
|
124
|
+
for (const row of rows) {
|
|
125
|
+
const categories: unknown =
|
|
126
|
+
options.semanticMetadata || options.excludeMetadata
|
|
127
|
+
? JSON.parse(row.categories ?? "[]")
|
|
128
|
+
: [];
|
|
129
|
+
if (
|
|
130
|
+
!Array.isArray(categories) ||
|
|
131
|
+
!categories.every((value): value is string => typeof value === "string")
|
|
132
|
+
) {
|
|
133
|
+
denied.add(row.id);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (options.semanticMetadata) {
|
|
137
|
+
const wanted = options.categories?.map((value) => value.toLowerCase());
|
|
138
|
+
if (
|
|
139
|
+
(options.author &&
|
|
140
|
+
!(row.author ?? "")
|
|
141
|
+
.toLowerCase()
|
|
142
|
+
.includes(options.author.toLowerCase())) ||
|
|
143
|
+
(wanted?.length &&
|
|
144
|
+
![row.content_type ?? "", ...categories].some((value) =>
|
|
145
|
+
wanted.includes(value.toLowerCase())
|
|
146
|
+
))
|
|
147
|
+
) {
|
|
148
|
+
denied.add(row.id);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (
|
|
153
|
+
matchesExcludedText(
|
|
154
|
+
[
|
|
155
|
+
row.title ?? "",
|
|
156
|
+
row.path,
|
|
157
|
+
row.text ?? "",
|
|
158
|
+
...(options.excludeMetadata
|
|
159
|
+
? [row.author ?? "", row.content_type ?? "", ...categories]
|
|
160
|
+
: []),
|
|
161
|
+
],
|
|
162
|
+
options.exclude ?? []
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
denied.add(row.id);
|
|
166
|
+
}
|
|
167
|
+
conditions.push("d.id NOT IN (SELECT value FROM json_each(?))");
|
|
168
|
+
params.push(JSON.stringify([...denied]));
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
sql: `SELECT d.id, d.mirror_hash FROM documents d WHERE ${conditions.join(" AND ")}`,
|
|
172
|
+
params,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
3
|
+
import type { DocEdgeConfidence, DocEdgeSource } from "../types";
|
|
4
|
+
|
|
5
|
+
export interface DesiredGraphEdge {
|
|
6
|
+
sourceId: number;
|
|
7
|
+
targetId: number;
|
|
8
|
+
edgeType: string;
|
|
9
|
+
confidence: DocEdgeConfidence;
|
|
10
|
+
source: DocEdgeSource;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Apply an exact scoped set without replacing retained edge identities/timestamps.
|
|
14
|
+
* Synchronous staging and application run in one transaction, including nested
|
|
15
|
+
* projection transactions. The temporary table is connection-local scratch only. */
|
|
16
|
+
export function applyGraphEdges(
|
|
17
|
+
db: Database,
|
|
18
|
+
edges: DesiredGraphEdge[],
|
|
19
|
+
sources: DocEdgeSource[],
|
|
20
|
+
sourceIds?: number[]
|
|
21
|
+
): number {
|
|
22
|
+
return db.transaction(() => {
|
|
23
|
+
db.exec(`CREATE TEMP TABLE IF NOT EXISTS graph_edge_application (
|
|
24
|
+
src_doc_id INTEGER NOT NULL, dst_doc_id INTEGER NOT NULL,
|
|
25
|
+
edge_type TEXT NOT NULL, confidence TEXT NOT NULL, source TEXT NOT NULL,
|
|
26
|
+
PRIMARY KEY(src_doc_id, dst_doc_id, edge_type, source)
|
|
27
|
+
)`);
|
|
28
|
+
db.run("DELETE FROM graph_edge_application");
|
|
29
|
+
const stage = db.query(`INSERT INTO graph_edge_application
|
|
30
|
+
(src_doc_id, dst_doc_id, edge_type, confidence, source) VALUES (?, ?, ?, ?, ?)
|
|
31
|
+
ON CONFLICT(src_doc_id, dst_doc_id, edge_type, source) DO UPDATE SET confidence=excluded.confidence`);
|
|
32
|
+
for (const edge of edges)
|
|
33
|
+
stage.run(
|
|
34
|
+
edge.sourceId,
|
|
35
|
+
edge.targetId,
|
|
36
|
+
edge.edgeType,
|
|
37
|
+
edge.confidence,
|
|
38
|
+
edge.source
|
|
39
|
+
);
|
|
40
|
+
const same =
|
|
41
|
+
"s.src_doc_id=e.src_doc_id AND s.dst_doc_id=e.dst_doc_id AND s.edge_type=e.edge_type AND s.source=e.source";
|
|
42
|
+
db.run(
|
|
43
|
+
`DELETE FROM doc_edges AS e
|
|
44
|
+
WHERE e.source IN (SELECT value FROM json_each(?))
|
|
45
|
+
${sourceIds ? "AND e.src_doc_id IN (SELECT value FROM json_each(?))" : ""}
|
|
46
|
+
AND NOT EXISTS (SELECT 1 FROM graph_edge_application s WHERE ${same})`,
|
|
47
|
+
sourceIds
|
|
48
|
+
? [JSON.stringify(sources), JSON.stringify(sourceIds)]
|
|
49
|
+
: [JSON.stringify(sources)]
|
|
50
|
+
);
|
|
51
|
+
db.run(`UPDATE doc_edges AS e SET confidence=(
|
|
52
|
+
SELECT s.confidence FROM graph_edge_application s WHERE ${same})
|
|
53
|
+
WHERE e.src_doc_id IN (SELECT src_doc_id FROM graph_edge_application)
|
|
54
|
+
AND EXISTS (SELECT 1 FROM graph_edge_application s WHERE ${same} AND s.confidence IS NOT e.confidence)`);
|
|
55
|
+
db.run(`INSERT INTO doc_edges(src_doc_id, dst_doc_id, edge_type, confidence, source)
|
|
56
|
+
SELECT s.src_doc_id, s.dst_doc_id, s.edge_type, s.confidence, s.source
|
|
57
|
+
FROM graph_edge_application s
|
|
58
|
+
WHERE NOT EXISTS (SELECT 1 FROM doc_edges e WHERE ${same})
|
|
59
|
+
ORDER BY s.rowid`);
|
|
60
|
+
// Bun run().changes includes trigger side effects; SQLite changes() does not.
|
|
61
|
+
return (
|
|
62
|
+
db.query<{ count: number }, []>("SELECT changes() AS count").get()
|
|
63
|
+
?.count ?? 0
|
|
64
|
+
);
|
|
65
|
+
})();
|
|
66
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
GraphProjectionState,
|
|
5
|
+
GraphReferenceDocument,
|
|
6
|
+
GraphReferenceInventory,
|
|
7
|
+
GraphReferenceStore,
|
|
8
|
+
} from "../types";
|
|
9
|
+
|
|
10
|
+
import { buildWikiMatchExpression } from "../../core/graph-resolver";
|
|
11
|
+
import { normalizeWikiName } from "../../core/links";
|
|
12
|
+
|
|
13
|
+
const SNAPSHOT_COLUMNS = `document_id AS documentId, collection, rel_path AS relPath,
|
|
14
|
+
docid, uri, title, mirror_hash AS mirrorHash, source_hash AS sourceHash,
|
|
15
|
+
content_type AS contentType`;
|
|
16
|
+
|
|
17
|
+
/** Raw references preserve resolver precedence; resolution belongs to ingestion.
|
|
18
|
+
* Keep old snapshots through deletion/rename until the affected closure is consumed.
|
|
19
|
+
* No second parsed-link inventory: use doc_links alongside these frontmatter rows. */
|
|
20
|
+
export function createGraphReferenceStore(db: Database): GraphReferenceStore {
|
|
21
|
+
function state(
|
|
22
|
+
version: number,
|
|
23
|
+
configFingerprint: string
|
|
24
|
+
): GraphProjectionState {
|
|
25
|
+
const row = db
|
|
26
|
+
.query<
|
|
27
|
+
{
|
|
28
|
+
epoch: number;
|
|
29
|
+
version: number | null;
|
|
30
|
+
configFingerprint: string | null;
|
|
31
|
+
dirty: number;
|
|
32
|
+
inProgress: number;
|
|
33
|
+
},
|
|
34
|
+
[]
|
|
35
|
+
>(`SELECT epoch, version, config_fingerprint AS configFingerprint, dirty, in_progress AS inProgress
|
|
36
|
+
FROM graph_projection_state WHERE id = 1`)
|
|
37
|
+
.get();
|
|
38
|
+
if (!row)
|
|
39
|
+
throw new Error("Missing graph projection state; rebuild required");
|
|
40
|
+
return {
|
|
41
|
+
...row,
|
|
42
|
+
dirty: row.dirty !== 0,
|
|
43
|
+
inProgress: row.inProgress !== 0,
|
|
44
|
+
complete:
|
|
45
|
+
row.dirty === 0 &&
|
|
46
|
+
row.inProgress === 0 &&
|
|
47
|
+
row.version === version &&
|
|
48
|
+
row.configFingerprint === configFingerprint,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
state,
|
|
53
|
+
begin(version, configFingerprint) {
|
|
54
|
+
if (!Number.isSafeInteger(version) || version < 1 || !configFingerprint)
|
|
55
|
+
throw new Error("Invalid graph projection identity");
|
|
56
|
+
return db.transaction(() => {
|
|
57
|
+
const previous = state(version, configFingerprint);
|
|
58
|
+
if (
|
|
59
|
+
previous.version !== version ||
|
|
60
|
+
previous.configFingerprint !== configFingerprint
|
|
61
|
+
)
|
|
62
|
+
db.run("DELETE FROM graph_reference_documents");
|
|
63
|
+
db.run(
|
|
64
|
+
`UPDATE graph_projection_state SET epoch = epoch + 1, dirty = 1, in_progress = 1, version = ?, config_fingerprint = ? WHERE id = 1`,
|
|
65
|
+
[version, configFingerprint]
|
|
66
|
+
);
|
|
67
|
+
return state(version, configFingerprint).epoch;
|
|
68
|
+
})();
|
|
69
|
+
},
|
|
70
|
+
readInventory() {
|
|
71
|
+
const docs = db
|
|
72
|
+
.query<GraphReferenceDocument, []>(
|
|
73
|
+
`SELECT ${SNAPSHOT_COLUMNS} FROM graph_reference_documents ORDER BY document_id`
|
|
74
|
+
)
|
|
75
|
+
.all();
|
|
76
|
+
const refs = db
|
|
77
|
+
.query<{ sourceId: number; edgeType: string; target: string }, []>(
|
|
78
|
+
`SELECT source_doc_id AS sourceId, edge_type AS edgeType, target FROM graph_frontmatter_references ORDER BY source_doc_id, ordinal`
|
|
79
|
+
)
|
|
80
|
+
.all();
|
|
81
|
+
const inventories = new Map<number, GraphReferenceInventory>(
|
|
82
|
+
docs.map((document) => [
|
|
83
|
+
document.documentId,
|
|
84
|
+
{ document, references: [] },
|
|
85
|
+
])
|
|
86
|
+
);
|
|
87
|
+
for (const ref of refs)
|
|
88
|
+
inventories
|
|
89
|
+
.get(ref.sourceId)
|
|
90
|
+
?.references.push({ edgeType: ref.edgeType, target: ref.target });
|
|
91
|
+
return [...inventories.values()];
|
|
92
|
+
},
|
|
93
|
+
incomingLinkSources(identities) {
|
|
94
|
+
if (identities.length === 0) return [];
|
|
95
|
+
const targets = identities.map((d) => ({
|
|
96
|
+
collection: d.collection,
|
|
97
|
+
rel_path: d.relPath,
|
|
98
|
+
title: d.title,
|
|
99
|
+
wiki_title: normalizeWikiName(
|
|
100
|
+
d.title ?? d.relPath.split("/").pop() ?? d.relPath
|
|
101
|
+
),
|
|
102
|
+
wiki_rel: normalizeWikiName(d.relPath),
|
|
103
|
+
wiki_stem: normalizeWikiName(d.relPath.replace(/\.[^/.]+$/, "")),
|
|
104
|
+
}));
|
|
105
|
+
return db
|
|
106
|
+
.query<{ sourceId: number }, [string]>(`
|
|
107
|
+
WITH targets AS (
|
|
108
|
+
SELECT json_extract(value, '$.collection') AS collection,
|
|
109
|
+
json_extract(value, '$.rel_path') AS rel_path,
|
|
110
|
+
json_extract(value, '$.title') AS title,
|
|
111
|
+
json_extract(value, '$.wiki_title') AS wiki_title,
|
|
112
|
+
json_extract(value, '$.wiki_rel') AS wiki_rel,
|
|
113
|
+
json_extract(value, '$.wiki_stem') AS wiki_stem
|
|
114
|
+
FROM json_each(?)
|
|
115
|
+
)
|
|
116
|
+
SELECT DISTINCT dl.source_doc_id AS sourceId
|
|
117
|
+
FROM doc_links dl JOIN documents src ON src.id = dl.source_doc_id AND src.active = 1
|
|
118
|
+
JOIN targets t ON (
|
|
119
|
+
(t.collection = COALESCE(dl.target_collection, src.collection) AND (
|
|
120
|
+
(dl.link_type = 'markdown' AND dl.target_ref_norm = t.rel_path) OR
|
|
121
|
+
(dl.link_type = 'wiki' AND ${buildWikiMatchExpression("t", "dl.target_ref_norm")})
|
|
122
|
+
)) OR
|
|
123
|
+
(dl.link_type = 'wiki' AND dl.target_ref_norm IN (t.wiki_title, t.wiki_rel, t.wiki_stem))
|
|
124
|
+
OR (dl.link_type = 'markdown' AND t.collection = src.collection AND dl.target_ref_norm = t.rel_path)
|
|
125
|
+
)
|
|
126
|
+
ORDER BY dl.source_doc_id
|
|
127
|
+
`)
|
|
128
|
+
.all(JSON.stringify(targets))
|
|
129
|
+
.map((row) => row.sourceId);
|
|
130
|
+
},
|
|
131
|
+
writeInventory({ document: d, references }) {
|
|
132
|
+
db.transaction(() => {
|
|
133
|
+
db.run("UPDATE graph_projection_state SET dirty = 1 WHERE id = 1");
|
|
134
|
+
db.run(
|
|
135
|
+
`INSERT INTO graph_reference_documents
|
|
136
|
+
(document_id, collection, rel_path, docid, uri, title, mirror_hash, source_hash, content_type)
|
|
137
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
138
|
+
ON CONFLICT(document_id) DO UPDATE SET collection=excluded.collection, rel_path=excluded.rel_path,
|
|
139
|
+
docid=excluded.docid, uri=excluded.uri, title=excluded.title, mirror_hash=excluded.mirror_hash,
|
|
140
|
+
source_hash=excluded.source_hash, content_type=excluded.content_type`,
|
|
141
|
+
[
|
|
142
|
+
d.documentId,
|
|
143
|
+
d.collection,
|
|
144
|
+
d.relPath,
|
|
145
|
+
d.docid,
|
|
146
|
+
d.uri,
|
|
147
|
+
d.title,
|
|
148
|
+
d.mirrorHash,
|
|
149
|
+
d.sourceHash,
|
|
150
|
+
d.contentType,
|
|
151
|
+
]
|
|
152
|
+
);
|
|
153
|
+
db.run(
|
|
154
|
+
"DELETE FROM graph_frontmatter_references WHERE source_doc_id = ?",
|
|
155
|
+
[d.documentId]
|
|
156
|
+
);
|
|
157
|
+
for (const [ordinal, ref] of references.entries()) {
|
|
158
|
+
db.run(
|
|
159
|
+
"INSERT INTO graph_frontmatter_references(source_doc_id, ordinal, edge_type, target) VALUES (?, ?, ?, ?)",
|
|
160
|
+
[d.documentId, ordinal, ref.edgeType, ref.target]
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
})();
|
|
164
|
+
},
|
|
165
|
+
complete(expectedEpoch) {
|
|
166
|
+
db.transaction(() => {
|
|
167
|
+
const row = db
|
|
168
|
+
.query<{ epoch: number; version: number | null }, []>(
|
|
169
|
+
"SELECT epoch, version FROM graph_projection_state WHERE id = 1"
|
|
170
|
+
)
|
|
171
|
+
.get();
|
|
172
|
+
if (!row?.version || row.epoch !== expectedEpoch)
|
|
173
|
+
throw new Error("Graph inputs changed during projection");
|
|
174
|
+
const missing = db
|
|
175
|
+
.query<{ id: number }, []>(`SELECT d.id FROM documents d
|
|
176
|
+
LEFT JOIN graph_reference_documents r ON r.document_id = d.id
|
|
177
|
+
WHERE d.active = 1 AND (r.document_id IS NULL
|
|
178
|
+
OR r.collection IS NOT d.collection OR r.rel_path IS NOT d.rel_path
|
|
179
|
+
OR r.docid IS NOT d.docid OR r.uri IS NOT d.uri OR r.title IS NOT d.title
|
|
180
|
+
OR r.mirror_hash IS NOT d.mirror_hash OR r.source_hash IS NOT d.source_hash
|
|
181
|
+
OR r.content_type IS NOT d.content_type) LIMIT 1`)
|
|
182
|
+
.get();
|
|
183
|
+
if (missing)
|
|
184
|
+
throw new Error("Graph reference inventory is incomplete or stale");
|
|
185
|
+
db.run(
|
|
186
|
+
"DELETE FROM graph_reference_documents WHERE document_id NOT IN (SELECT id FROM documents WHERE active = 1)"
|
|
187
|
+
);
|
|
188
|
+
db.run(
|
|
189
|
+
"UPDATE graph_projection_state SET dirty = 0, in_progress = 0 WHERE id = 1"
|
|
190
|
+
);
|
|
191
|
+
})();
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/** Conservative input invalidation for providers without verified variants. */
|
|
2
|
+
import type { Database } from "bun:sqlite";
|
|
3
|
+
|
|
4
|
+
import { formatDocForEmbedding } from "../../pipeline/contextual";
|
|
5
|
+
|
|
6
|
+
type TitleRow = { title: string | null };
|
|
7
|
+
export type LegacyTitleSnapshot = Map<string, (string | null)[]>;
|
|
8
|
+
|
|
9
|
+
function activeTitle(db: Database, mirror: string): TitleRow | null {
|
|
10
|
+
return db
|
|
11
|
+
.query<TitleRow, [string]>(`
|
|
12
|
+
SELECT title FROM documents WHERE mirror_hash = ? AND active = 1
|
|
13
|
+
ORDER BY id LIMIT 1
|
|
14
|
+
`)
|
|
15
|
+
.get(mirror);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Must be captured within the same transaction as the document mutation. */
|
|
19
|
+
export function snapshotLegacyTitles(
|
|
20
|
+
db: Database,
|
|
21
|
+
mirrors: (string | null | undefined)[]
|
|
22
|
+
): LegacyTitleSnapshot {
|
|
23
|
+
const snapshot: LegacyTitleSnapshot = new Map();
|
|
24
|
+
for (const mirror of new Set(mirrors)) {
|
|
25
|
+
if (!mirror) continue;
|
|
26
|
+
const active = activeTitle(db, mirror);
|
|
27
|
+
// All inactive titles must agree before a restored source can reuse an
|
|
28
|
+
// unproven legacy row. There is no persisted legacy input provenance.
|
|
29
|
+
const titles = active
|
|
30
|
+
? [active.title]
|
|
31
|
+
: db
|
|
32
|
+
.query<TitleRow, [string]>(
|
|
33
|
+
"SELECT DISTINCT title FROM documents WHERE mirror_hash = ?"
|
|
34
|
+
)
|
|
35
|
+
.all(mirror)
|
|
36
|
+
.map((row) => row.title);
|
|
37
|
+
snapshot.set(mirror, titles);
|
|
38
|
+
}
|
|
39
|
+
return snapshot;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Only legacy vectors are invalidated; canonical chunks and variants survive. */
|
|
43
|
+
export function reconcileLegacyTitles(
|
|
44
|
+
db: Database,
|
|
45
|
+
before: LegacyTitleSnapshot
|
|
46
|
+
): void {
|
|
47
|
+
for (const [mirror, titles] of before) {
|
|
48
|
+
const next = activeTitle(db, mirror);
|
|
49
|
+
// Retain vectors when the final owner disappears, for identical restore.
|
|
50
|
+
if (!next || (titles.length === 1 && titles[0] === next.title)) continue;
|
|
51
|
+
const rows = db
|
|
52
|
+
.query<{ seq: number; model: string; text: string }, [string]>(`
|
|
53
|
+
SELECT v.seq, v.model, c.text FROM content_vectors v
|
|
54
|
+
JOIN content_chunks c ON c.mirror_hash = v.mirror_hash AND c.seq = v.seq
|
|
55
|
+
WHERE v.mirror_hash = ?
|
|
56
|
+
`)
|
|
57
|
+
.all(mirror);
|
|
58
|
+
for (const row of rows) {
|
|
59
|
+
const input = formatDocForEmbedding(
|
|
60
|
+
row.text,
|
|
61
|
+
next.title ?? undefined,
|
|
62
|
+
row.model
|
|
63
|
+
);
|
|
64
|
+
if (
|
|
65
|
+
titles.length &&
|
|
66
|
+
titles.every(
|
|
67
|
+
(title) =>
|
|
68
|
+
formatDocForEmbedding(row.text, title ?? undefined, row.model) ===
|
|
69
|
+
input
|
|
70
|
+
)
|
|
71
|
+
)
|
|
72
|
+
continue;
|
|
73
|
+
db.run(
|
|
74
|
+
"DELETE FROM content_vectors WHERE mirror_hash = ? AND seq = ? AND model = ?",
|
|
75
|
+
[mirror, row.seq, row.model]
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
package/src/store/types.ts
CHANGED
|
@@ -615,22 +615,65 @@ export interface IngestErrorInput {
|
|
|
615
615
|
// Search Types
|
|
616
616
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
617
617
|
|
|
618
|
-
/**
|
|
619
|
-
export interface
|
|
620
|
-
|
|
621
|
-
|
|
618
|
+
/** Existing SQL owner filters shared by lexical and chunk candidate selection. */
|
|
619
|
+
export interface GraphReferenceDocument {
|
|
620
|
+
documentId: number;
|
|
621
|
+
collection: string;
|
|
622
|
+
relPath: string;
|
|
623
|
+
docid: string;
|
|
624
|
+
uri: string;
|
|
625
|
+
title: string | null;
|
|
626
|
+
mirrorHash: string | null;
|
|
627
|
+
sourceHash: string;
|
|
628
|
+
contentType: string | null;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
export interface GraphFrontmatterReference {
|
|
632
|
+
edgeType: string;
|
|
633
|
+
target: string;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
export interface GraphReferenceInventory {
|
|
637
|
+
document: GraphReferenceDocument;
|
|
638
|
+
references: GraphFrontmatterReference[];
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
export interface GraphProjectionState {
|
|
642
|
+
epoch: number;
|
|
643
|
+
version: number | null;
|
|
644
|
+
configFingerprint: string | null;
|
|
645
|
+
dirty: boolean;
|
|
646
|
+
/** Interrupted projection requires full recovery; ordinary input dirtiness does not. */
|
|
647
|
+
inProgress: boolean;
|
|
648
|
+
complete: boolean;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/** Synchronous internal operations compose with the adapter transaction. */
|
|
652
|
+
export interface GraphReferenceStore {
|
|
653
|
+
state(version: number, configFingerprint: string): GraphProjectionState;
|
|
654
|
+
begin(version: number, configFingerprint: string): number;
|
|
655
|
+
readInventory(): GraphReferenceInventory[];
|
|
656
|
+
/** Active parsed-link sources matching any old/new target candidate. */
|
|
657
|
+
incomingLinkSources(identities: GraphReferenceDocument[]): number[];
|
|
658
|
+
writeInventory(inventory: GraphReferenceInventory): void;
|
|
659
|
+
complete(expectedEpoch: number): void;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
export interface DocumentEligibilityOptions {
|
|
663
|
+
/** Internal owner eligibility; public lexical language remains reserved. */
|
|
664
|
+
chunkLanguage?: string;
|
|
665
|
+
/** Include author, content type and categories in whole-owner exclusion. */
|
|
666
|
+
excludeMetadata?: boolean;
|
|
667
|
+
/** Internal vector/hybrid JavaScript metadata matching semantics. */
|
|
668
|
+
semanticMetadata?: boolean;
|
|
669
|
+
/** Internal caller allowlist; undefined is unrestricted, empty denies all. */
|
|
670
|
+
allowedMirrorHashes?: string[];
|
|
671
|
+
/** Whole-document title/path/chunk exclusions, applied before the budget. */
|
|
672
|
+
exclude?: string[];
|
|
622
673
|
/** Filter by collection */
|
|
623
674
|
collection?: string;
|
|
624
675
|
/** Internal exact relative-path boundary applied before ranking and LIMIT. */
|
|
625
676
|
relPathPrefix?: string;
|
|
626
|
-
/**
|
|
627
|
-
* Language hint (reserved for future use).
|
|
628
|
-
* Note: FTS5 snowball tokenizer is language-aware at index time,
|
|
629
|
-
* so runtime language filtering is not currently implemented.
|
|
630
|
-
*/
|
|
631
|
-
language?: string;
|
|
632
|
-
/** Include snippet with highlights */
|
|
633
|
-
snippet?: boolean;
|
|
634
677
|
/** Filter to docs with ANY of these tags */
|
|
635
678
|
tagsAny?: string[];
|
|
636
679
|
/** Filter to docs with ALL of these tags */
|
|
@@ -654,6 +697,19 @@ export interface FtsSearchOptions {
|
|
|
654
697
|
* `supersedes` pointing at them). Applied inside the candidate subquery.
|
|
655
698
|
*/
|
|
656
699
|
excludeSuperseded?: boolean;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
export interface FtsSearchOptions extends DocumentEligibilityOptions {
|
|
703
|
+
/** Max eligible ranked results to return (filters run before this budget) */
|
|
704
|
+
limit?: number;
|
|
705
|
+
/**
|
|
706
|
+
* Language hint (reserved for future use).
|
|
707
|
+
* Note: FTS5 snowball tokenizer is language-aware at index time,
|
|
708
|
+
* so runtime language filtering is not currently implemented.
|
|
709
|
+
*/
|
|
710
|
+
language?: string;
|
|
711
|
+
/** Include snippet with highlights */
|
|
712
|
+
snippet?: boolean;
|
|
657
713
|
/** Match documents containing ANY positive term instead of ALL of them. */
|
|
658
714
|
anyTerm?: boolean;
|
|
659
715
|
}
|
|
@@ -1998,6 +2054,15 @@ export interface StorePort {
|
|
|
1998
2054
|
mirrorHashes: string[]
|
|
1999
2055
|
): Promise<StoreResult<Map<string, ChunkRow[]>>>;
|
|
2000
2056
|
|
|
2057
|
+
/**
|
|
2058
|
+
* Optional exact (mirrorHash, seq) batch hydration. Same mapping as
|
|
2059
|
+
* getChunksBatch, but only requested sequences; missing pairs are omitted.
|
|
2060
|
+
* Callers fall back to whole-hash batching when this capability is absent.
|
|
2061
|
+
*/
|
|
2062
|
+
getChunksBySequenceBatch?(
|
|
2063
|
+
keys: { mirrorHash: string; seq: number }[]
|
|
2064
|
+
): Promise<StoreResult<Map<string, ChunkRow[]>>>;
|
|
2065
|
+
|
|
2001
2066
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2002
2067
|
// FTS Search
|
|
2003
2068
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -2112,6 +2177,9 @@ export interface StorePort {
|
|
|
2112
2177
|
*/
|
|
2113
2178
|
getLinksForDoc(documentId: number): Promise<StoreResult<DocLinkRow[]>>;
|
|
2114
2179
|
|
|
2180
|
+
/** Missing capability or stale state requires full graph reconciliation. */
|
|
2181
|
+
graphReferenceStore?(): GraphReferenceStore;
|
|
2182
|
+
|
|
2115
2183
|
/**
|
|
2116
2184
|
* Get backlinks pointing to a document.
|
|
2117
2185
|
* Uses target_ref_norm for matching (wiki=normalized title with path fallbacks, markdown=rel_path).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
3
|
+
import type { VectorSearchOptions } from "./types";
|
|
4
|
+
|
|
5
|
+
import { buildEligibleDocumentQuery } from "../sqlite/eligibility";
|
|
6
|
+
|
|
7
|
+
/** Correlated canonical chunk domain; owner predicates stay in the shared builder. */
|
|
8
|
+
export function buildEligibleVectorQuery(
|
|
9
|
+
db: Database,
|
|
10
|
+
options: VectorSearchOptions
|
|
11
|
+
): { sql: string; params: (string | number)[] } {
|
|
12
|
+
const eligibility = options.eligibility ?? {};
|
|
13
|
+
const owners = buildEligibleDocumentQuery(
|
|
14
|
+
{ ...eligibility, semanticMetadata: true },
|
|
15
|
+
db
|
|
16
|
+
);
|
|
17
|
+
const conditions = [
|
|
18
|
+
"cc.mirror_hash = v.mirror_hash",
|
|
19
|
+
"cc.seq = v.seq",
|
|
20
|
+
`cc.mirror_hash IN (SELECT mirror_hash FROM (${owners.sql}))`,
|
|
21
|
+
];
|
|
22
|
+
const params = [...owners.params];
|
|
23
|
+
if (eligibility.language) {
|
|
24
|
+
conditions.push("cc.language = ?");
|
|
25
|
+
params.push(eligibility.language);
|
|
26
|
+
}
|
|
27
|
+
// Caller scope intersects owner scope; an explicitly empty array denies all.
|
|
28
|
+
if (options.allowedMirrorHashes !== undefined) {
|
|
29
|
+
conditions.push("cc.mirror_hash IN (SELECT value FROM json_each(?))");
|
|
30
|
+
params.push(JSON.stringify(options.allowedMirrorHashes));
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
sql: `SELECT 1 FROM content_chunks cc WHERE ${conditions.join(" AND ")}`,
|
|
34
|
+
params,
|
|
35
|
+
};
|
|
36
|
+
}
|