@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { KeyGeneratorRegistry } from "../keys/registry.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/backend/memory-backend.ts
|
|
4
|
+
/**
|
|
5
|
+
* MemoryIndexBackend — zero-dependency in-memory backend for the Index Provider.
|
|
6
|
+
*
|
|
7
|
+
* Uses plain Maps for storage and simple word matching for FTS.
|
|
8
|
+
* Suitable for testing and small data sets.
|
|
9
|
+
*/
|
|
10
|
+
var MemoryIndexBackend = class {
|
|
11
|
+
entries = /* @__PURE__ */ new Map();
|
|
12
|
+
anchorData = /* @__PURE__ */ new Map();
|
|
13
|
+
anchorKeys = /* @__PURE__ */ new Map();
|
|
14
|
+
summaries = /* @__PURE__ */ new Map();
|
|
15
|
+
embeddings = /* @__PURE__ */ new Map();
|
|
16
|
+
keyRegistry;
|
|
17
|
+
constructor(keyRegistry) {
|
|
18
|
+
this.keyRegistry = keyRegistry ?? new KeyGeneratorRegistry();
|
|
19
|
+
}
|
|
20
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
21
|
+
setKeyRegistry(registry) {
|
|
22
|
+
this.keyRegistry = registry;
|
|
23
|
+
}
|
|
24
|
+
async initialize() {}
|
|
25
|
+
async close() {
|
|
26
|
+
this.entries.clear();
|
|
27
|
+
this.anchorData.clear();
|
|
28
|
+
this.anchorKeys.clear();
|
|
29
|
+
this.summaries.clear();
|
|
30
|
+
this.embeddings.clear();
|
|
31
|
+
}
|
|
32
|
+
async writeEntry(entry) {
|
|
33
|
+
this.entries.set(entry.entryPath, { ...entry });
|
|
34
|
+
}
|
|
35
|
+
async getEntry(path) {
|
|
36
|
+
return this.entries.get(path) ?? null;
|
|
37
|
+
}
|
|
38
|
+
async removeEntry(path) {
|
|
39
|
+
this.entries.delete(path);
|
|
40
|
+
}
|
|
41
|
+
async listEntries(filter) {
|
|
42
|
+
return Array.from(this.entries.values()).filter((e) => {
|
|
43
|
+
if (filter.domain && e.domain !== filter.domain) return false;
|
|
44
|
+
if (filter.state && e.state !== filter.state) return false;
|
|
45
|
+
if (filter.extractorId && e.extractorId !== filter.extractorId) return false;
|
|
46
|
+
if (filter.scope && !e.entryPath.startsWith(filter.scope)) return false;
|
|
47
|
+
if (filter.boundDomains && filter.boundDomains.length > 0) {
|
|
48
|
+
if (!filter.boundDomains.includes(e.domain)) return false;
|
|
49
|
+
}
|
|
50
|
+
return true;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
async markStale(filter) {
|
|
54
|
+
let count = 0;
|
|
55
|
+
for (const entry of this.entries.values()) {
|
|
56
|
+
if (entry.state !== "active") continue;
|
|
57
|
+
if (filter.domain && entry.domain !== filter.domain) continue;
|
|
58
|
+
if (filter.extractorId && entry.extractorId !== filter.extractorId) continue;
|
|
59
|
+
if (filter.scope && !entry.entryPath.startsWith(filter.scope)) continue;
|
|
60
|
+
if (filter.boundDomains && filter.boundDomains.length > 0 && !filter.boundDomains.includes(entry.domain)) continue;
|
|
61
|
+
if (filter.anchorType) {
|
|
62
|
+
if (!(this.anchorData.get(entry.entryPath) ?? []).some((a) => a.type === filter.anchorType)) continue;
|
|
63
|
+
}
|
|
64
|
+
entry.state = "stale";
|
|
65
|
+
count++;
|
|
66
|
+
}
|
|
67
|
+
return count;
|
|
68
|
+
}
|
|
69
|
+
async countEntries(filter) {
|
|
70
|
+
return (await this.listEntries(filter)).length;
|
|
71
|
+
}
|
|
72
|
+
async writeAnchors(path, anchors) {
|
|
73
|
+
this.anchorData.set(path, anchors.map((a) => ({ ...a })));
|
|
74
|
+
for (const [key, entries] of this.anchorKeys) {
|
|
75
|
+
entries.delete(path);
|
|
76
|
+
if (entries.size === 0) this.anchorKeys.delete(key);
|
|
77
|
+
}
|
|
78
|
+
for (const anchor of anchors) {
|
|
79
|
+
const keys = this.keyRegistry.generateKeys(anchor);
|
|
80
|
+
for (const key of keys) {
|
|
81
|
+
if (!this.anchorKeys.has(key)) this.anchorKeys.set(key, /* @__PURE__ */ new Map());
|
|
82
|
+
this.anchorKeys.get(key).set(path, anchor.confidence);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async queryAnchors(query) {
|
|
87
|
+
if (query.keys.length === 0) return [];
|
|
88
|
+
const domainsSet = query.domains && query.domains.length > 0 ? new Set(query.domains) : null;
|
|
89
|
+
const groups = query.keys.map((kg) => {
|
|
90
|
+
const matches = /* @__PURE__ */ new Map();
|
|
91
|
+
for (const rawValue of kg.values) {
|
|
92
|
+
const candidates = new Set([rawValue]);
|
|
93
|
+
const synthetic = {
|
|
94
|
+
type: kg.type ?? "exact",
|
|
95
|
+
values: [rawValue],
|
|
96
|
+
confidence: 1
|
|
97
|
+
};
|
|
98
|
+
for (const k of this.keyRegistry.generateKeys(synthetic)) candidates.add(k);
|
|
99
|
+
for (const key of candidates) {
|
|
100
|
+
const entries = this.anchorKeys.get(key);
|
|
101
|
+
if (!entries) continue;
|
|
102
|
+
for (const [path, conf] of entries) {
|
|
103
|
+
if (query.scope && !path.startsWith(query.scope)) continue;
|
|
104
|
+
if (query.pathPrefix && !path.startsWith(query.pathPrefix)) continue;
|
|
105
|
+
if (domainsSet) {
|
|
106
|
+
const entry = this.entries.get(path);
|
|
107
|
+
if (!entry || !domainsSet.has(entry.domain)) continue;
|
|
108
|
+
}
|
|
109
|
+
if (query.minConfidence != null && conf < query.minConfidence) continue;
|
|
110
|
+
const current = matches.get(path);
|
|
111
|
+
matches.set(path, current != null ? Math.min(current, conf) : conf);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return matches;
|
|
116
|
+
});
|
|
117
|
+
if (groups.length === 0) return [];
|
|
118
|
+
let result = new Map(groups[0]);
|
|
119
|
+
for (let i = 1; i < groups.length; i++) {
|
|
120
|
+
const next = /* @__PURE__ */ new Map();
|
|
121
|
+
for (const [path, score] of result) {
|
|
122
|
+
const otherScore = groups[i].get(path);
|
|
123
|
+
if (otherScore != null) next.set(path, Math.min(score, otherScore));
|
|
124
|
+
}
|
|
125
|
+
result = next;
|
|
126
|
+
}
|
|
127
|
+
let results = Array.from(result).map(([entryPath, score]) => ({
|
|
128
|
+
entryPath,
|
|
129
|
+
score
|
|
130
|
+
}));
|
|
131
|
+
if (query.metaFilter && Object.keys(query.metaFilter).length > 0) results = results.filter((r) => {
|
|
132
|
+
return matchesMeta(this.entries.get(r.entryPath)?.meta, query.metaFilter);
|
|
133
|
+
});
|
|
134
|
+
results.sort((a, b) => b.score - a.score);
|
|
135
|
+
if (query.limit) results = results.slice(0, query.limit);
|
|
136
|
+
return results;
|
|
137
|
+
}
|
|
138
|
+
async removeAnchors(path) {
|
|
139
|
+
this.anchorData.delete(path);
|
|
140
|
+
for (const [key, entries] of this.anchorKeys) {
|
|
141
|
+
entries.delete(path);
|
|
142
|
+
if (entries.size === 0) this.anchorKeys.delete(key);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async getAnchorData(path) {
|
|
146
|
+
return (this.anchorData.get(path) ?? []).map((a) => ({
|
|
147
|
+
entryPath: path,
|
|
148
|
+
anchorType: a.type,
|
|
149
|
+
confidence: a.confidence,
|
|
150
|
+
rawAnchor: JSON.stringify(a)
|
|
151
|
+
}));
|
|
152
|
+
}
|
|
153
|
+
async writeSummary(path, summary) {
|
|
154
|
+
this.summaries.set(path, summary);
|
|
155
|
+
}
|
|
156
|
+
async getSummary(path) {
|
|
157
|
+
return this.summaries.get(path) ?? null;
|
|
158
|
+
}
|
|
159
|
+
async queryFTS(text, limit, metaFilter, opts) {
|
|
160
|
+
const isOr = /\bOR\b/.test(text);
|
|
161
|
+
const terms = text.toLowerCase().split(/\s+/).filter((t) => t.length > 0 && t !== "or");
|
|
162
|
+
if (terms.length === 0) return [];
|
|
163
|
+
const results = [];
|
|
164
|
+
for (const [path, summary] of this.summaries) {
|
|
165
|
+
const entry = this.entries.get(path);
|
|
166
|
+
if (opts?.domain && entry?.domain !== opts.domain) continue;
|
|
167
|
+
if (opts?.domains && opts.domains.length > 0 && (!entry || !opts.domains.includes(entry.domain))) continue;
|
|
168
|
+
if (opts?.scope && !path.startsWith(opts.scope)) continue;
|
|
169
|
+
if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;
|
|
170
|
+
const lowerSummary = summary.toLowerCase();
|
|
171
|
+
const matchCount = terms.filter((t) => lowerSummary.includes(t)).length;
|
|
172
|
+
if (isOr ? matchCount > 0 : matchCount === terms.length) {
|
|
173
|
+
if (metaFilter && Object.keys(metaFilter).length > 0) {
|
|
174
|
+
if (!matchesMeta(entry?.meta, metaFilter)) continue;
|
|
175
|
+
}
|
|
176
|
+
results.push({
|
|
177
|
+
entryPath: path,
|
|
178
|
+
rank: matchCount / terms.length
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
results.sort((a, b) => b.rank - a.rank);
|
|
183
|
+
if (limit != null) return results.slice(0, limit);
|
|
184
|
+
return results;
|
|
185
|
+
}
|
|
186
|
+
async removeSummary(path) {
|
|
187
|
+
this.summaries.delete(path);
|
|
188
|
+
}
|
|
189
|
+
async writeEmbedding(path, embedding) {
|
|
190
|
+
this.embeddings.set(path, [...embedding]);
|
|
191
|
+
}
|
|
192
|
+
async queryVector(embedding, limit, opts) {
|
|
193
|
+
const results = [];
|
|
194
|
+
for (const [path, stored] of this.embeddings) {
|
|
195
|
+
const entry = this.entries.get(path);
|
|
196
|
+
if (!entry) continue;
|
|
197
|
+
if (opts?.domain && entry.domain !== opts.domain) continue;
|
|
198
|
+
if (opts?.domains && opts.domains.length > 0 && !opts.domains.includes(entry.domain)) continue;
|
|
199
|
+
if (opts?.scope && !path.startsWith(opts.scope)) continue;
|
|
200
|
+
if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;
|
|
201
|
+
const similarity = cosineSimilarity(embedding, stored);
|
|
202
|
+
results.push({
|
|
203
|
+
entryPath: path,
|
|
204
|
+
similarity
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
results.sort((a, b) => b.similarity - a.similarity);
|
|
208
|
+
if (limit != null) return results.slice(0, limit);
|
|
209
|
+
return results;
|
|
210
|
+
}
|
|
211
|
+
async removeEmbedding(path) {
|
|
212
|
+
this.embeddings.delete(path);
|
|
213
|
+
}
|
|
214
|
+
embeddingCapability() {
|
|
215
|
+
return {
|
|
216
|
+
supported: true,
|
|
217
|
+
dimension: 0
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
async writeEntriesBatch(entries) {
|
|
221
|
+
for (const entry of entries) await this.writeEntry(entry);
|
|
222
|
+
}
|
|
223
|
+
async writeAnchorsBatch(items) {
|
|
224
|
+
for (const item of items) await this.writeAnchors(item.path, item.anchors);
|
|
225
|
+
}
|
|
226
|
+
async writeSummariesBatch(items) {
|
|
227
|
+
for (const item of items) await this.writeSummary(item.path, item.summary);
|
|
228
|
+
}
|
|
229
|
+
async purgeEntries(paths, opts) {
|
|
230
|
+
const skipVector = opts?.skipVector === true;
|
|
231
|
+
let count = 0;
|
|
232
|
+
for (const path of paths) if (this.entries.has(path)) {
|
|
233
|
+
await this.removeAnchors(path);
|
|
234
|
+
this.summaries.delete(path);
|
|
235
|
+
if (!skipVector) this.embeddings.delete(path);
|
|
236
|
+
this.entries.delete(path);
|
|
237
|
+
count++;
|
|
238
|
+
}
|
|
239
|
+
return count;
|
|
240
|
+
}
|
|
241
|
+
async queryCombined(keys, text, opts) {
|
|
242
|
+
const anchorResults = keys.length > 0 ? await this.queryAnchors({
|
|
243
|
+
keys,
|
|
244
|
+
scope: opts?.scope,
|
|
245
|
+
minConfidence: opts?.minConfidence,
|
|
246
|
+
limit: opts?.limit
|
|
247
|
+
}) : [];
|
|
248
|
+
const anchorPaths = new Set(anchorResults.map((r) => r.entryPath));
|
|
249
|
+
const ftsResults = text ? await this.queryFTS(text, opts?.limit) : [];
|
|
250
|
+
const results = anchorResults.map((r) => {
|
|
251
|
+
const fts = ftsResults.find((f) => f.entryPath === r.entryPath);
|
|
252
|
+
return {
|
|
253
|
+
entryPath: r.entryPath,
|
|
254
|
+
anchorScore: r.score,
|
|
255
|
+
ftsRank: fts?.rank,
|
|
256
|
+
score: r.score + (fts?.rank ?? 0)
|
|
257
|
+
};
|
|
258
|
+
});
|
|
259
|
+
for (const f of ftsResults) if (!anchorPaths.has(f.entryPath)) results.push({
|
|
260
|
+
entryPath: f.entryPath,
|
|
261
|
+
ftsRank: f.rank,
|
|
262
|
+
score: f.rank
|
|
263
|
+
});
|
|
264
|
+
results.sort((a, b) => b.score - a.score);
|
|
265
|
+
if (opts?.limit != null) return results.slice(0, opts.limit);
|
|
266
|
+
return results;
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
/**
|
|
270
|
+
* Check if an entry's meta matches the given filter.
|
|
271
|
+
* Each field in metaFilter is AND-ed. Array values are OR-ed within a field.
|
|
272
|
+
*
|
|
273
|
+
* NOTE: Uses strict equality (===) for value comparison. This works correctly
|
|
274
|
+
* for primitive values (string, number, boolean, null). Nested objects or arrays
|
|
275
|
+
* as meta values would need deep comparison to match correctly.
|
|
276
|
+
*/
|
|
277
|
+
function matchesMeta(meta, filter) {
|
|
278
|
+
if (!meta) return false;
|
|
279
|
+
for (const [key, expected] of Object.entries(filter)) {
|
|
280
|
+
const actual = meta[key];
|
|
281
|
+
if (Array.isArray(expected)) {
|
|
282
|
+
if (expected.length === 0) return false;
|
|
283
|
+
if (!expected.includes(actual)) return false;
|
|
284
|
+
} else if (actual !== expected) return false;
|
|
285
|
+
}
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
/** Cosine similarity between two vectors. Returns 0 for zero-length vectors. */
|
|
289
|
+
function cosineSimilarity(a, b) {
|
|
290
|
+
const len = Math.min(a.length, b.length);
|
|
291
|
+
let dot = 0;
|
|
292
|
+
let magA = 0;
|
|
293
|
+
let magB = 0;
|
|
294
|
+
for (let i = 0; i < len; i++) {
|
|
295
|
+
dot += a[i] * b[i];
|
|
296
|
+
magA += a[i] * a[i];
|
|
297
|
+
magB += b[i] * b[i];
|
|
298
|
+
}
|
|
299
|
+
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
300
|
+
return denom === 0 ? 0 : dot / denom;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
//#endregion
|
|
304
|
+
export { MemoryIndexBackend };
|
|
305
|
+
//# sourceMappingURL=memory-backend.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-backend.mjs","names":[],"sources":["../../src/backend/memory-backend.ts"],"sourcesContent":["/**\n * MemoryIndexBackend — zero-dependency in-memory backend for the Index Provider.\n *\n * Uses plain Maps for storage and simple word matching for FTS.\n * Suitable for testing and small data sets.\n */\nimport { KeyGeneratorRegistry } from \"../keys/registry.js\";\nimport type {\n Anchor,\n AnchorData,\n AnchorQuery,\n AnchorResult,\n CombinedResult,\n EntryFilter,\n FTSResult,\n IndexedEntry,\n QueryOpts,\n StaleFilter,\n VectorResult,\n} from \"../types.js\";\nimport type { IndexBackend, RemoveIndexOptions } from \"./types.js\";\n\nexport class MemoryIndexBackend implements IndexBackend {\n private entries = new Map<string, IndexedEntry>();\n private anchorData = new Map<string, Anchor[]>();\n private anchorKeys = new Map<string, Map<string, number>>(); // index_key → Map<entry_path, confidence>\n private summaries = new Map<string, string>();\n private embeddings = new Map<string, number[]>();\n private keyRegistry: KeyGeneratorRegistry;\n\n constructor(keyRegistry?: KeyGeneratorRegistry) {\n this.keyRegistry = keyRegistry ?? new KeyGeneratorRegistry();\n }\n\n /** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */\n setKeyRegistry(registry: KeyGeneratorRegistry): void {\n this.keyRegistry = registry;\n }\n\n async initialize(): Promise<void> {\n // No-op for in-memory backend\n }\n\n async close(): Promise<void> {\n this.entries.clear();\n this.anchorData.clear();\n this.anchorKeys.clear();\n this.summaries.clear();\n this.embeddings.clear();\n }\n\n // --- Entry metadata ---\n\n async writeEntry(entry: IndexedEntry): Promise<void> {\n this.entries.set(entry.entryPath, { ...entry });\n }\n\n async getEntry(path: string): Promise<IndexedEntry | null> {\n return this.entries.get(path) ?? null;\n }\n\n async removeEntry(path: string): Promise<void> {\n this.entries.delete(path);\n }\n\n async listEntries(filter: EntryFilter): Promise<IndexedEntry[]> {\n return Array.from(this.entries.values()).filter((e) => {\n if (filter.domain && e.domain !== filter.domain) return false;\n if (filter.state && e.state !== filter.state) return false;\n if (filter.extractorId && e.extractorId !== filter.extractorId) return false;\n if (filter.scope && !e.entryPath.startsWith(filter.scope)) return false;\n // Cross-blocklet boundary (parity with sqlite/d1). Memory is per-instance\n // so cross-USER can't arise here, but reindex/cleanup/verify honor\n // boundDomains uniformly across all three backends.\n if (filter.boundDomains && filter.boundDomains.length > 0) {\n if (!filter.boundDomains.includes(e.domain)) return false;\n }\n return true;\n });\n }\n\n async markStale(filter: StaleFilter): Promise<number> {\n let count = 0;\n for (const entry of this.entries.values()) {\n if (entry.state !== \"active\") continue;\n if (filter.domain && entry.domain !== filter.domain) continue;\n if (filter.extractorId && entry.extractorId !== filter.extractorId) continue;\n if (filter.scope && !entry.entryPath.startsWith(filter.scope)) continue;\n if (\n filter.boundDomains &&\n filter.boundDomains.length > 0 &&\n !filter.boundDomains.includes(entry.domain)\n )\n continue;\n if (filter.anchorType) {\n const anchors = this.anchorData.get(entry.entryPath) ?? [];\n if (!anchors.some((a) => a.type === filter.anchorType)) continue;\n }\n entry.state = \"stale\";\n count++;\n }\n return count;\n }\n\n async countEntries(filter: EntryFilter): Promise<number> {\n return (await this.listEntries(filter)).length;\n }\n\n // --- Anchor operations ---\n\n async writeAnchors(path: string, anchors: Anchor[]): Promise<void> {\n this.anchorData.set(\n path,\n anchors.map((a) => ({ ...a })),\n );\n\n // Clear existing keys for this path\n for (const [key, entries] of this.anchorKeys) {\n entries.delete(path);\n if (entries.size === 0) this.anchorKeys.delete(key);\n }\n\n // Generate and store new keys\n for (const anchor of anchors) {\n const keys = this.keyRegistry.generateKeys(anchor);\n for (const key of keys) {\n if (!this.anchorKeys.has(key)) this.anchorKeys.set(key, new Map());\n this.anchorKeys.get(key)!.set(path, anchor.confidence);\n }\n }\n }\n\n async queryAnchors(query: AnchorQuery): Promise<AnchorResult[]> {\n if (query.keys.length === 0) return [];\n\n // `boundDomains` fallback (issue #936 P4 leak fix) — restrict matches to\n // entries whose domain is in the bound set. Parity with queryFTS/\n // queryVector below, which already filter by domain.\n const domainsSet = query.domains && query.domains.length > 0 ? new Set(query.domains) : null;\n\n // For each key group, find matching entries with min confidence.\n const groups = query.keys.map((kg) => {\n const matches = new Map<string, number>();\n for (const rawValue of kg.values) {\n // Match the SAME candidate-key set the SQL backends (d1/sqlite) do: the\n // raw value AND every key the registry generates for it. This keeps all\n // three backends' anchor matching identical (conformance parity) and\n // makes both a full-key query (`values:[\"topic:ai\"]`, the client facet\n // convention) and a bare-value query (`values:[\"ai\"]`) resolve — the\n // generator normalizes the bare value to the canonical `<type>:<value>`.\n const candidates = new Set<string>([rawValue]);\n const synthetic = { type: kg.type ?? \"exact\", values: [rawValue], confidence: 1 } as Anchor;\n for (const k of this.keyRegistry.generateKeys(synthetic)) candidates.add(k);\n for (const key of candidates) {\n const entries = this.anchorKeys.get(key);\n if (!entries) continue;\n for (const [path, conf] of entries) {\n if (query.scope && !path.startsWith(query.scope)) continue;\n if (query.pathPrefix && !path.startsWith(query.pathPrefix)) continue;\n if (domainsSet) {\n const entry = this.entries.get(path);\n if (!entry || !domainsSet.has(entry.domain)) continue;\n }\n if (query.minConfidence != null && conf < query.minConfidence) continue;\n const current = matches.get(path);\n matches.set(path, current != null ? Math.min(current, conf) : conf);\n }\n }\n }\n return matches;\n });\n\n // Intersect all groups (AND logic)\n if (groups.length === 0) return [];\n let result = new Map(groups[0]!);\n for (let i = 1; i < groups.length; i++) {\n const next = new Map<string, number>();\n for (const [path, score] of result) {\n const otherScore = groups[i]!.get(path);\n if (otherScore != null) {\n next.set(path, Math.min(score, otherScore));\n }\n }\n result = next;\n }\n\n // Apply metaFilter BEFORE sorting/limiting\n let results = Array.from(result).map(([entryPath, score]) => ({ entryPath, score }));\n if (query.metaFilter && Object.keys(query.metaFilter).length > 0) {\n results = results.filter((r) => {\n const entry = this.entries.get(r.entryPath);\n return matchesMeta(entry?.meta, query.metaFilter!);\n });\n }\n\n // Sort by score DESC, apply limit\n results.sort((a, b) => b.score - a.score);\n if (query.limit) results = results.slice(0, query.limit);\n return results;\n }\n\n async removeAnchors(path: string): Promise<void> {\n this.anchorData.delete(path);\n for (const [key, entries] of this.anchorKeys) {\n entries.delete(path);\n if (entries.size === 0) this.anchorKeys.delete(key);\n }\n }\n\n async getAnchorData(path: string): Promise<AnchorData[]> {\n const anchors = this.anchorData.get(path) ?? [];\n return anchors.map((a) => ({\n entryPath: path,\n anchorType: a.type,\n confidence: a.confidence,\n rawAnchor: JSON.stringify(a),\n }));\n }\n\n // --- FTS operations (simple word matching) ---\n\n async writeSummary(path: string, summary: string): Promise<void> {\n this.summaries.set(path, summary);\n }\n\n async getSummary(path: string): Promise<string | null> {\n return this.summaries.get(path) ?? null;\n }\n\n async queryFTS(\n text: string,\n limit?: number,\n metaFilter?: Record<string, unknown>,\n opts?: { domain?: string; domains?: string[]; scope?: string; pathPrefix?: string },\n ): Promise<FTSResult[]> {\n // Query text is pre-built by transformFtsQuery (#933): and-mode is\n // space-joined (\"a b\"), or-mode is infix-OR (\"a OR b\", #1002). Honor the\n // OR operator so memory or-recall matches on ANY term — parity with the\n // SQLite/D1 backends (whose FTS5 MATCH does the same). Without this, an\n // uppercase \"OR\" reaches here as a literal term and silently collapses\n // or-mode back to AND-all.\n const isOr = /\\bOR\\b/.test(text);\n const terms = text\n .toLowerCase()\n .split(/\\s+/)\n .filter((t) => t.length > 0 && t !== \"or\");\n if (terms.length === 0) return [];\n\n const results: FTSResult[] = [];\n for (const [path, summary] of this.summaries) {\n // Domain/scope isolation parity with SQLiteIndexBackend — see\n // queryVector for context.\n const entry = this.entries.get(path);\n if (opts?.domain && entry?.domain !== opts.domain) continue;\n if (\n opts?.domains &&\n opts.domains.length > 0 &&\n (!entry || !opts.domains.includes(entry.domain))\n )\n continue;\n if (opts?.scope && !path.startsWith(opts.scope)) continue;\n if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;\n\n const lowerSummary = summary.toLowerCase();\n const matchCount = terms.filter((t) => lowerSummary.includes(t)).length;\n if (isOr ? matchCount > 0 : matchCount === terms.length) {\n // Apply metaFilter BEFORE adding to results (and before limit)\n if (metaFilter && Object.keys(metaFilter).length > 0) {\n if (!matchesMeta(entry?.meta, metaFilter)) continue;\n }\n results.push({ entryPath: path, rank: matchCount / terms.length });\n }\n }\n\n results.sort((a, b) => b.rank - a.rank);\n if (limit != null) return results.slice(0, limit);\n return results;\n }\n\n async removeSummary(path: string): Promise<void> {\n this.summaries.delete(path);\n }\n\n // --- Embedding operations ---\n\n async writeEmbedding(path: string, embedding: number[]): Promise<void> {\n this.embeddings.set(path, [...embedding]);\n }\n\n async queryVector(\n embedding: number[],\n limit?: number,\n opts?: { domain?: string; domains?: string[]; scope?: string; pathPrefix?: string },\n ): Promise<VectorResult[]> {\n const results: VectorResult[] = [];\n for (const [path, stored] of this.embeddings) {\n // Scope/domain filter parity with SQLiteIndexBackend — orphan\n // embeddings (those without a matching entry) drop out so a\n // multi-tenant Memory backend doesn't leak across-domain\n // similarities into recall.\n const entry = this.entries.get(path);\n if (!entry) continue;\n if (opts?.domain && entry.domain !== opts.domain) continue;\n if (opts?.domains && opts.domains.length > 0 && !opts.domains.includes(entry.domain))\n continue;\n if (opts?.scope && !path.startsWith(opts.scope)) continue;\n if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;\n const similarity = cosineSimilarity(embedding, stored);\n results.push({ entryPath: path, similarity });\n }\n results.sort((a, b) => b.similarity - a.similarity);\n if (limit != null) return results.slice(0, limit);\n return results;\n }\n\n async removeEmbedding(path: string): Promise<void> {\n this.embeddings.delete(path);\n }\n\n embeddingCapability(): { supported: boolean; dimension: number } {\n return { supported: true, dimension: 0 }; // dimension 0 = any dimension accepted\n }\n\n // --- Batch operations ---\n\n async writeEntriesBatch(entries: IndexedEntry[]): Promise<void> {\n for (const entry of entries) {\n await this.writeEntry(entry);\n }\n }\n\n async writeAnchorsBatch(items: { path: string; anchors: Anchor[] }[]): Promise<void> {\n for (const item of items) {\n await this.writeAnchors(item.path, item.anchors);\n }\n }\n\n async writeSummariesBatch(items: { path: string; summary: string }[]): Promise<void> {\n for (const item of items) {\n await this.writeSummary(item.path, item.summary);\n }\n }\n\n async purgeEntries(paths: string[], opts?: RemoveIndexOptions): Promise<number> {\n const skipVector = opts?.skipVector === true;\n let count = 0;\n for (const path of paths) {\n if (this.entries.has(path)) {\n await this.removeAnchors(path);\n this.summaries.delete(path);\n // Phase 4 offline reset: skipVector leaves the embedding intact.\n if (!skipVector) this.embeddings.delete(path);\n this.entries.delete(path);\n count++;\n }\n }\n return count;\n }\n\n // --- Combined query ---\n\n async queryCombined(\n keys: { type: string; values: string[] }[],\n text?: string,\n opts?: QueryOpts,\n ): Promise<CombinedResult[]> {\n const anchorResults =\n keys.length > 0\n ? await this.queryAnchors({\n keys,\n scope: opts?.scope,\n minConfidence: opts?.minConfidence,\n limit: opts?.limit,\n })\n : [];\n\n const anchorPaths = new Set(anchorResults.map((r) => r.entryPath));\n\n const ftsResults = text ? await this.queryFTS(text, opts?.limit) : [];\n\n const results: CombinedResult[] = anchorResults.map((r) => {\n const fts = ftsResults.find((f) => f.entryPath === r.entryPath);\n return {\n entryPath: r.entryPath,\n anchorScore: r.score,\n ftsRank: fts?.rank,\n score: r.score + (fts?.rank ?? 0),\n };\n });\n\n for (const f of ftsResults) {\n if (!anchorPaths.has(f.entryPath)) {\n results.push({\n entryPath: f.entryPath,\n ftsRank: f.rank,\n score: f.rank,\n });\n }\n }\n\n results.sort((a, b) => b.score - a.score);\n if (opts?.limit != null) return results.slice(0, opts.limit);\n return results;\n }\n}\n\n/**\n * Check if an entry's meta matches the given filter.\n * Each field in metaFilter is AND-ed. Array values are OR-ed within a field.\n *\n * NOTE: Uses strict equality (===) for value comparison. This works correctly\n * for primitive values (string, number, boolean, null). Nested objects or arrays\n * as meta values would need deep comparison to match correctly.\n */\nfunction matchesMeta(\n meta: Record<string, unknown> | undefined,\n filter: Record<string, unknown>,\n): boolean {\n if (!meta) return false;\n for (const [key, expected] of Object.entries(filter)) {\n const actual = meta[key];\n if (Array.isArray(expected)) {\n if (expected.length === 0) return false;\n if (!expected.includes(actual)) return false;\n } else {\n if (actual !== expected) return false;\n }\n }\n return true;\n}\n\n/** Cosine similarity between two vectors. Returns 0 for zero-length vectors. */\nfunction cosineSimilarity(a: number[], b: number[]): number {\n const len = Math.min(a.length, b.length);\n let dot = 0;\n let magA = 0;\n let magB = 0;\n for (let i = 0; i < len; i++) {\n dot += a[i]! * b[i]!;\n magA += a[i]! * a[i]!;\n magB += b[i]! * b[i]!;\n }\n const denom = Math.sqrt(magA) * Math.sqrt(magB);\n return denom === 0 ? 0 : dot / denom;\n}\n"],"mappings":";;;;;;;;;AAsBA,IAAa,qBAAb,MAAwD;CACtD,AAAQ,0BAAU,IAAI,KAA2B;CACjD,AAAQ,6BAAa,IAAI,KAAuB;CAChD,AAAQ,6BAAa,IAAI,KAAkC;CAC3D,AAAQ,4BAAY,IAAI,KAAqB;CAC7C,AAAQ,6BAAa,IAAI,KAAuB;CAChD,AAAQ;CAER,YAAY,aAAoC;AAC9C,OAAK,cAAc,eAAe,IAAI,sBAAsB;;;CAI9D,eAAe,UAAsC;AACnD,OAAK,cAAc;;CAGrB,MAAM,aAA4B;CAIlC,MAAM,QAAuB;AAC3B,OAAK,QAAQ,OAAO;AACpB,OAAK,WAAW,OAAO;AACvB,OAAK,WAAW,OAAO;AACvB,OAAK,UAAU,OAAO;AACtB,OAAK,WAAW,OAAO;;CAKzB,MAAM,WAAW,OAAoC;AACnD,OAAK,QAAQ,IAAI,MAAM,WAAW,EAAE,GAAG,OAAO,CAAC;;CAGjD,MAAM,SAAS,MAA4C;AACzD,SAAO,KAAK,QAAQ,IAAI,KAAK,IAAI;;CAGnC,MAAM,YAAY,MAA6B;AAC7C,OAAK,QAAQ,OAAO,KAAK;;CAG3B,MAAM,YAAY,QAA8C;AAC9D,SAAO,MAAM,KAAK,KAAK,QAAQ,QAAQ,CAAC,CAAC,QAAQ,MAAM;AACrD,OAAI,OAAO,UAAU,EAAE,WAAW,OAAO,OAAQ,QAAO;AACxD,OAAI,OAAO,SAAS,EAAE,UAAU,OAAO,MAAO,QAAO;AACrD,OAAI,OAAO,eAAe,EAAE,gBAAgB,OAAO,YAAa,QAAO;AACvE,OAAI,OAAO,SAAS,CAAC,EAAE,UAAU,WAAW,OAAO,MAAM,CAAE,QAAO;AAIlE,OAAI,OAAO,gBAAgB,OAAO,aAAa,SAAS,GACtD;QAAI,CAAC,OAAO,aAAa,SAAS,EAAE,OAAO,CAAE,QAAO;;AAEtD,UAAO;IACP;;CAGJ,MAAM,UAAU,QAAsC;EACpD,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,QAAQ,QAAQ,EAAE;AACzC,OAAI,MAAM,UAAU,SAAU;AAC9B,OAAI,OAAO,UAAU,MAAM,WAAW,OAAO,OAAQ;AACrD,OAAI,OAAO,eAAe,MAAM,gBAAgB,OAAO,YAAa;AACpE,OAAI,OAAO,SAAS,CAAC,MAAM,UAAU,WAAW,OAAO,MAAM,CAAE;AAC/D,OACE,OAAO,gBACP,OAAO,aAAa,SAAS,KAC7B,CAAC,OAAO,aAAa,SAAS,MAAM,OAAO,CAE3C;AACF,OAAI,OAAO,YAET;QAAI,EADY,KAAK,WAAW,IAAI,MAAM,UAAU,IAAI,EAAE,EAC7C,MAAM,MAAM,EAAE,SAAS,OAAO,WAAW,CAAE;;AAE1D,SAAM,QAAQ;AACd;;AAEF,SAAO;;CAGT,MAAM,aAAa,QAAsC;AACvD,UAAQ,MAAM,KAAK,YAAY,OAAO,EAAE;;CAK1C,MAAM,aAAa,MAAc,SAAkC;AACjE,OAAK,WAAW,IACd,MACA,QAAQ,KAAK,OAAO,EAAE,GAAG,GAAG,EAAE,CAC/B;AAGD,OAAK,MAAM,CAAC,KAAK,YAAY,KAAK,YAAY;AAC5C,WAAQ,OAAO,KAAK;AACpB,OAAI,QAAQ,SAAS,EAAG,MAAK,WAAW,OAAO,IAAI;;AAIrD,OAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,OAAO,KAAK,YAAY,aAAa,OAAO;AAClD,QAAK,MAAM,OAAO,MAAM;AACtB,QAAI,CAAC,KAAK,WAAW,IAAI,IAAI,CAAE,MAAK,WAAW,IAAI,qBAAK,IAAI,KAAK,CAAC;AAClE,SAAK,WAAW,IAAI,IAAI,CAAE,IAAI,MAAM,OAAO,WAAW;;;;CAK5D,MAAM,aAAa,OAA6C;AAC9D,MAAI,MAAM,KAAK,WAAW,EAAG,QAAO,EAAE;EAKtC,MAAM,aAAa,MAAM,WAAW,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI,MAAM,QAAQ,GAAG;EAGxF,MAAM,SAAS,MAAM,KAAK,KAAK,OAAO;GACpC,MAAM,0BAAU,IAAI,KAAqB;AACzC,QAAK,MAAM,YAAY,GAAG,QAAQ;IAOhC,MAAM,aAAa,IAAI,IAAY,CAAC,SAAS,CAAC;IAC9C,MAAM,YAAY;KAAE,MAAM,GAAG,QAAQ;KAAS,QAAQ,CAAC,SAAS;KAAE,YAAY;KAAG;AACjF,SAAK,MAAM,KAAK,KAAK,YAAY,aAAa,UAAU,CAAE,YAAW,IAAI,EAAE;AAC3E,SAAK,MAAM,OAAO,YAAY;KAC5B,MAAM,UAAU,KAAK,WAAW,IAAI,IAAI;AACxC,SAAI,CAAC,QAAS;AACd,UAAK,MAAM,CAAC,MAAM,SAAS,SAAS;AAClC,UAAI,MAAM,SAAS,CAAC,KAAK,WAAW,MAAM,MAAM,CAAE;AAClD,UAAI,MAAM,cAAc,CAAC,KAAK,WAAW,MAAM,WAAW,CAAE;AAC5D,UAAI,YAAY;OACd,MAAM,QAAQ,KAAK,QAAQ,IAAI,KAAK;AACpC,WAAI,CAAC,SAAS,CAAC,WAAW,IAAI,MAAM,OAAO,CAAE;;AAE/C,UAAI,MAAM,iBAAiB,QAAQ,OAAO,MAAM,cAAe;MAC/D,MAAM,UAAU,QAAQ,IAAI,KAAK;AACjC,cAAQ,IAAI,MAAM,WAAW,OAAO,KAAK,IAAI,SAAS,KAAK,GAAG,KAAK;;;;AAIzE,UAAO;IACP;AAGF,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE;EAClC,IAAI,SAAS,IAAI,IAAI,OAAO,GAAI;AAChC,OAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;GACtC,MAAM,uBAAO,IAAI,KAAqB;AACtC,QAAK,MAAM,CAAC,MAAM,UAAU,QAAQ;IAClC,MAAM,aAAa,OAAO,GAAI,IAAI,KAAK;AACvC,QAAI,cAAc,KAChB,MAAK,IAAI,MAAM,KAAK,IAAI,OAAO,WAAW,CAAC;;AAG/C,YAAS;;EAIX,IAAI,UAAU,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,YAAY;GAAE;GAAW;GAAO,EAAE;AACpF,MAAI,MAAM,cAAc,OAAO,KAAK,MAAM,WAAW,CAAC,SAAS,EAC7D,WAAU,QAAQ,QAAQ,MAAM;AAE9B,UAAO,YADO,KAAK,QAAQ,IAAI,EAAE,UAAU,EACjB,MAAM,MAAM,WAAY;IAClD;AAIJ,UAAQ,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;AACzC,MAAI,MAAM,MAAO,WAAU,QAAQ,MAAM,GAAG,MAAM,MAAM;AACxD,SAAO;;CAGT,MAAM,cAAc,MAA6B;AAC/C,OAAK,WAAW,OAAO,KAAK;AAC5B,OAAK,MAAM,CAAC,KAAK,YAAY,KAAK,YAAY;AAC5C,WAAQ,OAAO,KAAK;AACpB,OAAI,QAAQ,SAAS,EAAG,MAAK,WAAW,OAAO,IAAI;;;CAIvD,MAAM,cAAc,MAAqC;AAEvD,UADgB,KAAK,WAAW,IAAI,KAAK,IAAI,EAAE,EAChC,KAAK,OAAO;GACzB,WAAW;GACX,YAAY,EAAE;GACd,YAAY,EAAE;GACd,WAAW,KAAK,UAAU,EAAE;GAC7B,EAAE;;CAKL,MAAM,aAAa,MAAc,SAAgC;AAC/D,OAAK,UAAU,IAAI,MAAM,QAAQ;;CAGnC,MAAM,WAAW,MAAsC;AACrD,SAAO,KAAK,UAAU,IAAI,KAAK,IAAI;;CAGrC,MAAM,SACJ,MACA,OACA,YACA,MACsB;EAOtB,MAAM,OAAO,SAAS,KAAK,KAAK;EAChC,MAAM,QAAQ,KACX,aAAa,CACb,MAAM,MAAM,CACZ,QAAQ,MAAM,EAAE,SAAS,KAAK,MAAM,KAAK;AAC5C,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE;EAEjC,MAAM,UAAuB,EAAE;AAC/B,OAAK,MAAM,CAAC,MAAM,YAAY,KAAK,WAAW;GAG5C,MAAM,QAAQ,KAAK,QAAQ,IAAI,KAAK;AACpC,OAAI,MAAM,UAAU,OAAO,WAAW,KAAK,OAAQ;AACnD,OACE,MAAM,WACN,KAAK,QAAQ,SAAS,MACrB,CAAC,SAAS,CAAC,KAAK,QAAQ,SAAS,MAAM,OAAO,EAE/C;AACF,OAAI,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,CAAE;AACjD,OAAI,MAAM,cAAc,CAAC,KAAK,WAAW,KAAK,WAAW,CAAE;GAE3D,MAAM,eAAe,QAAQ,aAAa;GAC1C,MAAM,aAAa,MAAM,QAAQ,MAAM,aAAa,SAAS,EAAE,CAAC,CAAC;AACjE,OAAI,OAAO,aAAa,IAAI,eAAe,MAAM,QAAQ;AAEvD,QAAI,cAAc,OAAO,KAAK,WAAW,CAAC,SAAS,GACjD;SAAI,CAAC,YAAY,OAAO,MAAM,WAAW,CAAE;;AAE7C,YAAQ,KAAK;KAAE,WAAW;KAAM,MAAM,aAAa,MAAM;KAAQ,CAAC;;;AAItE,UAAQ,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK;AACvC,MAAI,SAAS,KAAM,QAAO,QAAQ,MAAM,GAAG,MAAM;AACjD,SAAO;;CAGT,MAAM,cAAc,MAA6B;AAC/C,OAAK,UAAU,OAAO,KAAK;;CAK7B,MAAM,eAAe,MAAc,WAAoC;AACrE,OAAK,WAAW,IAAI,MAAM,CAAC,GAAG,UAAU,CAAC;;CAG3C,MAAM,YACJ,WACA,OACA,MACyB;EACzB,MAAM,UAA0B,EAAE;AAClC,OAAK,MAAM,CAAC,MAAM,WAAW,KAAK,YAAY;GAK5C,MAAM,QAAQ,KAAK,QAAQ,IAAI,KAAK;AACpC,OAAI,CAAC,MAAO;AACZ,OAAI,MAAM,UAAU,MAAM,WAAW,KAAK,OAAQ;AAClD,OAAI,MAAM,WAAW,KAAK,QAAQ,SAAS,KAAK,CAAC,KAAK,QAAQ,SAAS,MAAM,OAAO,CAClF;AACF,OAAI,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,CAAE;AACjD,OAAI,MAAM,cAAc,CAAC,KAAK,WAAW,KAAK,WAAW,CAAE;GAC3D,MAAM,aAAa,iBAAiB,WAAW,OAAO;AACtD,WAAQ,KAAK;IAAE,WAAW;IAAM;IAAY,CAAC;;AAE/C,UAAQ,MAAM,GAAG,MAAM,EAAE,aAAa,EAAE,WAAW;AACnD,MAAI,SAAS,KAAM,QAAO,QAAQ,MAAM,GAAG,MAAM;AACjD,SAAO;;CAGT,MAAM,gBAAgB,MAA6B;AACjD,OAAK,WAAW,OAAO,KAAK;;CAG9B,sBAAiE;AAC/D,SAAO;GAAE,WAAW;GAAM,WAAW;GAAG;;CAK1C,MAAM,kBAAkB,SAAwC;AAC9D,OAAK,MAAM,SAAS,QAClB,OAAM,KAAK,WAAW,MAAM;;CAIhC,MAAM,kBAAkB,OAA6D;AACnF,OAAK,MAAM,QAAQ,MACjB,OAAM,KAAK,aAAa,KAAK,MAAM,KAAK,QAAQ;;CAIpD,MAAM,oBAAoB,OAA2D;AACnF,OAAK,MAAM,QAAQ,MACjB,OAAM,KAAK,aAAa,KAAK,MAAM,KAAK,QAAQ;;CAIpD,MAAM,aAAa,OAAiB,MAA4C;EAC9E,MAAM,aAAa,MAAM,eAAe;EACxC,IAAI,QAAQ;AACZ,OAAK,MAAM,QAAQ,MACjB,KAAI,KAAK,QAAQ,IAAI,KAAK,EAAE;AAC1B,SAAM,KAAK,cAAc,KAAK;AAC9B,QAAK,UAAU,OAAO,KAAK;AAE3B,OAAI,CAAC,WAAY,MAAK,WAAW,OAAO,KAAK;AAC7C,QAAK,QAAQ,OAAO,KAAK;AACzB;;AAGJ,SAAO;;CAKT,MAAM,cACJ,MACA,MACA,MAC2B;EAC3B,MAAM,gBACJ,KAAK,SAAS,IACV,MAAM,KAAK,aAAa;GACtB;GACA,OAAO,MAAM;GACb,eAAe,MAAM;GACrB,OAAO,MAAM;GACd,CAAC,GACF,EAAE;EAER,MAAM,cAAc,IAAI,IAAI,cAAc,KAAK,MAAM,EAAE,UAAU,CAAC;EAElE,MAAM,aAAa,OAAO,MAAM,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,EAAE;EAErE,MAAM,UAA4B,cAAc,KAAK,MAAM;GACzD,MAAM,MAAM,WAAW,MAAM,MAAM,EAAE,cAAc,EAAE,UAAU;AAC/D,UAAO;IACL,WAAW,EAAE;IACb,aAAa,EAAE;IACf,SAAS,KAAK;IACd,OAAO,EAAE,SAAS,KAAK,QAAQ;IAChC;IACD;AAEF,OAAK,MAAM,KAAK,WACd,KAAI,CAAC,YAAY,IAAI,EAAE,UAAU,CAC/B,SAAQ,KAAK;GACX,WAAW,EAAE;GACb,SAAS,EAAE;GACX,OAAO,EAAE;GACV,CAAC;AAIN,UAAQ,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;AACzC,MAAI,MAAM,SAAS,KAAM,QAAO,QAAQ,MAAM,GAAG,KAAK,MAAM;AAC5D,SAAO;;;;;;;;;;;AAYX,SAAS,YACP,MACA,QACS;AACT,KAAI,CAAC,KAAM,QAAO;AAClB,MAAK,MAAM,CAAC,KAAK,aAAa,OAAO,QAAQ,OAAO,EAAE;EACpD,MAAM,SAAS,KAAK;AACpB,MAAI,MAAM,QAAQ,SAAS,EAAE;AAC3B,OAAI,SAAS,WAAW,EAAG,QAAO;AAClC,OAAI,CAAC,SAAS,SAAS,OAAO,CAAE,QAAO;aAEnC,WAAW,SAAU,QAAO;;AAGpC,QAAO;;;AAIT,SAAS,iBAAiB,GAAa,GAAqB;CAC1D,MAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,OAAO;CACxC,IAAI,MAAM;CACV,IAAI,OAAO;CACX,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,SAAO,EAAE,KAAM,EAAE;AACjB,UAAQ,EAAE,KAAM,EAAE;AAClB,UAAQ,EAAE,KAAM,EAAE;;CAEpB,MAAM,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK,KAAK,KAAK;AAC/C,QAAO,UAAU,IAAI,IAAI,MAAM"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
|
|
2
|
+
//#region src/backend/prefix-range.ts
|
|
3
|
+
/**
|
|
4
|
+
* Half-open `[lo, hi)` key range covering every string that starts with
|
|
5
|
+
* `prefix`, so a `(… , entry_path)` index/PK can bound a subtree scan by an
|
|
6
|
+
* indexed range seek instead of a residual filter.
|
|
7
|
+
*
|
|
8
|
+
* Why not `entry_path LIKE 'prefix%'`: SQLite does NOT push a case-insensitive
|
|
9
|
+
* LIKE (the default) into a range seek against a BINARY-collation column, so it
|
|
10
|
+
* reads EVERY row for the leading equality columns and residual-filters the
|
|
11
|
+
* LIKE — measured on staging D1 as 3,494 rows_read for a selective anchor
|
|
12
|
+
* prefix that a range seek serves in 1 (issue #1120). A literal range is also
|
|
13
|
+
* more correct: `_` and `%` inside the prefix act as LIKE wildcards, whereas an
|
|
14
|
+
* `entry_path` prefix is documented as a LITERAL path prefix (QueryOpts.pathPrefix).
|
|
15
|
+
*
|
|
16
|
+
* Caller must pass a NON-empty prefix (the empty prefix is "no filter" — skip
|
|
17
|
+
* the range entirely). Mirrors `did-space/sqlite-resolve-store.ts` `prefixRange`.
|
|
18
|
+
*/
|
|
19
|
+
function entryPathPrefixRange(prefix) {
|
|
20
|
+
const last = prefix.charCodeAt(prefix.length - 1);
|
|
21
|
+
return [prefix, prefix.slice(0, -1) + String.fromCharCode(last + 1)];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
//#endregion
|
|
25
|
+
exports.entryPathPrefixRange = entryPathPrefixRange;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
//#region src/backend/prefix-range.ts
|
|
2
|
+
/**
|
|
3
|
+
* Half-open `[lo, hi)` key range covering every string that starts with
|
|
4
|
+
* `prefix`, so a `(… , entry_path)` index/PK can bound a subtree scan by an
|
|
5
|
+
* indexed range seek instead of a residual filter.
|
|
6
|
+
*
|
|
7
|
+
* Why not `entry_path LIKE 'prefix%'`: SQLite does NOT push a case-insensitive
|
|
8
|
+
* LIKE (the default) into a range seek against a BINARY-collation column, so it
|
|
9
|
+
* reads EVERY row for the leading equality columns and residual-filters the
|
|
10
|
+
* LIKE — measured on staging D1 as 3,494 rows_read for a selective anchor
|
|
11
|
+
* prefix that a range seek serves in 1 (issue #1120). A literal range is also
|
|
12
|
+
* more correct: `_` and `%` inside the prefix act as LIKE wildcards, whereas an
|
|
13
|
+
* `entry_path` prefix is documented as a LITERAL path prefix (QueryOpts.pathPrefix).
|
|
14
|
+
*
|
|
15
|
+
* Caller must pass a NON-empty prefix (the empty prefix is "no filter" — skip
|
|
16
|
+
* the range entirely). Mirrors `did-space/sqlite-resolve-store.ts` `prefixRange`.
|
|
17
|
+
*/
|
|
18
|
+
function entryPathPrefixRange(prefix) {
|
|
19
|
+
const last = prefix.charCodeAt(prefix.length - 1);
|
|
20
|
+
return [prefix, prefix.slice(0, -1) + String.fromCharCode(last + 1)];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
//#endregion
|
|
24
|
+
export { entryPathPrefixRange };
|
|
25
|
+
//# sourceMappingURL=prefix-range.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prefix-range.mjs","names":[],"sources":["../../src/backend/prefix-range.ts"],"sourcesContent":["/**\n * Half-open `[lo, hi)` key range covering every string that starts with\n * `prefix`, so a `(… , entry_path)` index/PK can bound a subtree scan by an\n * indexed range seek instead of a residual filter.\n *\n * Why not `entry_path LIKE 'prefix%'`: SQLite does NOT push a case-insensitive\n * LIKE (the default) into a range seek against a BINARY-collation column, so it\n * reads EVERY row for the leading equality columns and residual-filters the\n * LIKE — measured on staging D1 as 3,494 rows_read for a selective anchor\n * prefix that a range seek serves in 1 (issue #1120). A literal range is also\n * more correct: `_` and `%` inside the prefix act as LIKE wildcards, whereas an\n * `entry_path` prefix is documented as a LITERAL path prefix (QueryOpts.pathPrefix).\n *\n * Caller must pass a NON-empty prefix (the empty prefix is \"no filter\" — skip\n * the range entirely). Mirrors `did-space/sqlite-resolve-store.ts` `prefixRange`.\n */\nexport function entryPathPrefixRange(prefix: string): [lo: string, hi: string] {\n const last = prefix.charCodeAt(prefix.length - 1);\n const hi = prefix.slice(0, -1) + String.fromCharCode(last + 1);\n return [prefix, hi];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAgBA,SAAgB,qBAAqB,QAA0C;CAC7E,MAAM,OAAO,OAAO,WAAW,OAAO,SAAS,EAAE;AAEjD,QAAO,CAAC,QADG,OAAO,MAAM,GAAG,GAAG,GAAG,OAAO,aAAa,OAAO,EAAE,CAC3C"}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
let drizzle_orm = require("drizzle-orm");
|
|
2
|
+
|
|
3
|
+
//#region src/backend/schema.ts
|
|
4
|
+
const DEFAULT_FTS_TOKENIZER = "unicode61 remove_diacritics 2";
|
|
5
|
+
/**
|
|
6
|
+
* Schema version. Bumped to 2 when the per-caller tenant `scope` column landed
|
|
7
|
+
* on every data table (cross-user isolation, mirroring the D1 backend). Bumped
|
|
8
|
+
* to 3 when `idx_index_cursor.served_at` landed for scheduler fairness. A v1
|
|
9
|
+
* file has NO `scope` column and cannot be back-filled (the tenant scope is not
|
|
10
|
+
* recoverable from `entry_path`), so `migrateToScopedSchema` DROPS the derived
|
|
11
|
+
* index (data tables + follower cursors) and lets the changelog follower
|
|
12
|
+
* re-index from seq 0 — the index is a rebuildable derived model.
|
|
13
|
+
*/
|
|
14
|
+
const SCHEMA_VERSION = 3;
|
|
15
|
+
async function createSchema(db, opts = {}) {
|
|
16
|
+
const tokenizer = opts.ftsTokenizer ?? DEFAULT_FTS_TOKENIZER;
|
|
17
|
+
await migrateToScopedSchema(db);
|
|
18
|
+
await renameAnchorTables(db);
|
|
19
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
20
|
+
CREATE TABLE IF NOT EXISTS ds_indexed_entry (
|
|
21
|
+
scope TEXT NOT NULL,
|
|
22
|
+
entry_path TEXT NOT NULL,
|
|
23
|
+
source_path TEXT,
|
|
24
|
+
source_hash TEXT,
|
|
25
|
+
domain TEXT NOT NULL,
|
|
26
|
+
state TEXT NOT NULL DEFAULT 'active' CHECK (state IN ('active', 'stale', 'removed')),
|
|
27
|
+
extractor_id TEXT,
|
|
28
|
+
indexed_at TEXT NOT NULL,
|
|
29
|
+
updated_at TEXT NOT NULL,
|
|
30
|
+
anchor_count INTEGER DEFAULT 0,
|
|
31
|
+
meta TEXT,
|
|
32
|
+
PRIMARY KEY (scope, entry_path)
|
|
33
|
+
)
|
|
34
|
+
`));
|
|
35
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_entry_domain ON ds_indexed_entry(scope, domain)`));
|
|
36
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_entry_state ON ds_indexed_entry(scope, state)`));
|
|
37
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_entry_extractor ON ds_indexed_entry(scope, extractor_id)`));
|
|
38
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_entry_source ON ds_indexed_entry(scope, source_path)`));
|
|
39
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
40
|
+
CREATE TABLE IF NOT EXISTS idx_anchor_data (
|
|
41
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
42
|
+
scope TEXT NOT NULL,
|
|
43
|
+
entry_path TEXT NOT NULL,
|
|
44
|
+
anchor_type TEXT NOT NULL,
|
|
45
|
+
confidence REAL NOT NULL,
|
|
46
|
+
raw_anchor TEXT NOT NULL
|
|
47
|
+
)
|
|
48
|
+
`));
|
|
49
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
50
|
+
CREATE TABLE IF NOT EXISTS idx_anchor_key (
|
|
51
|
+
scope TEXT NOT NULL,
|
|
52
|
+
index_key TEXT,
|
|
53
|
+
entry_path TEXT,
|
|
54
|
+
anchor_type TEXT,
|
|
55
|
+
confidence REAL,
|
|
56
|
+
PRIMARY KEY (scope, index_key, entry_path, anchor_type)
|
|
57
|
+
) WITHOUT ROWID
|
|
58
|
+
`));
|
|
59
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_ad_path ON idx_anchor_data(scope, entry_path)`));
|
|
60
|
+
await db.run(drizzle_orm.sql.raw(`CREATE INDEX IF NOT EXISTS idx_ak_path ON idx_anchor_key(scope, entry_path)`));
|
|
61
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
62
|
+
CREATE TABLE IF NOT EXISTS idx_embedding_vector (
|
|
63
|
+
scope TEXT NOT NULL,
|
|
64
|
+
entry_path TEXT NOT NULL,
|
|
65
|
+
embedding TEXT,
|
|
66
|
+
embedding_blob BLOB,
|
|
67
|
+
dimension INTEGER NOT NULL,
|
|
68
|
+
PRIMARY KEY (scope, entry_path)
|
|
69
|
+
)
|
|
70
|
+
`));
|
|
71
|
+
if ((await db.all(drizzle_orm.sql.raw("SELECT name FROM sqlite_master WHERE type='table' AND name='idx_fts'"))).length === 0) await db.run(drizzle_orm.sql.raw(`
|
|
72
|
+
CREATE VIRTUAL TABLE idx_fts USING fts5(
|
|
73
|
+
scope UNINDEXED,
|
|
74
|
+
entry_path UNINDEXED,
|
|
75
|
+
summary,
|
|
76
|
+
tokenize='${tokenizer.replace(/'/g, "''")}'
|
|
77
|
+
)
|
|
78
|
+
`));
|
|
79
|
+
await ensureFtsRowidMapping(db);
|
|
80
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
81
|
+
CREATE TABLE IF NOT EXISTS idx_index_cursor (
|
|
82
|
+
consumer_id TEXT NOT NULL PRIMARY KEY,
|
|
83
|
+
last_seq INTEGER NOT NULL DEFAULT 0,
|
|
84
|
+
watermarks TEXT,
|
|
85
|
+
updated_at INTEGER NOT NULL,
|
|
86
|
+
served_at INTEGER,
|
|
87
|
+
instance_did TEXT,
|
|
88
|
+
domains TEXT
|
|
89
|
+
)
|
|
90
|
+
`));
|
|
91
|
+
await ensureCursorServedAt(db);
|
|
92
|
+
await setSchemaVersion(db, SCHEMA_VERSION);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Drop the v1 (scope-less) derived index in place so `createSchema` can rebuild
|
|
96
|
+
* it with the tenant `scope` column. A v1 file is detected by the ABSENCE of a
|
|
97
|
+
* `scope` column on `ds_indexed_entry` (the version table doesn't exist yet).
|
|
98
|
+
*
|
|
99
|
+
* Data tables AND the follower cursors are dropped together: keeping the
|
|
100
|
+
* cursors would leave the follower believing it is already caught up
|
|
101
|
+
* (`last_seq` high) against now-empty tables, so it would never re-index. With
|
|
102
|
+
* the cursors gone, `initCursor` recreates them at seq 0 and the follower
|
|
103
|
+
* re-materializes every tenant's index from the changelog.
|
|
104
|
+
*
|
|
105
|
+
* Idempotent: on a fresh db (no `ds_indexed_entry` yet) and on an already-v2 db
|
|
106
|
+
* it is a no-op. The one-time drop on a shared file rebuilds the index for ALL
|
|
107
|
+
* tenants at once; acceptable because the index is a derived model.
|
|
108
|
+
*/
|
|
109
|
+
async function migrateToScopedSchema(db) {
|
|
110
|
+
if ((await db.all(drizzle_orm.sql.raw("SELECT name FROM sqlite_master WHERE type='table' AND name='ds_indexed_entry'"))).length === 0) return;
|
|
111
|
+
if ((await db.all(drizzle_orm.sql.raw(`PRAGMA table_info(ds_indexed_entry)`))).some((c) => c.name === "scope")) return;
|
|
112
|
+
for (const table of [
|
|
113
|
+
"ds_indexed_entry",
|
|
114
|
+
"arc_anchor_data",
|
|
115
|
+
"arc_anchor_key",
|
|
116
|
+
"idx_anchor_data",
|
|
117
|
+
"idx_anchor_key",
|
|
118
|
+
"idx_embedding_vector",
|
|
119
|
+
"idx_fts",
|
|
120
|
+
"idx_fts_rowid",
|
|
121
|
+
"idx_index_cursor"
|
|
122
|
+
]) await db.run(drizzle_orm.sql.raw(`DROP TABLE IF EXISTS ${table}`));
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Rename the anchor tables from their legacy `arc_` prefix to the correct
|
|
126
|
+
* `idx_` module prefix. These tables are owned by the index provider — the same
|
|
127
|
+
* module as idx_fts / idx_embedding_vector — and were mis-prefixed `arc_`, a
|
|
128
|
+
* prefix reserved for `runtimes/*` Arc-system state (scheduler). Data-preserving:
|
|
129
|
+
* `ALTER TABLE … RENAME` keeps every row and its secondary indexes, so no
|
|
130
|
+
* re-index / follower-cursor reset is needed.
|
|
131
|
+
*
|
|
132
|
+
* Idempotent and safe on fresh / already-migrated / interrupted dbs: renames
|
|
133
|
+
* only when the legacy table exists and the target does not. If BOTH exist (an
|
|
134
|
+
* interrupted prior run), the legacy copy is dropped in favour of the
|
|
135
|
+
* already-created idx_ table. Must run AFTER migrateToScopedSchema (a v1 db has
|
|
136
|
+
* its anchor tables dropped there) and BEFORE the CREATE TABLE IF NOT EXISTS in
|
|
137
|
+
* createSchema (which would otherwise create an empty idx_ table alongside the
|
|
138
|
+
* populated arc_ one).
|
|
139
|
+
*/
|
|
140
|
+
async function renameAnchorTables(db) {
|
|
141
|
+
const pairs = [["arc_anchor_data", "idx_anchor_data"], ["arc_anchor_key", "idx_anchor_key"]];
|
|
142
|
+
const names = pairs.flat().map((n) => `'${n}'`);
|
|
143
|
+
const rows = await db.all(drizzle_orm.sql.raw(`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${names.join(", ")})`));
|
|
144
|
+
const present = new Set(rows.map((r) => r.name));
|
|
145
|
+
for (const [legacy, next] of pairs) {
|
|
146
|
+
if (!present.has(legacy)) continue;
|
|
147
|
+
if (present.has(next)) await db.run(drizzle_orm.sql.raw(`DROP TABLE IF EXISTS ${legacy}`));
|
|
148
|
+
else await db.run(drizzle_orm.sql.raw(`ALTER TABLE ${legacy} RENAME TO ${next}`));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Create the `idx_fts_rowid` mapping table the first time this db sees it,
|
|
153
|
+
* backfilling it from any pre-existing `idx_fts` rows (issue #1103). The
|
|
154
|
+
* backfill maps every existing FTS row to its own rowid so a later `fts_write`
|
|
155
|
+
* REPLACEs in place instead of leaving a duplicate for the same
|
|
156
|
+
* (scope, entry_path) — silently doubling search results for that entry.
|
|
157
|
+
*
|
|
158
|
+
* Race-safe and idempotent (matters on shared backends where two contexts may
|
|
159
|
+
* init at once, and there is no enclosing transaction):
|
|
160
|
+
* - `CREATE TABLE IF NOT EXISTS` never throws on a concurrent create.
|
|
161
|
+
* - The one-time backfill runs only while the mapping is still empty (a cheap
|
|
162
|
+
* LIMIT-1 probe, NOT a scan of `idx_fts`), so steady state costs nothing.
|
|
163
|
+
* - `INSERT OR IGNORE` makes the backfill itself idempotent, so a racing
|
|
164
|
+
* backfill (or a re-run) can't collide on the PK/UNIQUE.
|
|
165
|
+
*/
|
|
166
|
+
async function ensureFtsRowidMapping(db) {
|
|
167
|
+
await db.run(drizzle_orm.sql.raw(`
|
|
168
|
+
CREATE TABLE IF NOT EXISTS idx_fts_rowid (
|
|
169
|
+
fts_rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
170
|
+
scope TEXT NOT NULL,
|
|
171
|
+
entry_path TEXT NOT NULL,
|
|
172
|
+
UNIQUE (scope, entry_path)
|
|
173
|
+
)
|
|
174
|
+
`));
|
|
175
|
+
if ((await db.all(drizzle_orm.sql.raw("SELECT 1 AS one FROM idx_fts_rowid LIMIT 1"))).length > 0) return;
|
|
176
|
+
await db.run(drizzle_orm.sql.raw(`INSERT OR IGNORE INTO idx_fts_rowid (fts_rowid, scope, entry_path) SELECT rowid, scope, entry_path FROM idx_fts`));
|
|
177
|
+
}
|
|
178
|
+
async function ensureCursorServedAt(db) {
|
|
179
|
+
if ((await db.all(drizzle_orm.sql.raw("PRAGMA table_info(idx_index_cursor)"))).some((c) => c.name === "served_at")) return;
|
|
180
|
+
await db.run(drizzle_orm.sql.raw("ALTER TABLE idx_index_cursor ADD COLUMN served_at INTEGER"));
|
|
181
|
+
}
|
|
182
|
+
async function setSchemaVersion(db, version) {
|
|
183
|
+
await db.run(drizzle_orm.sql.raw(`PRAGMA user_version = ${version}`));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
//#endregion
|
|
187
|
+
exports.createSchema = createSchema;
|