@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
const require_registry = require('../keys/registry.cjs');
|
|
2
|
+
|
|
3
|
+
//#region src/backend/memory-backend.ts
|
|
4
|
+
/**
|
|
5
|
+
* MemoryIndexBackend — zero-dependency in-memory backend for the Index Provider.
|
|
6
|
+
*
|
|
7
|
+
* Uses plain Maps for storage and simple word matching for FTS.
|
|
8
|
+
* Suitable for testing and small data sets.
|
|
9
|
+
*/
|
|
10
|
+
var MemoryIndexBackend = class {
|
|
11
|
+
entries = /* @__PURE__ */ new Map();
|
|
12
|
+
anchorData = /* @__PURE__ */ new Map();
|
|
13
|
+
anchorKeys = /* @__PURE__ */ new Map();
|
|
14
|
+
summaries = /* @__PURE__ */ new Map();
|
|
15
|
+
embeddings = /* @__PURE__ */ new Map();
|
|
16
|
+
keyRegistry;
|
|
17
|
+
constructor(keyRegistry) {
|
|
18
|
+
this.keyRegistry = keyRegistry ?? new require_registry.KeyGeneratorRegistry();
|
|
19
|
+
}
|
|
20
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
21
|
+
setKeyRegistry(registry) {
|
|
22
|
+
this.keyRegistry = registry;
|
|
23
|
+
}
|
|
24
|
+
async initialize() {}
|
|
25
|
+
async close() {
|
|
26
|
+
this.entries.clear();
|
|
27
|
+
this.anchorData.clear();
|
|
28
|
+
this.anchorKeys.clear();
|
|
29
|
+
this.summaries.clear();
|
|
30
|
+
this.embeddings.clear();
|
|
31
|
+
}
|
|
32
|
+
async writeEntry(entry) {
|
|
33
|
+
this.entries.set(entry.entryPath, { ...entry });
|
|
34
|
+
}
|
|
35
|
+
async getEntry(path) {
|
|
36
|
+
return this.entries.get(path) ?? null;
|
|
37
|
+
}
|
|
38
|
+
async removeEntry(path) {
|
|
39
|
+
this.entries.delete(path);
|
|
40
|
+
}
|
|
41
|
+
async listEntries(filter) {
|
|
42
|
+
return Array.from(this.entries.values()).filter((e) => {
|
|
43
|
+
if (filter.domain && e.domain !== filter.domain) return false;
|
|
44
|
+
if (filter.state && e.state !== filter.state) return false;
|
|
45
|
+
if (filter.extractorId && e.extractorId !== filter.extractorId) return false;
|
|
46
|
+
if (filter.scope && !e.entryPath.startsWith(filter.scope)) return false;
|
|
47
|
+
if (filter.boundDomains && filter.boundDomains.length > 0) {
|
|
48
|
+
if (!filter.boundDomains.includes(e.domain)) return false;
|
|
49
|
+
}
|
|
50
|
+
return true;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
async markStale(filter) {
|
|
54
|
+
let count = 0;
|
|
55
|
+
for (const entry of this.entries.values()) {
|
|
56
|
+
if (entry.state !== "active") continue;
|
|
57
|
+
if (filter.domain && entry.domain !== filter.domain) continue;
|
|
58
|
+
if (filter.extractorId && entry.extractorId !== filter.extractorId) continue;
|
|
59
|
+
if (filter.scope && !entry.entryPath.startsWith(filter.scope)) continue;
|
|
60
|
+
if (filter.boundDomains && filter.boundDomains.length > 0 && !filter.boundDomains.includes(entry.domain)) continue;
|
|
61
|
+
if (filter.anchorType) {
|
|
62
|
+
if (!(this.anchorData.get(entry.entryPath) ?? []).some((a) => a.type === filter.anchorType)) continue;
|
|
63
|
+
}
|
|
64
|
+
entry.state = "stale";
|
|
65
|
+
count++;
|
|
66
|
+
}
|
|
67
|
+
return count;
|
|
68
|
+
}
|
|
69
|
+
async countEntries(filter) {
|
|
70
|
+
return (await this.listEntries(filter)).length;
|
|
71
|
+
}
|
|
72
|
+
async writeAnchors(path, anchors) {
|
|
73
|
+
this.anchorData.set(path, anchors.map((a) => ({ ...a })));
|
|
74
|
+
for (const [key, entries] of this.anchorKeys) {
|
|
75
|
+
entries.delete(path);
|
|
76
|
+
if (entries.size === 0) this.anchorKeys.delete(key);
|
|
77
|
+
}
|
|
78
|
+
for (const anchor of anchors) {
|
|
79
|
+
const keys = this.keyRegistry.generateKeys(anchor);
|
|
80
|
+
for (const key of keys) {
|
|
81
|
+
if (!this.anchorKeys.has(key)) this.anchorKeys.set(key, /* @__PURE__ */ new Map());
|
|
82
|
+
this.anchorKeys.get(key).set(path, anchor.confidence);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async queryAnchors(query) {
|
|
87
|
+
if (query.keys.length === 0) return [];
|
|
88
|
+
const domainsSet = query.domains && query.domains.length > 0 ? new Set(query.domains) : null;
|
|
89
|
+
const groups = query.keys.map((kg) => {
|
|
90
|
+
const matches = /* @__PURE__ */ new Map();
|
|
91
|
+
for (const rawValue of kg.values) {
|
|
92
|
+
const candidates = new Set([rawValue]);
|
|
93
|
+
const synthetic = {
|
|
94
|
+
type: kg.type ?? "exact",
|
|
95
|
+
values: [rawValue],
|
|
96
|
+
confidence: 1
|
|
97
|
+
};
|
|
98
|
+
for (const k of this.keyRegistry.generateKeys(synthetic)) candidates.add(k);
|
|
99
|
+
for (const key of candidates) {
|
|
100
|
+
const entries = this.anchorKeys.get(key);
|
|
101
|
+
if (!entries) continue;
|
|
102
|
+
for (const [path, conf] of entries) {
|
|
103
|
+
if (query.scope && !path.startsWith(query.scope)) continue;
|
|
104
|
+
if (query.pathPrefix && !path.startsWith(query.pathPrefix)) continue;
|
|
105
|
+
if (domainsSet) {
|
|
106
|
+
const entry = this.entries.get(path);
|
|
107
|
+
if (!entry || !domainsSet.has(entry.domain)) continue;
|
|
108
|
+
}
|
|
109
|
+
if (query.minConfidence != null && conf < query.minConfidence) continue;
|
|
110
|
+
const current = matches.get(path);
|
|
111
|
+
matches.set(path, current != null ? Math.min(current, conf) : conf);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return matches;
|
|
116
|
+
});
|
|
117
|
+
if (groups.length === 0) return [];
|
|
118
|
+
let result = new Map(groups[0]);
|
|
119
|
+
for (let i = 1; i < groups.length; i++) {
|
|
120
|
+
const next = /* @__PURE__ */ new Map();
|
|
121
|
+
for (const [path, score] of result) {
|
|
122
|
+
const otherScore = groups[i].get(path);
|
|
123
|
+
if (otherScore != null) next.set(path, Math.min(score, otherScore));
|
|
124
|
+
}
|
|
125
|
+
result = next;
|
|
126
|
+
}
|
|
127
|
+
let results = Array.from(result).map(([entryPath, score]) => ({
|
|
128
|
+
entryPath,
|
|
129
|
+
score
|
|
130
|
+
}));
|
|
131
|
+
if (query.metaFilter && Object.keys(query.metaFilter).length > 0) results = results.filter((r) => {
|
|
132
|
+
return matchesMeta(this.entries.get(r.entryPath)?.meta, query.metaFilter);
|
|
133
|
+
});
|
|
134
|
+
results.sort((a, b) => b.score - a.score);
|
|
135
|
+
if (query.limit) results = results.slice(0, query.limit);
|
|
136
|
+
return results;
|
|
137
|
+
}
|
|
138
|
+
async removeAnchors(path) {
|
|
139
|
+
this.anchorData.delete(path);
|
|
140
|
+
for (const [key, entries] of this.anchorKeys) {
|
|
141
|
+
entries.delete(path);
|
|
142
|
+
if (entries.size === 0) this.anchorKeys.delete(key);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async getAnchorData(path) {
|
|
146
|
+
return (this.anchorData.get(path) ?? []).map((a) => ({
|
|
147
|
+
entryPath: path,
|
|
148
|
+
anchorType: a.type,
|
|
149
|
+
confidence: a.confidence,
|
|
150
|
+
rawAnchor: JSON.stringify(a)
|
|
151
|
+
}));
|
|
152
|
+
}
|
|
153
|
+
async writeSummary(path, summary) {
|
|
154
|
+
this.summaries.set(path, summary);
|
|
155
|
+
}
|
|
156
|
+
async getSummary(path) {
|
|
157
|
+
return this.summaries.get(path) ?? null;
|
|
158
|
+
}
|
|
159
|
+
async queryFTS(text, limit, metaFilter, opts) {
|
|
160
|
+
const isOr = /\bOR\b/.test(text);
|
|
161
|
+
const terms = text.toLowerCase().split(/\s+/).filter((t) => t.length > 0 && t !== "or");
|
|
162
|
+
if (terms.length === 0) return [];
|
|
163
|
+
const results = [];
|
|
164
|
+
for (const [path, summary] of this.summaries) {
|
|
165
|
+
const entry = this.entries.get(path);
|
|
166
|
+
if (opts?.domain && entry?.domain !== opts.domain) continue;
|
|
167
|
+
if (opts?.domains && opts.domains.length > 0 && (!entry || !opts.domains.includes(entry.domain))) continue;
|
|
168
|
+
if (opts?.scope && !path.startsWith(opts.scope)) continue;
|
|
169
|
+
if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;
|
|
170
|
+
const lowerSummary = summary.toLowerCase();
|
|
171
|
+
const matchCount = terms.filter((t) => lowerSummary.includes(t)).length;
|
|
172
|
+
if (isOr ? matchCount > 0 : matchCount === terms.length) {
|
|
173
|
+
if (metaFilter && Object.keys(metaFilter).length > 0) {
|
|
174
|
+
if (!matchesMeta(entry?.meta, metaFilter)) continue;
|
|
175
|
+
}
|
|
176
|
+
results.push({
|
|
177
|
+
entryPath: path,
|
|
178
|
+
rank: matchCount / terms.length
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
results.sort((a, b) => b.rank - a.rank);
|
|
183
|
+
if (limit != null) return results.slice(0, limit);
|
|
184
|
+
return results;
|
|
185
|
+
}
|
|
186
|
+
async removeSummary(path) {
|
|
187
|
+
this.summaries.delete(path);
|
|
188
|
+
}
|
|
189
|
+
async writeEmbedding(path, embedding) {
|
|
190
|
+
this.embeddings.set(path, [...embedding]);
|
|
191
|
+
}
|
|
192
|
+
async queryVector(embedding, limit, opts) {
|
|
193
|
+
const results = [];
|
|
194
|
+
for (const [path, stored] of this.embeddings) {
|
|
195
|
+
const entry = this.entries.get(path);
|
|
196
|
+
if (!entry) continue;
|
|
197
|
+
if (opts?.domain && entry.domain !== opts.domain) continue;
|
|
198
|
+
if (opts?.domains && opts.domains.length > 0 && !opts.domains.includes(entry.domain)) continue;
|
|
199
|
+
if (opts?.scope && !path.startsWith(opts.scope)) continue;
|
|
200
|
+
if (opts?.pathPrefix && !path.startsWith(opts.pathPrefix)) continue;
|
|
201
|
+
const similarity = cosineSimilarity(embedding, stored);
|
|
202
|
+
results.push({
|
|
203
|
+
entryPath: path,
|
|
204
|
+
similarity
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
results.sort((a, b) => b.similarity - a.similarity);
|
|
208
|
+
if (limit != null) return results.slice(0, limit);
|
|
209
|
+
return results;
|
|
210
|
+
}
|
|
211
|
+
async removeEmbedding(path) {
|
|
212
|
+
this.embeddings.delete(path);
|
|
213
|
+
}
|
|
214
|
+
embeddingCapability() {
|
|
215
|
+
return {
|
|
216
|
+
supported: true,
|
|
217
|
+
dimension: 0
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
async writeEntriesBatch(entries) {
|
|
221
|
+
for (const entry of entries) await this.writeEntry(entry);
|
|
222
|
+
}
|
|
223
|
+
async writeAnchorsBatch(items) {
|
|
224
|
+
for (const item of items) await this.writeAnchors(item.path, item.anchors);
|
|
225
|
+
}
|
|
226
|
+
async writeSummariesBatch(items) {
|
|
227
|
+
for (const item of items) await this.writeSummary(item.path, item.summary);
|
|
228
|
+
}
|
|
229
|
+
async purgeEntries(paths, opts) {
|
|
230
|
+
const skipVector = opts?.skipVector === true;
|
|
231
|
+
let count = 0;
|
|
232
|
+
for (const path of paths) if (this.entries.has(path)) {
|
|
233
|
+
await this.removeAnchors(path);
|
|
234
|
+
this.summaries.delete(path);
|
|
235
|
+
if (!skipVector) this.embeddings.delete(path);
|
|
236
|
+
this.entries.delete(path);
|
|
237
|
+
count++;
|
|
238
|
+
}
|
|
239
|
+
return count;
|
|
240
|
+
}
|
|
241
|
+
async queryCombined(keys, text, opts) {
|
|
242
|
+
const anchorResults = keys.length > 0 ? await this.queryAnchors({
|
|
243
|
+
keys,
|
|
244
|
+
scope: opts?.scope,
|
|
245
|
+
minConfidence: opts?.minConfidence,
|
|
246
|
+
limit: opts?.limit
|
|
247
|
+
}) : [];
|
|
248
|
+
const anchorPaths = new Set(anchorResults.map((r) => r.entryPath));
|
|
249
|
+
const ftsResults = text ? await this.queryFTS(text, opts?.limit) : [];
|
|
250
|
+
const results = anchorResults.map((r) => {
|
|
251
|
+
const fts = ftsResults.find((f) => f.entryPath === r.entryPath);
|
|
252
|
+
return {
|
|
253
|
+
entryPath: r.entryPath,
|
|
254
|
+
anchorScore: r.score,
|
|
255
|
+
ftsRank: fts?.rank,
|
|
256
|
+
score: r.score + (fts?.rank ?? 0)
|
|
257
|
+
};
|
|
258
|
+
});
|
|
259
|
+
for (const f of ftsResults) if (!anchorPaths.has(f.entryPath)) results.push({
|
|
260
|
+
entryPath: f.entryPath,
|
|
261
|
+
ftsRank: f.rank,
|
|
262
|
+
score: f.rank
|
|
263
|
+
});
|
|
264
|
+
results.sort((a, b) => b.score - a.score);
|
|
265
|
+
if (opts?.limit != null) return results.slice(0, opts.limit);
|
|
266
|
+
return results;
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
/**
|
|
270
|
+
* Check if an entry's meta matches the given filter.
|
|
271
|
+
* Each field in metaFilter is AND-ed. Array values are OR-ed within a field.
|
|
272
|
+
*
|
|
273
|
+
* NOTE: Uses strict equality (===) for value comparison. This works correctly
|
|
274
|
+
* for primitive values (string, number, boolean, null). Nested objects or arrays
|
|
275
|
+
* as meta values would need deep comparison to match correctly.
|
|
276
|
+
*/
|
|
277
|
+
function matchesMeta(meta, filter) {
|
|
278
|
+
if (!meta) return false;
|
|
279
|
+
for (const [key, expected] of Object.entries(filter)) {
|
|
280
|
+
const actual = meta[key];
|
|
281
|
+
if (Array.isArray(expected)) {
|
|
282
|
+
if (expected.length === 0) return false;
|
|
283
|
+
if (!expected.includes(actual)) return false;
|
|
284
|
+
} else if (actual !== expected) return false;
|
|
285
|
+
}
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
/** Cosine similarity between two vectors. Returns 0 for zero-length vectors. */
|
|
289
|
+
function cosineSimilarity(a, b) {
|
|
290
|
+
const len = Math.min(a.length, b.length);
|
|
291
|
+
let dot = 0;
|
|
292
|
+
let magA = 0;
|
|
293
|
+
let magB = 0;
|
|
294
|
+
for (let i = 0; i < len; i++) {
|
|
295
|
+
dot += a[i] * b[i];
|
|
296
|
+
magA += a[i] * a[i];
|
|
297
|
+
magB += b[i] * b[i];
|
|
298
|
+
}
|
|
299
|
+
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
300
|
+
return denom === 0 ? 0 : dot / denom;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
//#endregion
|
|
304
|
+
exports.MemoryIndexBackend = MemoryIndexBackend;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { IndexBackend, RemoveIndexOptions } from "./types.cjs";
|
|
2
|
+
import { Anchor, AnchorData, AnchorQuery, AnchorResult, CombinedResult, EntryFilter, FTSResult, IndexedEntry, QueryOpts, StaleFilter, VectorResult } from "../types.cjs";
|
|
3
|
+
import { KeyGeneratorRegistry } from "../keys/registry.cjs";
|
|
4
|
+
|
|
5
|
+
//#region src/backend/memory-backend.d.ts
|
|
6
|
+
declare class MemoryIndexBackend implements IndexBackend {
|
|
7
|
+
private entries;
|
|
8
|
+
private anchorData;
|
|
9
|
+
private anchorKeys;
|
|
10
|
+
private summaries;
|
|
11
|
+
private embeddings;
|
|
12
|
+
private keyRegistry;
|
|
13
|
+
constructor(keyRegistry?: KeyGeneratorRegistry);
|
|
14
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
15
|
+
setKeyRegistry(registry: KeyGeneratorRegistry): void;
|
|
16
|
+
initialize(): Promise<void>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
writeEntry(entry: IndexedEntry): Promise<void>;
|
|
19
|
+
getEntry(path: string): Promise<IndexedEntry | null>;
|
|
20
|
+
removeEntry(path: string): Promise<void>;
|
|
21
|
+
listEntries(filter: EntryFilter): Promise<IndexedEntry[]>;
|
|
22
|
+
markStale(filter: StaleFilter): Promise<number>;
|
|
23
|
+
countEntries(filter: EntryFilter): Promise<number>;
|
|
24
|
+
writeAnchors(path: string, anchors: Anchor[]): Promise<void>;
|
|
25
|
+
queryAnchors(query: AnchorQuery): Promise<AnchorResult[]>;
|
|
26
|
+
removeAnchors(path: string): Promise<void>;
|
|
27
|
+
getAnchorData(path: string): Promise<AnchorData[]>;
|
|
28
|
+
writeSummary(path: string, summary: string): Promise<void>;
|
|
29
|
+
getSummary(path: string): Promise<string | null>;
|
|
30
|
+
queryFTS(text: string, limit?: number, metaFilter?: Record<string, unknown>, opts?: {
|
|
31
|
+
domain?: string;
|
|
32
|
+
domains?: string[];
|
|
33
|
+
scope?: string;
|
|
34
|
+
pathPrefix?: string;
|
|
35
|
+
}): Promise<FTSResult[]>;
|
|
36
|
+
removeSummary(path: string): Promise<void>;
|
|
37
|
+
writeEmbedding(path: string, embedding: number[]): Promise<void>;
|
|
38
|
+
queryVector(embedding: number[], limit?: number, opts?: {
|
|
39
|
+
domain?: string;
|
|
40
|
+
domains?: string[];
|
|
41
|
+
scope?: string;
|
|
42
|
+
pathPrefix?: string;
|
|
43
|
+
}): Promise<VectorResult[]>;
|
|
44
|
+
removeEmbedding(path: string): Promise<void>;
|
|
45
|
+
embeddingCapability(): {
|
|
46
|
+
supported: boolean;
|
|
47
|
+
dimension: number;
|
|
48
|
+
};
|
|
49
|
+
writeEntriesBatch(entries: IndexedEntry[]): Promise<void>;
|
|
50
|
+
writeAnchorsBatch(items: {
|
|
51
|
+
path: string;
|
|
52
|
+
anchors: Anchor[];
|
|
53
|
+
}[]): Promise<void>;
|
|
54
|
+
writeSummariesBatch(items: {
|
|
55
|
+
path: string;
|
|
56
|
+
summary: string;
|
|
57
|
+
}[]): Promise<void>;
|
|
58
|
+
purgeEntries(paths: string[], opts?: RemoveIndexOptions): Promise<number>;
|
|
59
|
+
queryCombined(keys: {
|
|
60
|
+
type: string;
|
|
61
|
+
values: string[];
|
|
62
|
+
}[], text?: string, opts?: QueryOpts): Promise<CombinedResult[]>;
|
|
63
|
+
}
|
|
64
|
+
//#endregion
|
|
65
|
+
export { MemoryIndexBackend };
|
|
66
|
+
//# sourceMappingURL=memory-backend.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-backend.d.cts","names":[],"sources":["../../src/backend/memory-backend.ts"],"mappings":";;;;;cAsBa,kBAAA,YAA8B,YAAA;EAAA,QACjC,OAAA;EAAA,QACA,UAAA;EAAA,QACA,UAAA;EAAA,QACA,SAAA;EAAA,QACA,UAAA;EAAA,QACA,WAAA;cAEI,WAAA,GAAc,oBAAA;EAmCA;EA9B1B,cAAA,CAAe,QAAA,EAAU,oBAAA;EAInB,UAAA,CAAA,GAAc,OAAA;EAId,KAAA,CAAA,GAAS,OAAA;EAUT,UAAA,CAAW,KAAA,EAAO,YAAA,GAAe,OAAA;EAIjC,QAAA,CAAS,IAAA,WAAe,OAAA,CAAQ,YAAA;EAIhC,WAAA,CAAY,IAAA,WAAe,OAAA;EAI3B,WAAA,CAAY,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,YAAA;EAgB1C,SAAA,CAAU,MAAA,EAAQ,WAAA,GAAc,OAAA;EAuBhC,YAAA,CAAa,MAAA,EAAQ,WAAA,GAAc,OAAA;EAMnC,YAAA,CAAa,IAAA,UAAc,OAAA,EAAS,MAAA,KAAW,OAAA;EAsB/C,YAAA,CAAa,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;EAqE1C,aAAA,CAAc,IAAA,WAAe,OAAA;EAQ7B,aAAA,CAAc,IAAA,WAAe,OAAA,CAAQ,UAAA;EAYrC,YAAA,CAAa,IAAA,UAAc,OAAA,WAAkB,OAAA;EAI7C,UAAA,CAAW,IAAA,WAAe,OAAA;EAI1B,QAAA,CACJ,IAAA,UACA,KAAA,WACA,UAAA,GAAa,MAAA,mBACb,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,SAAA;EA6CL,aAAA,CAAc,IAAA,WAAe,OAAA;EAM7B,cAAA,CAAe,IAAA,UAAc,SAAA,aAAsB,OAAA;EAInD,WAAA,CACJ,SAAA,YACA,KAAA,WACA,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,YAAA;EAsBL,eAAA,CAAgB,IAAA,WAAe,OAAA;EAIrC,mBAAA,CAAA;IAAyB,SAAA;IAAoB,SAAA;EAAA;EAMvC,iBAAA,CAAkB,OAAA,EAAS,YAAA,KAAiB,OAAA;EAM5C,iBAAA,CAAkB,KAAA;IAAS,IAAA;IAAc,OAAA,EAAS,MAAA;EAAA,MAAe,OAAA;EAMjE,mBAAA,CAAoB,KAAA;IAAS,IAAA;IAAc,OAAA;EAAA,MAAsB,OAAA;EAMjE,YAAA,CAAa,KAAA,YAAiB,IAAA,GAAO,kBAAA,GAAqB,OAAA;EAkB1D,aAAA,CACJ,IAAA;IAAQ,IAAA;IAAc,MAAA;EAAA,KACtB,IAAA,WACA,IAAA,GAAO,SAAA,GACN,OAAA,CAAQ,cAAA;AAAA"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { IndexBackend, RemoveIndexOptions } from "./types.mjs";
|
|
2
|
+
import { Anchor, AnchorData, AnchorQuery, AnchorResult, CombinedResult, EntryFilter, FTSResult, IndexedEntry, QueryOpts, StaleFilter, VectorResult } from "../types.mjs";
|
|
3
|
+
import { KeyGeneratorRegistry } from "../keys/registry.mjs";
|
|
4
|
+
|
|
5
|
+
//#region src/backend/memory-backend.d.ts
|
|
6
|
+
declare class MemoryIndexBackend implements IndexBackend {
|
|
7
|
+
private entries;
|
|
8
|
+
private anchorData;
|
|
9
|
+
private anchorKeys;
|
|
10
|
+
private summaries;
|
|
11
|
+
private embeddings;
|
|
12
|
+
private keyRegistry;
|
|
13
|
+
constructor(keyRegistry?: KeyGeneratorRegistry);
|
|
14
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
15
|
+
setKeyRegistry(registry: KeyGeneratorRegistry): void;
|
|
16
|
+
initialize(): Promise<void>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
writeEntry(entry: IndexedEntry): Promise<void>;
|
|
19
|
+
getEntry(path: string): Promise<IndexedEntry | null>;
|
|
20
|
+
removeEntry(path: string): Promise<void>;
|
|
21
|
+
listEntries(filter: EntryFilter): Promise<IndexedEntry[]>;
|
|
22
|
+
markStale(filter: StaleFilter): Promise<number>;
|
|
23
|
+
countEntries(filter: EntryFilter): Promise<number>;
|
|
24
|
+
writeAnchors(path: string, anchors: Anchor[]): Promise<void>;
|
|
25
|
+
queryAnchors(query: AnchorQuery): Promise<AnchorResult[]>;
|
|
26
|
+
removeAnchors(path: string): Promise<void>;
|
|
27
|
+
getAnchorData(path: string): Promise<AnchorData[]>;
|
|
28
|
+
writeSummary(path: string, summary: string): Promise<void>;
|
|
29
|
+
getSummary(path: string): Promise<string | null>;
|
|
30
|
+
queryFTS(text: string, limit?: number, metaFilter?: Record<string, unknown>, opts?: {
|
|
31
|
+
domain?: string;
|
|
32
|
+
domains?: string[];
|
|
33
|
+
scope?: string;
|
|
34
|
+
pathPrefix?: string;
|
|
35
|
+
}): Promise<FTSResult[]>;
|
|
36
|
+
removeSummary(path: string): Promise<void>;
|
|
37
|
+
writeEmbedding(path: string, embedding: number[]): Promise<void>;
|
|
38
|
+
queryVector(embedding: number[], limit?: number, opts?: {
|
|
39
|
+
domain?: string;
|
|
40
|
+
domains?: string[];
|
|
41
|
+
scope?: string;
|
|
42
|
+
pathPrefix?: string;
|
|
43
|
+
}): Promise<VectorResult[]>;
|
|
44
|
+
removeEmbedding(path: string): Promise<void>;
|
|
45
|
+
embeddingCapability(): {
|
|
46
|
+
supported: boolean;
|
|
47
|
+
dimension: number;
|
|
48
|
+
};
|
|
49
|
+
writeEntriesBatch(entries: IndexedEntry[]): Promise<void>;
|
|
50
|
+
writeAnchorsBatch(items: {
|
|
51
|
+
path: string;
|
|
52
|
+
anchors: Anchor[];
|
|
53
|
+
}[]): Promise<void>;
|
|
54
|
+
writeSummariesBatch(items: {
|
|
55
|
+
path: string;
|
|
56
|
+
summary: string;
|
|
57
|
+
}[]): Promise<void>;
|
|
58
|
+
purgeEntries(paths: string[], opts?: RemoveIndexOptions): Promise<number>;
|
|
59
|
+
queryCombined(keys: {
|
|
60
|
+
type: string;
|
|
61
|
+
values: string[];
|
|
62
|
+
}[], text?: string, opts?: QueryOpts): Promise<CombinedResult[]>;
|
|
63
|
+
}
|
|
64
|
+
//#endregion
|
|
65
|
+
export { MemoryIndexBackend };
|
|
66
|
+
//# sourceMappingURL=memory-backend.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-backend.d.mts","names":[],"sources":["../../src/backend/memory-backend.ts"],"mappings":";;;;;cAsBa,kBAAA,YAA8B,YAAA;EAAA,QACjC,OAAA;EAAA,QACA,UAAA;EAAA,QACA,UAAA;EAAA,QACA,SAAA;EAAA,QACA,UAAA;EAAA,QACA,WAAA;cAEI,WAAA,GAAc,oBAAA;EAmCA;EA9B1B,cAAA,CAAe,QAAA,EAAU,oBAAA;EAInB,UAAA,CAAA,GAAc,OAAA;EAId,KAAA,CAAA,GAAS,OAAA;EAUT,UAAA,CAAW,KAAA,EAAO,YAAA,GAAe,OAAA;EAIjC,QAAA,CAAS,IAAA,WAAe,OAAA,CAAQ,YAAA;EAIhC,WAAA,CAAY,IAAA,WAAe,OAAA;EAI3B,WAAA,CAAY,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,YAAA;EAgB1C,SAAA,CAAU,MAAA,EAAQ,WAAA,GAAc,OAAA;EAuBhC,YAAA,CAAa,MAAA,EAAQ,WAAA,GAAc,OAAA;EAMnC,YAAA,CAAa,IAAA,UAAc,OAAA,EAAS,MAAA,KAAW,OAAA;EAsB/C,YAAA,CAAa,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;EAqE1C,aAAA,CAAc,IAAA,WAAe,OAAA;EAQ7B,aAAA,CAAc,IAAA,WAAe,OAAA,CAAQ,UAAA;EAYrC,YAAA,CAAa,IAAA,UAAc,OAAA,WAAkB,OAAA;EAI7C,UAAA,CAAW,IAAA,WAAe,OAAA;EAI1B,QAAA,CACJ,IAAA,UACA,KAAA,WACA,UAAA,GAAa,MAAA,mBACb,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,SAAA;EA6CL,aAAA,CAAc,IAAA,WAAe,OAAA;EAM7B,cAAA,CAAe,IAAA,UAAc,SAAA,aAAsB,OAAA;EAInD,WAAA,CACJ,SAAA,YACA,KAAA,WACA,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,YAAA;EAsBL,eAAA,CAAgB,IAAA,WAAe,OAAA;EAIrC,mBAAA,CAAA;IAAyB,SAAA;IAAoB,SAAA;EAAA;EAMvC,iBAAA,CAAkB,OAAA,EAAS,YAAA,KAAiB,OAAA;EAM5C,iBAAA,CAAkB,KAAA;IAAS,IAAA;IAAc,OAAA,EAAS,MAAA;EAAA,MAAe,OAAA;EAMjE,mBAAA,CAAoB,KAAA;IAAS,IAAA;IAAc,OAAA;EAAA,MAAsB,OAAA;EAMjE,YAAA,CAAa,KAAA,YAAiB,IAAA,GAAO,kBAAA,GAAqB,OAAA;EAkB1D,aAAA,CACJ,IAAA;IAAQ,IAAA;IAAc,MAAA;EAAA,KACtB,IAAA,WACA,IAAA,GAAO,SAAA,GACN,OAAA,CAAQ,cAAA;AAAA"}
|