@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { CursorRow, CursorScopeRegistration, IndexBackend, IndexedDocumentWrite, RemoveIndexOptions } from "./types.cjs";
|
|
2
|
+
import { Anchor, AnchorData, AnchorQuery, AnchorResult, CombinedResult, EntryFilter, FTSResult, IndexedEntry, QueryOpts, StaleFilter, VectorResult } from "../types.cjs";
|
|
3
|
+
import { KeyGeneratorRegistry } from "../keys/registry.cjs";
|
|
4
|
+
import { VectorBackend } from "./vector-backend.cjs";
|
|
5
|
+
|
|
6
|
+
//#region src/backend/d1-index-backend.d.ts
|
|
7
|
+
declare class D1IndexBackend implements IndexBackend {
|
|
8
|
+
private db;
|
|
9
|
+
private readonly d1;
|
|
10
|
+
private readonly scope;
|
|
11
|
+
private keyRegistry;
|
|
12
|
+
private readonly ftsTokenizer;
|
|
13
|
+
private readonly _injectedVectorBackend?;
|
|
14
|
+
private _vectorBackend;
|
|
15
|
+
constructor(d1: unknown, scope: string, keyRegistry?: KeyGeneratorRegistry, opts?: {
|
|
16
|
+
ftsTokenizer?: string;
|
|
17
|
+
vectorBackend?: VectorBackend;
|
|
18
|
+
});
|
|
19
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
20
|
+
setKeyRegistry(registry: KeyGeneratorRegistry): void;
|
|
21
|
+
initialize(): Promise<void>;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
private d1BatchBinding;
|
|
24
|
+
/**
|
|
25
|
+
* Rename the anchor tables `arc_anchor_*` → `idx_anchor_*` in place, preserving
|
|
26
|
+
* rows. Probe-guarded (renames only when the legacy table exists and the target
|
|
27
|
+
* does not) so it is idempotent and safe on fresh / already-migrated / racing
|
|
28
|
+
* isolates. Concurrent isolates may both attempt the ALTER on the same D1 —
|
|
29
|
+
* the loser's `no such table` / `already exists` is swallowed.
|
|
30
|
+
*
|
|
31
|
+
* ensureSchema runs on the cold-start (first-request-per-isolate) path, so the
|
|
32
|
+
* existence check is a SINGLE `sqlite_master` read for all four names rather
|
|
33
|
+
* than one per pair — one O(1) probe, no per-request cost (steady state is
|
|
34
|
+
* gated out by the `schemaEnsured` WeakSet).
|
|
35
|
+
*/
|
|
36
|
+
private renameAnchorTables;
|
|
37
|
+
private ensureSchema;
|
|
38
|
+
/**
|
|
39
|
+
* Create `idx_fts_rowid` the first time this D1 sees it, backfilling from
|
|
40
|
+
* any pre-existing `idx_fts` rows (issue #1103). The backfill maps every
|
|
41
|
+
* existing FTS row to its own rowid so a later rewrite REPLACEs in place
|
|
42
|
+
* instead of leaving a duplicate for the same (scope, entry_path).
|
|
43
|
+
*
|
|
44
|
+
* Race-safe across concurrent cold isolates — D1 auto-commits each statement
|
|
45
|
+
* (no transaction), so two isolates can run this at once on a fresh D1:
|
|
46
|
+
* - `CREATE TABLE IF NOT EXISTS` never throws on a concurrent create.
|
|
47
|
+
* - The one-time backfill runs only while the mapping is still empty (a
|
|
48
|
+
* cheap LIMIT-1 probe, NOT a scan), so steady state costs nothing extra.
|
|
49
|
+
* - `INSERT OR IGNORE` makes the backfill itself idempotent, so two
|
|
50
|
+
* isolates racing it (or a re-run) can't collide on the PK/UNIQUE.
|
|
51
|
+
*/
|
|
52
|
+
private ensureFtsRowidMapping;
|
|
53
|
+
private supportsCursorServedAt;
|
|
54
|
+
writeEntry(entry: IndexedEntry): Promise<void>;
|
|
55
|
+
writeIndexedDocument(document: IndexedDocumentWrite): Promise<void>;
|
|
56
|
+
getEntry(path: string): Promise<IndexedEntry | null>;
|
|
57
|
+
removeEntry(path: string): Promise<void>;
|
|
58
|
+
removeIndexedDocument(path: string, opts?: RemoveIndexOptions): Promise<void>;
|
|
59
|
+
listEntries(filter: EntryFilter): Promise<IndexedEntry[]>;
|
|
60
|
+
private entryConditions;
|
|
61
|
+
markStale(filter: StaleFilter): Promise<number>;
|
|
62
|
+
countEntries(filter: EntryFilter): Promise<number>;
|
|
63
|
+
writeAnchors(path: string, anchors: Anchor[]): Promise<void>;
|
|
64
|
+
/**
|
|
65
|
+
* Run `<prefix>(row),(row),…` as multi-row INSERT statement(s), chunked so
|
|
66
|
+
* each stays under D1's 100-bound-parameter-per-statement limit (5 columns per
|
|
67
|
+
* row → 18 rows ≈ 90 params). No-op on an empty row list. Portable across all
|
|
68
|
+
* db backends (plain `db.run`, unlike the driver-specific db.batch).
|
|
69
|
+
*/
|
|
70
|
+
private multiRowInsert;
|
|
71
|
+
queryAnchors(query: AnchorQuery): Promise<AnchorResult[]>;
|
|
72
|
+
removeAnchors(path: string): Promise<void>;
|
|
73
|
+
getAnchorData(path: string): Promise<AnchorData[]>;
|
|
74
|
+
writeSummary(path: string, summary: string): Promise<void>;
|
|
75
|
+
getSummary(path: string): Promise<string | null>;
|
|
76
|
+
queryFTS(text: string, limit?: number, metaFilter?: Record<string, unknown>, opts?: {
|
|
77
|
+
domain?: string;
|
|
78
|
+
domains?: string[];
|
|
79
|
+
scope?: string;
|
|
80
|
+
pathPrefix?: string;
|
|
81
|
+
}): Promise<FTSResult[]>;
|
|
82
|
+
removeSummary(path: string): Promise<void>;
|
|
83
|
+
writeEmbedding(path: string, embedding: number[], domain?: string): Promise<void>;
|
|
84
|
+
queryVector(embedding: number[], limit?: number, opts?: {
|
|
85
|
+
domain?: string;
|
|
86
|
+
domains?: string[];
|
|
87
|
+
scope?: string;
|
|
88
|
+
pathPrefix?: string;
|
|
89
|
+
}): Promise<VectorResult[]>;
|
|
90
|
+
removeEmbedding(path: string): Promise<void>;
|
|
91
|
+
embeddingCapability(): {
|
|
92
|
+
supported: boolean;
|
|
93
|
+
dimension: number;
|
|
94
|
+
} | null;
|
|
95
|
+
readCursor(consumerId: string): Promise<CursorRow | null>;
|
|
96
|
+
/**
|
|
97
|
+
* Set a cursor row's `last_seq` (+ watermarks/updated_at/served_at).
|
|
98
|
+
*
|
|
99
|
+
* NOTE (index-follower-dirty-path §4.2 cutover): the index FOLLOWER no longer
|
|
100
|
+
* calls this — after the changelog replay was deleted, `last_seq` is a DEAD
|
|
101
|
+
* COLUMN (nothing advances it during drain; scheduling rides `served_at` via
|
|
102
|
+
* `markCursorServed`). `writeCursor` now has NO production caller (the ops
|
|
103
|
+
* scripts write cursor rows via raw SQL, not this method); it is retained for
|
|
104
|
+
* the monotonic-guard test suite + as a stable backend primitive, so the
|
|
105
|
+
* `MAX(last_seq, excluded.last_seq)` guard (issue #935 BLOCK fix #2) stays
|
|
106
|
+
* correct there; watermarks/updated_at/served_at update unconditionally.
|
|
107
|
+
*/
|
|
108
|
+
writeCursor(consumerId: string, lastSeq: number, watermarks?: Record<string, number>): Promise<void>;
|
|
109
|
+
markCursorServed(consumerId: string): Promise<void>;
|
|
110
|
+
initCursor(consumerId: string, registration?: {
|
|
111
|
+
instanceDid?: string;
|
|
112
|
+
domainsJson?: string;
|
|
113
|
+
}): Promise<void>;
|
|
114
|
+
/**
|
|
115
|
+
* See {@link IndexBackend.listCursorScopes}. Deliberately NOT filtered by
|
|
116
|
+
* `this.scope` — this is the one cursor operation that must see EVERY
|
|
117
|
+
* scope in the D1, so the CF cron can discover work registered by ANY
|
|
118
|
+
* caller in ANY prior isolate, not just the scope this backend instance
|
|
119
|
+
* happens to be constructed for.
|
|
120
|
+
*
|
|
121
|
+
* `consumerId` is a FAMILY prefix, not an exact match (design.md §4.4
|
|
122
|
+
* addendum, owner decision 2026-07-05): every real cursor row's
|
|
123
|
+
* `consumer_id` is scope-qualified via `deriveFollowerConsumerId(scope)`
|
|
124
|
+
* ("index:{scope}") so a single shared backend/database can host multiple
|
|
125
|
+
* scopes without their cursor rows colliding — see follower.ts. Discovery
|
|
126
|
+
* doesn't know the scope in advance (that's its whole job), so it matches
|
|
127
|
+
* every row whose `consumer_id` starts with `"{consumerId}:"` instead of
|
|
128
|
+
* one exact value. Callers keep passing the bare `FOLLOWER_CONSUMER_ID`
|
|
129
|
+
* ("index") — this method appends the `:` separator internally.
|
|
130
|
+
*/
|
|
131
|
+
listCursorScopes(consumerId: string): Promise<CursorScopeRegistration[]>;
|
|
132
|
+
writeEntriesBatch(entries: IndexedEntry[]): Promise<void>;
|
|
133
|
+
writeAnchorsBatch(items: {
|
|
134
|
+
path: string;
|
|
135
|
+
anchors: Anchor[];
|
|
136
|
+
}[]): Promise<void>;
|
|
137
|
+
writeSummariesBatch(items: {
|
|
138
|
+
path: string;
|
|
139
|
+
summary: string;
|
|
140
|
+
}[]): Promise<void>;
|
|
141
|
+
purgeEntries(paths: string[], opts?: RemoveIndexOptions): Promise<number>;
|
|
142
|
+
queryCombined(keys: {
|
|
143
|
+
type: string;
|
|
144
|
+
values: string[];
|
|
145
|
+
}[], text?: string, opts?: QueryOpts): Promise<CombinedResult[]>;
|
|
146
|
+
}
|
|
147
|
+
//#endregion
|
|
148
|
+
export { D1IndexBackend };
|
|
149
|
+
//# sourceMappingURL=d1-index-backend.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"d1-index-backend.d.cts","names":[],"sources":["../../src/backend/d1-index-backend.ts"],"mappings":";;;;;;cAkEa,cAAA,YAA0B,YAAA;EAAA,QAC7B,EAAA;EAAA,iBACS,EAAA;EAAA,iBACA,KAAA;EAAA,QACT,WAAA;EAAA,iBACS,YAAA;EAAA,iBACA,sBAAA;EAAA,QACT,cAAA;cAGN,EAAA,WACA,KAAA,UACA,WAAA,GAAc,oBAAA,EACd,IAAA;IAAQ,YAAA;IAAuB,aAAA,GAAgB,aAAA;EAAA;EAmPZ;EAzOrC,cAAA,CAAe,QAAA,EAAU,oBAAA;EAInB,UAAA,CAAA,GAAc,OAAA;EAWd,KAAA,CAAA,GAAS,OAAA;EAAA,QAIP,cAAA;EA4UyC;;;;;;;;;;;;EAAA,QAxTnC,kBAAA;EAAA,QA2BA,YAAA;EAgkBqB;;;;;;;;;;;;;;EAAA,QAzcrB,qBAAA;EAAA,QAuBA,sBAAA;EAiBR,UAAA,CAAW,KAAA,EAAO,YAAA,GAAe,OAAA;EAQjC,oBAAA,CAAqB,QAAA,EAAU,oBAAA,GAAuB,OAAA;EAwGtD,QAAA,CAAS,IAAA,WAAe,OAAA,CAAQ,YAAA;EAQhC,WAAA,CAAY,IAAA,WAAe,OAAA;EAM3B,qBAAA,CAAsB,IAAA,UAAc,IAAA,GAAO,kBAAA,GAAqB,OAAA;EA8ChE,WAAA,CAAY,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,YAAA;EAAA,QAQxC,eAAA;EAgBF,SAAA,CAAU,MAAA,EAAQ,WAAA,GAAc,OAAA;EAmChC,YAAA,CAAa,MAAA,EAAQ,WAAA,GAAc,OAAA;EAUnC,YAAA,CAAa,IAAA,UAAc,OAAA,EAAS,MAAA,KAAW,OAAA;EAscV;;;;;;EAAA,QA3Z7B,cAAA;EAUR,YAAA,CAAa,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;EA2H1C,aAAA,CAAc,IAAA,WAAe,OAAA;EAS7B,aAAA,CAAc,IAAA,WAAe,OAAA,CAAQ,UAAA;EAcrC,YAAA,CAAa,IAAA,UAAc,OAAA,WAAkB,OAAA;EAO7C,UAAA,CAAW,IAAA,WAAe,OAAA;EAO1B,QAAA,CACJ,IAAA,UACA,KAAA,WACA,UAAA,GAAa,MAAA,mBACb,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,SAAA;EA2DL,aAAA,CAAc,IAAA,WAAe,OAAA;EAO7B,cAAA,CAAe,IAAA,UAAc,SAAA,YAAqB,MAAA,YAAkB,OAAA;EAIpE,WAAA,CACJ,SAAA,YACA,KAAA,WACA,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,YAAA;EAQL,eAAA,CAAgB,IAAA,WAAe,OAAA;EAIrC,mBAAA,CAAA;IAAyB,SAAA;IAAoB,SAAA;EAAA;EAMvC,UAAA,CAAW,UAAA,WAAqB,OAAA,CAAQ,SAAA;EArvBtC;;;;;;;;;;;;EAsxBF,WAAA,CACJ,UAAA,UACA,OAAA,UACA,UAAA,GAAa,MAAA,mBACZ,OAAA;EAyBG,gBAAA,CAAiB,UAAA,WAAqB,OAAA;EAOtC,UAAA,CACJ,UAAA,UACA,YAAA;IAAiB,WAAA;IAAsB,WAAA;EAAA,IACtC,OAAA;EAvfe;;;;;;;;;;;;;;;;;EAuhBZ,gBAAA,CAAiB,UAAA,WAAqB,OAAA,CAAQ,uBAAA;EA4B9C,iBAAA,CAAkB,OAAA,EAAS,YAAA,KAAiB,OAAA;EAI5C,iBAAA,CAAkB,KAAA;IAAS,IAAA;IAAc,OAAA,EAAS,MAAA;EAAA,MAAe,OAAA;EAIjE,mBAAA,CAAoB,KAAA;IAAS,IAAA;IAAc,OAAA;EAAA,MAAsB,OAAA;EAIjE,YAAA,CAAa,KAAA,YAAiB,IAAA,GAAO,kBAAA,GAAqB,OAAA;EA6C1D,aAAA,CACJ,IAAA;IAAQ,IAAA;IAAc,MAAA;EAAA,KACtB,IAAA,WACA,IAAA,GAAO,SAAA,GACN,OAAA,CAAQ,cAAA;AAAA"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { CursorRow, CursorScopeRegistration, IndexBackend, IndexedDocumentWrite, RemoveIndexOptions } from "./types.mjs";
|
|
2
|
+
import { Anchor, AnchorData, AnchorQuery, AnchorResult, CombinedResult, EntryFilter, FTSResult, IndexedEntry, QueryOpts, StaleFilter, VectorResult } from "../types.mjs";
|
|
3
|
+
import { KeyGeneratorRegistry } from "../keys/registry.mjs";
|
|
4
|
+
import { VectorBackend } from "./vector-backend.mjs";
|
|
5
|
+
|
|
6
|
+
//#region src/backend/d1-index-backend.d.ts
|
|
7
|
+
declare class D1IndexBackend implements IndexBackend {
|
|
8
|
+
private db;
|
|
9
|
+
private readonly d1;
|
|
10
|
+
private readonly scope;
|
|
11
|
+
private keyRegistry;
|
|
12
|
+
private readonly ftsTokenizer;
|
|
13
|
+
private readonly _injectedVectorBackend?;
|
|
14
|
+
private _vectorBackend;
|
|
15
|
+
constructor(d1: unknown, scope: string, keyRegistry?: KeyGeneratorRegistry, opts?: {
|
|
16
|
+
ftsTokenizer?: string;
|
|
17
|
+
vectorBackend?: VectorBackend;
|
|
18
|
+
});
|
|
19
|
+
/** See `IndexBackend.setKeyRegistry` — wires an injected backend to the provider's shared registry. */
|
|
20
|
+
setKeyRegistry(registry: KeyGeneratorRegistry): void;
|
|
21
|
+
initialize(): Promise<void>;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
private d1BatchBinding;
|
|
24
|
+
/**
|
|
25
|
+
* Rename the anchor tables `arc_anchor_*` → `idx_anchor_*` in place, preserving
|
|
26
|
+
* rows. Probe-guarded (renames only when the legacy table exists and the target
|
|
27
|
+
* does not) so it is idempotent and safe on fresh / already-migrated / racing
|
|
28
|
+
* isolates. Concurrent isolates may both attempt the ALTER on the same D1 —
|
|
29
|
+
* the loser's `no such table` / `already exists` is swallowed.
|
|
30
|
+
*
|
|
31
|
+
* ensureSchema runs on the cold-start (first-request-per-isolate) path, so the
|
|
32
|
+
* existence check is a SINGLE `sqlite_master` read for all four names rather
|
|
33
|
+
* than one per pair — one O(1) probe, no per-request cost (steady state is
|
|
34
|
+
* gated out by the `schemaEnsured` WeakSet).
|
|
35
|
+
*/
|
|
36
|
+
private renameAnchorTables;
|
|
37
|
+
private ensureSchema;
|
|
38
|
+
/**
|
|
39
|
+
* Create `idx_fts_rowid` the first time this D1 sees it, backfilling from
|
|
40
|
+
* any pre-existing `idx_fts` rows (issue #1103). The backfill maps every
|
|
41
|
+
* existing FTS row to its own rowid so a later rewrite REPLACEs in place
|
|
42
|
+
* instead of leaving a duplicate for the same (scope, entry_path).
|
|
43
|
+
*
|
|
44
|
+
* Race-safe across concurrent cold isolates — D1 auto-commits each statement
|
|
45
|
+
* (no transaction), so two isolates can run this at once on a fresh D1:
|
|
46
|
+
* - `CREATE TABLE IF NOT EXISTS` never throws on a concurrent create.
|
|
47
|
+
* - The one-time backfill runs only while the mapping is still empty (a
|
|
48
|
+
* cheap LIMIT-1 probe, NOT a scan), so steady state costs nothing extra.
|
|
49
|
+
* - `INSERT OR IGNORE` makes the backfill itself idempotent, so two
|
|
50
|
+
* isolates racing it (or a re-run) can't collide on the PK/UNIQUE.
|
|
51
|
+
*/
|
|
52
|
+
private ensureFtsRowidMapping;
|
|
53
|
+
private supportsCursorServedAt;
|
|
54
|
+
writeEntry(entry: IndexedEntry): Promise<void>;
|
|
55
|
+
writeIndexedDocument(document: IndexedDocumentWrite): Promise<void>;
|
|
56
|
+
getEntry(path: string): Promise<IndexedEntry | null>;
|
|
57
|
+
removeEntry(path: string): Promise<void>;
|
|
58
|
+
removeIndexedDocument(path: string, opts?: RemoveIndexOptions): Promise<void>;
|
|
59
|
+
listEntries(filter: EntryFilter): Promise<IndexedEntry[]>;
|
|
60
|
+
private entryConditions;
|
|
61
|
+
markStale(filter: StaleFilter): Promise<number>;
|
|
62
|
+
countEntries(filter: EntryFilter): Promise<number>;
|
|
63
|
+
writeAnchors(path: string, anchors: Anchor[]): Promise<void>;
|
|
64
|
+
/**
|
|
65
|
+
* Run `<prefix>(row),(row),…` as multi-row INSERT statement(s), chunked so
|
|
66
|
+
* each stays under D1's 100-bound-parameter-per-statement limit (5 columns per
|
|
67
|
+
* row → 18 rows ≈ 90 params). No-op on an empty row list. Portable across all
|
|
68
|
+
* db backends (plain `db.run`, unlike the driver-specific db.batch).
|
|
69
|
+
*/
|
|
70
|
+
private multiRowInsert;
|
|
71
|
+
queryAnchors(query: AnchorQuery): Promise<AnchorResult[]>;
|
|
72
|
+
removeAnchors(path: string): Promise<void>;
|
|
73
|
+
getAnchorData(path: string): Promise<AnchorData[]>;
|
|
74
|
+
writeSummary(path: string, summary: string): Promise<void>;
|
|
75
|
+
getSummary(path: string): Promise<string | null>;
|
|
76
|
+
queryFTS(text: string, limit?: number, metaFilter?: Record<string, unknown>, opts?: {
|
|
77
|
+
domain?: string;
|
|
78
|
+
domains?: string[];
|
|
79
|
+
scope?: string;
|
|
80
|
+
pathPrefix?: string;
|
|
81
|
+
}): Promise<FTSResult[]>;
|
|
82
|
+
removeSummary(path: string): Promise<void>;
|
|
83
|
+
writeEmbedding(path: string, embedding: number[], domain?: string): Promise<void>;
|
|
84
|
+
queryVector(embedding: number[], limit?: number, opts?: {
|
|
85
|
+
domain?: string;
|
|
86
|
+
domains?: string[];
|
|
87
|
+
scope?: string;
|
|
88
|
+
pathPrefix?: string;
|
|
89
|
+
}): Promise<VectorResult[]>;
|
|
90
|
+
removeEmbedding(path: string): Promise<void>;
|
|
91
|
+
embeddingCapability(): {
|
|
92
|
+
supported: boolean;
|
|
93
|
+
dimension: number;
|
|
94
|
+
} | null;
|
|
95
|
+
readCursor(consumerId: string): Promise<CursorRow | null>;
|
|
96
|
+
/**
|
|
97
|
+
* Set a cursor row's `last_seq` (+ watermarks/updated_at/served_at).
|
|
98
|
+
*
|
|
99
|
+
* NOTE (index-follower-dirty-path §4.2 cutover): the index FOLLOWER no longer
|
|
100
|
+
* calls this — after the changelog replay was deleted, `last_seq` is a DEAD
|
|
101
|
+
* COLUMN (nothing advances it during drain; scheduling rides `served_at` via
|
|
102
|
+
* `markCursorServed`). `writeCursor` now has NO production caller (the ops
|
|
103
|
+
* scripts write cursor rows via raw SQL, not this method); it is retained for
|
|
104
|
+
* the monotonic-guard test suite + as a stable backend primitive, so the
|
|
105
|
+
* `MAX(last_seq, excluded.last_seq)` guard (issue #935 BLOCK fix #2) stays
|
|
106
|
+
* correct there; watermarks/updated_at/served_at update unconditionally.
|
|
107
|
+
*/
|
|
108
|
+
writeCursor(consumerId: string, lastSeq: number, watermarks?: Record<string, number>): Promise<void>;
|
|
109
|
+
markCursorServed(consumerId: string): Promise<void>;
|
|
110
|
+
initCursor(consumerId: string, registration?: {
|
|
111
|
+
instanceDid?: string;
|
|
112
|
+
domainsJson?: string;
|
|
113
|
+
}): Promise<void>;
|
|
114
|
+
/**
|
|
115
|
+
* See {@link IndexBackend.listCursorScopes}. Deliberately NOT filtered by
|
|
116
|
+
* `this.scope` — this is the one cursor operation that must see EVERY
|
|
117
|
+
* scope in the D1, so the CF cron can discover work registered by ANY
|
|
118
|
+
* caller in ANY prior isolate, not just the scope this backend instance
|
|
119
|
+
* happens to be constructed for.
|
|
120
|
+
*
|
|
121
|
+
* `consumerId` is a FAMILY prefix, not an exact match (design.md §4.4
|
|
122
|
+
* addendum, owner decision 2026-07-05): every real cursor row's
|
|
123
|
+
* `consumer_id` is scope-qualified via `deriveFollowerConsumerId(scope)`
|
|
124
|
+
* ("index:{scope}") so a single shared backend/database can host multiple
|
|
125
|
+
* scopes without their cursor rows colliding — see follower.ts. Discovery
|
|
126
|
+
* doesn't know the scope in advance (that's its whole job), so it matches
|
|
127
|
+
* every row whose `consumer_id` starts with `"{consumerId}:"` instead of
|
|
128
|
+
* one exact value. Callers keep passing the bare `FOLLOWER_CONSUMER_ID`
|
|
129
|
+
* ("index") — this method appends the `:` separator internally.
|
|
130
|
+
*/
|
|
131
|
+
listCursorScopes(consumerId: string): Promise<CursorScopeRegistration[]>;
|
|
132
|
+
writeEntriesBatch(entries: IndexedEntry[]): Promise<void>;
|
|
133
|
+
writeAnchorsBatch(items: {
|
|
134
|
+
path: string;
|
|
135
|
+
anchors: Anchor[];
|
|
136
|
+
}[]): Promise<void>;
|
|
137
|
+
writeSummariesBatch(items: {
|
|
138
|
+
path: string;
|
|
139
|
+
summary: string;
|
|
140
|
+
}[]): Promise<void>;
|
|
141
|
+
purgeEntries(paths: string[], opts?: RemoveIndexOptions): Promise<number>;
|
|
142
|
+
queryCombined(keys: {
|
|
143
|
+
type: string;
|
|
144
|
+
values: string[];
|
|
145
|
+
}[], text?: string, opts?: QueryOpts): Promise<CombinedResult[]>;
|
|
146
|
+
}
|
|
147
|
+
//#endregion
|
|
148
|
+
export { D1IndexBackend };
|
|
149
|
+
//# sourceMappingURL=d1-index-backend.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"d1-index-backend.d.mts","names":[],"sources":["../../src/backend/d1-index-backend.ts"],"mappings":";;;;;;cAkEa,cAAA,YAA0B,YAAA;EAAA,QAC7B,EAAA;EAAA,iBACS,EAAA;EAAA,iBACA,KAAA;EAAA,QACT,WAAA;EAAA,iBACS,YAAA;EAAA,iBACA,sBAAA;EAAA,QACT,cAAA;cAGN,EAAA,WACA,KAAA,UACA,WAAA,GAAc,oBAAA,EACd,IAAA;IAAQ,YAAA;IAAuB,aAAA,GAAgB,aAAA;EAAA;EAmPZ;EAzOrC,cAAA,CAAe,QAAA,EAAU,oBAAA;EAInB,UAAA,CAAA,GAAc,OAAA;EAWd,KAAA,CAAA,GAAS,OAAA;EAAA,QAIP,cAAA;EA4UyC;;;;;;;;;;;;EAAA,QAxTnC,kBAAA;EAAA,QA2BA,YAAA;EAgkBqB;;;;;;;;;;;;;;EAAA,QAzcrB,qBAAA;EAAA,QAuBA,sBAAA;EAiBR,UAAA,CAAW,KAAA,EAAO,YAAA,GAAe,OAAA;EAQjC,oBAAA,CAAqB,QAAA,EAAU,oBAAA,GAAuB,OAAA;EAwGtD,QAAA,CAAS,IAAA,WAAe,OAAA,CAAQ,YAAA;EAQhC,WAAA,CAAY,IAAA,WAAe,OAAA;EAM3B,qBAAA,CAAsB,IAAA,UAAc,IAAA,GAAO,kBAAA,GAAqB,OAAA;EA8ChE,WAAA,CAAY,MAAA,EAAQ,WAAA,GAAc,OAAA,CAAQ,YAAA;EAAA,QAQxC,eAAA;EAgBF,SAAA,CAAU,MAAA,EAAQ,WAAA,GAAc,OAAA;EAmChC,YAAA,CAAa,MAAA,EAAQ,WAAA,GAAc,OAAA;EAUnC,YAAA,CAAa,IAAA,UAAc,OAAA,EAAS,MAAA,KAAW,OAAA;EAscV;;;;;;EAAA,QA3Z7B,cAAA;EAUR,YAAA,CAAa,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;EA2H1C,aAAA,CAAc,IAAA,WAAe,OAAA;EAS7B,aAAA,CAAc,IAAA,WAAe,OAAA,CAAQ,UAAA;EAcrC,YAAA,CAAa,IAAA,UAAc,OAAA,WAAkB,OAAA;EAO7C,UAAA,CAAW,IAAA,WAAe,OAAA;EAO1B,QAAA,CACJ,IAAA,UACA,KAAA,WACA,UAAA,GAAa,MAAA,mBACb,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,SAAA;EA2DL,aAAA,CAAc,IAAA,WAAe,OAAA;EAO7B,cAAA,CAAe,IAAA,UAAc,SAAA,YAAqB,MAAA,YAAkB,OAAA;EAIpE,WAAA,CACJ,SAAA,YACA,KAAA,WACA,IAAA;IAAS,MAAA;IAAiB,OAAA;IAAoB,KAAA;IAAgB,UAAA;EAAA,IAC7D,OAAA,CAAQ,YAAA;EAQL,eAAA,CAAgB,IAAA,WAAe,OAAA;EAIrC,mBAAA,CAAA;IAAyB,SAAA;IAAoB,SAAA;EAAA;EAMvC,UAAA,CAAW,UAAA,WAAqB,OAAA,CAAQ,SAAA;EArvBtC;;;;;;;;;;;;EAsxBF,WAAA,CACJ,UAAA,UACA,OAAA,UACA,UAAA,GAAa,MAAA,mBACZ,OAAA;EAyBG,gBAAA,CAAiB,UAAA,WAAqB,OAAA;EAOtC,UAAA,CACJ,UAAA,UACA,YAAA;IAAiB,WAAA;IAAsB,WAAA;EAAA,IACtC,OAAA;EAvfe;;;;;;;;;;;;;;;;;EAuhBZ,gBAAA,CAAiB,UAAA,WAAqB,OAAA,CAAQ,uBAAA;EA4B9C,iBAAA,CAAkB,OAAA,EAAS,YAAA,KAAiB,OAAA;EAI5C,iBAAA,CAAkB,KAAA;IAAS,IAAA;IAAc,OAAA,EAAS,MAAA;EAAA,MAAe,OAAA;EAIjE,mBAAA,CAAoB,KAAA;IAAS,IAAA;IAAc,OAAA;EAAA,MAAsB,OAAA;EAIjE,YAAA,CAAa,KAAA,YAAiB,IAAA,GAAO,kBAAA,GAAqB,OAAA;EA6C1D,aAAA,CACJ,IAAA;IAAQ,IAAA;IAAc,MAAA;EAAA,KACtB,IAAA,WACA,IAAA,GAAO,SAAA,GACN,OAAA,CAAQ,cAAA;AAAA"}
|