@askdb/rag 0.2.0-beta.23 → 0.2.0-beta.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +14 -10
  2. package/bin/askdb-rag.js +8 -1
  3. package/dist/chunker/chunker.d.ts +5 -1
  4. package/dist/chunker/chunker.d.ts.map +1 -1
  5. package/dist/chunker/chunker.js +218 -125
  6. package/dist/chunker/chunker.js.map +1 -1
  7. package/dist/chunker/ids.d.ts +15 -0
  8. package/dist/chunker/ids.d.ts.map +1 -0
  9. package/dist/chunker/ids.js +22 -0
  10. package/dist/chunker/ids.js.map +1 -0
  11. package/dist/chunker/index.d.ts +1 -0
  12. package/dist/chunker/index.d.ts.map +1 -1
  13. package/dist/chunker/index.js +1 -0
  14. package/dist/chunker/index.js.map +1 -1
  15. package/dist/chunker/sources.d.ts.map +1 -1
  16. package/dist/chunker/sources.js +1 -0
  17. package/dist/chunker/sources.js.map +1 -1
  18. package/dist/embedders/ai-sdk.d.ts +15 -0
  19. package/dist/embedders/ai-sdk.d.ts.map +1 -1
  20. package/dist/embedders/ai-sdk.js +9 -0
  21. package/dist/embedders/ai-sdk.js.map +1 -1
  22. package/dist/embedders/dimensions.d.ts +9 -0
  23. package/dist/embedders/dimensions.d.ts.map +1 -0
  24. package/dist/embedders/dimensions.js +18 -0
  25. package/dist/embedders/dimensions.js.map +1 -0
  26. package/dist/embedders/openai.d.ts +1 -1
  27. package/dist/embedders/openai.js +1 -1
  28. package/dist/index.d.ts +5 -4
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +4 -3
  31. package/dist/index.js.map +1 -1
  32. package/dist/indexer/hash.d.ts +7 -0
  33. package/dist/indexer/hash.d.ts.map +1 -1
  34. package/dist/indexer/hash.js +13 -0
  35. package/dist/indexer/hash.js.map +1 -1
  36. package/dist/indexer/index.d.ts +10 -4
  37. package/dist/indexer/index.d.ts.map +1 -1
  38. package/dist/indexer/index.js +167 -30
  39. package/dist/indexer/index.js.map +1 -1
  40. package/dist/indexer/lock-file.d.ts +85 -4
  41. package/dist/indexer/lock-file.d.ts.map +1 -1
  42. package/dist/indexer/lock-file.js +108 -9
  43. package/dist/indexer/lock-file.js.map +1 -1
  44. package/dist/log-events.d.ts +11 -0
  45. package/dist/log-events.d.ts.map +1 -1
  46. package/dist/log-events.js +11 -0
  47. package/dist/log-events.js.map +1 -1
  48. package/dist/stores/file.d.ts +7 -1
  49. package/dist/stores/file.d.ts.map +1 -1
  50. package/dist/stores/file.js +52 -10
  51. package/dist/stores/file.js.map +1 -1
  52. package/dist/stores/memory.d.ts.map +1 -1
  53. package/dist/stores/memory.js +31 -3
  54. package/dist/stores/memory.js.map +1 -1
  55. package/dist/stores/pgvector.d.ts +36 -3
  56. package/dist/stores/pgvector.d.ts.map +1 -1
  57. package/dist/stores/pgvector.js +125 -17
  58. package/dist/stores/pgvector.js.map +1 -1
  59. package/dist/types.d.ts +69 -9
  60. package/dist/types.d.ts.map +1 -1
  61. package/package.json +10 -12
  62. package/dist/bin.d.ts +0 -3
  63. package/dist/bin.d.ts.map +0 -1
  64. package/dist/bin.js +0 -7
  65. package/dist/bin.js.map +0 -1
  66. package/dist/cli.d.ts +0 -11
  67. package/dist/cli.d.ts.map +0 -1
  68. package/dist/cli.js +0 -356
  69. package/dist/cli.js.map +0 -1
@@ -1,10 +1,10 @@
1
- import { chunkSchema } from "../chunker/index.js";
1
+ import { chunkIdPrefix, chunkSchema } from "../chunker/index.js";
2
2
  import { AskDbRagLogEvent } from "../log-events.js";
3
- import { chunkContentHash } from "./hash.js";
4
- import { readLockFile, writeLockFile, } from "./lock-file.js";
3
+ import { chunkContentHash, storedVectorHash } from "./hash.js";
4
+ import { SCHEMA_LOCK_VERSION, inspectLockFile, sameEmbedderId, writeLockFile, } from "./lock-file.js";
5
5
  const DEFAULT_BATCH_SIZE = 64;
6
6
  export async function buildSchemaIndex(options) {
7
- const { embedder, store, embedderId, lockFilePath, chunkOptions, batchSize = DEFAULT_BATCH_SIZE, logger, correlationId, onProgress, } = options;
7
+ const { embedder, store, embedderId, chunkOptions, batchSize = DEFAULT_BATCH_SIZE, logger, correlationId, onProgress, } = options;
8
8
  const sources = normalizeIndexSources(options);
9
9
  const baseLogContext = correlationId
10
10
  ? { correlationId }
@@ -28,25 +28,66 @@ export async function buildSchemaIndex(options) {
28
28
  count: chunkResult.stats.sensitiveIncluded,
29
29
  }, "sensitive describable-layer chunks INCLUDED (opt-in)");
30
30
  }
31
- // 3. Decide which chunks need (re-)embedding via lock-file/store hashes.
32
- const previousLock = lockFilePath ? readLockFile(lockFilePath) : undefined;
33
- const storeHashes = (await store.hashesByPrefix?.("chunk:")) ?? {};
34
- const previousHashes = { ...storeHashes, ...(previousLock?.hashes ?? {}) };
35
- const previousEmbedderId = previousLock?.embedderId;
36
- const embedderChanged = embedderId !== undefined &&
37
- previousEmbedderId !== undefined &&
38
- embedderId !== previousEmbedderId;
31
+ // 3. Decide which chunks need (re-)embedding.
32
+ //
33
+ // Stores that can report stored hashes are the source of truth: a chunk is
34
+ // skipped only when the store already holds, for its id, the hash of the
35
+ // same text embedded by the same embedder (`storedVectorHash`). Stores that
36
+ // cannot report hashes fall back to the lock's text hashes, guarded by the
37
+ // lock's embedder id, store identity, and dimensions.
38
+ const schemaId = sources.schema.schemaId;
39
+ const idPrefix = chunkIdPrefix(schemaId);
40
+ const descriptor = store.describe?.();
41
+ // An ephemeral store (memory) outlives no process, so the lock, which
42
+ // describes the persisted index, is neither read nor written for it.
43
+ const lockFilePath = descriptor?.ephemeral === true ? undefined : options.lockFilePath;
44
+ const lockState = lockFilePath
45
+ ? inspectLockFile(lockFilePath)
46
+ : { status: "missing" };
47
+ if (lockState.status === "ok" && lockState.lock.schemaId !== schemaId) {
48
+ // Renamed, or a lock copied from another schema's directory: its ids may
49
+ // belong to a schema that still shares this store, so nothing is deleted
50
+ // on its word.
51
+ logger?.info({
52
+ ...baseLogContext,
53
+ event: AskDbRagLogEvent.LockSchemaMismatch,
54
+ lockSchemaId: lockState.lock.schemaId,
55
+ schemaId,
56
+ }, "schema.lock.json was written for another schema id; reindexing without it and pruning none of its ids");
57
+ }
58
+ const previousLock = lockState.status === "ok" && lockState.lock.schemaId === schemaId
59
+ ? lockState.lock
60
+ : undefined;
61
+ const storeHashes = store.hashesByPrefix
62
+ ? await store.hashesByPrefix(idPrefix)
63
+ : undefined;
64
+ const fullReindexReason = decideFullReindex({
65
+ force: options.force === true,
66
+ lockFilePath,
67
+ lockState,
68
+ previousLock,
69
+ embedderId,
70
+ descriptor,
71
+ storeReportsHashes: storeHashes !== undefined,
72
+ });
73
+ // Whether or not it decided the full reindex (`force` or an outdated lock
74
+ // may come first): it also drops the previous lock's width below.
75
+ const embedderChanged = previousLock !== undefined && !sameEmbedderId(previousLock.embedderId, embedderId);
76
+ const previousHashes = storeHashes ?? previousLock?.hashes ?? {};
77
+ // `newHashes` (text only) go to the lock; `vectorHashes` (text + embedder
78
+ // id) go to the store with each vector.
39
79
  const newHashes = {};
80
+ const vectorHashes = {};
40
81
  const toEmbed = [];
41
82
  let reused = 0;
42
83
  for (const c of chunks) {
43
- const hash = chunkContentHash(c.text);
44
- newHashes[c.id] = hash;
45
- if (!embedderChanged && previousHashes[c.id] === hash) {
84
+ newHashes[c.id] = chunkContentHash(c.text);
85
+ vectorHashes[c.id] = storedVectorHash(c.text, embedderId);
86
+ const current = storeHashes !== undefined ? vectorHashes[c.id] : newHashes[c.id];
87
+ if (fullReindexReason === undefined && previousHashes[c.id] === current) {
88
+ // Same text, same embedder, and (for hash-reporting stores) verified
89
+ // present in the store — keep the stored vector.
46
90
  reused++;
47
- // Chunk text unchanged AND embedder unchanged — assume the store still
48
- // holds the embedding from a prior run. The retriever uses store.query
49
- // directly so we don't need to round-trip the vector.
50
91
  continue;
51
92
  }
52
93
  toEmbed.push(c);
@@ -58,6 +99,7 @@ export async function buildSchemaIndex(options) {
58
99
  toEmbed: toEmbed.length,
59
100
  reused,
60
101
  embedderChanged,
102
+ ...(fullReindexReason ? { fullReindexReason } : {}),
61
103
  }, "rag indexing started");
62
104
  onProgress?.({
63
105
  kind: "started",
@@ -70,20 +112,40 @@ export async function buildSchemaIndex(options) {
70
112
  ...baseLogContext,
71
113
  event: AskDbRagLogEvent.ChunksReused,
72
114
  count: reused,
73
- }, "rag chunks reused from lock file");
115
+ }, "rag chunks reused (unchanged and already stored)");
74
116
  }
75
117
  // 4. Embed in batches and upsert.
118
+ if (lockFilePath && previousLock && embedderChanged && toEmbed.length > 0) {
119
+ // Until this run finishes, the store holds two models' vectors: mark the
120
+ // lock so a query guard (checkIndexMatches) refuses it if the run fails.
121
+ writeLockFile(lockFilePath, { ...previousLock, incomplete: true, updatedAt: new Date().toISOString() });
122
+ }
76
123
  let embeddedCount = 0;
124
+ let observedDimensions;
77
125
  for (let i = 0; i < toEmbed.length; i += batchSize) {
78
126
  const batch = toEmbed.slice(i, i + batchSize);
79
127
  const vectors = await embedder(batch.map((c) => c.text));
80
128
  if (vectors.length !== batch.length) {
81
129
  throw new Error(`Embedder returned ${vectors.length} vectors for ${batch.length} inputs.`);
82
130
  }
131
+ for (const v of vectors) {
132
+ observedDimensions ??= v.length;
133
+ if (v.length !== observedDimensions) {
134
+ throw new Error(`Embedder returned vectors of inconsistent dimensions (${observedDimensions} and ${v.length}).`);
135
+ }
136
+ }
137
+ if (descriptor?.dimensions !== undefined &&
138
+ observedDimensions !== undefined &&
139
+ observedDimensions !== descriptor.dimensions) {
140
+ throw new Error(`Embedder returned ${observedDimensions}-dimension vectors but the ${descriptor.kind} store ` +
141
+ `is set up for ${descriptor.dimensions}. Use a store set up for ${observedDimensions}, or an ` +
142
+ `embedder that produces ${descriptor.dimensions}-dimension vectors.` +
143
+ (descriptor.widthHint ? ` ${descriptor.widthHint}` : ""));
144
+ }
83
145
  await store.upsert(batch.map((c, idx) => ({
84
146
  id: c.id,
85
147
  vector: vectors[idx],
86
- hash: newHashes[c.id],
148
+ hash: vectorHashes[c.id],
87
149
  payload: {
88
150
  id: c.id,
89
151
  type: c.type,
@@ -108,23 +170,51 @@ export async function buildSchemaIndex(options) {
108
170
  total: toEmbed.length,
109
171
  });
110
172
  }
111
- // 5. Drop chunks present in the previous lock but absent now (artifact pruning).
112
- const previousIds = new Set(Object.keys(previousHashes));
173
+ // 5. Drop this schema's chunks that no longer exist (artifact pruning).
174
+ // Scoped to this schema: other schemas sharing the store are never touched.
113
175
  const currentIds = new Set(chunks.map((c) => c.id));
114
- const orphaned = [];
115
- for (const id of previousIds) {
116
- if (!currentIds.has(id))
117
- orphaned.push(id);
176
+ const candidates = new Set();
177
+ if (store.idsBySchema) {
178
+ // Exact: matched on payload schemaId (also finds older-format ids).
179
+ for (const id of await store.idsBySchema(schemaId))
180
+ candidates.add(id);
181
+ }
182
+ else {
183
+ // Without `idsBySchema` only the ids this schema's own lock lists are
184
+ // pruned. The id prefix isn't safe here: it can match other schemas' ids
185
+ // in the older, unscoped format (`chunk:table:` is both the prefix of a
186
+ // schema named `table` and the start of every old table-chunk id).
187
+ logger?.info({
188
+ ...baseLogContext,
189
+ event: AskDbRagLogEvent.OrphanCleanupLimited,
190
+ }, "store has no idsBySchema; only orphaned ids listed in the previous lock are pruned");
191
+ }
192
+ for (const id of Object.keys(previousLock?.hashes ?? {})) {
193
+ if (id.startsWith(idPrefix))
194
+ candidates.add(id);
195
+ }
196
+ if (lockState.status === "outdated" && lockState.schemaId === schemaId) {
197
+ // Older-format (unscoped) ids this schema wrote under the previous lock.
198
+ for (const id of Object.keys(lockState.hashes))
199
+ candidates.add(id);
118
200
  }
201
+ const orphaned = [...candidates].filter((id) => !currentIds.has(id)).sort();
119
202
  if (orphaned.length > 0) {
120
203
  await store.delete(orphaned);
121
204
  }
122
205
  // 6. Persist lock file.
123
206
  if (lockFilePath) {
207
+ // The width of the vectors in the store: what this run embedded, else (nothing was
208
+ // re-embedded with the same embedder) what the previous run recorded, else what the
209
+ // store reports.
210
+ const dimensions = observedDimensions ??
211
+ (embedderChanged ? undefined : (previousLock?.dimensions ?? descriptor?.dimensions));
124
212
  const lock = {
125
- version: 1,
126
- schemaId: sources.schema.schemaId,
127
- embedderId,
213
+ version: SCHEMA_LOCK_VERSION,
214
+ schemaId,
215
+ ...(embedderId !== undefined ? { embedderId } : {}),
216
+ ...(dimensions !== undefined ? { dimensions } : {}),
217
+ ...(descriptor ? { store: storeIdentity(descriptor) } : {}),
128
218
  hashes: newHashes,
129
219
  updatedAt: new Date().toISOString(),
130
220
  };
@@ -163,6 +253,53 @@ export async function buildSchemaIndex(options) {
163
253
  chunks,
164
254
  };
165
255
  }
256
+ function decideFullReindex(args) {
257
+ if (args.force)
258
+ return "force";
259
+ const lock = args.previousLock;
260
+ if (args.lockFilePath !== undefined) {
261
+ if (args.lockState.status === "outdated")
262
+ return "lock-outdated";
263
+ // No usable lock → the embedder that produced any stored vectors is
264
+ // unknown, so nothing can be trusted.
265
+ if (!lock)
266
+ return "lock-missing";
267
+ }
268
+ if (!lock)
269
+ return undefined;
270
+ // Undefined vs defined counts as a change: we can't prove it's the same model.
271
+ if (!sameEmbedderId(lock.embedderId, args.embedderId))
272
+ return "embedder-changed";
273
+ if (lock.dimensions !== undefined &&
274
+ args.descriptor?.dimensions !== undefined &&
275
+ lock.dimensions !== args.descriptor.dimensions) {
276
+ return "dimensions-changed";
277
+ }
278
+ // The lock's hashes only describe the store they were written to. Stores
279
+ // that report their own hashes don't depend on it.
280
+ if (!args.storeReportsHashes) {
281
+ // An interrupted embedder switch left vectors the lock's hashes don't
282
+ // describe; only a store that reports its own hashes can sort them out.
283
+ if (lock.incomplete)
284
+ return "lock-incomplete";
285
+ // Without `describe()` nothing shows this is the store the lock was
286
+ // written to (a fresh, empty instance looks the same), so trust nothing.
287
+ if (!args.descriptor)
288
+ return "store-unidentified";
289
+ const current = storeIdentity(args.descriptor);
290
+ const previous = lock.store;
291
+ if (current.kind !== previous?.kind ||
292
+ (current.location ?? null) !== (previous?.location ?? null)) {
293
+ return "store-changed";
294
+ }
295
+ }
296
+ return undefined;
297
+ }
298
+ function storeIdentity(descriptor) {
299
+ return descriptor.location !== undefined
300
+ ? { kind: descriptor.kind, location: descriptor.location }
301
+ : { kind: descriptor.kind };
302
+ }
166
303
  function normalizeIndexSources(options) {
167
304
  const input = options.schema ?? options.sources;
168
305
  if (!input) {
@@ -191,6 +328,6 @@ export function createRetriever(args) {
191
328
  return results;
192
329
  };
193
330
  }
194
- export { readLockFile, writeLockFile, } from "./lock-file.js";
331
+ export { checkIndexMatches, readLockFile, writeLockFile, SCHEMA_LOCK_VERSION, } from "./lock-file.js";
195
332
  export { chunkContentHash } from "./hash.js";
196
333
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAoB,MAAM,qBAAqB,CAAC;AAGpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EACL,YAAY,EACZ,aAAa,GAEd,MAAM,gBAAgB,CAAC;AAgDxB,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAgC;IAEhC,MAAM,EACJ,QAAQ,EACR,KAAK,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,SAAS,GAAG,kBAAkB,EAC9B,MAAM,EACN,aAAa,EACb,UAAU,GACX,GAAG,OAAO,CAAC;IACZ,MAAM,OAAO,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,cAAc,GAA4B,aAAa;QAC3D,CAAC,CAAC,EAAE,aAAa,EAAE;QACnB,CAAC,CAAC,EAAE,CAAC;IAEP,WAAW;IACX,MAAM,WAAW,GAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;IAElC,sCAAsC;IACtC,IAAI,WAAW,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,uBAAuB;YAC/C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SAC3C,EACD,6CAA6C,CAC9C,CAAC;IACJ,CAAC;IACD,IACE,YAAY,EAAE,2BAA2B,KAAK,IAAI;QAClD,WAAW,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,EACvC,CAAC;QACD,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,uBAAuB;YAC/C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SAC3C,EACD,sDAAsD,CACvD,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,WAAW,GAAG,CAAC,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,MAAM,cAAc,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,YAAY,EAAE,UAAU,CAAC;IACpD,MAAM,eAAe,GACnB,UAAU,KAAK,SAAS;QACxB,kBAAkB,KAAK,SAAS;QAChC,UAAU,KAAK,kBAAkB,CAAC;IAEpC,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAY,EAAE,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;YACtD,MAAM,EAAE,CAAC;YACT,uEAAuE;YACvE,uEAAuE;YACvE,sDAAsD;YACtD,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,IAAI,CACV;QACE,GAAG,cAAc;QACjB,KAAK,EAAE,gBAAgB,CAAC,eAAe;QACvC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,MAAM;QACN,eAAe;KAChB,EACD,sBAAsB,CACvB,CAAC;IACF,UAAU,EAAE,CAAC;QACX,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,MAAM;KACP,CAAC,CAAC;IAEH,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,YAAY;YACpC,KAAK,EAAE,MAAM;SACd,EACD,kCAAkC,CACnC,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,qBAAqB,OAAO,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,UAAU,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,CAChB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YACrB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC;YACpB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE;gBACP,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB;SACF,CAAC,CAAC,CACJ,CAAC;QACF,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,EAAE,IAAI,CACV;gBACE,GAAG,cAAc;gBACjB,KAAK,EAAE,gBAAgB,CAAC,YAAY;gBACpC,SAAS,EAAE,CAAC,CAAC,IAAI;gBACjB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;aACzB,EACD,mBAAmB,CACpB,CAAC;QACJ,CAAC;QACD,UAAU,EAAE,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,wBAAwB;IACxB,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,IAAI,GAAmB;YAC3B,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;YACjC,UAAU;YACV,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;QACF,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,EAAE,IAAI,CACV;QACE,GAAG,cAAc;QACjB,KAAK,EAAE,gBAAgB,CAAC,iBAAiB;QACzC,QAAQ,EAAE,aAAa;QACvB,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,MAAM;KACrB,EACD,wBAAwB,CACzB,CAAC;IACF,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAErE,yDAAyD;IACzD,MAAM,SAAS,GAAc,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACjE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,kBAAkB;YAC1C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,CAAC;YACD,WAAW,EAAE,OAAO,CAAC,MAAM;SAC5B,EACD,yBAAyB,CAC1B,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,aAAa,EAAE,aAAa;YAC5B,YAAY,EAAE,MAAM;YACpB,iBAAiB,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;YACtD,iBAAiB,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SACvD;QACD,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAgC;IAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAChD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,IAAI,KAAK,IAAI,QAAQ,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACzD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAK/B;IACC,OAAO,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,IAAI,CACf;YACE,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,KAAK,EAAE,gBAAgB,CAAC,kBAAkB;YAC1C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,CAAC;YACD,WAAW,EAAE,OAAO,CAAC,MAAM;SAC5B,EACD,yBAAyB,CAC1B,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED,OAAO,EACL,YAAY,EACZ,aAAa,GAEd,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,WAAW,EAAoB,MAAM,qBAAqB,CAAC;AAGnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAQpD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,aAAa,GAGd,MAAM,gBAAgB,CAAC;AAsDxB,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAgC;IAEhC,MAAM,EACJ,QAAQ,EACR,KAAK,EACL,UAAU,EACV,YAAY,EACZ,SAAS,GAAG,kBAAkB,EAC9B,MAAM,EACN,aAAa,EACb,UAAU,GACX,GAAG,OAAO,CAAC;IACZ,MAAM,OAAO,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,cAAc,GAA4B,aAAa;QAC3D,CAAC,CAAC,EAAE,aAAa,EAAE;QACnB,CAAC,CAAC,EAAE,CAAC;IAEP,WAAW;IACX,MAAM,WAAW,GAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;IAElC,sCAAsC;IACtC,IAAI,WAAW,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,uBAAuB;YAC/C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SAC3C,EACD,6CAA6C,CAC9C,CAAC;IACJ,CAAC;IACD,IACE,YAAY,EAAE,2BAA2B,KAAK,IAAI;QAClD,WAAW,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,EACvC,CAAC;QACD,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,uBAAuB;YAC/C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SAC3C,EACD,sDAAsD,CACvD,CAAC;IACJ,CAAC;IAED,8CAA8C;IAC9C,EAAE;IACF,2EAA2E;IAC3E,yEAAyE;IACzE,4EAA4E;IAC5E,2EAA2E;IAC3E,sDAAsD;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;IACtC,sEAAsE;IACtE,qEAAqE;IACrE,MAAM,YAAY,GAAG,UAAU,EAAE,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;IACvF,MAAM,SAAS,GAAuB,YAAY;QAChD,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC;QAC/B,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC1B,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,eAAe;QACf,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,kBAAkB;YAC1C,YAAY,EAAE,SAAS,CAAC,IAAI,CAAC,QAAQ;YACrC,QAAQ;SACT,EACD,uGAAuG,CACxG,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAChB,SAAS,CAAC,MAAM,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;QAC/D,CAAC,CAAC,SAAS,CAAC,IAAI;QAChB,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,WAAW,GAAG,KAAK,CAAC,cAAc;QACtC,CAAC,CAAC,MAAM,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC;QACtC,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;QAC1C,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI;QAC7B,YAAY;QACZ,SAAS;QACT,YAAY;QACZ,UAAU;QACV,UAAU;QACV,kBAAkB,EAAE,WAAW,KAAK,SAAS;KAC9C,CAAC,CAAC;IACH,0EAA0E;IAC1E,kEAAkE;IAClE,MAAM,eAAe,GACnB,YAAY,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrF,MAAM,cAAc,GAClB,WAAW,IAAI,YAAY,EAAE,MAAM,IAAI,EAAE,CAAC;IAE5C,0EAA0E;IAC1E,wCAAwC;IACxC,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,OAAO,GAAY,EAAE,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjF,IAAI,iBAAiB,KAAK,SAAS,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;YACxE,qEAAqE;YACrE,iDAAiD;YACjD,MAAM,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,IAAI,CACV;QACE,GAAG,cAAc;QACjB,KAAK,EAAE,gBAAgB,CAAC,eAAe;QACvC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,MAAM;QACN,eAAe;QACf,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,EACD,sBAAsB,CACvB,CAAC;IACF,UAAU,EAAE,CAAC;QACX,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,MAAM;KACP,CAAC,CAAC;IAEH,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,YAAY;YACpC,KAAK,EAAE,MAAM;SACd,EACD,kDAAkD,CACnD,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,YAAY,IAAI,YAAY,IAAI,eAAe,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1E,yEAAyE;QACzE,yEAAyE;QACzE,aAAa,CAAC,YAAY,EAAE,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,kBAAsC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,qBAAqB,OAAO,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,UAAU,CAC1E,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,kBAAkB,KAAK,CAAC,CAAC,MAAM,CAAC;YAChC,IAAI,CAAC,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CACb,yDAAyD,kBAAkB,QAAQ,CAAC,CAAC,MAAM,IAAI,CAChG,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IACE,UAAU,EAAE,UAAU,KAAK,SAAS;YACpC,kBAAkB,KAAK,SAAS;YAChC,kBAAkB,KAAK,UAAU,CAAC,UAAU,EAC5C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,qBAAqB,kBAAkB,8BAA8B,UAAU,CAAC,IAAI,SAAS;gBAC3F,iBAAiB,UAAU,CAAC,UAAU,4BAA4B,kBAAkB,UAAU;gBAC9F,0BAA0B,UAAU,CAAC,UAAU,qBAAqB;gBACpE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC3D,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,CAChB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YACrB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC;YACpB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE;gBACP,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB;SACF,CAAC,CAAC,CACJ,CAAC;QACF,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,EAAE,IAAI,CACV;gBACE,GAAG,cAAc;gBACjB,KAAK,EAAE,gBAAgB,CAAC,YAAY;gBACpC,SAAS,EAAE,CAAC,CAAC,IAAI;gBACjB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;aACzB,EACD,mBAAmB,CACpB,CAAC;QACJ,CAAC;QACD,UAAU,EAAE,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,oEAAoE;QACpE,KAAK,MAAM,EAAE,IAAI,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;SAAM,CAAC;QACN,sEAAsE;QACtE,yEAAyE;QACzE,wEAAwE;QACxE,mEAAmE;QACnE,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,oBAAoB;SAC7C,EACD,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QACzD,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,UAAU,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACvE,yEAAyE;QACzE,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,wBAAwB;IACxB,IAAI,YAAY,EAAE,CAAC;QACjB,mFAAmF;QACnF,oFAAoF;QACpF,iBAAiB;QACjB,MAAM,UAAU,GACd,kBAAkB;YAClB,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,UAAU,IAAI,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAmB;YAC3B,OAAO,EAAE,mBAAmB;YAC5B,QAAQ;YACR,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;QACF,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,EAAE,IAAI,CACV;QACE,GAAG,cAAc;QACjB,KAAK,EAAE,gBAAgB,CAAC,iBAAiB;QACzC,QAAQ,EAAE,aAAa;QACvB,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,MAAM;KACrB,EACD,wBAAwB,CACzB,CAAC;IACF,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAErE,yDAAyD;IACzD,MAAM,SAAS,GAAc,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QACjE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,EAAE,IAAI,CACV;YACE,GAAG,cAAc;YACjB,KAAK,EAAE,gBAAgB,CAAC,kBAAkB;YAC1C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,CAAC;YACD,WAAW,EAAE,OAAO,CAAC,MAAM;SAC5B,EACD,yBAAyB,CAC1B,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,aAAa,EAAE,aAAa;YAC5B,YAAY,EAAE,MAAM;YACpB,iBAAiB,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;YACtD,iBAAiB,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;SACvD;QACD,MAAM;KACP,CAAC;AACJ,CAAC;AAaD,SAAS,iBAAiB,CAAC,IAQ1B;IACC,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,OAAO,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC;IAC/B,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,UAAU;YAAE,OAAO,eAAe,CAAC;QACjE,oEAAoE;QACpE,sCAAsC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,cAAc,CAAC;IACnC,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,+EAA+E;IAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACjF,IACE,IAAI,CAAC,UAAU,KAAK,SAAS;QAC7B,IAAI,CAAC,UAAU,EAAE,UAAU,KAAK,SAAS;QACzC,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU,EAC9C,CAAC;QACD,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IACD,yEAAyE;IACzE,mDAAmD;IACnD,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC7B,sEAAsE;QACtE,wEAAwE;QACxE,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,iBAAiB,CAAC;QAC9C,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO,oBAAoB,CAAC;QAClD,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC;QAC5B,IACE,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,IAAI;YAC/B,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,EAC3D,CAAC;YACD,OAAO,eAAe,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,UAAiC;IACtD,OAAO,UAAU,CAAC,QAAQ,KAAK,SAAS;QACtC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE;QAC1D,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAgC;IAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAChD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,IAAI,KAAK,IAAI,QAAQ,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACzD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAK/B;IACC,OAAO,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,EAAE,IAAI,CACf;YACE,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,KAAK,EAAE,gBAAgB,CAAC,kBAAkB;YAC1C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,CAAC;YACD,WAAW,EAAE,OAAO,CAAC,MAAM;SAC5B,EACD,yBAAyB,CAC1B,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,aAAa,EACb,mBAAmB,GAGpB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC"}
@@ -1,23 +1,104 @@
1
+ /** Current `schema.lock.json` format version. */
2
+ export declare const SCHEMA_LOCK_VERSION = 2;
1
3
  /**
2
- * `schema.lock.json` shape — machine-managed pointer used by the indexer to
3
- * skip re-embedding chunks whose content hasn't changed.
4
+ * `schema.lock.json` shape — machine-managed bookkeeping used by the indexer
5
+ * to skip re-embedding chunks whose content hasn't changed.
4
6
  *
5
7
  * Lives next to the v2 artifact (`<schemaId>.schema/schema.lock.json`).
8
+ *
9
+ * The lock is advisory: stores that can report stored content hashes
10
+ * (`VectorStore.hashesByPrefix` — all built-in stores) are the source of truth
11
+ * for what is already embedded; the lock only guards embedder identity for
12
+ * them. Stores that cannot report hashes fall back to `hashes` here, guarded
13
+ * by `store`, `dimensions`, and `embedderId`.
14
+ *
15
+ * Version history:
16
+ * - `1` — unscoped chunk ids (`chunk:table:public.orders`), no store identity.
17
+ * - `2` — schema-scoped chunk ids (`chunk:<schemaId>:table:public.orders`),
18
+ * records `store` identity and `dimensions`. A v1 lock triggers a one-time
19
+ * full reindex.
6
20
  */
7
21
  export type SchemaLockFile = {
8
22
  /** File-format version. Bump on breaking shape changes. */
9
- version: 1;
23
+ version: typeof SCHEMA_LOCK_VERSION;
10
24
  /** Schema id this lock is bound to. */
11
25
  schemaId: string;
12
26
  /** Embedder id the embeddings were produced with (e.g. `openai:text-embedding-3-small`). */
13
27
  embedderId?: string;
14
- /** Vector dimensions (informational; the store enforces). */
28
+ /** Width of the vectors in the store, learned from the embedder's output. */
15
29
  dimensions?: number;
30
+ /** Identity of the store the embeddings were written to (from `VectorStore.describe()`). */
31
+ store?: {
32
+ kind: string;
33
+ location?: string;
34
+ };
16
35
  /** chunkId → SHA-256 content hash of the chunk's text. */
17
36
  hashes: Record<string, string>;
18
37
  /** ISO 8601 timestamp of last successful index. */
19
38
  updatedAt?: string;
39
+ /**
40
+ * Set while a run that switches the embedder is writing vectors, and
41
+ * cleared when it finishes. A lock that still carries it after the run
42
+ * ended describes a store holding two models' vectors, so it never
43
+ * matches a query (see {@link checkIndexMatches}).
44
+ */
45
+ incomplete?: true;
46
+ };
47
+ /**
48
+ * Whether two embedder ids name the same embedder. Unset matches only unset:
49
+ * the indexer and {@link checkIndexMatches} share this rule.
50
+ */
51
+ export declare function sameEmbedderId(a: string | undefined, b: string | undefined): boolean;
52
+ /** Result of {@link inspectLockFile}. */
53
+ export type LockFileInspection = {
54
+ status: "missing";
55
+ } | {
56
+ status: "invalid";
57
+ }
58
+ /** Written by an older `@askdb/rag`. `hashes` keys are that version's chunk ids. */
59
+ | {
60
+ status: "outdated";
61
+ version: unknown;
62
+ schemaId?: string;
63
+ hashes: Record<string, string>;
64
+ } | {
65
+ status: "ok";
66
+ lock: SchemaLockFile;
20
67
  };
68
+ /**
69
+ * Read a lock file of the **current** version. Returns `undefined` when the
70
+ * file is missing, unparseable, or written in an older format.
71
+ */
21
72
  export declare function readLockFile(path: string): SchemaLockFile | undefined;
73
+ /**
74
+ * Read a lock file and report why it is (un)usable. Used by the indexer to
75
+ * detect older-format locks so it can reindex and clean up old chunk ids.
76
+ */
77
+ export declare function inspectLockFile(path: string): LockFileInspection;
22
78
  export declare function writeLockFile(path: string, lock: SchemaLockFile): void;
79
+ /** Result of {@link checkIndexMatches}. */
80
+ export type IndexMatch = {
81
+ ok: true;
82
+ /** The lock compared against; `undefined` when there is none for this schema (nothing to check). */
83
+ lock: SchemaLockFile | undefined;
84
+ } | {
85
+ ok: false;
86
+ reason: "lock-outdated" | "index-incomplete" | "embedder-changed" | "dimensions-changed";
87
+ message: string;
88
+ };
89
+ /**
90
+ * Whether an index described by `schema.lock.json` can be queried with this
91
+ * embedder: the rule `askdb rag query` applies, for any host that queries a
92
+ * persisted index (e.g. through `createRetriever`). The embedder id must be
93
+ * the one the lock records (unset matches only unset, as in the indexer),
94
+ * and when both widths are known they must match. A lock left `incomplete`
95
+ * by an interrupted embedder switch never matches. A missing or unreadable
96
+ * lock, or one for another schema, has nothing to compare and passes.
97
+ */
98
+ export declare function checkIndexMatches(args: {
99
+ lockFilePath: string;
100
+ schemaId: string;
101
+ embedderId: string | undefined;
102
+ dimensions?: number;
103
+ }): IndexMatch;
23
104
  //# sourceMappingURL=lock-file.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"lock-file.d.ts","sourceRoot":"","sources":["../../src/indexer/lock-file.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,2DAA2D;IAC3D,OAAO,EAAE,CAAC,CAAC;IACX,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAqBrE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,IAAI,CAQtE"}
1
+ {"version":3,"file":"lock-file.d.ts","sourceRoot":"","sources":["../../src/indexer/lock-file.ts"],"names":[],"mappings":"AAGA,iDAAiD;AACjD,eAAO,MAAM,mBAAmB,IAAI,CAAC;AAErC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,2DAA2D;IAC3D,OAAO,EAAE,OAAO,mBAAmB,CAAC;IACpC,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4FAA4F;IAC5F,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEpF;AAED,yCAAyC;AACzC,MAAM,MAAM,kBAAkB,GAC1B;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,GACrB;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE;AACvB,oFAAoF;GAClF;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC3F;IAAE,MAAM,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3C;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAGrE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,CA8BhE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,IAAI,CAetE;AAOD,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAClB;IACE,EAAE,EAAE,IAAI,CAAC;IACT,oGAAoG;IACpG,IAAI,EAAE,cAAc,GAAG,SAAS,CAAC;CAClC,GACD;IACE,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,eAAe,GAAG,kBAAkB,GAAG,kBAAkB,GAAG,oBAAoB,CAAC;IACzF,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEN;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,UAAU,CA0Cb"}
@@ -1,25 +1,61 @@
1
- import { readFileSync, writeFileSync } from "node:fs";
1
+ import { randomBytes } from "node:crypto";
2
+ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
3
+ /** Current `schema.lock.json` format version. */
4
+ export const SCHEMA_LOCK_VERSION = 2;
5
+ /**
6
+ * Whether two embedder ids name the same embedder. Unset matches only unset:
7
+ * the indexer and {@link checkIndexMatches} share this rule.
8
+ */
9
+ export function sameEmbedderId(a, b) {
10
+ return (a ?? null) === (b ?? null);
11
+ }
12
+ /**
13
+ * Read a lock file of the **current** version. Returns `undefined` when the
14
+ * file is missing, unparseable, or written in an older format.
15
+ */
2
16
  export function readLockFile(path) {
17
+ const inspected = inspectLockFile(path);
18
+ return inspected.status === "ok" ? inspected.lock : undefined;
19
+ }
20
+ /**
21
+ * Read a lock file and report why it is (un)usable. Used by the indexer to
22
+ * detect older-format locks so it can reindex and clean up old chunk ids.
23
+ */
24
+ export function inspectLockFile(path) {
3
25
  let raw;
4
26
  try {
5
27
  raw = readFileSync(path, "utf8");
6
28
  }
7
29
  catch {
8
- return undefined;
30
+ return { status: "missing" };
9
31
  }
10
32
  let parsed;
11
33
  try {
12
34
  parsed = JSON.parse(raw);
13
35
  }
14
36
  catch {
15
- return undefined;
37
+ return { status: "invalid" };
16
38
  }
17
- if (typeof parsed !== "object" ||
18
- parsed === null ||
19
- parsed.version !== 1) {
20
- return undefined;
39
+ if (typeof parsed !== "object" || parsed === null)
40
+ return { status: "invalid" };
41
+ const obj = parsed;
42
+ const hashes = isStringRecord(obj.hashes) ? obj.hashes : undefined;
43
+ if (obj.version !== SCHEMA_LOCK_VERSION) {
44
+ // Only an older lock is "outdated" (its ids get cleaned up). A newer one
45
+ // was written by a later @askdb/rag whose id format this one can't know.
46
+ const older = typeof obj.version === "number" && obj.version < SCHEMA_LOCK_VERSION;
47
+ if (!hashes || !older)
48
+ return { status: "invalid" };
49
+ return {
50
+ status: "outdated",
51
+ version: obj.version,
52
+ schemaId: typeof obj.schemaId === "string" ? obj.schemaId : undefined,
53
+ hashes,
54
+ };
21
55
  }
22
- return parsed;
56
+ if (!hashes || typeof obj.schemaId !== "string")
57
+ return { status: "invalid" };
58
+ return { status: "ok", lock: parsed };
23
59
  }
24
60
  export function writeLockFile(path, lock) {
25
61
  // Sort keys for deterministic on-disk output.
@@ -28,6 +64,69 @@ export function writeLockFile(path, lock) {
28
64
  sortedHashes[k] = lock.hashes[k];
29
65
  }
30
66
  const out = { ...lock, hashes: sortedHashes };
31
- writeFileSync(path, JSON.stringify(out, null, 2) + "\n", "utf8");
67
+ // Temp file + rename, so a crash never leaves a half-written lock.
68
+ const tmp = `${path}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`;
69
+ try {
70
+ writeFileSync(tmp, JSON.stringify(out, null, 2) + "\n", "utf8");
71
+ renameSync(tmp, path);
72
+ }
73
+ finally {
74
+ rmSync(tmp, { force: true });
75
+ }
76
+ }
77
+ function isStringRecord(value) {
78
+ if (typeof value !== "object" || value === null || Array.isArray(value))
79
+ return false;
80
+ return Object.values(value).every((v) => typeof v === "string");
81
+ }
82
+ /**
83
+ * Whether an index described by `schema.lock.json` can be queried with this
84
+ * embedder: the rule `askdb rag query` applies, for any host that queries a
85
+ * persisted index (e.g. through `createRetriever`). The embedder id must be
86
+ * the one the lock records (unset matches only unset, as in the indexer),
87
+ * and when both widths are known they must match. A lock left `incomplete`
88
+ * by an interrupted embedder switch never matches. A missing or unreadable
89
+ * lock, or one for another schema, has nothing to compare and passes.
90
+ */
91
+ export function checkIndexMatches(args) {
92
+ const inspected = inspectLockFile(args.lockFilePath);
93
+ if (inspected.status === "outdated" && (inspected.schemaId ?? args.schemaId) === args.schemaId) {
94
+ return {
95
+ ok: false,
96
+ reason: "lock-outdated",
97
+ message: "schema.lock.json was written by an older @askdb/rag (unscoped chunk ids); rebuild the index.",
98
+ };
99
+ }
100
+ if (inspected.status !== "ok" || inspected.lock.schemaId !== args.schemaId) {
101
+ return { ok: true, lock: undefined };
102
+ }
103
+ const lock = inspected.lock;
104
+ if (lock.incomplete) {
105
+ return {
106
+ ok: false,
107
+ reason: "index-incomplete",
108
+ message: "The last index build switched embedders and didn't finish, so the store holds two models' vectors; rebuild the index.",
109
+ };
110
+ }
111
+ // Equal widths don't make two models' vectors comparable.
112
+ if (!sameEmbedderId(lock.embedderId, args.embedderId)) {
113
+ const built = lock.embedderId === undefined
114
+ ? "without an embedder id, so its embedder is unknown,"
115
+ : `with embedder "${lock.embedderId}"`;
116
+ const using = args.embedderId === undefined ? "no embedder id" : `"${args.embedderId}"`;
117
+ return {
118
+ ok: false,
119
+ reason: "embedder-changed",
120
+ message: `The index was built ${built} but this query uses ${using}.`,
121
+ };
122
+ }
123
+ if (lock.dimensions !== undefined && args.dimensions !== undefined && lock.dimensions !== args.dimensions) {
124
+ return {
125
+ ok: false,
126
+ reason: "dimensions-changed",
127
+ message: `The index holds ${lock.dimensions}-dimension embeddings but this query uses ${args.dimensions}.`,
128
+ };
129
+ }
130
+ return { ok: true, lock };
32
131
  }
33
132
  //# sourceMappingURL=lock-file.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"lock-file.js","sourceRoot":"","sources":["../../src/indexer/lock-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAuBtD,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACd,MAAgC,CAAC,OAAO,KAAK,CAAC,EAC/C,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAwB,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,IAAoB;IAC9D,8CAA8C;IAC9C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAChD,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,GAAG,GAAmB,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9D,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;AACnE,CAAC"}
1
+ {"version":3,"file":"lock-file.js","sourceRoot":"","sources":["../../src/indexer/lock-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE1E,iDAAiD;AACjD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AA4CrC;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,CAAqB,EAAE,CAAqB;IACzE,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;AACrC,CAAC;AAUD;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,OAAO,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/B,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/B,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAChF,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,IAAI,GAAG,CAAC,OAAO,KAAK,mBAAmB,EAAE,CAAC;QACxC,yEAAyE;QACzE,yEAAyE;QACzE,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,GAAG,mBAAmB,CAAC;QACnF,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACpD,OAAO;YACL,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACrE,MAAM;SACP,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC9E,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAwB,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,IAAoB;IAC9D,8CAA8C;IAC9C,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAChD,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,GAAG,GAAmB,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9D,mEAAmE;IACnE,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3E,IAAI,CAAC;QACH,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAChE,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;AAClE,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAKjC;IACC,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,SAAS,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/F,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,eAAe;YACvB,OAAO,EAAE,8FAA8F;SACxG,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;IAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,kBAAkB;YAC1B,OAAO,EACL,uHAAuH;SAC1H,CAAC;IACJ,CAAC;IACD,0DAA0D;IAC1D,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GACT,IAAI,CAAC,UAAU,KAAK,SAAS;YAC3B,CAAC,CAAC,qDAAqD;YACvD,CAAC,CAAC,kBAAkB,IAAI,CAAC,UAAU,GAAG,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC;QACxF,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,kBAAkB;YAC1B,OAAO,EAAE,uBAAuB,KAAK,wBAAwB,KAAK,GAAG;SACtE,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1G,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,oBAAoB;YAC5B,OAAO,EAAE,mBAAmB,IAAI,CAAC,UAAU,6CAA6C,IAAI,CAAC,UAAU,GAAG;SAC3G,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC"}
@@ -16,6 +16,17 @@ export declare const AskDbRagLogEvent: {
16
16
  readonly SensitiveChunksExcluded: "askdb.rag.sensitive_chunks_excluded";
17
17
  /** Counts only — opt-in `includeSensitiveDescribable: true` is in effect. */
18
18
  readonly SensitiveChunksIncluded: "askdb.rag.sensitive_chunks_included";
19
+ /**
20
+ * The store has no `idsBySchema`, so orphan cleanup only prunes ids listed
21
+ * in the previous `schema.lock.json` (never other schemas' chunks).
22
+ */
23
+ readonly OrphanCleanupLimited: "askdb.rag.orphan_cleanup_limited";
24
+ /**
25
+ * `schema.lock.json` names another schema id (a renamed schema, or a lock
26
+ * copied from another schema's directory). The run reindexes without it
27
+ * and deletes none of the ids it lists.
28
+ */
29
+ readonly LockSchemaMismatch: "askdb.rag.lock_schema_mismatch";
19
30
  /** Retrieval ran for a question; counts only. */
20
31
  readonly RetrievalCompleted: "askdb.rag.retrieval_completed";
21
32
  };
@@ -1 +1 @@
1
- {"version":3,"file":"log-events.d.ts","sourceRoot":"","sources":["../src/log-events.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,gBAAgB;IAC3B,qDAAqD;;IAErD,gDAAgD;;IAEhD,sEAAsE;;IAEtE,kCAAkC;;IAElC,+GAA+G;;IAE/G,6EAA6E;;IAE7E,iDAAiD;;CAEzC,CAAC;AAEX,MAAM,MAAM,oBAAoB,GAC9B,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAC"}
1
+ {"version":3,"file":"log-events.d.ts","sourceRoot":"","sources":["../src/log-events.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,gBAAgB;IAC3B,qDAAqD;;IAErD,gDAAgD;;IAEhD,sEAAsE;;IAEtE,kCAAkC;;IAElC,+GAA+G;;IAE/G,6EAA6E;;IAE7E;;;OAGG;;IAEH;;;;OAIG;;IAEH,iDAAiD;;CAEzC,CAAC;AAEX,MAAM,MAAM,oBAAoB,GAC9B,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAC"}
@@ -16,6 +16,17 @@ export const AskDbRagLogEvent = {
16
16
  SensitiveChunksExcluded: "askdb.rag.sensitive_chunks_excluded",
17
17
  /** Counts only — opt-in `includeSensitiveDescribable: true` is in effect. */
18
18
  SensitiveChunksIncluded: "askdb.rag.sensitive_chunks_included",
19
+ /**
20
+ * The store has no `idsBySchema`, so orphan cleanup only prunes ids listed
21
+ * in the previous `schema.lock.json` (never other schemas' chunks).
22
+ */
23
+ OrphanCleanupLimited: "askdb.rag.orphan_cleanup_limited",
24
+ /**
25
+ * `schema.lock.json` names another schema id (a renamed schema, or a lock
26
+ * copied from another schema's directory). The run reindexes without it
27
+ * and deletes none of the ids it lists.
28
+ */
29
+ LockSchemaMismatch: "askdb.rag.lock_schema_mismatch",
19
30
  /** Retrieval ran for a question; counts only. */
20
31
  RetrievalCompleted: "askdb.rag.retrieval_completed",
21
32
  };
@@ -1 +1 @@
1
- {"version":3,"file":"log-events.js","sourceRoot":"","sources":["../src/log-events.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,qDAAqD;IACrD,eAAe,EAAE,4BAA4B;IAC7C,gDAAgD;IAChD,YAAY,EAAE,yBAAyB;IACvC,sEAAsE;IACtE,YAAY,EAAE,yBAAyB;IACvC,kCAAkC;IAClC,iBAAiB,EAAE,8BAA8B;IACjD,+GAA+G;IAC/G,uBAAuB,EAAE,qCAAqC;IAC9D,6EAA6E;IAC7E,uBAAuB,EAAE,qCAAqC;IAC9D,iDAAiD;IACjD,kBAAkB,EAAE,+BAA+B;CAC3C,CAAC"}
1
+ {"version":3,"file":"log-events.js","sourceRoot":"","sources":["../src/log-events.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,qDAAqD;IACrD,eAAe,EAAE,4BAA4B;IAC7C,gDAAgD;IAChD,YAAY,EAAE,yBAAyB;IACvC,sEAAsE;IACtE,YAAY,EAAE,yBAAyB;IACvC,kCAAkC;IAClC,iBAAiB,EAAE,8BAA8B;IACjD,+GAA+G;IAC/G,uBAAuB,EAAE,qCAAqC;IAC9D,6EAA6E;IAC7E,uBAAuB,EAAE,qCAAqC;IAC9D;;;OAGG;IACH,oBAAoB,EAAE,kCAAkC;IACxD;;;;OAIG;IACH,kBAAkB,EAAE,gCAAgC;IACpD,iDAAiD;IACjD,kBAAkB,EAAE,+BAA+B;CAC3C,CAAC"}