@apibara/plugin-sqlite 2.1.0-beta.5 → 2.1.0-beta.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/index.cjs +87 -35
  2. package/dist/index.cjs.map +1 -0
  3. package/dist/index.d.cts +2 -1
  4. package/dist/index.d.mts +2 -1
  5. package/dist/index.d.ts +2 -1
  6. package/dist/index.mjs +88 -36
  7. package/dist/index.mjs.map +1 -0
  8. package/package.json +4 -3
  9. package/src/index.ts +54 -14
  10. package/src/kv.ts +42 -22
  11. package/src/persistence.ts +29 -0
package/dist/index.cjs CHANGED
@@ -2,9 +2,9 @@
2
2
 
3
3
  const indexer = require('@apibara/indexer');
4
4
  const plugins = require('@apibara/indexer/plugins');
5
- const protocol = require('@apibara/protocol');
6
5
  const internal = require('@apibara/indexer/internal');
7
6
  const plugins$1 = require('@apibara/indexer/internal/plugins');
7
+ const protocol = require('@apibara/protocol');
8
8
 
9
9
  class SqliteStorageError extends Error {
10
10
  constructor(message, options) {
@@ -49,45 +49,56 @@ function initializeKeyValueStore(db) {
49
49
  db.exec(statements$1.createTable);
50
50
  }
51
51
  class KeyValueStore {
52
- constructor(db, endCursor, finality, serialize, deserialize) {
52
+ constructor(db, endCursor, finality, serialize, deserialize, indexerId) {
53
53
  this.db = db;
54
54
  this.endCursor = endCursor;
55
55
  this.finality = finality;
56
56
  this.serialize = serialize;
57
57
  this.deserialize = deserialize;
58
+ this.indexerId = indexerId;
58
59
  assertInTransaction(db);
59
60
  }
60
61
  get(key) {
61
- const row = this.db.prepare(statements$1.get).get(key);
62
+ const row = this.db.prepare(statements$1.get).get(key, this.indexerId);
62
63
  return row ? this.deserialize(row.v) : void 0;
63
64
  }
64
65
  put(key, value) {
65
- this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key);
66
- this.db.prepare(statements$1.insertIntoKvs).run(
66
+ this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key, this.indexerId);
67
+ this.db.prepare(
68
+ statements$1.insertIntoKvs
69
+ ).run(
67
70
  Number(this.endCursor.orderKey),
68
71
  key,
69
- this.serialize(value)
72
+ this.serialize(value),
73
+ this.indexerId
70
74
  );
71
75
  }
72
76
  del(key) {
73
- this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key);
77
+ this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key, this.indexerId);
74
78
  }
75
79
  }
76
- function finalizeKV(db, cursor) {
80
+ function finalizeKV(db, cursor, indexerId) {
77
81
  assertInTransaction(db);
78
82
  db.prepare(statements$1.finalize).run(
79
- Number(cursor.orderKey)
83
+ Number(cursor.orderKey),
84
+ indexerId
80
85
  );
81
86
  }
82
- function invalidateKV(db, cursor) {
87
+ function invalidateKV(db, cursor, indexerId) {
83
88
  assertInTransaction(db);
84
89
  db.prepare(statements$1.invalidateDelete).run(
85
- Number(cursor.orderKey)
90
+ Number(cursor.orderKey),
91
+ indexerId
86
92
  );
87
93
  db.prepare(statements$1.invalidateUpdate).run(
88
- Number(cursor.orderKey)
94
+ Number(cursor.orderKey),
95
+ indexerId
89
96
  );
90
97
  }
98
+ function cleanupKV(db, indexerId) {
99
+ assertInTransaction(db);
100
+ db.prepare(statements$1.cleanup).run(indexerId);
101
+ }
91
102
  const statements$1 = {
92
103
  createTable: `
93
104
  CREATE TABLE IF NOT EXISTS kvs (
@@ -95,33 +106,37 @@ const statements$1 = {
95
106
  to_block INTEGER,
96
107
  k TEXT NOT NULL,
97
108
  v BLOB NOT NULL,
98
- PRIMARY KEY (from_block, k)
109
+ id TEXT NOT NULL,
110
+ PRIMARY KEY (from_block, k, id)
99
111
  );`,
100
112
  get: `
101
113
  SELECT v
102
114
  FROM kvs
103
- WHERE k = ? AND to_block IS NULL`,
115
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
104
116
  updateToBlock: `
105
117
  UPDATE kvs
106
118
  SET to_block = ?
107
- WHERE k = ? AND to_block IS NULL`,
119
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
108
120
  insertIntoKvs: `
109
- INSERT INTO kvs (from_block, to_block, k, v)
110
- VALUES (?, NULL, ?, ?)`,
121
+ INSERT INTO kvs (from_block, to_block, k, v, id)
122
+ VALUES (?, NULL, ?, ?, ?)`,
111
123
  del: `
112
124
  UPDATE kvs
113
125
  SET to_block = ?
114
- WHERE k = ? AND to_block IS NULL`,
126
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
115
127
  finalize: `
116
128
  DELETE FROM kvs
117
- WHERE to_block <= ?`,
129
+ WHERE to_block <= ? AND id = ?`,
118
130
  invalidateDelete: `
119
131
  DELETE FROM kvs
120
- WHERE from_block > ?`,
132
+ WHERE from_block > ? AND id = ?`,
121
133
  invalidateUpdate: `
122
134
  UPDATE kvs
123
135
  SET to_block = NULL
124
- WHERE to_block > ?`
136
+ WHERE to_block > ? AND id = ?`,
137
+ cleanup: `
138
+ DELETE FROM kvs
139
+ WHERE id = ?`
125
140
  };
126
141
 
127
142
  function initializePersistentState(db) {
@@ -189,6 +204,12 @@ function invalidateState(props) {
189
204
  Number(cursor.orderKey)
190
205
  );
191
206
  }
207
+ function resetPersistence(props) {
208
+ const { db, indexerId } = props;
209
+ assertInTransaction(db);
210
+ db.prepare(statements.resetCheckpoint).run(indexerId);
211
+ db.prepare(statements.resetFilter).run(indexerId);
212
+ }
192
213
  const statements = {
193
214
  createCheckpointsTable: `
194
215
  CREATE TABLE IF NOT EXISTS checkpoints (
@@ -243,7 +264,13 @@ const statements = {
243
264
  invalidateFilterUpdate: `
244
265
  UPDATE filters
245
266
  SET to_block = NULL
246
- WHERE id = ? AND to_block > ?`
267
+ WHERE id = ? AND to_block > ?`,
268
+ resetCheckpoint: `
269
+ DELETE FROM checkpoints
270
+ WHERE id = ?`,
271
+ resetFilter: `
272
+ DELETE FROM filters
273
+ WHERE id = ?`
247
274
  };
248
275
 
249
276
  const KV_PROPERTY = "_kv_sqlite";
@@ -267,10 +294,14 @@ function sqliteStorage({
267
294
  }) {
268
295
  return plugins.defineIndexerPlugin((indexer) => {
269
296
  let indexerId = "";
270
- indexer.hooks.hook("run:before", async () => {
297
+ let prevFinality;
298
+ const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
299
+ indexer.hooks.hook("plugins:init", async () => {
271
300
  const { indexerName: indexerFileName, availableIndexers } = plugins$1.useInternalContext();
301
+ const logger = plugins.useLogger();
272
302
  indexerId = internal.generateIndexerId(indexerFileName, identifier);
273
303
  let retries = 0;
304
+ let cleanupApplied = false;
274
305
  while (retries <= MAX_RETRIES) {
275
306
  try {
276
307
  await withTransaction(database, async (db) => {
@@ -280,6 +311,18 @@ function sqliteStorage({
280
311
  if (enableKeyValueStore) {
281
312
  initializeKeyValueStore(db);
282
313
  }
314
+ if (alwaysReindex && !cleanupApplied) {
315
+ if (enableKeyValueStore) {
316
+ logger.warn("Reindexing: Cleaning up key-value store");
317
+ cleanupKV(db, indexerId);
318
+ }
319
+ if (enablePersistState) {
320
+ logger.warn("Reindexing: Resetting persistence state");
321
+ resetPersistence({ db, indexerId });
322
+ }
323
+ cleanupApplied = true;
324
+ logger.success("All data has been cleaned up for reindexing");
325
+ }
283
326
  });
284
327
  break;
285
328
  } catch (error) {
@@ -320,7 +363,7 @@ function sqliteStorage({
320
363
  invalidateState({ db, cursor, indexerId });
321
364
  }
322
365
  if (enableKeyValueStore) {
323
- invalidateKV(db, cursor);
366
+ invalidateKV(db, cursor, indexerId);
324
367
  }
325
368
  });
326
369
  });
@@ -339,7 +382,7 @@ function sqliteStorage({
339
382
  }
340
383
  });
341
384
  indexer.hooks.hook("message:finalize", async ({ message }) => {
342
- const { cursor } = message.finalize;
385
+ const { cursor } = message;
343
386
  if (!cursor) {
344
387
  throw new SqliteStorageError("finalized cursor is undefined");
345
388
  }
@@ -348,12 +391,12 @@ function sqliteStorage({
348
391
  finalizeState({ db, cursor, indexerId });
349
392
  }
350
393
  if (enableKeyValueStore) {
351
- finalizeKV(db, cursor);
394
+ finalizeKV(db, cursor, indexerId);
352
395
  }
353
396
  });
354
397
  });
355
398
  indexer.hooks.hook("message:invalidate", async ({ message }) => {
356
- const { cursor } = message.invalidate;
399
+ const { cursor } = message;
357
400
  if (!cursor) {
358
401
  throw new SqliteStorageError("invalidate cursor is undefined");
359
402
  }
@@ -362,37 +405,45 @@ function sqliteStorage({
362
405
  invalidateState({ db, cursor, indexerId });
363
406
  }
364
407
  if (enableKeyValueStore) {
365
- invalidateKV(db, cursor);
408
+ invalidateKV(db, cursor, indexerId);
366
409
  }
367
410
  });
368
411
  });
369
412
  indexer.hooks.hook("handler:middleware", ({ use }) => {
370
413
  use(async (ctx, next) => {
371
- if (!ctx.finality) {
414
+ const { endCursor, finality, cursor } = ctx;
415
+ if (!finality) {
372
416
  throw new SqliteStorageError("finality is undefined");
373
417
  }
374
- if (!ctx.endCursor || !protocol.isCursor(ctx.endCursor)) {
418
+ if (!endCursor) {
375
419
  throw new SqliteStorageError(
376
420
  "endCursor is undefined or not a cursor"
377
421
  );
378
422
  }
379
423
  await withTransaction(database, async (db) => {
424
+ if (prevFinality === "pending") {
425
+ if (enableKeyValueStore) {
426
+ invalidateKV(db, cursor, indexerId);
427
+ }
428
+ }
380
429
  if (enableKeyValueStore) {
381
430
  ctx[KV_PROPERTY] = new KeyValueStore(
382
431
  db,
383
- ctx.endCursor,
384
- ctx.finality,
432
+ endCursor,
433
+ finality,
385
434
  serializeFn,
386
- deserializeFn
435
+ deserializeFn,
436
+ indexerId
387
437
  );
388
438
  }
389
439
  await next();
390
- if (enablePersistState) {
391
- persistState({ db, endCursor: ctx.endCursor, indexerId });
440
+ if (enablePersistState && finality !== "pending") {
441
+ persistState({ db, endCursor, indexerId });
392
442
  }
393
443
  if (enableKeyValueStore) {
394
444
  delete ctx[KV_PROPERTY];
395
445
  }
446
+ prevFinality = finality;
396
447
  });
397
448
  });
398
449
  });
@@ -402,3 +453,4 @@ function sqliteStorage({
402
453
  exports.KeyValueStore = KeyValueStore;
403
454
  exports.sqliteStorage = sqliteStorage;
404
455
  exports.useSqliteKeyValueStore = useSqliteKeyValueStore;
456
+ //# sourceMappingURL=index.cjs.map
package/dist/index.cjs.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.cjs","sources":["../src/utils.ts","../src/kv.ts","../src/persistence.ts","../src/index.ts"],"sourcesContent":["import type { Database } from \"better-sqlite3\";\n\nexport type SerializeFn = <T>(value: T) => string;\nexport type DeserializeFn = <T>(value: string) => T;\n\nexport class SqliteStorageError extends Error {\n constructor(message: string, options?: ErrorOptions) {\n super(message, options);\n this.name = \"SqliteStorageError\";\n }\n}\n\nexport async function withTransaction(\n db: Database,\n cb: (db: Database) => Promise<void>,\n) {\n db.prepare(\"BEGIN TRANSACTION\").run();\n try {\n await cb(db);\n } catch (error) {\n db.prepare(\"ROLLBACK TRANSACTION\").run();\n throw error;\n }\n db.prepare(\"COMMIT TRANSACTION\").run();\n}\n\nexport function assertInTransaction(db: Database) {\n if (!db.inTransaction) {\n throw new SqliteStorageError(\"Database is not in transaction\");\n }\n}\n\nexport function deserialize<T>(str: string): T {\n return JSON.parse(str, (_, value) =>\n typeof value === \"string\" && value.match(/^\\d+n$/)\n ? BigInt(value.slice(0, -1))\n : value,\n ) as T;\n}\n\nexport function serialize<T>(obj: T): string {\n return JSON.stringify(\n obj,\n (_, value) => (typeof value === \"bigint\" ? `${value.toString()}n` : value),\n \"\\t\",\n );\n}\n\nexport function sleep(ms: number) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n","import type { Cursor, DataFinality } from \"@apibara/protocol\";\nimport type { Database } from \"better-sqlite3\";\n\nimport {\n type DeserializeFn,\n type SerializeFn,\n assertInTransaction,\n} from \"./utils\";\n\nexport function initializeKeyValueStore(db: Database) {\n assertInTransaction(db);\n db.exec(statements.createTable);\n}\n\nexport class KeyValueStore {\n constructor(\n private readonly db: Database,\n private readonly endCursor: Cursor,\n private readonly finality: DataFinality,\n private readonly serialize: SerializeFn,\n private readonly deserialize: DeserializeFn,\n private readonly indexerId: string,\n ) {\n assertInTransaction(db);\n }\n\n get<T>(key: string): T | undefined {\n const row = this.db\n .prepare<[string, string], KeyValueRow>(statements.get)\n .get(key, this.indexerId);\n\n return row ? this.deserialize(row.v) : undefined;\n }\n\n put<T>(key: string, value: T) {\n this.db\n .prepare<[number, string, string], KeyValueRow>(statements.updateToBlock)\n .run(Number(this.endCursor.orderKey), key, this.indexerId);\n\n this.db\n .prepare<[number, string, string, string], KeyValueRow>(\n statements.insertIntoKvs,\n )\n .run(\n Number(this.endCursor.orderKey),\n key,\n this.serialize(value as Record<string, unknown>),\n this.indexerId,\n );\n }\n\n del(key: string) {\n this.db\n .prepare<[number, string, string], KeyValueRow>(statements.del)\n .run(Number(this.endCursor.orderKey), key, this.indexerId);\n }\n}\n\nexport function finalizeKV(db: Database, cursor: Cursor, indexerId: string) {\n assertInTransaction(db);\n\n db.prepare<[number, string], KeyValueRow>(statements.finalize).run(\n Number(cursor.orderKey),\n indexerId,\n );\n}\n\nexport function invalidateKV(db: Database, cursor: Cursor, indexerId: string) {\n assertInTransaction(db);\n\n // Delete entries that started after the invalidation cursor\n db.prepare<[number, string], KeyValueRow>(statements.invalidateDelete).run(\n Number(cursor.orderKey),\n indexerId,\n );\n\n // Update entries that were supposed to end after the invalidation cursor\n db.prepare<[number, string], KeyValueRow>(statements.invalidateUpdate).run(\n Number(cursor.orderKey),\n indexerId,\n );\n}\n\nexport function cleanupKV(db: Database, indexerId: string) {\n assertInTransaction(db);\n\n db.prepare<[string], KeyValueRow>(statements.cleanup).run(indexerId);\n}\n\nexport type KeyValueRow = {\n from_block: number;\n to_block: number;\n k: string;\n v: string;\n id: string;\n};\n\nconst statements = {\n createTable: `\n CREATE TABLE IF NOT EXISTS kvs (\n from_block INTEGER NOT NULL,\n to_block INTEGER,\n k TEXT NOT NULL,\n v BLOB NOT NULL,\n id TEXT NOT NULL,\n PRIMARY KEY (from_block, k, id)\n );`,\n get: `\n SELECT v\n FROM kvs\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n updateToBlock: `\n UPDATE kvs\n SET to_block = ?\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n insertIntoKvs: `\n INSERT INTO kvs (from_block, to_block, k, v, id)\n VALUES (?, NULL, ?, ?, ?)`,\n del: `\n UPDATE kvs\n SET to_block = ?\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n finalize: `\n DELETE FROM kvs\n WHERE to_block <= ? AND id = ?`,\n invalidateDelete: `\n DELETE FROM kvs\n WHERE from_block > ? AND id = ?`,\n invalidateUpdate: `\n UPDATE kvs\n SET to_block = NULL\n WHERE to_block > ? AND id = ?`,\n cleanup: `\n DELETE FROM kvs\n WHERE id = ?`,\n} as const;\n","import { type Cursor, normalizeCursor } from \"@apibara/protocol\";\nimport type { Database } from \"better-sqlite3\";\n\nimport { assertInTransaction, deserialize, serialize } from \"./utils\";\n\nexport function initializePersistentState(db: Database) {\n assertInTransaction(db);\n db.exec(statements.createCheckpointsTable);\n db.exec(statements.createFiltersTable);\n}\n\nexport function persistState<TFilter>(props: {\n db: Database;\n endCursor: Cursor;\n filter?: TFilter;\n indexerId: string;\n}) {\n const { db, endCursor, filter, indexerId } = props;\n\n assertInTransaction(db);\n\n db.prepare(statements.putCheckpoint).run(\n indexerId,\n Number(endCursor.orderKey),\n endCursor.uniqueKey,\n );\n\n if (filter) {\n db.prepare(statements.updateFilterToBlock).run(\n Number(endCursor.orderKey),\n indexerId,\n );\n db.prepare(statements.insertFilter).run(\n indexerId,\n serialize(filter as Record<string, unknown>),\n Number(endCursor.orderKey),\n );\n }\n}\n\nexport function getState<TFilter>(props: {\n db: Database;\n indexerId: string;\n}) {\n const { db, indexerId } = props;\n assertInTransaction(db);\n const storedCursor = db\n .prepare<string, { order_key?: number; unique_key?: string }>(\n statements.getCheckpoint,\n )\n .get(indexerId);\n const storedFilter = db\n .prepare<string, { filter: string }>(statements.getFilter)\n .get(indexerId);\n\n let cursor: Cursor | undefined;\n let filter: TFilter | undefined;\n\n if (storedCursor?.order_key) {\n cursor = normalizeCursor({\n orderKey: BigInt(storedCursor.order_key),\n uniqueKey: storedCursor.unique_key ? storedCursor.unique_key : null,\n });\n }\n\n if (storedFilter) {\n filter = deserialize(storedFilter.filter) as TFilter;\n }\n\n return { cursor, filter };\n}\n\nexport function finalizeState(props: {\n db: Database;\n cursor: Cursor;\n indexerId: string;\n}) {\n const { cursor, db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string, number]>(statements.finalizeFilter).run(\n indexerId,\n Number(cursor.orderKey),\n );\n}\n\nexport function invalidateState(props: {\n db: Database;\n cursor: Cursor;\n indexerId: string;\n}) {\n const { cursor, db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string, number]>(statements.invalidateFilterDelete).run(\n indexerId,\n Number(cursor.orderKey),\n );\n db.prepare<[string, number]>(statements.invalidateFilterUpdate).run(\n indexerId,\n Number(cursor.orderKey),\n );\n}\n\nexport function resetPersistence(props: {\n db: Database;\n indexerId: string;\n}) {\n const { db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string]>(statements.resetCheckpoint).run(indexerId);\n db.prepare<[string]>(statements.resetFilter).run(indexerId);\n}\n\nexport type CheckpointRow = {\n id: string;\n order_key: number;\n unique_key: string | null;\n};\n\nexport type FilterRow = {\n id: string;\n filter: string;\n from_block: number;\n to_block: number | null;\n};\n\nconst statements = {\n createCheckpointsTable: `\n CREATE TABLE IF NOT EXISTS checkpoints (\n id TEXT NOT NULL PRIMARY KEY,\n order_key INTEGER,\n unique_key TEXT\n );`,\n createFiltersTable: `\n CREATE TABLE IF NOT EXISTS filters (\n id TEXT NOT NULL,\n filter BLOB NOT NULL,\n from_block INTEGER NOT NULL,\n to_block INTEGER,\n PRIMARY KEY (id, from_block)\n );`,\n getCheckpoint: `\n SELECT *\n FROM checkpoints\n WHERE id = ?`,\n putCheckpoint: `\n INSERT INTO checkpoints (id, order_key, unique_key)\n VALUES (?, ?, ?)\n ON CONFLICT(id) DO UPDATE SET\n order_key = excluded.order_key,\n unique_key = excluded.unique_key`,\n delCheckpoint: `\n DELETE FROM checkpoints\n WHERE id = ?`,\n getFilter: `\n SELECT *\n FROM filters\n WHERE id = ? AND to_block IS NULL`,\n updateFilterToBlock: `\n UPDATE filters\n SET to_block = ?\n WHERE id = ? AND to_block IS NULL`,\n insertFilter: `\n INSERT INTO filters (id, filter, from_block)\n VALUES (?, ?, ?)\n ON CONFLICT(id, from_block) DO UPDATE SET\n filter = excluded.filter,\n from_block = excluded.from_block`,\n delFilter: `\n DELETE FROM filters\n WHERE id = ?`,\n finalizeFilter: `\n DELETE FROM filters\n WHERE id = ? AND to_block <= ?`,\n invalidateFilterDelete: `\n DELETE FROM filters\n WHERE id = ? AND from_block > ?`,\n invalidateFilterUpdate: `\n UPDATE filters\n SET to_block = NULL\n WHERE id = ? AND to_block > ?`,\n resetCheckpoint: `\n DELETE FROM checkpoints\n WHERE id = ?`,\n resetFilter: `\n DELETE FROM filters\n WHERE id = ?`,\n};\n","import { useIndexerContext } from \"@apibara/indexer\";\nimport { defineIndexerPlugin, useLogger } from \"@apibara/indexer/plugins\";\nimport type { Cursor, DataFinality } from \"@apibara/protocol\";\nimport type { Database as SqliteDatabase } from \"better-sqlite3\";\n\nimport { generateIndexerId } from \"@apibara/indexer/internal\";\nimport { useInternalContext } from \"@apibara/indexer/internal/plugins\";\nimport {\n KeyValueStore,\n cleanupKV,\n finalizeKV,\n initializeKeyValueStore,\n invalidateKV,\n} from \"./kv\";\nimport {\n finalizeState,\n getState,\n initializePersistentState,\n invalidateState,\n persistState,\n resetPersistence,\n} from \"./persistence\";\nimport {\n type DeserializeFn,\n type SerializeFn,\n SqliteStorageError,\n assertInTransaction,\n deserialize,\n serialize,\n sleep,\n withTransaction,\n} from \"./utils\";\n\nconst KV_PROPERTY = \"_kv_sqlite\" as const;\nconst MAX_RETRIES = 5;\n\nexport { KeyValueStore } from \"./kv\";\n\nexport function useSqliteKeyValueStore(): KeyValueStore {\n const kv = useIndexerContext()[KV_PROPERTY] as KeyValueStore | undefined;\n if (!kv) {\n throw new SqliteStorageError(\n \"SQLite key-value store is not available. Did you forget to enable it?\",\n );\n }\n\n return kv;\n}\n\nexport type SqliteStorageOptions = {\n database: SqliteDatabase;\n keyValueStore?: boolean;\n persistState?: boolean;\n indexerName?: string;\n\n serialize?: SerializeFn;\n deserialize?: DeserializeFn;\n};\n\n/**\n * Creates a plugin that uses SQLite as the storage layer.\n *\n * Supports storing the indexer's state and provides a simple Key-Value store.\n * @param options.database - The SQLite database instance.\n * @param options.persistState - Whether to persist the indexer's state. Defaults to true.\n * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.\n * @param options.serialize - A function to serialize the value to the KV.\n * @param options.deserialize - A function to deserialize the value from the KV.\n * @param options.indexerName - The name of the indexer. Defaults value is 'default'.\n */\nexport function sqliteStorage<TFilter, TBlock>({\n database,\n persistState: enablePersistState = true,\n keyValueStore: enableKeyValueStore = true,\n serialize: serializeFn = serialize,\n deserialize: deserializeFn = deserialize,\n indexerName: identifier = \"default\",\n}: SqliteStorageOptions) {\n return defineIndexerPlugin<TFilter, TBlock>((indexer) => {\n let indexerId = \"\";\n let prevFinality: DataFinality | undefined;\n const alwaysReindex = process.env[\"APIBARA_ALWAYS_REINDEX\"] === \"true\";\n\n indexer.hooks.hook(\"plugins:init\", async () => {\n const { indexerName: indexerFileName, availableIndexers } =\n useInternalContext();\n\n const logger = useLogger();\n\n indexerId = generateIndexerId(indexerFileName, identifier);\n\n let retries = 0;\n\n let cleanupApplied = false;\n\n while (retries <= MAX_RETRIES) {\n try {\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n initializePersistentState(db);\n }\n\n if (enableKeyValueStore) {\n initializeKeyValueStore(db);\n }\n\n if (alwaysReindex && !cleanupApplied) {\n if (enableKeyValueStore) {\n logger.warn(\"Reindexing: Cleaning up key-value store\");\n cleanupKV(db, indexerId);\n }\n\n if (enablePersistState) {\n logger.warn(\"Reindexing: Resetting persistence state\");\n resetPersistence({ db, indexerId });\n }\n\n cleanupApplied = true;\n\n logger.success(\"All data has been cleaned up for reindexing\");\n }\n });\n break;\n } catch (error) {\n if (retries === MAX_RETRIES) {\n throw new SqliteStorageError(\n \"Initialization failed after 5 retries\",\n {\n cause: error,\n },\n );\n }\n await sleep(retries * 1000);\n retries++;\n }\n }\n });\n\n indexer.hooks.hook(\"connect:before\", async ({ request }) => {\n if (!enablePersistState) {\n return;\n }\n\n return await withTransaction(database, async (db) => {\n const { cursor, filter } = getState<TFilter>({ db, indexerId });\n\n if (cursor) {\n request.startingCursor = cursor;\n }\n\n if (filter) {\n request.filter[1] = filter;\n }\n });\n });\n\n indexer.hooks.hook(\"connect:after\", async ({ request }) => {\n // On restart, we need to invalidate data for blocks that were processed but not persisted.\n const cursor = request.startingCursor;\n\n if (!cursor) {\n return;\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n invalidateState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"connect:factory\", ({ request, endCursor }) => {\n if (!enablePersistState) {\n return;\n }\n\n // The connect factory hook is called while indexing a block, so the database should be in a transaction\n // created by the middleware.\n assertInTransaction(database);\n\n if (endCursor && request.filter[1]) {\n persistState({\n db: database,\n endCursor,\n indexerId,\n filter: request.filter[1],\n });\n }\n });\n\n indexer.hooks.hook(\"message:finalize\", async ({ message }) => {\n const { cursor } = message;\n\n if (!cursor) {\n throw new SqliteStorageError(\"finalized cursor is undefined\");\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n finalizeState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n finalizeKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"message:invalidate\", async ({ message }) => {\n const { cursor } = message;\n\n if (!cursor) {\n throw new SqliteStorageError(\"invalidate cursor is undefined\");\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n invalidateState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"handler:middleware\", ({ use }) => {\n use(async (ctx, next) => {\n const { endCursor, finality, cursor } = ctx as {\n cursor: Cursor;\n endCursor: Cursor;\n finality: DataFinality;\n };\n\n if (!finality) {\n throw new SqliteStorageError(\"finality is undefined\");\n }\n\n if (!endCursor) {\n throw new SqliteStorageError(\n \"endCursor is undefined or not a cursor\",\n );\n }\n\n await withTransaction(database, async (db) => {\n if (prevFinality === \"pending\") {\n // invalidate if previous block's finality was \"pending\"\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n }\n\n if (enableKeyValueStore) {\n ctx[KV_PROPERTY] = new KeyValueStore(\n db,\n endCursor,\n finality,\n serializeFn,\n deserializeFn,\n indexerId,\n );\n }\n\n await next();\n\n if (enablePersistState && finality !== \"pending\") {\n persistState({ db, endCursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n delete ctx[KV_PROPERTY];\n }\n\n prevFinality = finality;\n });\n });\n });\n });\n}\n"],"names":["statements","normalizeCursor","useIndexerContext","defineIndexerPlugin","useInternalContext","useLogger","generateIndexerId"],"mappings":";;;;;;;;AAKO,MAAM,2BAA2B,KAAM,CAAA;AAAA,EAC5C,WAAA,CAAY,SAAiB,OAAwB,EAAA;AACnD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA,CAAA;AACtB,IAAA,IAAA,CAAK,IAAO,GAAA,oBAAA,CAAA;AAAA,GACd;AACF,CAAA;AAEsB,eAAA,eAAA,CACpB,IACA,EACA,EAAA;AACA,EAAG,EAAA,CAAA,OAAA,CAAQ,mBAAmB,CAAA,CAAE,GAAI,EAAA,CAAA;AACpC,EAAI,IAAA;AACF,IAAA,MAAM,GAAG,EAAE,CAAA,CAAA;AAAA,WACJ,KAAO,EAAA;AACd,IAAG,EAAA,CAAA,OAAA,CAAQ,sBAAsB,CAAA,CAAE,GAAI,EAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACA,EAAG,EAAA,CAAA,OAAA,CAAQ,oBAAoB,CAAA,CAAE,GAAI,EAAA,CAAA;AACvC,CAAA;AAEO,SAAS,oBAAoB,EAAc,EAAA;AAChD,EAAI,IAAA,CAAC,GAAG,aAAe,EAAA;AACrB,IAAM,MAAA,IAAI,mBAAmB,gCAAgC,CAAA,CAAA;AAAA,GAC/D;AACF,CAAA;AAEO,SAAS,YAAe,GAAgB,EAAA;AAC7C,EAAA,OAAO,IAAK,CAAA,KAAA;AAAA,IAAM,GAAA;AAAA,IAAK,CAAC,CAAG,EAAA,KAAA,KACzB,OAAO,KAAA,KAAU,YAAY,KAAM,CAAA,KAAA,CAAM,QAAQ,CAAA,GAC7C,OAAO,KAAM,CAAA,KAAA,CAAM,CAAG,EAAA,CAAA,CAAE,CAAC,CACzB,GAAA,KAAA;AAAA,GACN,CAAA;AACF,CAAA;AAEO,SAAS,UAAa,GAAgB,EAAA;AAC3C,EAAA,OAAO,IAAK,CAAA,SAAA;AAAA,IACV,GAAA;AAAA,IACA,CAAC,CAAG,EAAA,KAAA,KAAW,OAAO,KAAA,KAAU,WAAW,CAAG,EAAA,KAAA,CAAM,QAAS,EAAC,CAAM,CAAA,CAAA,GAAA,KAAA;AAAA,IACpE,GAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEO,SAAS,MAAM,EAAY,EAAA;AAChC,EAAA,OAAO,IAAI,OAAQ,CAAA,CAAC,YAAY,UAAW,CAAA,OAAA,EAAS,EAAE,CAAC,CAAA,CAAA;AACzD;;ACzCO,SAAS,wBAAwB,EAAc,EAAA;AACpD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,IAAA,CAAKA,aAAW,WAAW,CAAA,CAAA;AAChC,CAAA;AAEO,MAAM,aAAc,CAAA;AAAA,EACzB,YACmB,EACA,EAAA,SAAA,EACA,QACA,EAAA,SAAA,EACA,aACA,SACjB,EAAA;AANiB,IAAA,IAAA,CAAA,EAAA,GAAA,EAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AACA,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AACA,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AAEjB,IAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAAA,GACxB;AAAA,EAEA,IAAO,GAA4B,EAAA;AACjC,IAAM,MAAA,GAAA,GAAM,IAAK,CAAA,EAAA,CACd,OAAuC,CAAAA,YAAA,CAAW,GAAG,CACrD,CAAA,GAAA,CAAI,GAAK,EAAA,IAAA,CAAK,SAAS,CAAA,CAAA;AAE1B,IAAA,OAAO,GAAM,GAAA,IAAA,CAAK,WAAY,CAAA,GAAA,CAAI,CAAC,CAAI,GAAA,KAAA,CAAA,CAAA;AAAA,GACzC;AAAA,EAEA,GAAA,CAAO,KAAa,KAAU,EAAA;AAC5B,IAAA,IAAA,CAAK,EACF,CAAA,OAAA,CAA+CA,YAAW,CAAA,aAAa,CACvE,CAAA,GAAA,CAAI,MAAO,CAAA,IAAA,CAAK,SAAU,CAAA,QAAQ,CAAG,EAAA,GAAA,EAAK,KAAK,SAAS,CAAA,CAAA;AAE3D,IAAA,IAAA,CAAK,EACF,CAAA,OAAA;AAAA,MACCA,YAAW,CAAA,aAAA;AAAA,KAEZ,CAAA,GAAA;AAAA,MACC,MAAA,CAAO,IAAK,CAAA,SAAA,CAAU,QAAQ,CAAA;AAAA,MAC9B,GAAA;AAAA,MACA,IAAA,CAAK,UAAU,KAAgC,CAAA;AAAA,MAC/C,IAAK,CAAA,SAAA;AAAA,KACP,CAAA;AAAA,GACJ;AAAA,EAEA,IAAI,GAAa,EAAA;AACf,IAAA,IAAA,CAAK,EACF,CAAA,OAAA,CAA+CA,YAAW,CAAA,GAAG,CAC7D,CAAA,GAAA,CAAI,MAAO,CAAA,IAAA,CAAK,SAAU,CAAA,QAAQ,CAAG,EAAA,GAAA,EAAK,KAAK,SAAS,CAAA,CAAA;AAAA,GAC7D;AACF,CAAA;AAEgB,SAAA,UAAA,CAAW,EAAc,EAAA,MAAA,EAAgB,SAAmB,EAAA;AAC1E,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,QAAQ,CAAE,CAAA,GAAA;AAAA,IAC7D,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEgB,SAAA,YAAA,CAAa,EAAc,EAAA,MAAA,EAAgB,SAAmB,EAAA;AAC5E,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAGtB,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,gBAAgB,CAAE,CAAA,GAAA;AAAA,IACrE,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AAGA,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,gBAAgB,CAAE,CAAA,GAAA;AAAA,IACrE,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEgB,SAAA,SAAA,CAAU,IAAc,SAAmB,EAAA;AACzD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAA,EAAA,CAAG,OAA+B,CAAAA,YAAA,CAAW,OAAO,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AACrE,CAAA;AAUA,MAAMA,YAAa,GAAA;AAAA,EACjB,WAAa,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA;AAAA,EASb,GAAK,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIL,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIf,aAAe,EAAA,CAAA;AAAA;AAAA,6BAAA,CAAA;AAAA,EAGf,GAAK,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIL,QAAU,EAAA,CAAA;AAAA;AAAA,kCAAA,CAAA;AAAA,EAGV,gBAAkB,EAAA,CAAA;AAAA;AAAA,mCAAA,CAAA;AAAA,EAGlB,gBAAkB,EAAA,CAAA;AAAA;AAAA;AAAA,iCAAA,CAAA;AAAA,EAIlB,OAAS,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAGX,CAAA;;AClIO,SAAS,0BAA0B,EAAc,EAAA;AACtD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,IAAA,CAAK,WAAW,sBAAsB,CAAA,CAAA;AACzC,EAAG,EAAA,CAAA,IAAA,CAAK,WAAW,kBAAkB,CAAA,CAAA;AACvC,CAAA;AAEO,SAAS,aAAsB,KAKnC,EAAA;AACD,EAAA,MAAM,EAAE,EAAA,EAAI,SAAW,EAAA,MAAA,EAAQ,WAAc,GAAA,KAAA,CAAA;AAE7C,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,aAAa,CAAE,CAAA,GAAA;AAAA,IACnC,SAAA;AAAA,IACA,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,IACzB,SAAU,CAAA,SAAA;AAAA,GACZ,CAAA;AAEA,EAAA,IAAI,MAAQ,EAAA;AACV,IAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,mBAAmB,CAAE,CAAA,GAAA;AAAA,MACzC,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,MACzB,SAAA;AAAA,KACF,CAAA;AACA,IAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,YAAY,CAAE,CAAA,GAAA;AAAA,MAClC,SAAA;AAAA,MACA,UAAU,MAAiC,CAAA;AAAA,MAC3C,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,KAC3B,CAAA;AAAA,GACF;AACF,CAAA;AAEO,SAAS,SAAkB,KAG/B,EAAA;AACD,EAAM,MAAA,EAAE,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAC1B,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAA,MAAM,eAAe,EAClB,CAAA,OAAA;AAAA,IACC,UAAW,CAAA,aAAA;AAAA,GACb,CACC,IAAI,SAAS,CAAA,CAAA;AAChB,EAAA,MAAM,eAAe,EAClB,CAAA,OAAA,CAAoC,WAAW,SAAS,CAAA,CACxD,IAAI,SAAS,CAAA,CAAA;AAEhB,EAAI,IAAA,MAAA,CAAA;AACJ,EAAI,IAAA,MAAA,CAAA;AAEJ,EAAA,IAAI,cAAc,SAAW,EAAA;AAC3B,IAAA,MAAA,GAASC,wBAAgB,CAAA;AAAA,MACvB,QAAA,EAAU,MAAO,CAAA,YAAA,CAAa,SAAS,CAAA;AAAA,MACvC,SAAW,EAAA,YAAA,CAAa,UAAa,GAAA,YAAA,CAAa,UAAa,GAAA,IAAA;AAAA,KAChE,CAAA,CAAA;AAAA,GACH;AAEA,EAAA,IAAI,YAAc,EAAA;AAChB,IAAS,MAAA,GAAA,WAAA,CAAY,aAAa,MAAM,CAAA,CAAA;AAAA,GAC1C;AAEA,EAAO,OAAA,EAAE,QAAQ,MAAO,EAAA,CAAA;AAC1B,CAAA;AAEO,SAAS,cAAc,KAI3B,EAAA;AACD,EAAA,MAAM,EAAE,MAAA,EAAQ,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAClC,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,cAAc,CAAE,CAAA,GAAA;AAAA,IACtD,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACF,CAAA;AAEO,SAAS,gBAAgB,KAI7B,EAAA;AACD,EAAA,MAAM,EAAE,MAAA,EAAQ,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAClC,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,sBAAsB,CAAE,CAAA,GAAA;AAAA,IAC9D,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACA,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,sBAAsB,CAAE,CAAA,GAAA;AAAA,IAC9D,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACF,CAAA;AAEO,SAAS,iBAAiB,KAG9B,EAAA;AACD,EAAM,MAAA,EAAE,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAC1B,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAA,EAAA,CAAG,OAAkB,CAAA,UAAA,CAAW,eAAe,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AAC9D,EAAA,EAAA,CAAG,OAAkB,CAAA,UAAA,CAAW,WAAW,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AAC5D,CAAA;AAeA,MAAM,UAAa,GAAA;AAAA,EACjB,sBAAwB,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA,CAAA;AAAA,EAMxB,kBAAoB,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA,CAAA;AAAA,EAQpB,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAIf,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sCAAA,CAAA;AAAA,EAMf,aAAe,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGf,SAAW,EAAA,CAAA;AAAA;AAAA;AAAA,qCAAA,CAAA;AAAA,EAIX,mBAAqB,EAAA,CAAA;AAAA;AAAA;AAAA,qCAAA,CAAA;AAAA,EAIrB,YAAc,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sCAAA,CAAA;AAAA,EAMd,SAAW,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGX,cAAgB,EAAA,CAAA;AAAA;AAAA,kCAAA,CAAA;AAAA,EAGhB,sBAAwB,EAAA,CAAA;AAAA;AAAA,mCAAA,CAAA;AAAA,EAGxB,sBAAwB,EAAA,CAAA;AAAA;AAAA;AAAA,iCAAA,CAAA;AAAA,EAIxB,eAAiB,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGjB,WAAa,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAGf,CAAA;;ACzJA,MAAM,WAAc,GAAA,YAAA,CAAA;AACpB,MAAM,WAAc,GAAA,CAAA,CAAA;AAIb,SAAS,sBAAwC,GAAA;AACtD,EAAM,MAAA,EAAA,GAAKC,yBAAkB,EAAA,CAAE,WAAW,CAAA,CAAA;AAC1C,EAAA,IAAI,CAAC,EAAI,EAAA;AACP,IAAA,MAAM,IAAI,kBAAA;AAAA,MACR,uEAAA;AAAA,KACF,CAAA;AAAA,GACF;AAEA,EAAO,OAAA,EAAA,CAAA;AACT,CAAA;AAuBO,SAAS,aAA+B,CAAA;AAAA,EAC7C,QAAA;AAAA,EACA,cAAc,kBAAqB,GAAA,IAAA;AAAA,EACnC,eAAe,mBAAsB,GAAA,IAAA;AAAA,EACrC,WAAW,WAAc,GAAA,SAAA;AAAA,EACzB,aAAa,aAAgB,GAAA,WAAA;AAAA,EAC7B,aAAa,UAAa,GAAA,SAAA;AAC5B,CAAyB,EAAA;AACvB,EAAO,OAAAC,2BAAA,CAAqC,CAAC,OAAY,KAAA;AACvD,IAAA,IAAI,SAAY,GAAA,EAAA,CAAA;AAChB,IAAI,IAAA,YAAA,CAAA;AACJ,IAAA,MAAM,aAAgB,GAAA,OAAA,CAAQ,GAAI,CAAA,wBAAwB,CAAM,KAAA,MAAA,CAAA;AAEhE,IAAQ,OAAA,CAAA,KAAA,CAAM,IAAK,CAAA,cAAA,EAAgB,YAAY;AAC7C,MAAA,MAAM,EAAE,WAAA,EAAa,eAAiB,EAAA,iBAAA,KACpCC,4BAAmB,EAAA,CAAA;AAErB,MAAA,MAAM,SAASC,iBAAU,EAAA,CAAA;AAEzB,MAAY,SAAA,GAAAC,0BAAA,CAAkB,iBAAiB,UAAU,CAAA,CAAA;AAEzD,MAAA,IAAI,OAAU,GAAA,CAAA,CAAA;AAEd,MAAA,IAAI,cAAiB,GAAA,KAAA,CAAA;AAErB,MAAA,OAAO,WAAW,WAAa,EAAA;AAC7B,QAAI,IAAA;AACF,UAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,YAAA,IAAI,kBAAoB,EAAA;AACtB,cAAA,yBAAA,CAA0B,EAAE,CAAA,CAAA;AAAA,aAC9B;AAEA,YAAA,IAAI,mBAAqB,EAAA;AACvB,cAAA,uBAAA,CAAwB,EAAE,CAAA,CAAA;AAAA,aAC5B;AAEA,YAAI,IAAA,aAAA,IAAiB,CAAC,cAAgB,EAAA;AACpC,cAAA,IAAI,mBAAqB,EAAA;AACvB,gBAAA,MAAA,CAAO,KAAK,yCAAyC,CAAA,CAAA;AACrD,gBAAA,SAAA,CAAU,IAAI,SAAS,CAAA,CAAA;AAAA,eACzB;AAEA,cAAA,IAAI,kBAAoB,EAAA;AACtB,gBAAA,MAAA,CAAO,KAAK,yCAAyC,CAAA,CAAA;AACrD,gBAAiB,gBAAA,CAAA,EAAE,EAAI,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,eACpC;AAEA,cAAiB,cAAA,GAAA,IAAA,CAAA;AAEjB,cAAA,MAAA,CAAO,QAAQ,6CAA6C,CAAA,CAAA;AAAA,aAC9D;AAAA,WACD,CAAA,CAAA;AACD,UAAA,MAAA;AAAA,iBACO,KAAO,EAAA;AACd,UAAA,IAAI,YAAY,WAAa,EAAA;AAC3B,YAAA,MAAM,IAAI,kBAAA;AAAA,cACR,uCAAA;AAAA,cACA;AAAA,gBACE,KAAO,EAAA,KAAA;AAAA,eACT;AAAA,aACF,CAAA;AAAA,WACF;AACA,UAAM,MAAA,KAAA,CAAM,UAAU,GAAI,CAAA,CAAA;AAC1B,UAAA,OAAA,EAAA,CAAA;AAAA,SACF;AAAA,OACF;AAAA,KACD,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,gBAAA,EAAkB,OAAO,EAAE,SAAc,KAAA;AAC1D,MAAA,IAAI,CAAC,kBAAoB,EAAA;AACvB,QAAA,OAAA;AAAA,OACF;AAEA,MAAA,OAAO,MAAM,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AACnD,QAAM,MAAA,EAAE,QAAQ,MAAO,EAAA,GAAI,SAAkB,EAAE,EAAA,EAAI,WAAW,CAAA,CAAA;AAE9D,QAAA,IAAI,MAAQ,EAAA;AACV,UAAA,OAAA,CAAQ,cAAiB,GAAA,MAAA,CAAA;AAAA,SAC3B;AAEA,QAAA,IAAI,MAAQ,EAAA;AACV,UAAQ,OAAA,CAAA,MAAA,CAAO,CAAC,CAAI,GAAA,MAAA,CAAA;AAAA,SACtB;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,eAAA,EAAiB,OAAO,EAAE,SAAc,KAAA;AAEzD,MAAA,MAAM,SAAS,OAAQ,CAAA,cAAA,CAAA;AAEvB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAA,OAAA;AAAA,OACF;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,eAAA,CAAgB,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SAC3C;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SACpC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,iBAAA,EAAmB,CAAC,EAAE,OAAA,EAAS,WAAgB,KAAA;AAChE,MAAA,IAAI,CAAC,kBAAoB,EAAA;AACvB,QAAA,OAAA;AAAA,OACF;AAIA,MAAA,mBAAA,CAAoB,QAAQ,CAAA,CAAA;AAE5B,MAAA,IAAI,SAAa,IAAA,OAAA,CAAQ,MAAO,CAAA,CAAC,CAAG,EAAA;AAClC,QAAa,YAAA,CAAA;AAAA,UACX,EAAI,EAAA,QAAA;AAAA,UACJ,SAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA,EAAQ,OAAQ,CAAA,MAAA,CAAO,CAAC,CAAA;AAAA,SACzB,CAAA,CAAA;AAAA,OACH;AAAA,KACD,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,kBAAA,EAAoB,OAAO,EAAE,SAAc,KAAA;AAC5D,MAAM,MAAA,EAAE,QAAW,GAAA,OAAA,CAAA;AAEnB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAM,MAAA,IAAI,mBAAmB,+BAA+B,CAAA,CAAA;AAAA,OAC9D;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,aAAA,CAAc,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SACzC;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAW,UAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SAClC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,oBAAA,EAAsB,OAAO,EAAE,SAAc,KAAA;AAC9D,MAAM,MAAA,EAAE,QAAW,GAAA,OAAA,CAAA;AAEnB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAM,MAAA,IAAI,mBAAmB,gCAAgC,CAAA,CAAA;AAAA,OAC/D;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,eAAA,CAAgB,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SAC3C;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SACpC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,oBAAA,EAAsB,CAAC,EAAE,KAAU,KAAA;AACpD,MAAI,GAAA,CAAA,OAAO,KAAK,IAAS,KAAA;AACvB,QAAA,MAAM,EAAE,SAAA,EAAW,QAAU,EAAA,MAAA,EAAW,GAAA,GAAA,CAAA;AAMxC,QAAA,IAAI,CAAC,QAAU,EAAA;AACb,UAAM,MAAA,IAAI,mBAAmB,uBAAuB,CAAA,CAAA;AAAA,SACtD;AAEA,QAAA,IAAI,CAAC,SAAW,EAAA;AACd,UAAA,MAAM,IAAI,kBAAA;AAAA,YACR,wCAAA;AAAA,WACF,CAAA;AAAA,SACF;AAEA,QAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,UAAA,IAAI,iBAAiB,SAAW,EAAA;AAE9B,YAAA,IAAI,mBAAqB,EAAA;AACvB,cAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,aACpC;AAAA,WACF;AAEA,UAAA,IAAI,mBAAqB,EAAA;AACvB,YAAI,GAAA,CAAA,WAAW,IAAI,IAAI,aAAA;AAAA,cACrB,EAAA;AAAA,cACA,SAAA;AAAA,cACA,QAAA;AAAA,cACA,WAAA;AAAA,cACA,aAAA;AAAA,cACA,SAAA;AAAA,aACF,CAAA;AAAA,WACF;AAEA,UAAA,MAAM,IAAK,EAAA,CAAA;AAEX,UAAI,IAAA,kBAAA,IAAsB,aAAa,SAAW,EAAA;AAChD,YAAA,YAAA,CAAa,EAAE,EAAA,EAAI,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,WAC3C;AAEA,UAAA,IAAI,mBAAqB,EAAA;AACvB,YAAA,OAAO,IAAI,WAAW,CAAA,CAAA;AAAA,WACxB;AAEA,UAAe,YAAA,GAAA,QAAA,CAAA;AAAA,SAChB,CAAA,CAAA;AAAA,OACF,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAAA,GACF,CAAA,CAAA;AACH;;;;;;"}
package/dist/index.d.cts CHANGED
@@ -11,7 +11,8 @@ declare class KeyValueStore {
11
11
  private readonly finality;
12
12
  private readonly serialize;
13
13
  private readonly deserialize;
14
- constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
14
+ private readonly indexerId;
15
+ constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn, indexerId: string);
15
16
  get<T>(key: string): T | undefined;
16
17
  put<T>(key: string, value: T): void;
17
18
  del(key: string): void;
package/dist/index.d.mts CHANGED
@@ -11,7 +11,8 @@ declare class KeyValueStore {
11
11
  private readonly finality;
12
12
  private readonly serialize;
13
13
  private readonly deserialize;
14
- constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
14
+ private readonly indexerId;
15
+ constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn, indexerId: string);
15
16
  get<T>(key: string): T | undefined;
16
17
  put<T>(key: string, value: T): void;
17
18
  del(key: string): void;
package/dist/index.d.ts CHANGED
@@ -11,7 +11,8 @@ declare class KeyValueStore {
11
11
  private readonly finality;
12
12
  private readonly serialize;
13
13
  private readonly deserialize;
14
- constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
14
+ private readonly indexerId;
15
+ constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn, indexerId: string);
15
16
  get<T>(key: string): T | undefined;
16
17
  put<T>(key: string, value: T): void;
17
18
  del(key: string): void;
package/dist/index.mjs CHANGED
@@ -1,8 +1,8 @@
1
1
  import { useIndexerContext } from '@apibara/indexer';
2
- import { defineIndexerPlugin } from '@apibara/indexer/plugins';
3
- import { normalizeCursor, isCursor } from '@apibara/protocol';
2
+ import { defineIndexerPlugin, useLogger } from '@apibara/indexer/plugins';
4
3
  import { generateIndexerId } from '@apibara/indexer/internal';
5
4
  import { useInternalContext } from '@apibara/indexer/internal/plugins';
5
+ import { normalizeCursor } from '@apibara/protocol';
6
6
 
7
7
  class SqliteStorageError extends Error {
8
8
  constructor(message, options) {
@@ -47,45 +47,56 @@ function initializeKeyValueStore(db) {
47
47
  db.exec(statements$1.createTable);
48
48
  }
49
49
  class KeyValueStore {
50
- constructor(db, endCursor, finality, serialize, deserialize) {
50
+ constructor(db, endCursor, finality, serialize, deserialize, indexerId) {
51
51
  this.db = db;
52
52
  this.endCursor = endCursor;
53
53
  this.finality = finality;
54
54
  this.serialize = serialize;
55
55
  this.deserialize = deserialize;
56
+ this.indexerId = indexerId;
56
57
  assertInTransaction(db);
57
58
  }
58
59
  get(key) {
59
- const row = this.db.prepare(statements$1.get).get(key);
60
+ const row = this.db.prepare(statements$1.get).get(key, this.indexerId);
60
61
  return row ? this.deserialize(row.v) : void 0;
61
62
  }
62
63
  put(key, value) {
63
- this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key);
64
- this.db.prepare(statements$1.insertIntoKvs).run(
64
+ this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key, this.indexerId);
65
+ this.db.prepare(
66
+ statements$1.insertIntoKvs
67
+ ).run(
65
68
  Number(this.endCursor.orderKey),
66
69
  key,
67
- this.serialize(value)
70
+ this.serialize(value),
71
+ this.indexerId
68
72
  );
69
73
  }
70
74
  del(key) {
71
- this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key);
75
+ this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key, this.indexerId);
72
76
  }
73
77
  }
74
- function finalizeKV(db, cursor) {
78
+ function finalizeKV(db, cursor, indexerId) {
75
79
  assertInTransaction(db);
76
80
  db.prepare(statements$1.finalize).run(
77
- Number(cursor.orderKey)
81
+ Number(cursor.orderKey),
82
+ indexerId
78
83
  );
79
84
  }
80
- function invalidateKV(db, cursor) {
85
+ function invalidateKV(db, cursor, indexerId) {
81
86
  assertInTransaction(db);
82
87
  db.prepare(statements$1.invalidateDelete).run(
83
- Number(cursor.orderKey)
88
+ Number(cursor.orderKey),
89
+ indexerId
84
90
  );
85
91
  db.prepare(statements$1.invalidateUpdate).run(
86
- Number(cursor.orderKey)
92
+ Number(cursor.orderKey),
93
+ indexerId
87
94
  );
88
95
  }
96
+ function cleanupKV(db, indexerId) {
97
+ assertInTransaction(db);
98
+ db.prepare(statements$1.cleanup).run(indexerId);
99
+ }
89
100
  const statements$1 = {
90
101
  createTable: `
91
102
  CREATE TABLE IF NOT EXISTS kvs (
@@ -93,33 +104,37 @@ const statements$1 = {
93
104
  to_block INTEGER,
94
105
  k TEXT NOT NULL,
95
106
  v BLOB NOT NULL,
96
- PRIMARY KEY (from_block, k)
107
+ id TEXT NOT NULL,
108
+ PRIMARY KEY (from_block, k, id)
97
109
  );`,
98
110
  get: `
99
111
  SELECT v
100
112
  FROM kvs
101
- WHERE k = ? AND to_block IS NULL`,
113
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
102
114
  updateToBlock: `
103
115
  UPDATE kvs
104
116
  SET to_block = ?
105
- WHERE k = ? AND to_block IS NULL`,
117
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
106
118
  insertIntoKvs: `
107
- INSERT INTO kvs (from_block, to_block, k, v)
108
- VALUES (?, NULL, ?, ?)`,
119
+ INSERT INTO kvs (from_block, to_block, k, v, id)
120
+ VALUES (?, NULL, ?, ?, ?)`,
109
121
  del: `
110
122
  UPDATE kvs
111
123
  SET to_block = ?
112
- WHERE k = ? AND to_block IS NULL`,
124
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
113
125
  finalize: `
114
126
  DELETE FROM kvs
115
- WHERE to_block <= ?`,
127
+ WHERE to_block <= ? AND id = ?`,
116
128
  invalidateDelete: `
117
129
  DELETE FROM kvs
118
- WHERE from_block > ?`,
130
+ WHERE from_block > ? AND id = ?`,
119
131
  invalidateUpdate: `
120
132
  UPDATE kvs
121
133
  SET to_block = NULL
122
- WHERE to_block > ?`
134
+ WHERE to_block > ? AND id = ?`,
135
+ cleanup: `
136
+ DELETE FROM kvs
137
+ WHERE id = ?`
123
138
  };
124
139
 
125
140
  function initializePersistentState(db) {
@@ -187,6 +202,12 @@ function invalidateState(props) {
187
202
  Number(cursor.orderKey)
188
203
  );
189
204
  }
205
+ function resetPersistence(props) {
206
+ const { db, indexerId } = props;
207
+ assertInTransaction(db);
208
+ db.prepare(statements.resetCheckpoint).run(indexerId);
209
+ db.prepare(statements.resetFilter).run(indexerId);
210
+ }
190
211
  const statements = {
191
212
  createCheckpointsTable: `
192
213
  CREATE TABLE IF NOT EXISTS checkpoints (
@@ -241,7 +262,13 @@ const statements = {
241
262
  invalidateFilterUpdate: `
242
263
  UPDATE filters
243
264
  SET to_block = NULL
244
- WHERE id = ? AND to_block > ?`
265
+ WHERE id = ? AND to_block > ?`,
266
+ resetCheckpoint: `
267
+ DELETE FROM checkpoints
268
+ WHERE id = ?`,
269
+ resetFilter: `
270
+ DELETE FROM filters
271
+ WHERE id = ?`
245
272
  };
246
273
 
247
274
  const KV_PROPERTY = "_kv_sqlite";
@@ -265,10 +292,14 @@ function sqliteStorage({
265
292
  }) {
266
293
  return defineIndexerPlugin((indexer) => {
267
294
  let indexerId = "";
268
- indexer.hooks.hook("run:before", async () => {
295
+ let prevFinality;
296
+ const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
297
+ indexer.hooks.hook("plugins:init", async () => {
269
298
  const { indexerName: indexerFileName, availableIndexers } = useInternalContext();
299
+ const logger = useLogger();
270
300
  indexerId = generateIndexerId(indexerFileName, identifier);
271
301
  let retries = 0;
302
+ let cleanupApplied = false;
272
303
  while (retries <= MAX_RETRIES) {
273
304
  try {
274
305
  await withTransaction(database, async (db) => {
@@ -278,6 +309,18 @@ function sqliteStorage({
278
309
  if (enableKeyValueStore) {
279
310
  initializeKeyValueStore(db);
280
311
  }
312
+ if (alwaysReindex && !cleanupApplied) {
313
+ if (enableKeyValueStore) {
314
+ logger.warn("Reindexing: Cleaning up key-value store");
315
+ cleanupKV(db, indexerId);
316
+ }
317
+ if (enablePersistState) {
318
+ logger.warn("Reindexing: Resetting persistence state");
319
+ resetPersistence({ db, indexerId });
320
+ }
321
+ cleanupApplied = true;
322
+ logger.success("All data has been cleaned up for reindexing");
323
+ }
281
324
  });
282
325
  break;
283
326
  } catch (error) {
@@ -318,7 +361,7 @@ function sqliteStorage({
318
361
  invalidateState({ db, cursor, indexerId });
319
362
  }
320
363
  if (enableKeyValueStore) {
321
- invalidateKV(db, cursor);
364
+ invalidateKV(db, cursor, indexerId);
322
365
  }
323
366
  });
324
367
  });
@@ -337,7 +380,7 @@ function sqliteStorage({
337
380
  }
338
381
  });
339
382
  indexer.hooks.hook("message:finalize", async ({ message }) => {
340
- const { cursor } = message.finalize;
383
+ const { cursor } = message;
341
384
  if (!cursor) {
342
385
  throw new SqliteStorageError("finalized cursor is undefined");
343
386
  }
@@ -346,12 +389,12 @@ function sqliteStorage({
346
389
  finalizeState({ db, cursor, indexerId });
347
390
  }
348
391
  if (enableKeyValueStore) {
349
- finalizeKV(db, cursor);
392
+ finalizeKV(db, cursor, indexerId);
350
393
  }
351
394
  });
352
395
  });
353
396
  indexer.hooks.hook("message:invalidate", async ({ message }) => {
354
- const { cursor } = message.invalidate;
397
+ const { cursor } = message;
355
398
  if (!cursor) {
356
399
  throw new SqliteStorageError("invalidate cursor is undefined");
357
400
  }
@@ -360,37 +403,45 @@ function sqliteStorage({
360
403
  invalidateState({ db, cursor, indexerId });
361
404
  }
362
405
  if (enableKeyValueStore) {
363
- invalidateKV(db, cursor);
406
+ invalidateKV(db, cursor, indexerId);
364
407
  }
365
408
  });
366
409
  });
367
410
  indexer.hooks.hook("handler:middleware", ({ use }) => {
368
411
  use(async (ctx, next) => {
369
- if (!ctx.finality) {
412
+ const { endCursor, finality, cursor } = ctx;
413
+ if (!finality) {
370
414
  throw new SqliteStorageError("finality is undefined");
371
415
  }
372
- if (!ctx.endCursor || !isCursor(ctx.endCursor)) {
416
+ if (!endCursor) {
373
417
  throw new SqliteStorageError(
374
418
  "endCursor is undefined or not a cursor"
375
419
  );
376
420
  }
377
421
  await withTransaction(database, async (db) => {
422
+ if (prevFinality === "pending") {
423
+ if (enableKeyValueStore) {
424
+ invalidateKV(db, cursor, indexerId);
425
+ }
426
+ }
378
427
  if (enableKeyValueStore) {
379
428
  ctx[KV_PROPERTY] = new KeyValueStore(
380
429
  db,
381
- ctx.endCursor,
382
- ctx.finality,
430
+ endCursor,
431
+ finality,
383
432
  serializeFn,
384
- deserializeFn
433
+ deserializeFn,
434
+ indexerId
385
435
  );
386
436
  }
387
437
  await next();
388
- if (enablePersistState) {
389
- persistState({ db, endCursor: ctx.endCursor, indexerId });
438
+ if (enablePersistState && finality !== "pending") {
439
+ persistState({ db, endCursor, indexerId });
390
440
  }
391
441
  if (enableKeyValueStore) {
392
442
  delete ctx[KV_PROPERTY];
393
443
  }
444
+ prevFinality = finality;
394
445
  });
395
446
  });
396
447
  });
@@ -398,3 +449,4 @@ function sqliteStorage({
398
449
  }
399
450
 
400
451
  export { KeyValueStore, sqliteStorage, useSqliteKeyValueStore };
452
+ //# sourceMappingURL=index.mjs.map
package/dist/index.mjs.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.mjs","sources":["../src/utils.ts","../src/kv.ts","../src/persistence.ts","../src/index.ts"],"sourcesContent":["import type { Database } from \"better-sqlite3\";\n\nexport type SerializeFn = <T>(value: T) => string;\nexport type DeserializeFn = <T>(value: string) => T;\n\nexport class SqliteStorageError extends Error {\n constructor(message: string, options?: ErrorOptions) {\n super(message, options);\n this.name = \"SqliteStorageError\";\n }\n}\n\nexport async function withTransaction(\n db: Database,\n cb: (db: Database) => Promise<void>,\n) {\n db.prepare(\"BEGIN TRANSACTION\").run();\n try {\n await cb(db);\n } catch (error) {\n db.prepare(\"ROLLBACK TRANSACTION\").run();\n throw error;\n }\n db.prepare(\"COMMIT TRANSACTION\").run();\n}\n\nexport function assertInTransaction(db: Database) {\n if (!db.inTransaction) {\n throw new SqliteStorageError(\"Database is not in transaction\");\n }\n}\n\nexport function deserialize<T>(str: string): T {\n return JSON.parse(str, (_, value) =>\n typeof value === \"string\" && value.match(/^\\d+n$/)\n ? BigInt(value.slice(0, -1))\n : value,\n ) as T;\n}\n\nexport function serialize<T>(obj: T): string {\n return JSON.stringify(\n obj,\n (_, value) => (typeof value === \"bigint\" ? `${value.toString()}n` : value),\n \"\\t\",\n );\n}\n\nexport function sleep(ms: number) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n","import type { Cursor, DataFinality } from \"@apibara/protocol\";\nimport type { Database } from \"better-sqlite3\";\n\nimport {\n type DeserializeFn,\n type SerializeFn,\n assertInTransaction,\n} from \"./utils\";\n\nexport function initializeKeyValueStore(db: Database) {\n assertInTransaction(db);\n db.exec(statements.createTable);\n}\n\nexport class KeyValueStore {\n constructor(\n private readonly db: Database,\n private readonly endCursor: Cursor,\n private readonly finality: DataFinality,\n private readonly serialize: SerializeFn,\n private readonly deserialize: DeserializeFn,\n private readonly indexerId: string,\n ) {\n assertInTransaction(db);\n }\n\n get<T>(key: string): T | undefined {\n const row = this.db\n .prepare<[string, string], KeyValueRow>(statements.get)\n .get(key, this.indexerId);\n\n return row ? this.deserialize(row.v) : undefined;\n }\n\n put<T>(key: string, value: T) {\n this.db\n .prepare<[number, string, string], KeyValueRow>(statements.updateToBlock)\n .run(Number(this.endCursor.orderKey), key, this.indexerId);\n\n this.db\n .prepare<[number, string, string, string], KeyValueRow>(\n statements.insertIntoKvs,\n )\n .run(\n Number(this.endCursor.orderKey),\n key,\n this.serialize(value as Record<string, unknown>),\n this.indexerId,\n );\n }\n\n del(key: string) {\n this.db\n .prepare<[number, string, string], KeyValueRow>(statements.del)\n .run(Number(this.endCursor.orderKey), key, this.indexerId);\n }\n}\n\nexport function finalizeKV(db: Database, cursor: Cursor, indexerId: string) {\n assertInTransaction(db);\n\n db.prepare<[number, string], KeyValueRow>(statements.finalize).run(\n Number(cursor.orderKey),\n indexerId,\n );\n}\n\nexport function invalidateKV(db: Database, cursor: Cursor, indexerId: string) {\n assertInTransaction(db);\n\n // Delete entries that started after the invalidation cursor\n db.prepare<[number, string], KeyValueRow>(statements.invalidateDelete).run(\n Number(cursor.orderKey),\n indexerId,\n );\n\n // Update entries that were supposed to end after the invalidation cursor\n db.prepare<[number, string], KeyValueRow>(statements.invalidateUpdate).run(\n Number(cursor.orderKey),\n indexerId,\n );\n}\n\nexport function cleanupKV(db: Database, indexerId: string) {\n assertInTransaction(db);\n\n db.prepare<[string], KeyValueRow>(statements.cleanup).run(indexerId);\n}\n\nexport type KeyValueRow = {\n from_block: number;\n to_block: number;\n k: string;\n v: string;\n id: string;\n};\n\nconst statements = {\n createTable: `\n CREATE TABLE IF NOT EXISTS kvs (\n from_block INTEGER NOT NULL,\n to_block INTEGER,\n k TEXT NOT NULL,\n v BLOB NOT NULL,\n id TEXT NOT NULL,\n PRIMARY KEY (from_block, k, id)\n );`,\n get: `\n SELECT v\n FROM kvs\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n updateToBlock: `\n UPDATE kvs\n SET to_block = ?\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n insertIntoKvs: `\n INSERT INTO kvs (from_block, to_block, k, v, id)\n VALUES (?, NULL, ?, ?, ?)`,\n del: `\n UPDATE kvs\n SET to_block = ?\n WHERE k = ? AND id = ? AND to_block IS NULL`,\n finalize: `\n DELETE FROM kvs\n WHERE to_block <= ? AND id = ?`,\n invalidateDelete: `\n DELETE FROM kvs\n WHERE from_block > ? AND id = ?`,\n invalidateUpdate: `\n UPDATE kvs\n SET to_block = NULL\n WHERE to_block > ? AND id = ?`,\n cleanup: `\n DELETE FROM kvs\n WHERE id = ?`,\n} as const;\n","import { type Cursor, normalizeCursor } from \"@apibara/protocol\";\nimport type { Database } from \"better-sqlite3\";\n\nimport { assertInTransaction, deserialize, serialize } from \"./utils\";\n\nexport function initializePersistentState(db: Database) {\n assertInTransaction(db);\n db.exec(statements.createCheckpointsTable);\n db.exec(statements.createFiltersTable);\n}\n\nexport function persistState<TFilter>(props: {\n db: Database;\n endCursor: Cursor;\n filter?: TFilter;\n indexerId: string;\n}) {\n const { db, endCursor, filter, indexerId } = props;\n\n assertInTransaction(db);\n\n db.prepare(statements.putCheckpoint).run(\n indexerId,\n Number(endCursor.orderKey),\n endCursor.uniqueKey,\n );\n\n if (filter) {\n db.prepare(statements.updateFilterToBlock).run(\n Number(endCursor.orderKey),\n indexerId,\n );\n db.prepare(statements.insertFilter).run(\n indexerId,\n serialize(filter as Record<string, unknown>),\n Number(endCursor.orderKey),\n );\n }\n}\n\nexport function getState<TFilter>(props: {\n db: Database;\n indexerId: string;\n}) {\n const { db, indexerId } = props;\n assertInTransaction(db);\n const storedCursor = db\n .prepare<string, { order_key?: number; unique_key?: string }>(\n statements.getCheckpoint,\n )\n .get(indexerId);\n const storedFilter = db\n .prepare<string, { filter: string }>(statements.getFilter)\n .get(indexerId);\n\n let cursor: Cursor | undefined;\n let filter: TFilter | undefined;\n\n if (storedCursor?.order_key) {\n cursor = normalizeCursor({\n orderKey: BigInt(storedCursor.order_key),\n uniqueKey: storedCursor.unique_key ? storedCursor.unique_key : null,\n });\n }\n\n if (storedFilter) {\n filter = deserialize(storedFilter.filter) as TFilter;\n }\n\n return { cursor, filter };\n}\n\nexport function finalizeState(props: {\n db: Database;\n cursor: Cursor;\n indexerId: string;\n}) {\n const { cursor, db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string, number]>(statements.finalizeFilter).run(\n indexerId,\n Number(cursor.orderKey),\n );\n}\n\nexport function invalidateState(props: {\n db: Database;\n cursor: Cursor;\n indexerId: string;\n}) {\n const { cursor, db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string, number]>(statements.invalidateFilterDelete).run(\n indexerId,\n Number(cursor.orderKey),\n );\n db.prepare<[string, number]>(statements.invalidateFilterUpdate).run(\n indexerId,\n Number(cursor.orderKey),\n );\n}\n\nexport function resetPersistence(props: {\n db: Database;\n indexerId: string;\n}) {\n const { db, indexerId } = props;\n assertInTransaction(db);\n db.prepare<[string]>(statements.resetCheckpoint).run(indexerId);\n db.prepare<[string]>(statements.resetFilter).run(indexerId);\n}\n\nexport type CheckpointRow = {\n id: string;\n order_key: number;\n unique_key: string | null;\n};\n\nexport type FilterRow = {\n id: string;\n filter: string;\n from_block: number;\n to_block: number | null;\n};\n\nconst statements = {\n createCheckpointsTable: `\n CREATE TABLE IF NOT EXISTS checkpoints (\n id TEXT NOT NULL PRIMARY KEY,\n order_key INTEGER,\n unique_key TEXT\n );`,\n createFiltersTable: `\n CREATE TABLE IF NOT EXISTS filters (\n id TEXT NOT NULL,\n filter BLOB NOT NULL,\n from_block INTEGER NOT NULL,\n to_block INTEGER,\n PRIMARY KEY (id, from_block)\n );`,\n getCheckpoint: `\n SELECT *\n FROM checkpoints\n WHERE id = ?`,\n putCheckpoint: `\n INSERT INTO checkpoints (id, order_key, unique_key)\n VALUES (?, ?, ?)\n ON CONFLICT(id) DO UPDATE SET\n order_key = excluded.order_key,\n unique_key = excluded.unique_key`,\n delCheckpoint: `\n DELETE FROM checkpoints\n WHERE id = ?`,\n getFilter: `\n SELECT *\n FROM filters\n WHERE id = ? AND to_block IS NULL`,\n updateFilterToBlock: `\n UPDATE filters\n SET to_block = ?\n WHERE id = ? AND to_block IS NULL`,\n insertFilter: `\n INSERT INTO filters (id, filter, from_block)\n VALUES (?, ?, ?)\n ON CONFLICT(id, from_block) DO UPDATE SET\n filter = excluded.filter,\n from_block = excluded.from_block`,\n delFilter: `\n DELETE FROM filters\n WHERE id = ?`,\n finalizeFilter: `\n DELETE FROM filters\n WHERE id = ? AND to_block <= ?`,\n invalidateFilterDelete: `\n DELETE FROM filters\n WHERE id = ? AND from_block > ?`,\n invalidateFilterUpdate: `\n UPDATE filters\n SET to_block = NULL\n WHERE id = ? AND to_block > ?`,\n resetCheckpoint: `\n DELETE FROM checkpoints\n WHERE id = ?`,\n resetFilter: `\n DELETE FROM filters\n WHERE id = ?`,\n};\n","import { useIndexerContext } from \"@apibara/indexer\";\nimport { defineIndexerPlugin, useLogger } from \"@apibara/indexer/plugins\";\nimport type { Cursor, DataFinality } from \"@apibara/protocol\";\nimport type { Database as SqliteDatabase } from \"better-sqlite3\";\n\nimport { generateIndexerId } from \"@apibara/indexer/internal\";\nimport { useInternalContext } from \"@apibara/indexer/internal/plugins\";\nimport {\n KeyValueStore,\n cleanupKV,\n finalizeKV,\n initializeKeyValueStore,\n invalidateKV,\n} from \"./kv\";\nimport {\n finalizeState,\n getState,\n initializePersistentState,\n invalidateState,\n persistState,\n resetPersistence,\n} from \"./persistence\";\nimport {\n type DeserializeFn,\n type SerializeFn,\n SqliteStorageError,\n assertInTransaction,\n deserialize,\n serialize,\n sleep,\n withTransaction,\n} from \"./utils\";\n\nconst KV_PROPERTY = \"_kv_sqlite\" as const;\nconst MAX_RETRIES = 5;\n\nexport { KeyValueStore } from \"./kv\";\n\nexport function useSqliteKeyValueStore(): KeyValueStore {\n const kv = useIndexerContext()[KV_PROPERTY] as KeyValueStore | undefined;\n if (!kv) {\n throw new SqliteStorageError(\n \"SQLite key-value store is not available. Did you forget to enable it?\",\n );\n }\n\n return kv;\n}\n\nexport type SqliteStorageOptions = {\n database: SqliteDatabase;\n keyValueStore?: boolean;\n persistState?: boolean;\n indexerName?: string;\n\n serialize?: SerializeFn;\n deserialize?: DeserializeFn;\n};\n\n/**\n * Creates a plugin that uses SQLite as the storage layer.\n *\n * Supports storing the indexer's state and provides a simple Key-Value store.\n * @param options.database - The SQLite database instance.\n * @param options.persistState - Whether to persist the indexer's state. Defaults to true.\n * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.\n * @param options.serialize - A function to serialize the value to the KV.\n * @param options.deserialize - A function to deserialize the value from the KV.\n * @param options.indexerName - The name of the indexer. Defaults value is 'default'.\n */\nexport function sqliteStorage<TFilter, TBlock>({\n database,\n persistState: enablePersistState = true,\n keyValueStore: enableKeyValueStore = true,\n serialize: serializeFn = serialize,\n deserialize: deserializeFn = deserialize,\n indexerName: identifier = \"default\",\n}: SqliteStorageOptions) {\n return defineIndexerPlugin<TFilter, TBlock>((indexer) => {\n let indexerId = \"\";\n let prevFinality: DataFinality | undefined;\n const alwaysReindex = process.env[\"APIBARA_ALWAYS_REINDEX\"] === \"true\";\n\n indexer.hooks.hook(\"plugins:init\", async () => {\n const { indexerName: indexerFileName, availableIndexers } =\n useInternalContext();\n\n const logger = useLogger();\n\n indexerId = generateIndexerId(indexerFileName, identifier);\n\n let retries = 0;\n\n let cleanupApplied = false;\n\n while (retries <= MAX_RETRIES) {\n try {\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n initializePersistentState(db);\n }\n\n if (enableKeyValueStore) {\n initializeKeyValueStore(db);\n }\n\n if (alwaysReindex && !cleanupApplied) {\n if (enableKeyValueStore) {\n logger.warn(\"Reindexing: Cleaning up key-value store\");\n cleanupKV(db, indexerId);\n }\n\n if (enablePersistState) {\n logger.warn(\"Reindexing: Resetting persistence state\");\n resetPersistence({ db, indexerId });\n }\n\n cleanupApplied = true;\n\n logger.success(\"All data has been cleaned up for reindexing\");\n }\n });\n break;\n } catch (error) {\n if (retries === MAX_RETRIES) {\n throw new SqliteStorageError(\n \"Initialization failed after 5 retries\",\n {\n cause: error,\n },\n );\n }\n await sleep(retries * 1000);\n retries++;\n }\n }\n });\n\n indexer.hooks.hook(\"connect:before\", async ({ request }) => {\n if (!enablePersistState) {\n return;\n }\n\n return await withTransaction(database, async (db) => {\n const { cursor, filter } = getState<TFilter>({ db, indexerId });\n\n if (cursor) {\n request.startingCursor = cursor;\n }\n\n if (filter) {\n request.filter[1] = filter;\n }\n });\n });\n\n indexer.hooks.hook(\"connect:after\", async ({ request }) => {\n // On restart, we need to invalidate data for blocks that were processed but not persisted.\n const cursor = request.startingCursor;\n\n if (!cursor) {\n return;\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n invalidateState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"connect:factory\", ({ request, endCursor }) => {\n if (!enablePersistState) {\n return;\n }\n\n // The connect factory hook is called while indexing a block, so the database should be in a transaction\n // created by the middleware.\n assertInTransaction(database);\n\n if (endCursor && request.filter[1]) {\n persistState({\n db: database,\n endCursor,\n indexerId,\n filter: request.filter[1],\n });\n }\n });\n\n indexer.hooks.hook(\"message:finalize\", async ({ message }) => {\n const { cursor } = message;\n\n if (!cursor) {\n throw new SqliteStorageError(\"finalized cursor is undefined\");\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n finalizeState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n finalizeKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"message:invalidate\", async ({ message }) => {\n const { cursor } = message;\n\n if (!cursor) {\n throw new SqliteStorageError(\"invalidate cursor is undefined\");\n }\n\n await withTransaction(database, async (db) => {\n if (enablePersistState) {\n invalidateState({ db, cursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n });\n });\n\n indexer.hooks.hook(\"handler:middleware\", ({ use }) => {\n use(async (ctx, next) => {\n const { endCursor, finality, cursor } = ctx as {\n cursor: Cursor;\n endCursor: Cursor;\n finality: DataFinality;\n };\n\n if (!finality) {\n throw new SqliteStorageError(\"finality is undefined\");\n }\n\n if (!endCursor) {\n throw new SqliteStorageError(\n \"endCursor is undefined or not a cursor\",\n );\n }\n\n await withTransaction(database, async (db) => {\n if (prevFinality === \"pending\") {\n // invalidate if previous block's finality was \"pending\"\n if (enableKeyValueStore) {\n invalidateKV(db, cursor, indexerId);\n }\n }\n\n if (enableKeyValueStore) {\n ctx[KV_PROPERTY] = new KeyValueStore(\n db,\n endCursor,\n finality,\n serializeFn,\n deserializeFn,\n indexerId,\n );\n }\n\n await next();\n\n if (enablePersistState && finality !== \"pending\") {\n persistState({ db, endCursor, indexerId });\n }\n\n if (enableKeyValueStore) {\n delete ctx[KV_PROPERTY];\n }\n\n prevFinality = finality;\n });\n });\n });\n });\n}\n"],"names":["statements"],"mappings":";;;;;;AAKO,MAAM,2BAA2B,KAAM,CAAA;AAAA,EAC5C,WAAA,CAAY,SAAiB,OAAwB,EAAA;AACnD,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA,CAAA;AACtB,IAAA,IAAA,CAAK,IAAO,GAAA,oBAAA,CAAA;AAAA,GACd;AACF,CAAA;AAEsB,eAAA,eAAA,CACpB,IACA,EACA,EAAA;AACA,EAAG,EAAA,CAAA,OAAA,CAAQ,mBAAmB,CAAA,CAAE,GAAI,EAAA,CAAA;AACpC,EAAI,IAAA;AACF,IAAA,MAAM,GAAG,EAAE,CAAA,CAAA;AAAA,WACJ,KAAO,EAAA;AACd,IAAG,EAAA,CAAA,OAAA,CAAQ,sBAAsB,CAAA,CAAE,GAAI,EAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACA,EAAG,EAAA,CAAA,OAAA,CAAQ,oBAAoB,CAAA,CAAE,GAAI,EAAA,CAAA;AACvC,CAAA;AAEO,SAAS,oBAAoB,EAAc,EAAA;AAChD,EAAI,IAAA,CAAC,GAAG,aAAe,EAAA;AACrB,IAAM,MAAA,IAAI,mBAAmB,gCAAgC,CAAA,CAAA;AAAA,GAC/D;AACF,CAAA;AAEO,SAAS,YAAe,GAAgB,EAAA;AAC7C,EAAA,OAAO,IAAK,CAAA,KAAA;AAAA,IAAM,GAAA;AAAA,IAAK,CAAC,CAAG,EAAA,KAAA,KACzB,OAAO,KAAA,KAAU,YAAY,KAAM,CAAA,KAAA,CAAM,QAAQ,CAAA,GAC7C,OAAO,KAAM,CAAA,KAAA,CAAM,CAAG,EAAA,CAAA,CAAE,CAAC,CACzB,GAAA,KAAA;AAAA,GACN,CAAA;AACF,CAAA;AAEO,SAAS,UAAa,GAAgB,EAAA;AAC3C,EAAA,OAAO,IAAK,CAAA,SAAA;AAAA,IACV,GAAA;AAAA,IACA,CAAC,CAAG,EAAA,KAAA,KAAW,OAAO,KAAA,KAAU,WAAW,CAAG,EAAA,KAAA,CAAM,QAAS,EAAC,CAAM,CAAA,CAAA,GAAA,KAAA;AAAA,IACpE,GAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEO,SAAS,MAAM,EAAY,EAAA;AAChC,EAAA,OAAO,IAAI,OAAQ,CAAA,CAAC,YAAY,UAAW,CAAA,OAAA,EAAS,EAAE,CAAC,CAAA,CAAA;AACzD;;ACzCO,SAAS,wBAAwB,EAAc,EAAA;AACpD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,IAAA,CAAKA,aAAW,WAAW,CAAA,CAAA;AAChC,CAAA;AAEO,MAAM,aAAc,CAAA;AAAA,EACzB,YACmB,EACA,EAAA,SAAA,EACA,QACA,EAAA,SAAA,EACA,aACA,SACjB,EAAA;AANiB,IAAA,IAAA,CAAA,EAAA,GAAA,EAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AACA,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AACA,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA,CAAA;AACA,IAAA,IAAA,CAAA,SAAA,GAAA,SAAA,CAAA;AAEjB,IAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAAA,GACxB;AAAA,EAEA,IAAO,GAA4B,EAAA;AACjC,IAAM,MAAA,GAAA,GAAM,IAAK,CAAA,EAAA,CACd,OAAuC,CAAAA,YAAA,CAAW,GAAG,CACrD,CAAA,GAAA,CAAI,GAAK,EAAA,IAAA,CAAK,SAAS,CAAA,CAAA;AAE1B,IAAA,OAAO,GAAM,GAAA,IAAA,CAAK,WAAY,CAAA,GAAA,CAAI,CAAC,CAAI,GAAA,KAAA,CAAA,CAAA;AAAA,GACzC;AAAA,EAEA,GAAA,CAAO,KAAa,KAAU,EAAA;AAC5B,IAAA,IAAA,CAAK,EACF,CAAA,OAAA,CAA+CA,YAAW,CAAA,aAAa,CACvE,CAAA,GAAA,CAAI,MAAO,CAAA,IAAA,CAAK,SAAU,CAAA,QAAQ,CAAG,EAAA,GAAA,EAAK,KAAK,SAAS,CAAA,CAAA;AAE3D,IAAA,IAAA,CAAK,EACF,CAAA,OAAA;AAAA,MACCA,YAAW,CAAA,aAAA;AAAA,KAEZ,CAAA,GAAA;AAAA,MACC,MAAA,CAAO,IAAK,CAAA,SAAA,CAAU,QAAQ,CAAA;AAAA,MAC9B,GAAA;AAAA,MACA,IAAA,CAAK,UAAU,KAAgC,CAAA;AAAA,MAC/C,IAAK,CAAA,SAAA;AAAA,KACP,CAAA;AAAA,GACJ;AAAA,EAEA,IAAI,GAAa,EAAA;AACf,IAAA,IAAA,CAAK,EACF,CAAA,OAAA,CAA+CA,YAAW,CAAA,GAAG,CAC7D,CAAA,GAAA,CAAI,MAAO,CAAA,IAAA,CAAK,SAAU,CAAA,QAAQ,CAAG,EAAA,GAAA,EAAK,KAAK,SAAS,CAAA,CAAA;AAAA,GAC7D;AACF,CAAA;AAEgB,SAAA,UAAA,CAAW,EAAc,EAAA,MAAA,EAAgB,SAAmB,EAAA;AAC1E,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,QAAQ,CAAE,CAAA,GAAA;AAAA,IAC7D,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEgB,SAAA,YAAA,CAAa,EAAc,EAAA,MAAA,EAAgB,SAAmB,EAAA;AAC5E,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAGtB,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,gBAAgB,CAAE,CAAA,GAAA;AAAA,IACrE,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AAGA,EAAG,EAAA,CAAA,OAAA,CAAuCA,YAAW,CAAA,gBAAgB,CAAE,CAAA,GAAA;AAAA,IACrE,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,IACtB,SAAA;AAAA,GACF,CAAA;AACF,CAAA;AAEgB,SAAA,SAAA,CAAU,IAAc,SAAmB,EAAA;AACzD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAA,EAAA,CAAG,OAA+B,CAAAA,YAAA,CAAW,OAAO,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AACrE,CAAA;AAUA,MAAMA,YAAa,GAAA;AAAA,EACjB,WAAa,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA;AAAA,EASb,GAAK,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIL,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIf,aAAe,EAAA,CAAA;AAAA;AAAA,6BAAA,CAAA;AAAA,EAGf,GAAK,EAAA,CAAA;AAAA;AAAA;AAAA,+CAAA,CAAA;AAAA,EAIL,QAAU,EAAA,CAAA;AAAA;AAAA,kCAAA,CAAA;AAAA,EAGV,gBAAkB,EAAA,CAAA;AAAA;AAAA,mCAAA,CAAA;AAAA,EAGlB,gBAAkB,EAAA,CAAA;AAAA;AAAA;AAAA,iCAAA,CAAA;AAAA,EAIlB,OAAS,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAGX,CAAA;;AClIO,SAAS,0BAA0B,EAAc,EAAA;AACtD,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,IAAA,CAAK,WAAW,sBAAsB,CAAA,CAAA;AACzC,EAAG,EAAA,CAAA,IAAA,CAAK,WAAW,kBAAkB,CAAA,CAAA;AACvC,CAAA;AAEO,SAAS,aAAsB,KAKnC,EAAA;AACD,EAAA,MAAM,EAAE,EAAA,EAAI,SAAW,EAAA,MAAA,EAAQ,WAAc,GAAA,KAAA,CAAA;AAE7C,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AAEtB,EAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,aAAa,CAAE,CAAA,GAAA;AAAA,IACnC,SAAA;AAAA,IACA,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,IACzB,SAAU,CAAA,SAAA;AAAA,GACZ,CAAA;AAEA,EAAA,IAAI,MAAQ,EAAA;AACV,IAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,mBAAmB,CAAE,CAAA,GAAA;AAAA,MACzC,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,MACzB,SAAA;AAAA,KACF,CAAA;AACA,IAAG,EAAA,CAAA,OAAA,CAAQ,UAAW,CAAA,YAAY,CAAE,CAAA,GAAA;AAAA,MAClC,SAAA;AAAA,MACA,UAAU,MAAiC,CAAA;AAAA,MAC3C,MAAA,CAAO,UAAU,QAAQ,CAAA;AAAA,KAC3B,CAAA;AAAA,GACF;AACF,CAAA;AAEO,SAAS,SAAkB,KAG/B,EAAA;AACD,EAAM,MAAA,EAAE,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAC1B,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAA,MAAM,eAAe,EAClB,CAAA,OAAA;AAAA,IACC,UAAW,CAAA,aAAA;AAAA,GACb,CACC,IAAI,SAAS,CAAA,CAAA;AAChB,EAAA,MAAM,eAAe,EAClB,CAAA,OAAA,CAAoC,WAAW,SAAS,CAAA,CACxD,IAAI,SAAS,CAAA,CAAA;AAEhB,EAAI,IAAA,MAAA,CAAA;AACJ,EAAI,IAAA,MAAA,CAAA;AAEJ,EAAA,IAAI,cAAc,SAAW,EAAA;AAC3B,IAAA,MAAA,GAAS,eAAgB,CAAA;AAAA,MACvB,QAAA,EAAU,MAAO,CAAA,YAAA,CAAa,SAAS,CAAA;AAAA,MACvC,SAAW,EAAA,YAAA,CAAa,UAAa,GAAA,YAAA,CAAa,UAAa,GAAA,IAAA;AAAA,KAChE,CAAA,CAAA;AAAA,GACH;AAEA,EAAA,IAAI,YAAc,EAAA;AAChB,IAAS,MAAA,GAAA,WAAA,CAAY,aAAa,MAAM,CAAA,CAAA;AAAA,GAC1C;AAEA,EAAO,OAAA,EAAE,QAAQ,MAAO,EAAA,CAAA;AAC1B,CAAA;AAEO,SAAS,cAAc,KAI3B,EAAA;AACD,EAAA,MAAM,EAAE,MAAA,EAAQ,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAClC,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,cAAc,CAAE,CAAA,GAAA;AAAA,IACtD,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACF,CAAA;AAEO,SAAS,gBAAgB,KAI7B,EAAA;AACD,EAAA,MAAM,EAAE,MAAA,EAAQ,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAClC,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,sBAAsB,CAAE,CAAA,GAAA;AAAA,IAC9D,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACA,EAAG,EAAA,CAAA,OAAA,CAA0B,UAAW,CAAA,sBAAsB,CAAE,CAAA,GAAA;AAAA,IAC9D,SAAA;AAAA,IACA,MAAA,CAAO,OAAO,QAAQ,CAAA;AAAA,GACxB,CAAA;AACF,CAAA;AAEO,SAAS,iBAAiB,KAG9B,EAAA;AACD,EAAM,MAAA,EAAE,EAAI,EAAA,SAAA,EAAc,GAAA,KAAA,CAAA;AAC1B,EAAA,mBAAA,CAAoB,EAAE,CAAA,CAAA;AACtB,EAAA,EAAA,CAAG,OAAkB,CAAA,UAAA,CAAW,eAAe,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AAC9D,EAAA,EAAA,CAAG,OAAkB,CAAA,UAAA,CAAW,WAAW,CAAA,CAAE,IAAI,SAAS,CAAA,CAAA;AAC5D,CAAA;AAeA,MAAM,UAAa,GAAA;AAAA,EACjB,sBAAwB,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA,CAAA;AAAA,EAMxB,kBAAoB,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA,CAAA;AAAA,EAQpB,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAIf,aAAe,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sCAAA,CAAA;AAAA,EAMf,aAAe,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGf,SAAW,EAAA,CAAA;AAAA;AAAA;AAAA,qCAAA,CAAA;AAAA,EAIX,mBAAqB,EAAA,CAAA;AAAA;AAAA;AAAA,qCAAA,CAAA;AAAA,EAIrB,YAAc,EAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sCAAA,CAAA;AAAA,EAMd,SAAW,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGX,cAAgB,EAAA,CAAA;AAAA;AAAA,kCAAA,CAAA;AAAA,EAGhB,sBAAwB,EAAA,CAAA;AAAA;AAAA,mCAAA,CAAA;AAAA,EAGxB,sBAAwB,EAAA,CAAA;AAAA;AAAA;AAAA,iCAAA,CAAA;AAAA,EAIxB,eAAiB,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAAA,EAGjB,WAAa,EAAA,CAAA;AAAA;AAAA,gBAAA,CAAA;AAGf,CAAA;;ACzJA,MAAM,WAAc,GAAA,YAAA,CAAA;AACpB,MAAM,WAAc,GAAA,CAAA,CAAA;AAIb,SAAS,sBAAwC,GAAA;AACtD,EAAM,MAAA,EAAA,GAAK,iBAAkB,EAAA,CAAE,WAAW,CAAA,CAAA;AAC1C,EAAA,IAAI,CAAC,EAAI,EAAA;AACP,IAAA,MAAM,IAAI,kBAAA;AAAA,MACR,uEAAA;AAAA,KACF,CAAA;AAAA,GACF;AAEA,EAAO,OAAA,EAAA,CAAA;AACT,CAAA;AAuBO,SAAS,aAA+B,CAAA;AAAA,EAC7C,QAAA;AAAA,EACA,cAAc,kBAAqB,GAAA,IAAA;AAAA,EACnC,eAAe,mBAAsB,GAAA,IAAA;AAAA,EACrC,WAAW,WAAc,GAAA,SAAA;AAAA,EACzB,aAAa,aAAgB,GAAA,WAAA;AAAA,EAC7B,aAAa,UAAa,GAAA,SAAA;AAC5B,CAAyB,EAAA;AACvB,EAAO,OAAA,mBAAA,CAAqC,CAAC,OAAY,KAAA;AACvD,IAAA,IAAI,SAAY,GAAA,EAAA,CAAA;AAChB,IAAI,IAAA,YAAA,CAAA;AACJ,IAAA,MAAM,aAAgB,GAAA,OAAA,CAAQ,GAAI,CAAA,wBAAwB,CAAM,KAAA,MAAA,CAAA;AAEhE,IAAQ,OAAA,CAAA,KAAA,CAAM,IAAK,CAAA,cAAA,EAAgB,YAAY;AAC7C,MAAA,MAAM,EAAE,WAAA,EAAa,eAAiB,EAAA,iBAAA,KACpC,kBAAmB,EAAA,CAAA;AAErB,MAAA,MAAM,SAAS,SAAU,EAAA,CAAA;AAEzB,MAAY,SAAA,GAAA,iBAAA,CAAkB,iBAAiB,UAAU,CAAA,CAAA;AAEzD,MAAA,IAAI,OAAU,GAAA,CAAA,CAAA;AAEd,MAAA,IAAI,cAAiB,GAAA,KAAA,CAAA;AAErB,MAAA,OAAO,WAAW,WAAa,EAAA;AAC7B,QAAI,IAAA;AACF,UAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,YAAA,IAAI,kBAAoB,EAAA;AACtB,cAAA,yBAAA,CAA0B,EAAE,CAAA,CAAA;AAAA,aAC9B;AAEA,YAAA,IAAI,mBAAqB,EAAA;AACvB,cAAA,uBAAA,CAAwB,EAAE,CAAA,CAAA;AAAA,aAC5B;AAEA,YAAI,IAAA,aAAA,IAAiB,CAAC,cAAgB,EAAA;AACpC,cAAA,IAAI,mBAAqB,EAAA;AACvB,gBAAA,MAAA,CAAO,KAAK,yCAAyC,CAAA,CAAA;AACrD,gBAAA,SAAA,CAAU,IAAI,SAAS,CAAA,CAAA;AAAA,eACzB;AAEA,cAAA,IAAI,kBAAoB,EAAA;AACtB,gBAAA,MAAA,CAAO,KAAK,yCAAyC,CAAA,CAAA;AACrD,gBAAiB,gBAAA,CAAA,EAAE,EAAI,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,eACpC;AAEA,cAAiB,cAAA,GAAA,IAAA,CAAA;AAEjB,cAAA,MAAA,CAAO,QAAQ,6CAA6C,CAAA,CAAA;AAAA,aAC9D;AAAA,WACD,CAAA,CAAA;AACD,UAAA,MAAA;AAAA,iBACO,KAAO,EAAA;AACd,UAAA,IAAI,YAAY,WAAa,EAAA;AAC3B,YAAA,MAAM,IAAI,kBAAA;AAAA,cACR,uCAAA;AAAA,cACA;AAAA,gBACE,KAAO,EAAA,KAAA;AAAA,eACT;AAAA,aACF,CAAA;AAAA,WACF;AACA,UAAM,MAAA,KAAA,CAAM,UAAU,GAAI,CAAA,CAAA;AAC1B,UAAA,OAAA,EAAA,CAAA;AAAA,SACF;AAAA,OACF;AAAA,KACD,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,gBAAA,EAAkB,OAAO,EAAE,SAAc,KAAA;AAC1D,MAAA,IAAI,CAAC,kBAAoB,EAAA;AACvB,QAAA,OAAA;AAAA,OACF;AAEA,MAAA,OAAO,MAAM,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AACnD,QAAM,MAAA,EAAE,QAAQ,MAAO,EAAA,GAAI,SAAkB,EAAE,EAAA,EAAI,WAAW,CAAA,CAAA;AAE9D,QAAA,IAAI,MAAQ,EAAA;AACV,UAAA,OAAA,CAAQ,cAAiB,GAAA,MAAA,CAAA;AAAA,SAC3B;AAEA,QAAA,IAAI,MAAQ,EAAA;AACV,UAAQ,OAAA,CAAA,MAAA,CAAO,CAAC,CAAI,GAAA,MAAA,CAAA;AAAA,SACtB;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,eAAA,EAAiB,OAAO,EAAE,SAAc,KAAA;AAEzD,MAAA,MAAM,SAAS,OAAQ,CAAA,cAAA,CAAA;AAEvB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAA,OAAA;AAAA,OACF;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,eAAA,CAAgB,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SAC3C;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SACpC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,iBAAA,EAAmB,CAAC,EAAE,OAAA,EAAS,WAAgB,KAAA;AAChE,MAAA,IAAI,CAAC,kBAAoB,EAAA;AACvB,QAAA,OAAA;AAAA,OACF;AAIA,MAAA,mBAAA,CAAoB,QAAQ,CAAA,CAAA;AAE5B,MAAA,IAAI,SAAa,IAAA,OAAA,CAAQ,MAAO,CAAA,CAAC,CAAG,EAAA;AAClC,QAAa,YAAA,CAAA;AAAA,UACX,EAAI,EAAA,QAAA;AAAA,UACJ,SAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA,EAAQ,OAAQ,CAAA,MAAA,CAAO,CAAC,CAAA;AAAA,SACzB,CAAA,CAAA;AAAA,OACH;AAAA,KACD,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,kBAAA,EAAoB,OAAO,EAAE,SAAc,KAAA;AAC5D,MAAM,MAAA,EAAE,QAAW,GAAA,OAAA,CAAA;AAEnB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAM,MAAA,IAAI,mBAAmB,+BAA+B,CAAA,CAAA;AAAA,OAC9D;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,aAAA,CAAc,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SACzC;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAW,UAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SAClC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,oBAAA,EAAsB,OAAO,EAAE,SAAc,KAAA;AAC9D,MAAM,MAAA,EAAE,QAAW,GAAA,OAAA,CAAA;AAEnB,MAAA,IAAI,CAAC,MAAQ,EAAA;AACX,QAAM,MAAA,IAAI,mBAAmB,gCAAgC,CAAA,CAAA;AAAA,OAC/D;AAEA,MAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,QAAA,IAAI,kBAAoB,EAAA;AACtB,UAAA,eAAA,CAAgB,EAAE,EAAA,EAAI,MAAQ,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,SAC3C;AAEA,QAAA,IAAI,mBAAqB,EAAA;AACvB,UAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,SACpC;AAAA,OACD,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAED,IAAA,OAAA,CAAQ,MAAM,IAAK,CAAA,oBAAA,EAAsB,CAAC,EAAE,KAAU,KAAA;AACpD,MAAI,GAAA,CAAA,OAAO,KAAK,IAAS,KAAA;AACvB,QAAA,MAAM,EAAE,SAAA,EAAW,QAAU,EAAA,MAAA,EAAW,GAAA,GAAA,CAAA;AAMxC,QAAA,IAAI,CAAC,QAAU,EAAA;AACb,UAAM,MAAA,IAAI,mBAAmB,uBAAuB,CAAA,CAAA;AAAA,SACtD;AAEA,QAAA,IAAI,CAAC,SAAW,EAAA;AACd,UAAA,MAAM,IAAI,kBAAA;AAAA,YACR,wCAAA;AAAA,WACF,CAAA;AAAA,SACF;AAEA,QAAM,MAAA,eAAA,CAAgB,QAAU,EAAA,OAAO,EAAO,KAAA;AAC5C,UAAA,IAAI,iBAAiB,SAAW,EAAA;AAE9B,YAAA,IAAI,mBAAqB,EAAA;AACvB,cAAa,YAAA,CAAA,EAAA,EAAI,QAAQ,SAAS,CAAA,CAAA;AAAA,aACpC;AAAA,WACF;AAEA,UAAA,IAAI,mBAAqB,EAAA;AACvB,YAAI,GAAA,CAAA,WAAW,IAAI,IAAI,aAAA;AAAA,cACrB,EAAA;AAAA,cACA,SAAA;AAAA,cACA,QAAA;AAAA,cACA,WAAA;AAAA,cACA,aAAA;AAAA,cACA,SAAA;AAAA,aACF,CAAA;AAAA,WACF;AAEA,UAAA,MAAM,IAAK,EAAA,CAAA;AAEX,UAAI,IAAA,kBAAA,IAAsB,aAAa,SAAW,EAAA;AAChD,YAAA,YAAA,CAAa,EAAE,EAAA,EAAI,SAAW,EAAA,SAAA,EAAW,CAAA,CAAA;AAAA,WAC3C;AAEA,UAAA,IAAI,mBAAqB,EAAA;AACvB,YAAA,OAAO,IAAI,WAAW,CAAA,CAAA;AAAA,WACxB;AAEA,UAAe,YAAA,GAAA,QAAA,CAAA;AAAA,SAChB,CAAA,CAAA;AAAA,OACF,CAAA,CAAA;AAAA,KACF,CAAA,CAAA;AAAA,GACF,CAAA,CAAA;AACH;;;;"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@apibara/plugin-sqlite",
3
- "version": "2.1.0-beta.5",
3
+ "version": "2.1.0-beta.50",
4
4
  "type": "module",
5
5
  "files": [
6
6
  "dist",
@@ -28,6 +28,7 @@
28
28
  "devDependencies": {
29
29
  "@types/better-sqlite3": "^7.6.11",
30
30
  "@types/node": "^20.14.0",
31
+ "better-sqlite3": "^9.0.0",
31
32
  "unbuild": "^2.0.0",
32
33
  "vitest": "^1.6.0"
33
34
  },
@@ -35,7 +36,7 @@
35
36
  "better-sqlite3": "^9.0.0"
36
37
  },
37
38
  "dependencies": {
38
- "@apibara/indexer": "2.1.0-beta.5",
39
- "@apibara/protocol": "2.1.0-beta.5"
39
+ "@apibara/indexer": "2.1.0-beta.51",
40
+ "@apibara/protocol": "2.1.0-beta.51"
40
41
  }
41
42
  }
package/src/index.ts CHANGED
@@ -1,12 +1,13 @@
1
1
  import { useIndexerContext } from "@apibara/indexer";
2
- import { defineIndexerPlugin } from "@apibara/indexer/plugins";
3
- import { isCursor } from "@apibara/protocol";
2
+ import { defineIndexerPlugin, useLogger } from "@apibara/indexer/plugins";
3
+ import type { Cursor, DataFinality } from "@apibara/protocol";
4
4
  import type { Database as SqliteDatabase } from "better-sqlite3";
5
5
 
6
6
  import { generateIndexerId } from "@apibara/indexer/internal";
7
7
  import { useInternalContext } from "@apibara/indexer/internal/plugins";
8
8
  import {
9
9
  KeyValueStore,
10
+ cleanupKV,
10
11
  finalizeKV,
11
12
  initializeKeyValueStore,
12
13
  invalidateKV,
@@ -17,6 +18,7 @@ import {
17
18
  initializePersistentState,
18
19
  invalidateState,
19
20
  persistState,
21
+ resetPersistence,
20
22
  } from "./persistence";
21
23
  import {
22
24
  type DeserializeFn,
@@ -76,15 +78,21 @@ export function sqliteStorage<TFilter, TBlock>({
76
78
  }: SqliteStorageOptions) {
77
79
  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
78
80
  let indexerId = "";
81
+ let prevFinality: DataFinality | undefined;
82
+ const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
79
83
 
80
- indexer.hooks.hook("run:before", async () => {
84
+ indexer.hooks.hook("plugins:init", async () => {
81
85
  const { indexerName: indexerFileName, availableIndexers } =
82
86
  useInternalContext();
83
87
 
88
+ const logger = useLogger();
89
+
84
90
  indexerId = generateIndexerId(indexerFileName, identifier);
85
91
 
86
92
  let retries = 0;
87
93
 
94
+ let cleanupApplied = false;
95
+
88
96
  while (retries <= MAX_RETRIES) {
89
97
  try {
90
98
  await withTransaction(database, async (db) => {
@@ -95,6 +103,22 @@ export function sqliteStorage<TFilter, TBlock>({
95
103
  if (enableKeyValueStore) {
96
104
  initializeKeyValueStore(db);
97
105
  }
106
+
107
+ if (alwaysReindex && !cleanupApplied) {
108
+ if (enableKeyValueStore) {
109
+ logger.warn("Reindexing: Cleaning up key-value store");
110
+ cleanupKV(db, indexerId);
111
+ }
112
+
113
+ if (enablePersistState) {
114
+ logger.warn("Reindexing: Resetting persistence state");
115
+ resetPersistence({ db, indexerId });
116
+ }
117
+
118
+ cleanupApplied = true;
119
+
120
+ logger.success("All data has been cleaned up for reindexing");
121
+ }
98
122
  });
99
123
  break;
100
124
  } catch (error) {
@@ -144,7 +168,7 @@ export function sqliteStorage<TFilter, TBlock>({
144
168
  }
145
169
 
146
170
  if (enableKeyValueStore) {
147
- invalidateKV(db, cursor);
171
+ invalidateKV(db, cursor, indexerId);
148
172
  }
149
173
  });
150
174
  });
@@ -169,7 +193,7 @@ export function sqliteStorage<TFilter, TBlock>({
169
193
  });
170
194
 
171
195
  indexer.hooks.hook("message:finalize", async ({ message }) => {
172
- const { cursor } = message.finalize;
196
+ const { cursor } = message;
173
197
 
174
198
  if (!cursor) {
175
199
  throw new SqliteStorageError("finalized cursor is undefined");
@@ -181,13 +205,13 @@ export function sqliteStorage<TFilter, TBlock>({
181
205
  }
182
206
 
183
207
  if (enableKeyValueStore) {
184
- finalizeKV(db, cursor);
208
+ finalizeKV(db, cursor, indexerId);
185
209
  }
186
210
  });
187
211
  });
188
212
 
189
213
  indexer.hooks.hook("message:invalidate", async ({ message }) => {
190
- const { cursor } = message.invalidate;
214
+ const { cursor } = message;
191
215
 
192
216
  if (!cursor) {
193
217
  throw new SqliteStorageError("invalidate cursor is undefined");
@@ -199,43 +223,59 @@ export function sqliteStorage<TFilter, TBlock>({
199
223
  }
200
224
 
201
225
  if (enableKeyValueStore) {
202
- invalidateKV(db, cursor);
226
+ invalidateKV(db, cursor, indexerId);
203
227
  }
204
228
  });
205
229
  });
206
230
 
207
231
  indexer.hooks.hook("handler:middleware", ({ use }) => {
208
232
  use(async (ctx, next) => {
209
- if (!ctx.finality) {
233
+ const { endCursor, finality, cursor } = ctx as {
234
+ cursor: Cursor;
235
+ endCursor: Cursor;
236
+ finality: DataFinality;
237
+ };
238
+
239
+ if (!finality) {
210
240
  throw new SqliteStorageError("finality is undefined");
211
241
  }
212
242
 
213
- if (!ctx.endCursor || !isCursor(ctx.endCursor)) {
243
+ if (!endCursor) {
214
244
  throw new SqliteStorageError(
215
245
  "endCursor is undefined or not a cursor",
216
246
  );
217
247
  }
218
248
 
219
249
  await withTransaction(database, async (db) => {
250
+ if (prevFinality === "pending") {
251
+ // invalidate if previous block's finality was "pending"
252
+ if (enableKeyValueStore) {
253
+ invalidateKV(db, cursor, indexerId);
254
+ }
255
+ }
256
+
220
257
  if (enableKeyValueStore) {
221
258
  ctx[KV_PROPERTY] = new KeyValueStore(
222
259
  db,
223
- ctx.endCursor,
224
- ctx.finality,
260
+ endCursor,
261
+ finality,
225
262
  serializeFn,
226
263
  deserializeFn,
264
+ indexerId,
227
265
  );
228
266
  }
229
267
 
230
268
  await next();
231
269
 
232
- if (enablePersistState) {
233
- persistState({ db, endCursor: ctx.endCursor, indexerId });
270
+ if (enablePersistState && finality !== "pending") {
271
+ persistState({ db, endCursor, indexerId });
234
272
  }
235
273
 
236
274
  if (enableKeyValueStore) {
237
275
  delete ctx[KV_PROPERTY];
238
276
  }
277
+
278
+ prevFinality = finality;
239
279
  });
240
280
  });
241
281
  });
package/src/kv.ts CHANGED
@@ -19,64 +19,80 @@ export class KeyValueStore {
19
19
  private readonly finality: DataFinality,
20
20
  private readonly serialize: SerializeFn,
21
21
  private readonly deserialize: DeserializeFn,
22
+ private readonly indexerId: string,
22
23
  ) {
23
24
  assertInTransaction(db);
24
25
  }
25
26
 
26
27
  get<T>(key: string): T | undefined {
27
- const row = this.db.prepare<string, KeyValueRow>(statements.get).get(key);
28
+ const row = this.db
29
+ .prepare<[string, string], KeyValueRow>(statements.get)
30
+ .get(key, this.indexerId);
28
31
 
29
32
  return row ? this.deserialize(row.v) : undefined;
30
33
  }
31
34
 
32
35
  put<T>(key: string, value: T) {
33
36
  this.db
34
- .prepare<[number, string], KeyValueRow>(statements.updateToBlock)
35
- .run(Number(this.endCursor.orderKey), key);
37
+ .prepare<[number, string, string], KeyValueRow>(statements.updateToBlock)
38
+ .run(Number(this.endCursor.orderKey), key, this.indexerId);
36
39
 
37
40
  this.db
38
- .prepare<[number, string, string], KeyValueRow>(statements.insertIntoKvs)
41
+ .prepare<[number, string, string, string], KeyValueRow>(
42
+ statements.insertIntoKvs,
43
+ )
39
44
  .run(
40
45
  Number(this.endCursor.orderKey),
41
46
  key,
42
47
  this.serialize(value as Record<string, unknown>),
48
+ this.indexerId,
43
49
  );
44
50
  }
45
51
 
46
52
  del(key: string) {
47
53
  this.db
48
- .prepare<[number, string], KeyValueRow>(statements.del)
49
- .run(Number(this.endCursor.orderKey), key);
54
+ .prepare<[number, string, string], KeyValueRow>(statements.del)
55
+ .run(Number(this.endCursor.orderKey), key, this.indexerId);
50
56
  }
51
57
  }
52
58
 
53
- export function finalizeKV(db: Database, cursor: Cursor) {
59
+ export function finalizeKV(db: Database, cursor: Cursor, indexerId: string) {
54
60
  assertInTransaction(db);
55
61
 
56
- db.prepare<[number], KeyValueRow>(statements.finalize).run(
62
+ db.prepare<[number, string], KeyValueRow>(statements.finalize).run(
57
63
  Number(cursor.orderKey),
64
+ indexerId,
58
65
  );
59
66
  }
60
67
 
61
- export function invalidateKV(db: Database, cursor: Cursor) {
68
+ export function invalidateKV(db: Database, cursor: Cursor, indexerId: string) {
62
69
  assertInTransaction(db);
63
70
 
64
71
  // Delete entries that started after the invalidation cursor
65
- db.prepare<[number], KeyValueRow>(statements.invalidateDelete).run(
72
+ db.prepare<[number, string], KeyValueRow>(statements.invalidateDelete).run(
66
73
  Number(cursor.orderKey),
74
+ indexerId,
67
75
  );
68
76
 
69
77
  // Update entries that were supposed to end after the invalidation cursor
70
- db.prepare<[number], KeyValueRow>(statements.invalidateUpdate).run(
78
+ db.prepare<[number, string], KeyValueRow>(statements.invalidateUpdate).run(
71
79
  Number(cursor.orderKey),
80
+ indexerId,
72
81
  );
73
82
  }
74
83
 
75
- type KeyValueRow = {
84
+ export function cleanupKV(db: Database, indexerId: string) {
85
+ assertInTransaction(db);
86
+
87
+ db.prepare<[string], KeyValueRow>(statements.cleanup).run(indexerId);
88
+ }
89
+
90
+ export type KeyValueRow = {
76
91
  from_block: number;
77
92
  to_block: number;
78
93
  k: string;
79
94
  v: string;
95
+ id: string;
80
96
  };
81
97
 
82
98
  const statements = {
@@ -86,31 +102,35 @@ const statements = {
86
102
  to_block INTEGER,
87
103
  k TEXT NOT NULL,
88
104
  v BLOB NOT NULL,
89
- PRIMARY KEY (from_block, k)
105
+ id TEXT NOT NULL,
106
+ PRIMARY KEY (from_block, k, id)
90
107
  );`,
91
108
  get: `
92
109
  SELECT v
93
110
  FROM kvs
94
- WHERE k = ? AND to_block IS NULL`,
111
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
95
112
  updateToBlock: `
96
113
  UPDATE kvs
97
114
  SET to_block = ?
98
- WHERE k = ? AND to_block IS NULL`,
115
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
99
116
  insertIntoKvs: `
100
- INSERT INTO kvs (from_block, to_block, k, v)
101
- VALUES (?, NULL, ?, ?)`,
117
+ INSERT INTO kvs (from_block, to_block, k, v, id)
118
+ VALUES (?, NULL, ?, ?, ?)`,
102
119
  del: `
103
120
  UPDATE kvs
104
121
  SET to_block = ?
105
- WHERE k = ? AND to_block IS NULL`,
122
+ WHERE k = ? AND id = ? AND to_block IS NULL`,
106
123
  finalize: `
107
124
  DELETE FROM kvs
108
- WHERE to_block <= ?`,
125
+ WHERE to_block <= ? AND id = ?`,
109
126
  invalidateDelete: `
110
127
  DELETE FROM kvs
111
- WHERE from_block > ?`,
128
+ WHERE from_block > ? AND id = ?`,
112
129
  invalidateUpdate: `
113
130
  UPDATE kvs
114
131
  SET to_block = NULL
115
- WHERE to_block > ?`,
116
- };
132
+ WHERE to_block > ? AND id = ?`,
133
+ cleanup: `
134
+ DELETE FROM kvs
135
+ WHERE id = ?`,
136
+ } as const;
package/src/persistence.ts CHANGED
@@ -100,6 +100,29 @@ export function invalidateState(props: {
100
100
  );
101
101
  }
102
102
 
103
+ export function resetPersistence(props: {
104
+ db: Database;
105
+ indexerId: string;
106
+ }) {
107
+ const { db, indexerId } = props;
108
+ assertInTransaction(db);
109
+ db.prepare<[string]>(statements.resetCheckpoint).run(indexerId);
110
+ db.prepare<[string]>(statements.resetFilter).run(indexerId);
111
+ }
112
+
113
+ export type CheckpointRow = {
114
+ id: string;
115
+ order_key: number;
116
+ unique_key: string | null;
117
+ };
118
+
119
+ export type FilterRow = {
120
+ id: string;
121
+ filter: string;
122
+ from_block: number;
123
+ to_block: number | null;
124
+ };
125
+
103
126
  const statements = {
104
127
  createCheckpointsTable: `
105
128
  CREATE TABLE IF NOT EXISTS checkpoints (
@@ -155,4 +178,10 @@ const statements = {
155
178
  UPDATE filters
156
179
  SET to_block = NULL
157
180
  WHERE id = ? AND to_block > ?`,
181
+ resetCheckpoint: `
182
+ DELETE FROM checkpoints
183
+ WHERE id = ?`,
184
+ resetFilter: `
185
+ DELETE FROM filters
186
+ WHERE id = ?`,
158
187
  };