@apibara/plugin-drizzle 2.1.0-beta.3 → 2.1.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +251 -64
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +165 -4
- package/dist/index.d.mts +165 -4
- package/dist/index.d.ts +165 -4
- package/dist/index.mjs +240 -55
- package/dist/index.mjs.map +1 -0
- package/dist/shared/plugin-drizzle.2d226351.mjs +6 -0
- package/dist/shared/plugin-drizzle.2d226351.mjs.map +1 -0
- package/dist/shared/plugin-drizzle.cae20704.cjs +10 -0
- package/dist/shared/plugin-drizzle.cae20704.cjs.map +1 -0
- package/dist/testing.cjs +14 -0
- package/dist/testing.cjs.map +1 -0
- package/dist/testing.d.cts +6 -0
- package/dist/testing.d.mts +6 -0
- package/dist/testing.d.ts +6 -0
- package/dist/testing.mjs +12 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +21 -6
- package/src/constants.ts +3 -0
- package/src/helper.ts +219 -0
- package/src/index.ts +142 -17
- package/src/persistence.ts +60 -18
- package/src/storage.ts +88 -23
- package/src/testing.ts +13 -0
- package/src/utils.ts +19 -0
package/dist/index.cjs
CHANGED
|
@@ -4,8 +4,9 @@ const indexer = require('@apibara/indexer');
|
|
|
4
4
|
const plugins = require('@apibara/indexer/plugins');
|
|
5
5
|
const internal = require('@apibara/indexer/internal');
|
|
6
6
|
const plugins$1 = require('@apibara/indexer/internal/plugins');
|
|
7
|
-
const
|
|
7
|
+
const constants = require('./shared/plugin-drizzle.cae20704.cjs');
|
|
8
8
|
const drizzleOrm = require('drizzle-orm');
|
|
9
|
+
const protocol = require('@apibara/protocol');
|
|
9
10
|
const pgCore = require('drizzle-orm/pg-core');
|
|
10
11
|
|
|
11
12
|
class DrizzleStorageError extends Error {
|
|
@@ -35,16 +36,89 @@ function serialize(obj) {
|
|
|
35
36
|
function sleep(ms) {
|
|
36
37
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
38
|
}
|
|
39
|
+
const getIdColumnForTable = (tableName, idColumn) => {
|
|
40
|
+
if (idColumn[tableName]) {
|
|
41
|
+
return idColumn[tableName];
|
|
42
|
+
}
|
|
43
|
+
return idColumn["*"];
|
|
44
|
+
};
|
|
38
45
|
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
46
|
+
function drizzle(options) {
|
|
47
|
+
const {
|
|
48
|
+
connectionString = process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://",
|
|
49
|
+
schema,
|
|
50
|
+
type = "pglite",
|
|
51
|
+
config,
|
|
52
|
+
poolConfig
|
|
53
|
+
} = options ?? {};
|
|
54
|
+
if (isPgliteConnectionString(connectionString) && type === "pglite") {
|
|
55
|
+
const { drizzle: drizzlePGLite } = require("drizzle-orm/pglite");
|
|
56
|
+
return drizzlePGLite({
|
|
57
|
+
schema,
|
|
58
|
+
connection: {
|
|
59
|
+
dataDir: connectionString || "memory://pglite"
|
|
60
|
+
},
|
|
61
|
+
...config || {}
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
const { Pool } = require("pg");
|
|
65
|
+
const { drizzle: drizzleNode } = require("drizzle-orm/node-postgres");
|
|
66
|
+
const pool = new Pool({
|
|
67
|
+
connectionString,
|
|
68
|
+
...poolConfig || {}
|
|
69
|
+
});
|
|
70
|
+
return drizzleNode(pool, { schema, ...config || {} });
|
|
71
|
+
}
|
|
72
|
+
async function migrate(db, options) {
|
|
73
|
+
const isPglite = isDrizzleKind(db, "PgliteDatabase");
|
|
74
|
+
try {
|
|
75
|
+
if (isPglite) {
|
|
76
|
+
const { migrate: migratePGLite } = require("drizzle-orm/pglite/migrator");
|
|
77
|
+
await migratePGLite(db, options);
|
|
78
|
+
} else {
|
|
79
|
+
const {
|
|
80
|
+
migrate: migrateNode
|
|
81
|
+
} = require("drizzle-orm/node-postgres/migrator");
|
|
82
|
+
await migrateNode(db, options);
|
|
83
|
+
}
|
|
84
|
+
} catch (error) {
|
|
85
|
+
throw new DrizzleStorageError(
|
|
86
|
+
"Failed to apply migrations! Please check if you have generated migrations using drizzle:generate",
|
|
87
|
+
{
|
|
88
|
+
cause: error
|
|
89
|
+
}
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function isPgliteConnectionString(conn) {
|
|
94
|
+
return conn.startsWith("memory://") || conn.startsWith("file://") || conn.startsWith("idb://");
|
|
95
|
+
}
|
|
96
|
+
function isDrizzleKind(value, entityKindValue) {
|
|
97
|
+
if (!value || typeof value !== "object") {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
let cls = Object.getPrototypeOf(value).constructor;
|
|
101
|
+
if (cls) {
|
|
102
|
+
while (cls) {
|
|
103
|
+
if (drizzleOrm.entityKind in cls && cls[drizzleOrm.entityKind] === entityKindValue) {
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
cls = Object.getPrototypeOf(cls);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const CHECKPOINTS_TABLE_NAME = "checkpoints";
|
|
113
|
+
const FILTERS_TABLE_NAME = "filters";
|
|
114
|
+
const SCHEMA_VERSION_TABLE_NAME = "schema_version";
|
|
115
|
+
const schema$1 = pgCore.pgSchema(constants.SCHEMA_NAME);
|
|
116
|
+
const checkpoints = schema$1.table(CHECKPOINTS_TABLE_NAME, {
|
|
43
117
|
id: pgCore.text("id").notNull().primaryKey(),
|
|
44
118
|
orderKey: pgCore.integer("order_key").notNull(),
|
|
45
119
|
uniqueKey: pgCore.text("unique_key")
|
|
46
120
|
});
|
|
47
|
-
const filters =
|
|
121
|
+
const filters = schema$1.table(
|
|
48
122
|
FILTERS_TABLE_NAME,
|
|
49
123
|
{
|
|
50
124
|
id: pgCore.text("id").notNull(),
|
|
@@ -58,7 +132,7 @@ const filters = pgCore.pgTable(
|
|
|
58
132
|
}
|
|
59
133
|
]
|
|
60
134
|
);
|
|
61
|
-
const schemaVersion =
|
|
135
|
+
const schemaVersion = schema$1.table(SCHEMA_VERSION_TABLE_NAME, {
|
|
62
136
|
k: pgCore.integer("k").notNull().primaryKey(),
|
|
63
137
|
version: pgCore.integer("version").notNull()
|
|
64
138
|
});
|
|
@@ -69,12 +143,19 @@ const MIGRATIONS = [
|
|
|
69
143
|
// Add more migration arrays for future versions
|
|
70
144
|
];
|
|
71
145
|
async function initializePersistentState(tx) {
|
|
72
|
-
await tx.execute(
|
|
73
|
-
|
|
146
|
+
await tx.execute(
|
|
147
|
+
drizzleOrm.sql.raw(`
|
|
148
|
+
CREATE SCHEMA IF NOT EXISTS ${constants.SCHEMA_NAME};
|
|
149
|
+
`)
|
|
150
|
+
);
|
|
151
|
+
await tx.execute(
|
|
152
|
+
drizzleOrm.sql.raw(`
|
|
153
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${SCHEMA_VERSION_TABLE_NAME} (
|
|
74
154
|
k INTEGER PRIMARY KEY,
|
|
75
155
|
version INTEGER NOT NULL
|
|
76
156
|
);
|
|
77
|
-
`)
|
|
157
|
+
`)
|
|
158
|
+
);
|
|
78
159
|
const versionRows = await tx.select().from(schemaVersion).where(drizzleOrm.eq(schemaVersion.k, 0));
|
|
79
160
|
const storedVersion = versionRows[0]?.version ?? -1;
|
|
80
161
|
if (storedVersion > CURRENT_SCHEMA_VERSION) {
|
|
@@ -84,22 +165,26 @@ async function initializePersistentState(tx) {
|
|
|
84
165
|
}
|
|
85
166
|
try {
|
|
86
167
|
if (storedVersion === -1) {
|
|
87
|
-
await tx.execute(
|
|
88
|
-
|
|
168
|
+
await tx.execute(
|
|
169
|
+
drizzleOrm.sql.raw(`
|
|
170
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${CHECKPOINTS_TABLE_NAME} (
|
|
89
171
|
id TEXT PRIMARY KEY,
|
|
90
172
|
order_key INTEGER NOT NULL,
|
|
91
173
|
unique_key TEXT
|
|
92
174
|
);
|
|
93
|
-
`)
|
|
94
|
-
|
|
95
|
-
|
|
175
|
+
`)
|
|
176
|
+
);
|
|
177
|
+
await tx.execute(
|
|
178
|
+
drizzleOrm.sql.raw(`
|
|
179
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${FILTERS_TABLE_NAME} (
|
|
96
180
|
id TEXT NOT NULL,
|
|
97
181
|
filter TEXT NOT NULL,
|
|
98
182
|
from_block INTEGER NOT NULL,
|
|
99
183
|
to_block INTEGER DEFAULT NULL,
|
|
100
184
|
PRIMARY KEY (id, from_block)
|
|
101
185
|
);
|
|
102
|
-
`)
|
|
186
|
+
`)
|
|
187
|
+
);
|
|
103
188
|
await tx.insert(schemaVersion).values({
|
|
104
189
|
k: 0,
|
|
105
190
|
version: CURRENT_SCHEMA_VERSION
|
|
@@ -134,7 +219,9 @@ async function persistState(props) {
|
|
|
134
219
|
target: checkpoints.id,
|
|
135
220
|
set: {
|
|
136
221
|
orderKey: Number(endCursor.orderKey),
|
|
137
|
-
|
|
222
|
+
// Explicitly set the unique key to `null` to indicate that it has been deleted
|
|
223
|
+
// Otherwise drizzle will not update its value.
|
|
224
|
+
uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null
|
|
138
225
|
}
|
|
139
226
|
});
|
|
140
227
|
if (filter) {
|
|
@@ -213,11 +300,24 @@ async function finalizeState(props) {
|
|
|
213
300
|
});
|
|
214
301
|
}
|
|
215
302
|
}
|
|
303
|
+
async function resetPersistence(props) {
|
|
304
|
+
const { tx, indexerId } = props;
|
|
305
|
+
try {
|
|
306
|
+
await tx.delete(checkpoints).where(drizzleOrm.eq(checkpoints.id, indexerId));
|
|
307
|
+
await tx.delete(filters).where(drizzleOrm.eq(filters.id, indexerId));
|
|
308
|
+
} catch (error) {
|
|
309
|
+
throw new DrizzleStorageError("Failed to reset persistence state", {
|
|
310
|
+
cause: error
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
}
|
|
216
314
|
|
|
315
|
+
const ROLLBACK_TABLE_NAME = "reorg_rollback";
|
|
316
|
+
const schema = pgCore.pgSchema(constants.SCHEMA_NAME);
|
|
217
317
|
function getReorgTriggerName(table, indexerId) {
|
|
218
318
|
return `${table}_reorg_${indexerId}`;
|
|
219
319
|
}
|
|
220
|
-
|
|
320
|
+
schema.table(ROLLBACK_TABLE_NAME, {
|
|
221
321
|
n: pgCore.serial("n").primaryKey(),
|
|
222
322
|
op: pgCore.char("op", { length: 1 }).$type().notNull(),
|
|
223
323
|
table_name: pgCore.text("table_name").notNull(),
|
|
@@ -228,9 +328,12 @@ pgCore.pgTable("__reorg_rollback", {
|
|
|
228
328
|
});
|
|
229
329
|
async function initializeReorgRollbackTable(tx, indexerId) {
|
|
230
330
|
try {
|
|
331
|
+
await tx.execute(`
|
|
332
|
+
CREATE SCHEMA IF NOT EXISTS ${constants.SCHEMA_NAME};
|
|
333
|
+
`);
|
|
231
334
|
await tx.execute(
|
|
232
335
|
drizzleOrm.sql.raw(`
|
|
233
|
-
CREATE TABLE IF NOT EXISTS
|
|
336
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(
|
|
234
337
|
n SERIAL PRIMARY KEY,
|
|
235
338
|
op CHAR(1) NOT NULL,
|
|
236
339
|
table_name TEXT NOT NULL,
|
|
@@ -243,7 +346,7 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
243
346
|
);
|
|
244
347
|
await tx.execute(
|
|
245
348
|
drizzleOrm.sql.raw(`
|
|
246
|
-
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON
|
|
349
|
+
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(indexer_id, cursor);
|
|
247
350
|
`)
|
|
248
351
|
);
|
|
249
352
|
} catch (error) {
|
|
@@ -254,24 +357,25 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
254
357
|
try {
|
|
255
358
|
await tx.execute(
|
|
256
359
|
drizzleOrm.sql.raw(`
|
|
257
|
-
CREATE OR REPLACE FUNCTION reorg_checkpoint()
|
|
360
|
+
CREATE OR REPLACE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint()
|
|
258
361
|
RETURNS TRIGGER AS $$
|
|
259
362
|
DECLARE
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
363
|
+
table_name TEXT := TG_ARGV[0]::TEXT;
|
|
364
|
+
id_col TEXT := TG_ARGV[1]::TEXT;
|
|
365
|
+
order_key INTEGER := TG_ARGV[2]::INTEGER;
|
|
366
|
+
indexer_id TEXT := TG_ARGV[3]::TEXT;
|
|
263
367
|
new_id_value TEXT := row_to_json(NEW.*)->>id_col;
|
|
264
368
|
old_id_value TEXT := row_to_json(OLD.*)->>id_col;
|
|
265
369
|
BEGIN
|
|
266
370
|
IF (TG_OP = 'DELETE') THEN
|
|
267
|
-
INSERT INTO
|
|
268
|
-
SELECT 'D',
|
|
371
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
372
|
+
SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id;
|
|
269
373
|
ELSIF (TG_OP = 'UPDATE') THEN
|
|
270
|
-
INSERT INTO
|
|
271
|
-
SELECT 'U',
|
|
374
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
375
|
+
SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id;
|
|
272
376
|
ELSIF (TG_OP = 'INSERT') THEN
|
|
273
|
-
INSERT INTO
|
|
274
|
-
SELECT 'I',
|
|
377
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
378
|
+
SELECT 'I', table_name, order_key, new_id_value, null, indexer_id;
|
|
275
379
|
END IF;
|
|
276
380
|
RETURN NULL;
|
|
277
381
|
END;
|
|
@@ -287,9 +391,10 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
287
391
|
);
|
|
288
392
|
}
|
|
289
393
|
}
|
|
290
|
-
async function registerTriggers(tx, tables, endCursor,
|
|
394
|
+
async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) {
|
|
291
395
|
try {
|
|
292
396
|
for (const table of tables) {
|
|
397
|
+
const tableIdColumn = getIdColumnForTable(table, idColumnMap);
|
|
293
398
|
await tx.execute(
|
|
294
399
|
drizzleOrm.sql.raw(
|
|
295
400
|
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
@@ -300,7 +405,7 @@ async function registerTriggers(tx, tables, endCursor, idColumn, indexerId) {
|
|
|
300
405
|
CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)}
|
|
301
406
|
AFTER INSERT OR UPDATE OR DELETE ON ${table}
|
|
302
407
|
DEFERRABLE INITIALLY DEFERRED
|
|
303
|
-
FOR EACH ROW EXECUTE FUNCTION reorg_checkpoint('${
|
|
408
|
+
FOR EACH ROW EXECUTE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}');
|
|
304
409
|
`)
|
|
305
410
|
);
|
|
306
411
|
}
|
|
@@ -325,11 +430,11 @@ async function removeTriggers(db, tables, indexerId) {
|
|
|
325
430
|
});
|
|
326
431
|
}
|
|
327
432
|
}
|
|
328
|
-
async function invalidate(tx, cursor,
|
|
433
|
+
async function invalidate(tx, cursor, idColumnMap, indexerId) {
|
|
329
434
|
const { rows: result } = await tx.execute(
|
|
330
435
|
drizzleOrm.sql.raw(`
|
|
331
436
|
WITH deleted AS (
|
|
332
|
-
DELETE FROM
|
|
437
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
333
438
|
WHERE cursor > ${Number(cursor.orderKey)}
|
|
334
439
|
AND indexer_id = '${indexerId}'
|
|
335
440
|
RETURNING *
|
|
@@ -343,6 +448,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
343
448
|
);
|
|
344
449
|
}
|
|
345
450
|
for (const op of result) {
|
|
451
|
+
const tableIdColumn = getIdColumnForTable(op.table_name, idColumnMap);
|
|
346
452
|
switch (op.op) {
|
|
347
453
|
case "I":
|
|
348
454
|
try {
|
|
@@ -352,7 +458,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
352
458
|
await tx.execute(
|
|
353
459
|
drizzleOrm.sql.raw(`
|
|
354
460
|
DELETE FROM ${op.table_name}
|
|
355
|
-
WHERE ${
|
|
461
|
+
WHERE ${tableIdColumn} = '${op.row_id}'
|
|
356
462
|
`)
|
|
357
463
|
);
|
|
358
464
|
} catch (error) {
|
|
@@ -392,7 +498,9 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
392
498
|
);
|
|
393
499
|
}
|
|
394
500
|
const rowValue = typeof op.row_value === "string" ? JSON.parse(op.row_value) : op.row_value;
|
|
395
|
-
const nonIdKeys = Object.keys(rowValue).filter(
|
|
501
|
+
const nonIdKeys = Object.keys(rowValue).filter(
|
|
502
|
+
(k) => k !== tableIdColumn
|
|
503
|
+
);
|
|
396
504
|
const fields = nonIdKeys.map((c) => `${c} = prev.${c}`).join(", ");
|
|
397
505
|
const query = drizzleOrm.sql.raw(`
|
|
398
506
|
UPDATE ${op.table_name}
|
|
@@ -400,7 +508,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
400
508
|
FROM (
|
|
401
509
|
SELECT * FROM json_populate_record(null::${op.table_name}, '${JSON.stringify(op.row_value)}'::json)
|
|
402
510
|
) as prev
|
|
403
|
-
WHERE ${op.table_name}.${
|
|
511
|
+
WHERE ${op.table_name}.${tableIdColumn} = '${op.row_id}'
|
|
404
512
|
`);
|
|
405
513
|
await tx.execute(query);
|
|
406
514
|
} catch (error) {
|
|
@@ -422,7 +530,7 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
422
530
|
try {
|
|
423
531
|
await tx.execute(
|
|
424
532
|
drizzleOrm.sql.raw(`
|
|
425
|
-
DELETE FROM
|
|
533
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
426
534
|
WHERE cursor <= ${Number(cursor.orderKey)}
|
|
427
535
|
AND indexer_id = '${indexerId}'
|
|
428
536
|
`)
|
|
@@ -433,52 +541,124 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
433
541
|
});
|
|
434
542
|
}
|
|
435
543
|
}
|
|
544
|
+
async function cleanupStorage(tx, tables, indexerId) {
|
|
545
|
+
try {
|
|
546
|
+
for (const table of tables) {
|
|
547
|
+
await tx.execute(
|
|
548
|
+
drizzleOrm.sql.raw(
|
|
549
|
+
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
550
|
+
)
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
await tx.execute(
|
|
554
|
+
drizzleOrm.sql.raw(`
|
|
555
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
556
|
+
WHERE indexer_id = '${indexerId}'
|
|
557
|
+
`)
|
|
558
|
+
);
|
|
559
|
+
for (const table of tables) {
|
|
560
|
+
try {
|
|
561
|
+
await tx.execute(drizzleOrm.sql.raw(`TRUNCATE TABLE ${table} CASCADE;`));
|
|
562
|
+
} catch (error) {
|
|
563
|
+
throw new DrizzleStorageError(`Failed to truncate table ${table}`, {
|
|
564
|
+
cause: error
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
} catch (error) {
|
|
569
|
+
throw new DrizzleStorageError("Failed to clean up storage", {
|
|
570
|
+
cause: error
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
}
|
|
436
574
|
|
|
437
|
-
const DRIZZLE_PROPERTY = "_drizzle";
|
|
438
575
|
const MAX_RETRIES = 5;
|
|
439
576
|
function useDrizzleStorage(_db) {
|
|
440
577
|
const context = indexer.useIndexerContext();
|
|
441
|
-
if (!context[DRIZZLE_PROPERTY]) {
|
|
578
|
+
if (!context[constants.DRIZZLE_PROPERTY]) {
|
|
442
579
|
throw new DrizzleStorageError(
|
|
443
580
|
"drizzle storage is not available. Did you register the plugin?"
|
|
444
581
|
);
|
|
445
582
|
}
|
|
446
|
-
return context[DRIZZLE_PROPERTY];
|
|
583
|
+
return context[constants.DRIZZLE_PROPERTY];
|
|
447
584
|
}
|
|
448
585
|
function drizzleStorage({
|
|
449
586
|
db,
|
|
450
587
|
persistState: enablePersistence = true,
|
|
451
588
|
indexerName: identifier = "default",
|
|
452
|
-
schema,
|
|
453
|
-
idColumn
|
|
589
|
+
schema: _schema,
|
|
590
|
+
idColumn,
|
|
591
|
+
migrate: migrateOptions
|
|
454
592
|
}) {
|
|
455
|
-
return plugins.defineIndexerPlugin((indexer) => {
|
|
593
|
+
return plugins.defineIndexerPlugin((indexer$1) => {
|
|
456
594
|
let tableNames = [];
|
|
457
595
|
let indexerId = "";
|
|
596
|
+
const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
|
|
597
|
+
let prevFinality;
|
|
598
|
+
const schema = _schema ?? db._.schema ?? {};
|
|
599
|
+
const idColumnMap = {
|
|
600
|
+
"*": typeof idColumn === "string" ? idColumn : "id",
|
|
601
|
+
...typeof idColumn === "object" ? idColumn : {}
|
|
602
|
+
};
|
|
458
603
|
try {
|
|
459
|
-
tableNames = Object.values(schema
|
|
460
|
-
(table) => table.dbName
|
|
461
|
-
);
|
|
604
|
+
tableNames = Object.values(schema).map((table) => table.dbName);
|
|
462
605
|
} catch (error) {
|
|
463
606
|
throw new DrizzleStorageError("Failed to get table names from schema", {
|
|
464
607
|
cause: error
|
|
465
608
|
});
|
|
466
609
|
}
|
|
467
|
-
|
|
468
|
-
const
|
|
610
|
+
for (const table of Object.values(schema)) {
|
|
611
|
+
const columns = table.columns;
|
|
612
|
+
const tableIdColumn = getIdColumnForTable(table.dbName, idColumnMap);
|
|
613
|
+
const columnExists = Object.values(columns).some(
|
|
614
|
+
(column) => column.name === tableIdColumn
|
|
615
|
+
);
|
|
616
|
+
if (!columnExists) {
|
|
617
|
+
throw new DrizzleStorageError(
|
|
618
|
+
`Column \`"${tableIdColumn}"\` does not exist in table \`"${table.dbName}"\`. Make sure the table has the specified column or provide a valid \`idColumn\` mapping to \`drizzleStorage\`.`
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
indexer$1.hooks.hook("run:before", async () => {
|
|
623
|
+
const internalContext = plugins$1.useInternalContext();
|
|
624
|
+
const context = indexer.useIndexerContext();
|
|
625
|
+
const logger = plugins.useLogger();
|
|
626
|
+
context[constants.DRIZZLE_STORAGE_DB_PROPERTY] = db;
|
|
627
|
+
const { indexerName: indexerFileName, availableIndexers } = internalContext;
|
|
469
628
|
indexerId = internal.generateIndexerId(indexerFileName, identifier);
|
|
470
629
|
let retries = 0;
|
|
630
|
+
let migrationsApplied = false;
|
|
631
|
+
let cleanupApplied = false;
|
|
471
632
|
while (retries <= MAX_RETRIES) {
|
|
472
633
|
try {
|
|
634
|
+
if (migrateOptions && !migrationsApplied) {
|
|
635
|
+
await migrate(db, migrateOptions);
|
|
636
|
+
migrationsApplied = true;
|
|
637
|
+
logger.success("Migrations applied");
|
|
638
|
+
}
|
|
473
639
|
await withTransaction(db, async (tx) => {
|
|
474
640
|
await initializeReorgRollbackTable(tx, indexerId);
|
|
475
641
|
if (enablePersistence) {
|
|
476
642
|
await initializePersistentState(tx);
|
|
477
643
|
}
|
|
644
|
+
if (alwaysReindex && !cleanupApplied) {
|
|
645
|
+
logger.warn(
|
|
646
|
+
`Reindexing: Deleting all data from tables - ${tableNames.join(", ")}`
|
|
647
|
+
);
|
|
648
|
+
await cleanupStorage(tx, tableNames, indexerId);
|
|
649
|
+
if (enablePersistence) {
|
|
650
|
+
await resetPersistence({ tx, indexerId });
|
|
651
|
+
}
|
|
652
|
+
cleanupApplied = true;
|
|
653
|
+
logger.success("Tables have been cleaned up for reindexing");
|
|
654
|
+
}
|
|
478
655
|
});
|
|
479
656
|
break;
|
|
480
657
|
} catch (error) {
|
|
481
658
|
if (retries === MAX_RETRIES) {
|
|
659
|
+
if (error instanceof DrizzleStorageError) {
|
|
660
|
+
throw error;
|
|
661
|
+
}
|
|
482
662
|
throw new DrizzleStorageError(
|
|
483
663
|
"Initialization failed after 5 retries",
|
|
484
664
|
{
|
|
@@ -491,7 +671,7 @@ function drizzleStorage({
|
|
|
491
671
|
}
|
|
492
672
|
}
|
|
493
673
|
});
|
|
494
|
-
indexer.hooks.hook("connect:before", async ({ request }) => {
|
|
674
|
+
indexer$1.hooks.hook("connect:before", async ({ request }) => {
|
|
495
675
|
if (!enablePersistence) {
|
|
496
676
|
return;
|
|
497
677
|
}
|
|
@@ -508,19 +688,19 @@ function drizzleStorage({
|
|
|
508
688
|
}
|
|
509
689
|
});
|
|
510
690
|
});
|
|
511
|
-
indexer.hooks.hook("connect:after", async ({ request }) => {
|
|
691
|
+
indexer$1.hooks.hook("connect:after", async ({ request }) => {
|
|
512
692
|
const cursor = request.startingCursor;
|
|
513
693
|
if (!cursor) {
|
|
514
694
|
return;
|
|
515
695
|
}
|
|
516
696
|
await withTransaction(db, async (tx) => {
|
|
517
|
-
await invalidate(tx, cursor,
|
|
697
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
518
698
|
if (enablePersistence) {
|
|
519
699
|
await invalidateState({ tx, cursor, indexerId });
|
|
520
700
|
}
|
|
521
701
|
});
|
|
522
702
|
});
|
|
523
|
-
indexer.hooks.hook("connect:factory", async ({ request, endCursor }) => {
|
|
703
|
+
indexer$1.hooks.hook("connect:factory", async ({ request, endCursor }) => {
|
|
524
704
|
if (!enablePersistence) {
|
|
525
705
|
return;
|
|
526
706
|
}
|
|
@@ -534,8 +714,8 @@ function drizzleStorage({
|
|
|
534
714
|
});
|
|
535
715
|
}
|
|
536
716
|
});
|
|
537
|
-
indexer.hooks.hook("message:finalize", async ({ message }) => {
|
|
538
|
-
const { cursor } = message
|
|
717
|
+
indexer$1.hooks.hook("message:finalize", async ({ message }) => {
|
|
718
|
+
const { cursor } = message;
|
|
539
719
|
if (!cursor) {
|
|
540
720
|
throw new DrizzleStorageError("Finalized Cursor is undefined");
|
|
541
721
|
}
|
|
@@ -546,45 +726,49 @@ function drizzleStorage({
|
|
|
546
726
|
}
|
|
547
727
|
});
|
|
548
728
|
});
|
|
549
|
-
indexer.hooks.hook("message:invalidate", async ({ message }) => {
|
|
550
|
-
const { cursor } = message
|
|
729
|
+
indexer$1.hooks.hook("message:invalidate", async ({ message }) => {
|
|
730
|
+
const { cursor } = message;
|
|
551
731
|
if (!cursor) {
|
|
552
732
|
throw new DrizzleStorageError("Invalidate Cursor is undefined");
|
|
553
733
|
}
|
|
554
734
|
await withTransaction(db, async (tx) => {
|
|
555
|
-
await invalidate(tx, cursor,
|
|
735
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
556
736
|
if (enablePersistence) {
|
|
557
737
|
await invalidateState({ tx, cursor, indexerId });
|
|
558
738
|
}
|
|
559
739
|
});
|
|
560
740
|
});
|
|
561
|
-
indexer.hooks.hook("handler:middleware", async ({ use }) => {
|
|
741
|
+
indexer$1.hooks.hook("handler:middleware", async ({ use }) => {
|
|
562
742
|
use(async (context, next) => {
|
|
563
743
|
try {
|
|
564
|
-
const { endCursor, finality } = context;
|
|
744
|
+
const { endCursor, finality, cursor } = context;
|
|
565
745
|
if (!endCursor) {
|
|
566
746
|
throw new DrizzleStorageError("End Cursor is undefined");
|
|
567
747
|
}
|
|
568
748
|
await withTransaction(db, async (tx) => {
|
|
569
|
-
context[DRIZZLE_PROPERTY] = { db: tx };
|
|
749
|
+
context[constants.DRIZZLE_PROPERTY] = { db: tx };
|
|
750
|
+
if (prevFinality === "pending") {
|
|
751
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
752
|
+
}
|
|
570
753
|
if (finality !== "finalized") {
|
|
571
754
|
await registerTriggers(
|
|
572
755
|
tx,
|
|
573
756
|
tableNames,
|
|
574
757
|
endCursor,
|
|
575
|
-
|
|
758
|
+
idColumnMap,
|
|
576
759
|
indexerId
|
|
577
760
|
);
|
|
578
761
|
}
|
|
579
762
|
await next();
|
|
580
|
-
delete context[DRIZZLE_PROPERTY];
|
|
581
|
-
if (enablePersistence) {
|
|
763
|
+
delete context[constants.DRIZZLE_PROPERTY];
|
|
764
|
+
if (enablePersistence && finality !== "pending") {
|
|
582
765
|
await persistState({
|
|
583
766
|
tx,
|
|
584
767
|
endCursor,
|
|
585
768
|
indexerId
|
|
586
769
|
});
|
|
587
770
|
}
|
|
771
|
+
prevFinality = finality;
|
|
588
772
|
});
|
|
589
773
|
if (finality !== "finalized") {
|
|
590
774
|
await removeTriggers(db, tableNames, indexerId);
|
|
@@ -600,5 +784,8 @@ function drizzleStorage({
|
|
|
600
784
|
});
|
|
601
785
|
}
|
|
602
786
|
|
|
787
|
+
exports.drizzle = drizzle;
|
|
603
788
|
exports.drizzleStorage = drizzleStorage;
|
|
789
|
+
exports.migrate = migrate;
|
|
604
790
|
exports.useDrizzleStorage = useDrizzleStorage;
|
|
791
|
+
//# sourceMappingURL=index.cjs.map
|