@apibara/plugin-drizzle 2.1.0-beta.4 → 2.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +261 -64
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +166 -4
- package/dist/index.d.mts +166 -4
- package/dist/index.d.ts +166 -4
- package/dist/index.mjs +250 -56
- package/dist/index.mjs.map +1 -0
- package/dist/shared/plugin-drizzle.2d226351.mjs +6 -0
- package/dist/shared/plugin-drizzle.2d226351.mjs.map +1 -0
- package/dist/shared/plugin-drizzle.cae20704.cjs +10 -0
- package/dist/shared/plugin-drizzle.cae20704.cjs.map +1 -0
- package/dist/testing.cjs +14 -0
- package/dist/testing.cjs.map +1 -0
- package/dist/testing.d.cts +6 -0
- package/dist/testing.d.mts +6 -0
- package/dist/testing.d.ts +6 -0
- package/dist/testing.mjs +12 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +21 -6
- package/src/constants.ts +3 -0
- package/src/helper.ts +219 -0
- package/src/index.ts +160 -18
- package/src/persistence.ts +60 -18
- package/src/storage.ts +88 -23
- package/src/testing.ts +13 -0
- package/src/utils.ts +19 -0
package/dist/index.cjs
CHANGED
|
@@ -4,8 +4,9 @@ const indexer = require('@apibara/indexer');
|
|
|
4
4
|
const plugins = require('@apibara/indexer/plugins');
|
|
5
5
|
const internal = require('@apibara/indexer/internal');
|
|
6
6
|
const plugins$1 = require('@apibara/indexer/internal/plugins');
|
|
7
|
-
const
|
|
7
|
+
const constants = require('./shared/plugin-drizzle.cae20704.cjs');
|
|
8
8
|
const drizzleOrm = require('drizzle-orm');
|
|
9
|
+
const protocol = require('@apibara/protocol');
|
|
9
10
|
const pgCore = require('drizzle-orm/pg-core');
|
|
10
11
|
|
|
11
12
|
class DrizzleStorageError extends Error {
|
|
@@ -35,16 +36,89 @@ function serialize(obj) {
|
|
|
35
36
|
function sleep(ms) {
|
|
36
37
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
38
|
}
|
|
39
|
+
const getIdColumnForTable = (tableName, idColumn) => {
|
|
40
|
+
if (idColumn[tableName]) {
|
|
41
|
+
return idColumn[tableName];
|
|
42
|
+
}
|
|
43
|
+
return idColumn["*"];
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
function drizzle(options) {
|
|
47
|
+
const {
|
|
48
|
+
connectionString = process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://",
|
|
49
|
+
schema,
|
|
50
|
+
type = "pglite",
|
|
51
|
+
config,
|
|
52
|
+
poolConfig
|
|
53
|
+
} = options ?? {};
|
|
54
|
+
if (isPgliteConnectionString(connectionString) && type === "pglite") {
|
|
55
|
+
const { drizzle: drizzlePGLite } = require("drizzle-orm/pglite");
|
|
56
|
+
return drizzlePGLite({
|
|
57
|
+
schema,
|
|
58
|
+
connection: {
|
|
59
|
+
dataDir: connectionString || "memory://pglite"
|
|
60
|
+
},
|
|
61
|
+
...config || {}
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
const { Pool } = require("pg");
|
|
65
|
+
const { drizzle: drizzleNode } = require("drizzle-orm/node-postgres");
|
|
66
|
+
const pool = new Pool({
|
|
67
|
+
connectionString,
|
|
68
|
+
...poolConfig || {}
|
|
69
|
+
});
|
|
70
|
+
return drizzleNode(pool, { schema, ...config || {} });
|
|
71
|
+
}
|
|
72
|
+
async function migrate(db, options) {
|
|
73
|
+
const isPglite = isDrizzleKind(db, "PgliteDatabase");
|
|
74
|
+
try {
|
|
75
|
+
if (isPglite) {
|
|
76
|
+
const { migrate: migratePGLite } = require("drizzle-orm/pglite/migrator");
|
|
77
|
+
await migratePGLite(db, options);
|
|
78
|
+
} else {
|
|
79
|
+
const {
|
|
80
|
+
migrate: migrateNode
|
|
81
|
+
} = require("drizzle-orm/node-postgres/migrator");
|
|
82
|
+
await migrateNode(db, options);
|
|
83
|
+
}
|
|
84
|
+
} catch (error) {
|
|
85
|
+
throw new DrizzleStorageError(
|
|
86
|
+
"Failed to apply migrations! Please check if you have generated migrations using drizzle:generate",
|
|
87
|
+
{
|
|
88
|
+
cause: error
|
|
89
|
+
}
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function isPgliteConnectionString(conn) {
|
|
94
|
+
return conn.startsWith("memory://") || conn.startsWith("file://") || conn.startsWith("idb://");
|
|
95
|
+
}
|
|
96
|
+
function isDrizzleKind(value, entityKindValue) {
|
|
97
|
+
if (!value || typeof value !== "object") {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
let cls = Object.getPrototypeOf(value).constructor;
|
|
101
|
+
if (cls) {
|
|
102
|
+
while (cls) {
|
|
103
|
+
if (drizzleOrm.entityKind in cls && cls[drizzleOrm.entityKind] === entityKindValue) {
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
cls = Object.getPrototypeOf(cls);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
38
111
|
|
|
39
|
-
const CHECKPOINTS_TABLE_NAME = "
|
|
40
|
-
const FILTERS_TABLE_NAME = "
|
|
41
|
-
const SCHEMA_VERSION_TABLE_NAME = "
|
|
42
|
-
const
|
|
112
|
+
const CHECKPOINTS_TABLE_NAME = "checkpoints";
|
|
113
|
+
const FILTERS_TABLE_NAME = "filters";
|
|
114
|
+
const SCHEMA_VERSION_TABLE_NAME = "schema_version";
|
|
115
|
+
const schema$1 = pgCore.pgSchema(constants.SCHEMA_NAME);
|
|
116
|
+
const checkpoints = schema$1.table(CHECKPOINTS_TABLE_NAME, {
|
|
43
117
|
id: pgCore.text("id").notNull().primaryKey(),
|
|
44
118
|
orderKey: pgCore.integer("order_key").notNull(),
|
|
45
119
|
uniqueKey: pgCore.text("unique_key")
|
|
46
120
|
});
|
|
47
|
-
const filters =
|
|
121
|
+
const filters = schema$1.table(
|
|
48
122
|
FILTERS_TABLE_NAME,
|
|
49
123
|
{
|
|
50
124
|
id: pgCore.text("id").notNull(),
|
|
@@ -58,7 +132,7 @@ const filters = pgCore.pgTable(
|
|
|
58
132
|
}
|
|
59
133
|
]
|
|
60
134
|
);
|
|
61
|
-
const schemaVersion =
|
|
135
|
+
const schemaVersion = schema$1.table(SCHEMA_VERSION_TABLE_NAME, {
|
|
62
136
|
k: pgCore.integer("k").notNull().primaryKey(),
|
|
63
137
|
version: pgCore.integer("version").notNull()
|
|
64
138
|
});
|
|
@@ -69,12 +143,19 @@ const MIGRATIONS = [
|
|
|
69
143
|
// Add more migration arrays for future versions
|
|
70
144
|
];
|
|
71
145
|
async function initializePersistentState(tx) {
|
|
72
|
-
await tx.execute(
|
|
73
|
-
|
|
146
|
+
await tx.execute(
|
|
147
|
+
drizzleOrm.sql.raw(`
|
|
148
|
+
CREATE SCHEMA IF NOT EXISTS ${constants.SCHEMA_NAME};
|
|
149
|
+
`)
|
|
150
|
+
);
|
|
151
|
+
await tx.execute(
|
|
152
|
+
drizzleOrm.sql.raw(`
|
|
153
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${SCHEMA_VERSION_TABLE_NAME} (
|
|
74
154
|
k INTEGER PRIMARY KEY,
|
|
75
155
|
version INTEGER NOT NULL
|
|
76
156
|
);
|
|
77
|
-
`)
|
|
157
|
+
`)
|
|
158
|
+
);
|
|
78
159
|
const versionRows = await tx.select().from(schemaVersion).where(drizzleOrm.eq(schemaVersion.k, 0));
|
|
79
160
|
const storedVersion = versionRows[0]?.version ?? -1;
|
|
80
161
|
if (storedVersion > CURRENT_SCHEMA_VERSION) {
|
|
@@ -84,22 +165,26 @@ async function initializePersistentState(tx) {
|
|
|
84
165
|
}
|
|
85
166
|
try {
|
|
86
167
|
if (storedVersion === -1) {
|
|
87
|
-
await tx.execute(
|
|
88
|
-
|
|
168
|
+
await tx.execute(
|
|
169
|
+
drizzleOrm.sql.raw(`
|
|
170
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${CHECKPOINTS_TABLE_NAME} (
|
|
89
171
|
id TEXT PRIMARY KEY,
|
|
90
172
|
order_key INTEGER NOT NULL,
|
|
91
173
|
unique_key TEXT
|
|
92
174
|
);
|
|
93
|
-
`)
|
|
94
|
-
|
|
95
|
-
|
|
175
|
+
`)
|
|
176
|
+
);
|
|
177
|
+
await tx.execute(
|
|
178
|
+
drizzleOrm.sql.raw(`
|
|
179
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${FILTERS_TABLE_NAME} (
|
|
96
180
|
id TEXT NOT NULL,
|
|
97
181
|
filter TEXT NOT NULL,
|
|
98
182
|
from_block INTEGER NOT NULL,
|
|
99
183
|
to_block INTEGER DEFAULT NULL,
|
|
100
184
|
PRIMARY KEY (id, from_block)
|
|
101
185
|
);
|
|
102
|
-
`)
|
|
186
|
+
`)
|
|
187
|
+
);
|
|
103
188
|
await tx.insert(schemaVersion).values({
|
|
104
189
|
k: 0,
|
|
105
190
|
version: CURRENT_SCHEMA_VERSION
|
|
@@ -134,7 +219,9 @@ async function persistState(props) {
|
|
|
134
219
|
target: checkpoints.id,
|
|
135
220
|
set: {
|
|
136
221
|
orderKey: Number(endCursor.orderKey),
|
|
137
|
-
|
|
222
|
+
// Explicitly set the unique key to `null` to indicate that it has been deleted
|
|
223
|
+
// Otherwise drizzle will not update its value.
|
|
224
|
+
uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null
|
|
138
225
|
}
|
|
139
226
|
});
|
|
140
227
|
if (filter) {
|
|
@@ -213,11 +300,24 @@ async function finalizeState(props) {
|
|
|
213
300
|
});
|
|
214
301
|
}
|
|
215
302
|
}
|
|
303
|
+
async function resetPersistence(props) {
|
|
304
|
+
const { tx, indexerId } = props;
|
|
305
|
+
try {
|
|
306
|
+
await tx.delete(checkpoints).where(drizzleOrm.eq(checkpoints.id, indexerId));
|
|
307
|
+
await tx.delete(filters).where(drizzleOrm.eq(filters.id, indexerId));
|
|
308
|
+
} catch (error) {
|
|
309
|
+
throw new DrizzleStorageError("Failed to reset persistence state", {
|
|
310
|
+
cause: error
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
}
|
|
216
314
|
|
|
315
|
+
const ROLLBACK_TABLE_NAME = "reorg_rollback";
|
|
316
|
+
const schema = pgCore.pgSchema(constants.SCHEMA_NAME);
|
|
217
317
|
function getReorgTriggerName(table, indexerId) {
|
|
218
318
|
return `${table}_reorg_${indexerId}`;
|
|
219
319
|
}
|
|
220
|
-
|
|
320
|
+
schema.table(ROLLBACK_TABLE_NAME, {
|
|
221
321
|
n: pgCore.serial("n").primaryKey(),
|
|
222
322
|
op: pgCore.char("op", { length: 1 }).$type().notNull(),
|
|
223
323
|
table_name: pgCore.text("table_name").notNull(),
|
|
@@ -228,9 +328,12 @@ pgCore.pgTable("__reorg_rollback", {
|
|
|
228
328
|
});
|
|
229
329
|
async function initializeReorgRollbackTable(tx, indexerId) {
|
|
230
330
|
try {
|
|
331
|
+
await tx.execute(`
|
|
332
|
+
CREATE SCHEMA IF NOT EXISTS ${constants.SCHEMA_NAME};
|
|
333
|
+
`);
|
|
231
334
|
await tx.execute(
|
|
232
335
|
drizzleOrm.sql.raw(`
|
|
233
|
-
CREATE TABLE IF NOT EXISTS
|
|
336
|
+
CREATE TABLE IF NOT EXISTS ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(
|
|
234
337
|
n SERIAL PRIMARY KEY,
|
|
235
338
|
op CHAR(1) NOT NULL,
|
|
236
339
|
table_name TEXT NOT NULL,
|
|
@@ -243,7 +346,7 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
243
346
|
);
|
|
244
347
|
await tx.execute(
|
|
245
348
|
drizzleOrm.sql.raw(`
|
|
246
|
-
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON
|
|
349
|
+
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(indexer_id, cursor);
|
|
247
350
|
`)
|
|
248
351
|
);
|
|
249
352
|
} catch (error) {
|
|
@@ -254,24 +357,25 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
254
357
|
try {
|
|
255
358
|
await tx.execute(
|
|
256
359
|
drizzleOrm.sql.raw(`
|
|
257
|
-
CREATE OR REPLACE FUNCTION reorg_checkpoint()
|
|
360
|
+
CREATE OR REPLACE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint()
|
|
258
361
|
RETURNS TRIGGER AS $$
|
|
259
362
|
DECLARE
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
363
|
+
table_name TEXT := TG_ARGV[0]::TEXT;
|
|
364
|
+
id_col TEXT := TG_ARGV[1]::TEXT;
|
|
365
|
+
order_key INTEGER := TG_ARGV[2]::INTEGER;
|
|
366
|
+
indexer_id TEXT := TG_ARGV[3]::TEXT;
|
|
263
367
|
new_id_value TEXT := row_to_json(NEW.*)->>id_col;
|
|
264
368
|
old_id_value TEXT := row_to_json(OLD.*)->>id_col;
|
|
265
369
|
BEGIN
|
|
266
370
|
IF (TG_OP = 'DELETE') THEN
|
|
267
|
-
INSERT INTO
|
|
268
|
-
SELECT 'D',
|
|
371
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
372
|
+
SELECT 'D', table_name, order_key, old_id_value, row_to_json(OLD.*), indexer_id;
|
|
269
373
|
ELSIF (TG_OP = 'UPDATE') THEN
|
|
270
|
-
INSERT INTO
|
|
271
|
-
SELECT 'U',
|
|
374
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
375
|
+
SELECT 'U', table_name, order_key, new_id_value, row_to_json(OLD.*), indexer_id;
|
|
272
376
|
ELSIF (TG_OP = 'INSERT') THEN
|
|
273
|
-
INSERT INTO
|
|
274
|
-
SELECT 'I',
|
|
377
|
+
INSERT INTO ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
378
|
+
SELECT 'I', table_name, order_key, new_id_value, null, indexer_id;
|
|
275
379
|
END IF;
|
|
276
380
|
RETURN NULL;
|
|
277
381
|
END;
|
|
@@ -287,9 +391,10 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
287
391
|
);
|
|
288
392
|
}
|
|
289
393
|
}
|
|
290
|
-
async function registerTriggers(tx, tables, endCursor,
|
|
394
|
+
async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) {
|
|
291
395
|
try {
|
|
292
396
|
for (const table of tables) {
|
|
397
|
+
const tableIdColumn = getIdColumnForTable(table, idColumnMap);
|
|
293
398
|
await tx.execute(
|
|
294
399
|
drizzleOrm.sql.raw(
|
|
295
400
|
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
@@ -300,7 +405,7 @@ async function registerTriggers(tx, tables, endCursor, idColumn, indexerId) {
|
|
|
300
405
|
CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)}
|
|
301
406
|
AFTER INSERT OR UPDATE OR DELETE ON ${table}
|
|
302
407
|
DEFERRABLE INITIALLY DEFERRED
|
|
303
|
-
FOR EACH ROW EXECUTE FUNCTION reorg_checkpoint('${
|
|
408
|
+
FOR EACH ROW EXECUTE FUNCTION ${constants.SCHEMA_NAME}.reorg_checkpoint('${table}', '${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}');
|
|
304
409
|
`)
|
|
305
410
|
);
|
|
306
411
|
}
|
|
@@ -325,11 +430,11 @@ async function removeTriggers(db, tables, indexerId) {
|
|
|
325
430
|
});
|
|
326
431
|
}
|
|
327
432
|
}
|
|
328
|
-
async function invalidate(tx, cursor,
|
|
433
|
+
async function invalidate(tx, cursor, idColumnMap, indexerId) {
|
|
329
434
|
const { rows: result } = await tx.execute(
|
|
330
435
|
drizzleOrm.sql.raw(`
|
|
331
436
|
WITH deleted AS (
|
|
332
|
-
DELETE FROM
|
|
437
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
333
438
|
WHERE cursor > ${Number(cursor.orderKey)}
|
|
334
439
|
AND indexer_id = '${indexerId}'
|
|
335
440
|
RETURNING *
|
|
@@ -343,6 +448,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
343
448
|
);
|
|
344
449
|
}
|
|
345
450
|
for (const op of result) {
|
|
451
|
+
const tableIdColumn = getIdColumnForTable(op.table_name, idColumnMap);
|
|
346
452
|
switch (op.op) {
|
|
347
453
|
case "I":
|
|
348
454
|
try {
|
|
@@ -352,7 +458,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
352
458
|
await tx.execute(
|
|
353
459
|
drizzleOrm.sql.raw(`
|
|
354
460
|
DELETE FROM ${op.table_name}
|
|
355
|
-
WHERE ${
|
|
461
|
+
WHERE ${tableIdColumn} = '${op.row_id}'
|
|
356
462
|
`)
|
|
357
463
|
);
|
|
358
464
|
} catch (error) {
|
|
@@ -392,7 +498,9 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
392
498
|
);
|
|
393
499
|
}
|
|
394
500
|
const rowValue = typeof op.row_value === "string" ? JSON.parse(op.row_value) : op.row_value;
|
|
395
|
-
const nonIdKeys = Object.keys(rowValue).filter(
|
|
501
|
+
const nonIdKeys = Object.keys(rowValue).filter(
|
|
502
|
+
(k) => k !== tableIdColumn
|
|
503
|
+
);
|
|
396
504
|
const fields = nonIdKeys.map((c) => `${c} = prev.${c}`).join(", ");
|
|
397
505
|
const query = drizzleOrm.sql.raw(`
|
|
398
506
|
UPDATE ${op.table_name}
|
|
@@ -400,7 +508,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
400
508
|
FROM (
|
|
401
509
|
SELECT * FROM json_populate_record(null::${op.table_name}, '${JSON.stringify(op.row_value)}'::json)
|
|
402
510
|
) as prev
|
|
403
|
-
WHERE ${op.table_name}.${
|
|
511
|
+
WHERE ${op.table_name}.${tableIdColumn} = '${op.row_id}'
|
|
404
512
|
`);
|
|
405
513
|
await tx.execute(query);
|
|
406
514
|
} catch (error) {
|
|
@@ -422,7 +530,7 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
422
530
|
try {
|
|
423
531
|
await tx.execute(
|
|
424
532
|
drizzleOrm.sql.raw(`
|
|
425
|
-
DELETE FROM
|
|
533
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
426
534
|
WHERE cursor <= ${Number(cursor.orderKey)}
|
|
427
535
|
AND indexer_id = '${indexerId}'
|
|
428
536
|
`)
|
|
@@ -433,52 +541,133 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
433
541
|
});
|
|
434
542
|
}
|
|
435
543
|
}
|
|
544
|
+
async function cleanupStorage(tx, tables, indexerId) {
|
|
545
|
+
try {
|
|
546
|
+
for (const table of tables) {
|
|
547
|
+
await tx.execute(
|
|
548
|
+
drizzleOrm.sql.raw(
|
|
549
|
+
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
550
|
+
)
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
await tx.execute(
|
|
554
|
+
drizzleOrm.sql.raw(`
|
|
555
|
+
DELETE FROM ${constants.SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
556
|
+
WHERE indexer_id = '${indexerId}'
|
|
557
|
+
`)
|
|
558
|
+
);
|
|
559
|
+
for (const table of tables) {
|
|
560
|
+
try {
|
|
561
|
+
await tx.execute(drizzleOrm.sql.raw(`TRUNCATE TABLE ${table} CASCADE;`));
|
|
562
|
+
} catch (error) {
|
|
563
|
+
throw new DrizzleStorageError(`Failed to truncate table ${table}`, {
|
|
564
|
+
cause: error
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
} catch (error) {
|
|
569
|
+
throw new DrizzleStorageError("Failed to clean up storage", {
|
|
570
|
+
cause: error
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
}
|
|
436
574
|
|
|
437
|
-
const DRIZZLE_PROPERTY = "_drizzle";
|
|
438
575
|
const MAX_RETRIES = 5;
|
|
439
576
|
function useDrizzleStorage(_db) {
|
|
440
577
|
const context = indexer.useIndexerContext();
|
|
441
|
-
if (!context[DRIZZLE_PROPERTY]) {
|
|
578
|
+
if (!context[constants.DRIZZLE_PROPERTY]) {
|
|
442
579
|
throw new DrizzleStorageError(
|
|
443
580
|
"drizzle storage is not available. Did you register the plugin?"
|
|
444
581
|
);
|
|
445
582
|
}
|
|
446
|
-
return context[DRIZZLE_PROPERTY];
|
|
583
|
+
return context[constants.DRIZZLE_PROPERTY];
|
|
584
|
+
}
|
|
585
|
+
function useTestDrizzleStorage() {
|
|
586
|
+
const context = indexer.useIndexerContext();
|
|
587
|
+
if (!context[constants.DRIZZLE_STORAGE_DB_PROPERTY]) {
|
|
588
|
+
throw new DrizzleStorageError(
|
|
589
|
+
"drizzle storage db is not available. Did you register the plugin?"
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
return context[constants.DRIZZLE_STORAGE_DB_PROPERTY];
|
|
447
593
|
}
|
|
448
594
|
function drizzleStorage({
|
|
449
595
|
db,
|
|
450
596
|
persistState: enablePersistence = true,
|
|
451
597
|
indexerName: identifier = "default",
|
|
452
|
-
schema,
|
|
453
|
-
idColumn
|
|
598
|
+
schema: _schema,
|
|
599
|
+
idColumn,
|
|
600
|
+
migrate: migrateOptions
|
|
454
601
|
}) {
|
|
455
|
-
return plugins.defineIndexerPlugin((indexer) => {
|
|
602
|
+
return plugins.defineIndexerPlugin((indexer$1) => {
|
|
456
603
|
let tableNames = [];
|
|
457
604
|
let indexerId = "";
|
|
605
|
+
const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
|
|
606
|
+
let prevFinality;
|
|
607
|
+
const schema = _schema ?? db._.schema ?? {};
|
|
608
|
+
const idColumnMap = {
|
|
609
|
+
"*": typeof idColumn === "string" ? idColumn : "id",
|
|
610
|
+
...typeof idColumn === "object" ? idColumn : {}
|
|
611
|
+
};
|
|
458
612
|
try {
|
|
459
|
-
tableNames = Object.values(schema
|
|
460
|
-
(table) => table.dbName
|
|
461
|
-
);
|
|
613
|
+
tableNames = Object.values(schema).map((table) => table.dbName);
|
|
462
614
|
} catch (error) {
|
|
463
615
|
throw new DrizzleStorageError("Failed to get table names from schema", {
|
|
464
616
|
cause: error
|
|
465
617
|
});
|
|
466
618
|
}
|
|
467
|
-
|
|
468
|
-
const
|
|
619
|
+
for (const table of Object.values(schema)) {
|
|
620
|
+
const columns = table.columns;
|
|
621
|
+
const tableIdColumn = getIdColumnForTable(table.dbName, idColumnMap);
|
|
622
|
+
const columnExists = Object.values(columns).some(
|
|
623
|
+
(column) => column.name === tableIdColumn
|
|
624
|
+
);
|
|
625
|
+
if (!columnExists) {
|
|
626
|
+
throw new DrizzleStorageError(
|
|
627
|
+
`Column \`"${tableIdColumn}"\` does not exist in table \`"${table.dbName}"\`. Make sure the table has the specified column or provide a valid \`idColumn\` mapping to \`drizzleStorage\`.`
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
indexer$1.hooks.hook("plugins:init", async () => {
|
|
632
|
+
const internalContext = plugins$1.useInternalContext();
|
|
633
|
+
const context = indexer.useIndexerContext();
|
|
634
|
+
const logger = plugins.useLogger();
|
|
635
|
+
context[constants.DRIZZLE_STORAGE_DB_PROPERTY] = db;
|
|
636
|
+
const { indexerName: indexerFileName, availableIndexers } = internalContext;
|
|
469
637
|
indexerId = internal.generateIndexerId(indexerFileName, identifier);
|
|
470
638
|
let retries = 0;
|
|
639
|
+
let migrationsApplied = false;
|
|
640
|
+
let cleanupApplied = false;
|
|
471
641
|
while (retries <= MAX_RETRIES) {
|
|
472
642
|
try {
|
|
643
|
+
if (migrateOptions && !migrationsApplied) {
|
|
644
|
+
await migrate(db, migrateOptions);
|
|
645
|
+
migrationsApplied = true;
|
|
646
|
+
logger.success("Migrations applied");
|
|
647
|
+
}
|
|
473
648
|
await withTransaction(db, async (tx) => {
|
|
474
649
|
await initializeReorgRollbackTable(tx, indexerId);
|
|
475
650
|
if (enablePersistence) {
|
|
476
651
|
await initializePersistentState(tx);
|
|
477
652
|
}
|
|
653
|
+
if (alwaysReindex && !cleanupApplied) {
|
|
654
|
+
logger.warn(
|
|
655
|
+
`Reindexing: Deleting all data from tables - ${tableNames.join(", ")}`
|
|
656
|
+
);
|
|
657
|
+
await cleanupStorage(tx, tableNames, indexerId);
|
|
658
|
+
if (enablePersistence) {
|
|
659
|
+
await resetPersistence({ tx, indexerId });
|
|
660
|
+
}
|
|
661
|
+
cleanupApplied = true;
|
|
662
|
+
logger.success("Tables have been cleaned up for reindexing");
|
|
663
|
+
}
|
|
478
664
|
});
|
|
479
665
|
break;
|
|
480
666
|
} catch (error) {
|
|
481
667
|
if (retries === MAX_RETRIES) {
|
|
668
|
+
if (error instanceof DrizzleStorageError) {
|
|
669
|
+
throw error;
|
|
670
|
+
}
|
|
482
671
|
throw new DrizzleStorageError(
|
|
483
672
|
"Initialization failed after 5 retries",
|
|
484
673
|
{
|
|
@@ -491,7 +680,7 @@ function drizzleStorage({
|
|
|
491
680
|
}
|
|
492
681
|
}
|
|
493
682
|
});
|
|
494
|
-
indexer.hooks.hook("connect:before", async ({ request }) => {
|
|
683
|
+
indexer$1.hooks.hook("connect:before", async ({ request }) => {
|
|
495
684
|
if (!enablePersistence) {
|
|
496
685
|
return;
|
|
497
686
|
}
|
|
@@ -508,19 +697,19 @@ function drizzleStorage({
|
|
|
508
697
|
}
|
|
509
698
|
});
|
|
510
699
|
});
|
|
511
|
-
indexer.hooks.hook("connect:after", async ({ request }) => {
|
|
700
|
+
indexer$1.hooks.hook("connect:after", async ({ request }) => {
|
|
512
701
|
const cursor = request.startingCursor;
|
|
513
702
|
if (!cursor) {
|
|
514
703
|
return;
|
|
515
704
|
}
|
|
516
705
|
await withTransaction(db, async (tx) => {
|
|
517
|
-
await invalidate(tx, cursor,
|
|
706
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
518
707
|
if (enablePersistence) {
|
|
519
708
|
await invalidateState({ tx, cursor, indexerId });
|
|
520
709
|
}
|
|
521
710
|
});
|
|
522
711
|
});
|
|
523
|
-
indexer.hooks.hook("connect:factory", async ({ request, endCursor }) => {
|
|
712
|
+
indexer$1.hooks.hook("connect:factory", async ({ request, endCursor }) => {
|
|
524
713
|
if (!enablePersistence) {
|
|
525
714
|
return;
|
|
526
715
|
}
|
|
@@ -534,8 +723,8 @@ function drizzleStorage({
|
|
|
534
723
|
});
|
|
535
724
|
}
|
|
536
725
|
});
|
|
537
|
-
indexer.hooks.hook("message:finalize", async ({ message }) => {
|
|
538
|
-
const { cursor } = message
|
|
726
|
+
indexer$1.hooks.hook("message:finalize", async ({ message }) => {
|
|
727
|
+
const { cursor } = message;
|
|
539
728
|
if (!cursor) {
|
|
540
729
|
throw new DrizzleStorageError("Finalized Cursor is undefined");
|
|
541
730
|
}
|
|
@@ -546,45 +735,49 @@ function drizzleStorage({
|
|
|
546
735
|
}
|
|
547
736
|
});
|
|
548
737
|
});
|
|
549
|
-
indexer.hooks.hook("message:invalidate", async ({ message }) => {
|
|
550
|
-
const { cursor } = message
|
|
738
|
+
indexer$1.hooks.hook("message:invalidate", async ({ message }) => {
|
|
739
|
+
const { cursor } = message;
|
|
551
740
|
if (!cursor) {
|
|
552
741
|
throw new DrizzleStorageError("Invalidate Cursor is undefined");
|
|
553
742
|
}
|
|
554
743
|
await withTransaction(db, async (tx) => {
|
|
555
|
-
await invalidate(tx, cursor,
|
|
744
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
556
745
|
if (enablePersistence) {
|
|
557
746
|
await invalidateState({ tx, cursor, indexerId });
|
|
558
747
|
}
|
|
559
748
|
});
|
|
560
749
|
});
|
|
561
|
-
indexer.hooks.hook("handler:middleware", async ({ use }) => {
|
|
750
|
+
indexer$1.hooks.hook("handler:middleware", async ({ use }) => {
|
|
562
751
|
use(async (context, next) => {
|
|
563
752
|
try {
|
|
564
|
-
const { endCursor, finality } = context;
|
|
753
|
+
const { endCursor, finality, cursor } = context;
|
|
565
754
|
if (!endCursor) {
|
|
566
755
|
throw new DrizzleStorageError("End Cursor is undefined");
|
|
567
756
|
}
|
|
568
757
|
await withTransaction(db, async (tx) => {
|
|
569
|
-
context[DRIZZLE_PROPERTY] = { db: tx };
|
|
758
|
+
context[constants.DRIZZLE_PROPERTY] = { db: tx };
|
|
759
|
+
if (prevFinality === "pending") {
|
|
760
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
761
|
+
}
|
|
570
762
|
if (finality !== "finalized") {
|
|
571
763
|
await registerTriggers(
|
|
572
764
|
tx,
|
|
573
765
|
tableNames,
|
|
574
766
|
endCursor,
|
|
575
|
-
|
|
767
|
+
idColumnMap,
|
|
576
768
|
indexerId
|
|
577
769
|
);
|
|
578
770
|
}
|
|
579
771
|
await next();
|
|
580
|
-
delete context[DRIZZLE_PROPERTY];
|
|
581
|
-
if (enablePersistence) {
|
|
772
|
+
delete context[constants.DRIZZLE_PROPERTY];
|
|
773
|
+
if (enablePersistence && finality !== "pending") {
|
|
582
774
|
await persistState({
|
|
583
775
|
tx,
|
|
584
776
|
endCursor,
|
|
585
777
|
indexerId
|
|
586
778
|
});
|
|
587
779
|
}
|
|
780
|
+
prevFinality = finality;
|
|
588
781
|
});
|
|
589
782
|
if (finality !== "finalized") {
|
|
590
783
|
await removeTriggers(db, tableNames, indexerId);
|
|
@@ -600,5 +793,9 @@ function drizzleStorage({
|
|
|
600
793
|
});
|
|
601
794
|
}
|
|
602
795
|
|
|
796
|
+
exports.drizzle = drizzle;
|
|
603
797
|
exports.drizzleStorage = drizzleStorage;
|
|
798
|
+
exports.migrate = migrate;
|
|
604
799
|
exports.useDrizzleStorage = useDrizzleStorage;
|
|
800
|
+
exports.useTestDrizzleStorage = useTestDrizzleStorage;
|
|
801
|
+
//# sourceMappingURL=index.cjs.map
|