@apibara/plugin-drizzle 2.1.0-beta.2 → 2.1.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +231 -55
- package/dist/index.d.cts +165 -4
- package/dist/index.d.mts +165 -4
- package/dist/index.d.ts +165 -4
- package/dist/index.mjs +217 -47
- package/dist/shared/plugin-drizzle.2d226351.mjs +5 -0
- package/dist/shared/plugin-drizzle.cae20704.cjs +9 -0
- package/dist/testing.cjs +13 -0
- package/dist/testing.d.cts +6 -0
- package/dist/testing.d.mts +6 -0
- package/dist/testing.d.ts +6 -0
- package/dist/testing.mjs +11 -0
- package/package.json +12 -6
- package/src/constants.ts +3 -0
- package/src/helper.ts +202 -0
- package/src/index.ts +140 -15
- package/src/persistence.ts +60 -18
- package/src/storage.ts +81 -17
- package/src/testing.ts +13 -0
- package/src/utils.ts +19 -0
package/dist/index.mjs
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import { useIndexerContext } from '@apibara/indexer';
|
|
2
|
-
import { defineIndexerPlugin } from '@apibara/indexer/plugins';
|
|
2
|
+
import { defineIndexerPlugin, useLogger } from '@apibara/indexer/plugins';
|
|
3
3
|
import { generateIndexerId } from '@apibara/indexer/internal';
|
|
4
4
|
import { useInternalContext } from '@apibara/indexer/internal/plugins';
|
|
5
|
+
import { S as SCHEMA_NAME, D as DRIZZLE_PROPERTY, a as DRIZZLE_STORAGE_DB_PROPERTY } from './shared/plugin-drizzle.2d226351.mjs';
|
|
6
|
+
import { PGlite } from '@electric-sql/pglite';
|
|
7
|
+
import { drizzle as drizzle$1 } from 'drizzle-orm/node-postgres';
|
|
8
|
+
import { migrate as migrate$2 } from 'drizzle-orm/node-postgres/migrator';
|
|
9
|
+
import { drizzle as drizzle$2 } from 'drizzle-orm/pglite';
|
|
10
|
+
import { migrate as migrate$1 } from 'drizzle-orm/pglite/migrator';
|
|
11
|
+
import pg from 'pg';
|
|
5
12
|
import { normalizeCursor } from '@apibara/protocol';
|
|
6
|
-
import { eq, and, isNull, gt, lt
|
|
7
|
-
import {
|
|
13
|
+
import { sql, eq, and, isNull, gt, lt } from 'drizzle-orm';
|
|
14
|
+
import { pgSchema, text, integer, primaryKey, serial, char, jsonb } from 'drizzle-orm/pg-core';
|
|
8
15
|
|
|
9
16
|
class DrizzleStorageError extends Error {
|
|
10
17
|
constructor(message, options) {
|
|
@@ -33,16 +40,70 @@ function serialize(obj) {
|
|
|
33
40
|
function sleep(ms) {
|
|
34
41
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
35
42
|
}
|
|
43
|
+
const getIdColumnForTable = (tableName, idColumn) => {
|
|
44
|
+
if (idColumn[tableName]) {
|
|
45
|
+
return idColumn[tableName];
|
|
46
|
+
}
|
|
47
|
+
return idColumn["*"];
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
function drizzle(options) {
|
|
51
|
+
const {
|
|
52
|
+
connectionString = process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://",
|
|
53
|
+
schema,
|
|
54
|
+
type = "pglite",
|
|
55
|
+
config,
|
|
56
|
+
poolConfig
|
|
57
|
+
} = options ?? {};
|
|
58
|
+
if (isPostgresConnectionString(connectionString) || type === "node-postgres") {
|
|
59
|
+
const pool = new pg.Pool({
|
|
60
|
+
connectionString,
|
|
61
|
+
...poolConfig || {}
|
|
62
|
+
});
|
|
63
|
+
return drizzle$1(pool, { schema, ...config || {} });
|
|
64
|
+
}
|
|
65
|
+
if (type === "pglite") {
|
|
66
|
+
return drizzle$2({
|
|
67
|
+
schema,
|
|
68
|
+
connection: {
|
|
69
|
+
dataDir: connectionString || "memory://pglite"
|
|
70
|
+
},
|
|
71
|
+
...config || {}
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
throw new Error("Invalid database type");
|
|
75
|
+
}
|
|
76
|
+
async function migrate(db, options) {
|
|
77
|
+
const isPglite = !!("$client" in db && db.$client instanceof PGlite);
|
|
78
|
+
try {
|
|
79
|
+
if (isPglite) {
|
|
80
|
+
await migrate$1(db, options);
|
|
81
|
+
} else {
|
|
82
|
+
await migrate$2(db, options);
|
|
83
|
+
}
|
|
84
|
+
} catch (error) {
|
|
85
|
+
throw new DrizzleStorageError(
|
|
86
|
+
"Failed to apply migrations! Please check if you have generated migrations using drizzle:generate",
|
|
87
|
+
{
|
|
88
|
+
cause: error
|
|
89
|
+
}
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function isPostgresConnectionString(conn) {
|
|
94
|
+
return conn.startsWith("postgres://") || conn.startsWith("postgresql://");
|
|
95
|
+
}
|
|
36
96
|
|
|
37
|
-
const CHECKPOINTS_TABLE_NAME = "
|
|
38
|
-
const FILTERS_TABLE_NAME = "
|
|
39
|
-
const SCHEMA_VERSION_TABLE_NAME = "
|
|
40
|
-
const
|
|
97
|
+
const CHECKPOINTS_TABLE_NAME = "checkpoints";
|
|
98
|
+
const FILTERS_TABLE_NAME = "filters";
|
|
99
|
+
const SCHEMA_VERSION_TABLE_NAME = "schema_version";
|
|
100
|
+
const schema$1 = pgSchema(SCHEMA_NAME);
|
|
101
|
+
const checkpoints = schema$1.table(CHECKPOINTS_TABLE_NAME, {
|
|
41
102
|
id: text("id").notNull().primaryKey(),
|
|
42
103
|
orderKey: integer("order_key").notNull(),
|
|
43
104
|
uniqueKey: text("unique_key")
|
|
44
105
|
});
|
|
45
|
-
const filters =
|
|
106
|
+
const filters = schema$1.table(
|
|
46
107
|
FILTERS_TABLE_NAME,
|
|
47
108
|
{
|
|
48
109
|
id: text("id").notNull(),
|
|
@@ -56,7 +117,7 @@ const filters = pgTable(
|
|
|
56
117
|
}
|
|
57
118
|
]
|
|
58
119
|
);
|
|
59
|
-
const schemaVersion =
|
|
120
|
+
const schemaVersion = schema$1.table(SCHEMA_VERSION_TABLE_NAME, {
|
|
60
121
|
k: integer("k").notNull().primaryKey(),
|
|
61
122
|
version: integer("version").notNull()
|
|
62
123
|
});
|
|
@@ -67,12 +128,19 @@ const MIGRATIONS = [
|
|
|
67
128
|
// Add more migration arrays for future versions
|
|
68
129
|
];
|
|
69
130
|
async function initializePersistentState(tx) {
|
|
70
|
-
await tx.execute(
|
|
71
|
-
|
|
131
|
+
await tx.execute(
|
|
132
|
+
sql.raw(`
|
|
133
|
+
CREATE SCHEMA IF NOT EXISTS ${SCHEMA_NAME};
|
|
134
|
+
`)
|
|
135
|
+
);
|
|
136
|
+
await tx.execute(
|
|
137
|
+
sql.raw(`
|
|
138
|
+
CREATE TABLE IF NOT EXISTS ${SCHEMA_NAME}.${SCHEMA_VERSION_TABLE_NAME} (
|
|
72
139
|
k INTEGER PRIMARY KEY,
|
|
73
140
|
version INTEGER NOT NULL
|
|
74
141
|
);
|
|
75
|
-
`)
|
|
142
|
+
`)
|
|
143
|
+
);
|
|
76
144
|
const versionRows = await tx.select().from(schemaVersion).where(eq(schemaVersion.k, 0));
|
|
77
145
|
const storedVersion = versionRows[0]?.version ?? -1;
|
|
78
146
|
if (storedVersion > CURRENT_SCHEMA_VERSION) {
|
|
@@ -82,22 +150,26 @@ async function initializePersistentState(tx) {
|
|
|
82
150
|
}
|
|
83
151
|
try {
|
|
84
152
|
if (storedVersion === -1) {
|
|
85
|
-
await tx.execute(
|
|
86
|
-
|
|
153
|
+
await tx.execute(
|
|
154
|
+
sql.raw(`
|
|
155
|
+
CREATE TABLE IF NOT EXISTS ${SCHEMA_NAME}.${CHECKPOINTS_TABLE_NAME} (
|
|
87
156
|
id TEXT PRIMARY KEY,
|
|
88
157
|
order_key INTEGER NOT NULL,
|
|
89
158
|
unique_key TEXT
|
|
90
159
|
);
|
|
91
|
-
`)
|
|
92
|
-
|
|
93
|
-
|
|
160
|
+
`)
|
|
161
|
+
);
|
|
162
|
+
await tx.execute(
|
|
163
|
+
sql.raw(`
|
|
164
|
+
CREATE TABLE IF NOT EXISTS ${SCHEMA_NAME}.${FILTERS_TABLE_NAME} (
|
|
94
165
|
id TEXT NOT NULL,
|
|
95
166
|
filter TEXT NOT NULL,
|
|
96
167
|
from_block INTEGER NOT NULL,
|
|
97
168
|
to_block INTEGER DEFAULT NULL,
|
|
98
169
|
PRIMARY KEY (id, from_block)
|
|
99
170
|
);
|
|
100
|
-
`)
|
|
171
|
+
`)
|
|
172
|
+
);
|
|
101
173
|
await tx.insert(schemaVersion).values({
|
|
102
174
|
k: 0,
|
|
103
175
|
version: CURRENT_SCHEMA_VERSION
|
|
@@ -132,7 +204,9 @@ async function persistState(props) {
|
|
|
132
204
|
target: checkpoints.id,
|
|
133
205
|
set: {
|
|
134
206
|
orderKey: Number(endCursor.orderKey),
|
|
135
|
-
|
|
207
|
+
// Explicitly set the unique key to `null` to indicate that it has been deleted
|
|
208
|
+
// Otherwise drizzle will not update its value.
|
|
209
|
+
uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null
|
|
136
210
|
}
|
|
137
211
|
});
|
|
138
212
|
if (filter) {
|
|
@@ -211,11 +285,24 @@ async function finalizeState(props) {
|
|
|
211
285
|
});
|
|
212
286
|
}
|
|
213
287
|
}
|
|
288
|
+
async function resetPersistence(props) {
|
|
289
|
+
const { tx, indexerId } = props;
|
|
290
|
+
try {
|
|
291
|
+
await tx.delete(checkpoints).where(eq(checkpoints.id, indexerId));
|
|
292
|
+
await tx.delete(filters).where(eq(filters.id, indexerId));
|
|
293
|
+
} catch (error) {
|
|
294
|
+
throw new DrizzleStorageError("Failed to reset persistence state", {
|
|
295
|
+
cause: error
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
}
|
|
214
299
|
|
|
300
|
+
const ROLLBACK_TABLE_NAME = "reorg_rollback";
|
|
301
|
+
const schema = pgSchema(SCHEMA_NAME);
|
|
215
302
|
function getReorgTriggerName(table, indexerId) {
|
|
216
303
|
return `${table}_reorg_${indexerId}`;
|
|
217
304
|
}
|
|
218
|
-
|
|
305
|
+
schema.table(ROLLBACK_TABLE_NAME, {
|
|
219
306
|
n: serial("n").primaryKey(),
|
|
220
307
|
op: char("op", { length: 1 }).$type().notNull(),
|
|
221
308
|
table_name: text("table_name").notNull(),
|
|
@@ -226,9 +313,12 @@ pgTable("__reorg_rollback", {
|
|
|
226
313
|
});
|
|
227
314
|
async function initializeReorgRollbackTable(tx, indexerId) {
|
|
228
315
|
try {
|
|
316
|
+
await tx.execute(`
|
|
317
|
+
CREATE SCHEMA IF NOT EXISTS ${SCHEMA_NAME};
|
|
318
|
+
`);
|
|
229
319
|
await tx.execute(
|
|
230
320
|
sql.raw(`
|
|
231
|
-
CREATE TABLE IF NOT EXISTS
|
|
321
|
+
CREATE TABLE IF NOT EXISTS ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(
|
|
232
322
|
n SERIAL PRIMARY KEY,
|
|
233
323
|
op CHAR(1) NOT NULL,
|
|
234
324
|
table_name TEXT NOT NULL,
|
|
@@ -241,7 +331,7 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
241
331
|
);
|
|
242
332
|
await tx.execute(
|
|
243
333
|
sql.raw(`
|
|
244
|
-
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON
|
|
334
|
+
CREATE INDEX IF NOT EXISTS idx_reorg_rollback_indexer_id_cursor ON ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(indexer_id, cursor);
|
|
245
335
|
`)
|
|
246
336
|
);
|
|
247
337
|
} catch (error) {
|
|
@@ -252,7 +342,7 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
252
342
|
try {
|
|
253
343
|
await tx.execute(
|
|
254
344
|
sql.raw(`
|
|
255
|
-
CREATE OR REPLACE FUNCTION reorg_checkpoint()
|
|
345
|
+
CREATE OR REPLACE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint()
|
|
256
346
|
RETURNS TRIGGER AS $$
|
|
257
347
|
DECLARE
|
|
258
348
|
id_col TEXT := TG_ARGV[0]::TEXT;
|
|
@@ -262,13 +352,13 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
262
352
|
old_id_value TEXT := row_to_json(OLD.*)->>id_col;
|
|
263
353
|
BEGIN
|
|
264
354
|
IF (TG_OP = 'DELETE') THEN
|
|
265
|
-
INSERT INTO
|
|
355
|
+
INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
266
356
|
SELECT 'D', TG_TABLE_NAME, order_key, old_id_value, row_to_json(OLD.*), indexer_id;
|
|
267
357
|
ELSIF (TG_OP = 'UPDATE') THEN
|
|
268
|
-
INSERT INTO
|
|
358
|
+
INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
269
359
|
SELECT 'U', TG_TABLE_NAME, order_key, new_id_value, row_to_json(OLD.*), indexer_id;
|
|
270
360
|
ELSIF (TG_OP = 'INSERT') THEN
|
|
271
|
-
INSERT INTO
|
|
361
|
+
INSERT INTO ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}(op, table_name, cursor, row_id, row_value, indexer_id)
|
|
272
362
|
SELECT 'I', TG_TABLE_NAME, order_key, new_id_value, null, indexer_id;
|
|
273
363
|
END IF;
|
|
274
364
|
RETURN NULL;
|
|
@@ -285,9 +375,10 @@ async function initializeReorgRollbackTable(tx, indexerId) {
|
|
|
285
375
|
);
|
|
286
376
|
}
|
|
287
377
|
}
|
|
288
|
-
async function registerTriggers(tx, tables, endCursor,
|
|
378
|
+
async function registerTriggers(tx, tables, endCursor, idColumnMap, indexerId) {
|
|
289
379
|
try {
|
|
290
380
|
for (const table of tables) {
|
|
381
|
+
const tableIdColumn = getIdColumnForTable(table, idColumnMap);
|
|
291
382
|
await tx.execute(
|
|
292
383
|
sql.raw(
|
|
293
384
|
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
@@ -298,7 +389,7 @@ async function registerTriggers(tx, tables, endCursor, idColumn, indexerId) {
|
|
|
298
389
|
CREATE CONSTRAINT TRIGGER ${getReorgTriggerName(table, indexerId)}
|
|
299
390
|
AFTER INSERT OR UPDATE OR DELETE ON ${table}
|
|
300
391
|
DEFERRABLE INITIALLY DEFERRED
|
|
301
|
-
FOR EACH ROW EXECUTE FUNCTION reorg_checkpoint('${
|
|
392
|
+
FOR EACH ROW EXECUTE FUNCTION ${SCHEMA_NAME}.reorg_checkpoint('${tableIdColumn}', ${Number(endCursor.orderKey)}, '${indexerId}');
|
|
302
393
|
`)
|
|
303
394
|
);
|
|
304
395
|
}
|
|
@@ -323,11 +414,11 @@ async function removeTriggers(db, tables, indexerId) {
|
|
|
323
414
|
});
|
|
324
415
|
}
|
|
325
416
|
}
|
|
326
|
-
async function invalidate(tx, cursor,
|
|
417
|
+
async function invalidate(tx, cursor, idColumnMap, indexerId) {
|
|
327
418
|
const { rows: result } = await tx.execute(
|
|
328
419
|
sql.raw(`
|
|
329
420
|
WITH deleted AS (
|
|
330
|
-
DELETE FROM
|
|
421
|
+
DELETE FROM ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
331
422
|
WHERE cursor > ${Number(cursor.orderKey)}
|
|
332
423
|
AND indexer_id = '${indexerId}'
|
|
333
424
|
RETURNING *
|
|
@@ -341,6 +432,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
341
432
|
);
|
|
342
433
|
}
|
|
343
434
|
for (const op of result) {
|
|
435
|
+
const tableIdColumn = getIdColumnForTable(op.table_name, idColumnMap);
|
|
344
436
|
switch (op.op) {
|
|
345
437
|
case "I":
|
|
346
438
|
try {
|
|
@@ -350,7 +442,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
350
442
|
await tx.execute(
|
|
351
443
|
sql.raw(`
|
|
352
444
|
DELETE FROM ${op.table_name}
|
|
353
|
-
WHERE ${
|
|
445
|
+
WHERE ${tableIdColumn} = '${op.row_id}'
|
|
354
446
|
`)
|
|
355
447
|
);
|
|
356
448
|
} catch (error) {
|
|
@@ -390,7 +482,9 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
390
482
|
);
|
|
391
483
|
}
|
|
392
484
|
const rowValue = typeof op.row_value === "string" ? JSON.parse(op.row_value) : op.row_value;
|
|
393
|
-
const nonIdKeys = Object.keys(rowValue).filter(
|
|
485
|
+
const nonIdKeys = Object.keys(rowValue).filter(
|
|
486
|
+
(k) => k !== tableIdColumn
|
|
487
|
+
);
|
|
394
488
|
const fields = nonIdKeys.map((c) => `${c} = prev.${c}`).join(", ");
|
|
395
489
|
const query = sql.raw(`
|
|
396
490
|
UPDATE ${op.table_name}
|
|
@@ -398,7 +492,7 @@ async function invalidate(tx, cursor, idColumn, indexerId) {
|
|
|
398
492
|
FROM (
|
|
399
493
|
SELECT * FROM json_populate_record(null::${op.table_name}, '${JSON.stringify(op.row_value)}'::json)
|
|
400
494
|
) as prev
|
|
401
|
-
WHERE ${op.table_name}.${
|
|
495
|
+
WHERE ${op.table_name}.${tableIdColumn} = '${op.row_id}'
|
|
402
496
|
`);
|
|
403
497
|
await tx.execute(query);
|
|
404
498
|
} catch (error) {
|
|
@@ -420,7 +514,7 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
420
514
|
try {
|
|
421
515
|
await tx.execute(
|
|
422
516
|
sql.raw(`
|
|
423
|
-
DELETE FROM
|
|
517
|
+
DELETE FROM ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
424
518
|
WHERE cursor <= ${Number(cursor.orderKey)}
|
|
425
519
|
AND indexer_id = '${indexerId}'
|
|
426
520
|
`)
|
|
@@ -431,8 +525,37 @@ async function finalize(tx, cursor, indexerId) {
|
|
|
431
525
|
});
|
|
432
526
|
}
|
|
433
527
|
}
|
|
528
|
+
async function cleanupStorage(tx, tables, indexerId) {
|
|
529
|
+
try {
|
|
530
|
+
for (const table of tables) {
|
|
531
|
+
await tx.execute(
|
|
532
|
+
sql.raw(
|
|
533
|
+
`DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
|
|
534
|
+
)
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
await tx.execute(
|
|
538
|
+
sql.raw(`
|
|
539
|
+
DELETE FROM ${SCHEMA_NAME}.${ROLLBACK_TABLE_NAME}
|
|
540
|
+
WHERE indexer_id = '${indexerId}'
|
|
541
|
+
`)
|
|
542
|
+
);
|
|
543
|
+
for (const table of tables) {
|
|
544
|
+
try {
|
|
545
|
+
await tx.execute(sql.raw(`TRUNCATE TABLE ${table} CASCADE;`));
|
|
546
|
+
} catch (error) {
|
|
547
|
+
throw new DrizzleStorageError(`Failed to truncate table ${table}`, {
|
|
548
|
+
cause: error
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
} catch (error) {
|
|
553
|
+
throw new DrizzleStorageError("Failed to clean up storage", {
|
|
554
|
+
cause: error
|
|
555
|
+
});
|
|
556
|
+
}
|
|
557
|
+
}
|
|
434
558
|
|
|
435
|
-
const DRIZZLE_PROPERTY = "_drizzle";
|
|
436
559
|
const MAX_RETRIES = 5;
|
|
437
560
|
function useDrizzleStorage(_db) {
|
|
438
561
|
const context = useIndexerContext();
|
|
@@ -447,36 +570,79 @@ function drizzleStorage({
|
|
|
447
570
|
db,
|
|
448
571
|
persistState: enablePersistence = true,
|
|
449
572
|
indexerName: identifier = "default",
|
|
450
|
-
schema,
|
|
451
|
-
idColumn
|
|
573
|
+
schema: _schema,
|
|
574
|
+
idColumn,
|
|
575
|
+
migrate: migrateOptions
|
|
452
576
|
}) {
|
|
453
577
|
return defineIndexerPlugin((indexer) => {
|
|
454
578
|
let tableNames = [];
|
|
455
579
|
let indexerId = "";
|
|
580
|
+
const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
|
|
581
|
+
let prevFinality;
|
|
582
|
+
const schema = _schema ?? db._.schema ?? {};
|
|
583
|
+
const idColumnMap = {
|
|
584
|
+
"*": typeof idColumn === "string" ? idColumn : "id",
|
|
585
|
+
...typeof idColumn === "object" ? idColumn : {}
|
|
586
|
+
};
|
|
456
587
|
try {
|
|
457
|
-
tableNames = Object.values(schema
|
|
458
|
-
(table) => table.dbName
|
|
459
|
-
);
|
|
588
|
+
tableNames = Object.values(schema).map((table) => table.dbName);
|
|
460
589
|
} catch (error) {
|
|
461
590
|
throw new DrizzleStorageError("Failed to get table names from schema", {
|
|
462
591
|
cause: error
|
|
463
592
|
});
|
|
464
593
|
}
|
|
594
|
+
for (const table of Object.values(schema)) {
|
|
595
|
+
const columns = table.columns;
|
|
596
|
+
const tableIdColumn = getIdColumnForTable(table.dbName, idColumnMap);
|
|
597
|
+
const columnExists = Object.values(columns).some(
|
|
598
|
+
(column) => column.name === tableIdColumn
|
|
599
|
+
);
|
|
600
|
+
if (!columnExists) {
|
|
601
|
+
throw new DrizzleStorageError(
|
|
602
|
+
`Column \`"${tableIdColumn}"\` does not exist in table \`"${table.dbName}"\`. Make sure the table has the specified column or provide a valid \`idColumn\` mapping to \`drizzleStorage\`.`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
465
606
|
indexer.hooks.hook("run:before", async () => {
|
|
466
|
-
const
|
|
607
|
+
const internalContext = useInternalContext();
|
|
608
|
+
const context = useIndexerContext();
|
|
609
|
+
const logger = useLogger();
|
|
610
|
+
context[DRIZZLE_STORAGE_DB_PROPERTY] = db;
|
|
611
|
+
const { indexerName: indexerFileName, availableIndexers } = internalContext;
|
|
467
612
|
indexerId = generateIndexerId(indexerFileName, identifier);
|
|
468
613
|
let retries = 0;
|
|
614
|
+
let migrationsApplied = false;
|
|
615
|
+
let cleanupApplied = false;
|
|
469
616
|
while (retries <= MAX_RETRIES) {
|
|
470
617
|
try {
|
|
618
|
+
if (migrateOptions && !migrationsApplied) {
|
|
619
|
+
await migrate(db, migrateOptions);
|
|
620
|
+
migrationsApplied = true;
|
|
621
|
+
logger.success("Migrations applied");
|
|
622
|
+
}
|
|
471
623
|
await withTransaction(db, async (tx) => {
|
|
472
624
|
await initializeReorgRollbackTable(tx, indexerId);
|
|
473
625
|
if (enablePersistence) {
|
|
474
626
|
await initializePersistentState(tx);
|
|
475
627
|
}
|
|
628
|
+
if (alwaysReindex && !cleanupApplied) {
|
|
629
|
+
logger.warn(
|
|
630
|
+
`Reindexing: Deleting all data from tables - ${tableNames.join(", ")}`
|
|
631
|
+
);
|
|
632
|
+
await cleanupStorage(tx, tableNames, indexerId);
|
|
633
|
+
if (enablePersistence) {
|
|
634
|
+
await resetPersistence({ tx, indexerId });
|
|
635
|
+
}
|
|
636
|
+
cleanupApplied = true;
|
|
637
|
+
logger.success("Tables have been cleaned up for reindexing");
|
|
638
|
+
}
|
|
476
639
|
});
|
|
477
640
|
break;
|
|
478
641
|
} catch (error) {
|
|
479
642
|
if (retries === MAX_RETRIES) {
|
|
643
|
+
if (error instanceof DrizzleStorageError) {
|
|
644
|
+
throw error;
|
|
645
|
+
}
|
|
480
646
|
throw new DrizzleStorageError(
|
|
481
647
|
"Initialization failed after 5 retries",
|
|
482
648
|
{
|
|
@@ -512,7 +678,7 @@ function drizzleStorage({
|
|
|
512
678
|
return;
|
|
513
679
|
}
|
|
514
680
|
await withTransaction(db, async (tx) => {
|
|
515
|
-
await invalidate(tx, cursor,
|
|
681
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
516
682
|
if (enablePersistence) {
|
|
517
683
|
await invalidateState({ tx, cursor, indexerId });
|
|
518
684
|
}
|
|
@@ -550,7 +716,7 @@ function drizzleStorage({
|
|
|
550
716
|
throw new DrizzleStorageError("Invalidate Cursor is undefined");
|
|
551
717
|
}
|
|
552
718
|
await withTransaction(db, async (tx) => {
|
|
553
|
-
await invalidate(tx, cursor,
|
|
719
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
554
720
|
if (enablePersistence) {
|
|
555
721
|
await invalidateState({ tx, cursor, indexerId });
|
|
556
722
|
}
|
|
@@ -559,30 +725,34 @@ function drizzleStorage({
|
|
|
559
725
|
indexer.hooks.hook("handler:middleware", async ({ use }) => {
|
|
560
726
|
use(async (context, next) => {
|
|
561
727
|
try {
|
|
562
|
-
const { endCursor, finality } = context;
|
|
728
|
+
const { endCursor, finality, cursor } = context;
|
|
563
729
|
if (!endCursor) {
|
|
564
730
|
throw new DrizzleStorageError("End Cursor is undefined");
|
|
565
731
|
}
|
|
566
732
|
await withTransaction(db, async (tx) => {
|
|
567
733
|
context[DRIZZLE_PROPERTY] = { db: tx };
|
|
734
|
+
if (prevFinality === "pending") {
|
|
735
|
+
await invalidate(tx, cursor, idColumnMap, indexerId);
|
|
736
|
+
}
|
|
568
737
|
if (finality !== "finalized") {
|
|
569
738
|
await registerTriggers(
|
|
570
739
|
tx,
|
|
571
740
|
tableNames,
|
|
572
741
|
endCursor,
|
|
573
|
-
|
|
742
|
+
idColumnMap,
|
|
574
743
|
indexerId
|
|
575
744
|
);
|
|
576
745
|
}
|
|
577
746
|
await next();
|
|
578
747
|
delete context[DRIZZLE_PROPERTY];
|
|
579
|
-
if (enablePersistence) {
|
|
748
|
+
if (enablePersistence && finality !== "pending") {
|
|
580
749
|
await persistState({
|
|
581
750
|
tx,
|
|
582
751
|
endCursor,
|
|
583
752
|
indexerId
|
|
584
753
|
});
|
|
585
754
|
}
|
|
755
|
+
prevFinality = finality;
|
|
586
756
|
});
|
|
587
757
|
if (finality !== "finalized") {
|
|
588
758
|
await removeTriggers(db, tableNames, indexerId);
|
|
@@ -598,4 +768,4 @@ function drizzleStorage({
|
|
|
598
768
|
});
|
|
599
769
|
}
|
|
600
770
|
|
|
601
|
-
export { drizzleStorage, useDrizzleStorage };
|
|
771
|
+
export { drizzle, drizzleStorage, migrate, useDrizzleStorage };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const DRIZZLE_PROPERTY = "_drizzle";
|
|
4
|
+
const DRIZZLE_STORAGE_DB_PROPERTY = "_drizzleStorageDB";
|
|
5
|
+
const SCHEMA_NAME = "airfoil";
|
|
6
|
+
|
|
7
|
+
exports.DRIZZLE_PROPERTY = DRIZZLE_PROPERTY;
|
|
8
|
+
exports.DRIZZLE_STORAGE_DB_PROPERTY = DRIZZLE_STORAGE_DB_PROPERTY;
|
|
9
|
+
exports.SCHEMA_NAME = SCHEMA_NAME;
|
package/dist/testing.cjs
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const constants = require('./shared/plugin-drizzle.cae20704.cjs');
|
|
4
|
+
|
|
5
|
+
function getTestDatabase(context) {
|
|
6
|
+
const db = context[constants.DRIZZLE_STORAGE_DB_PROPERTY];
|
|
7
|
+
if (!db) {
|
|
8
|
+
throw new Error("Drizzle database not found in context");
|
|
9
|
+
}
|
|
10
|
+
return db;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
exports.getTestDatabase = getTestDatabase;
|
package/dist/testing.mjs
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { a as DRIZZLE_STORAGE_DB_PROPERTY } from './shared/plugin-drizzle.2d226351.mjs';
|
|
2
|
+
|
|
3
|
+
function getTestDatabase(context) {
|
|
4
|
+
const db = context[DRIZZLE_STORAGE_DB_PROPERTY];
|
|
5
|
+
if (!db) {
|
|
6
|
+
throw new Error("Drizzle database not found in context");
|
|
7
|
+
}
|
|
8
|
+
return db;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export { getTestDatabase };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apibara/plugin-drizzle",
|
|
3
|
-
"version": "2.1.0-beta.
|
|
3
|
+
"version": "2.1.0-beta.21",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"files": [
|
|
6
6
|
"dist",
|
|
@@ -15,6 +15,12 @@
|
|
|
15
15
|
"import": "./dist/index.mjs",
|
|
16
16
|
"require": "./dist/index.cjs",
|
|
17
17
|
"default": "./dist/index.mjs"
|
|
18
|
+
},
|
|
19
|
+
"./testing": {
|
|
20
|
+
"types": "./dist/testing.d.ts",
|
|
21
|
+
"import": "./dist/testing.mjs",
|
|
22
|
+
"require": "./dist/testing.cjs",
|
|
23
|
+
"default": "./dist/testing.mjs"
|
|
18
24
|
}
|
|
19
25
|
},
|
|
20
26
|
"scripts": {
|
|
@@ -26,21 +32,21 @@
|
|
|
26
32
|
"test:ci": "vitest run"
|
|
27
33
|
},
|
|
28
34
|
"peerDependencies": {
|
|
29
|
-
"drizzle-orm": "
|
|
30
|
-
"pg": "
|
|
35
|
+
"drizzle-orm": "<1",
|
|
36
|
+
"pg": ">=8"
|
|
31
37
|
},
|
|
32
38
|
"devDependencies": {
|
|
33
39
|
"@electric-sql/pglite": "^0.2.17",
|
|
34
40
|
"@types/node": "^20.14.0",
|
|
35
41
|
"@types/pg": "^8.11.10",
|
|
36
|
-
"drizzle-orm": "^0.
|
|
42
|
+
"drizzle-orm": "^0.40.1",
|
|
37
43
|
"pg": "^8.13.1",
|
|
38
44
|
"unbuild": "^2.0.0",
|
|
39
45
|
"vitest": "^1.6.0"
|
|
40
46
|
},
|
|
41
47
|
"dependencies": {
|
|
42
|
-
"@apibara/indexer": "2.1.0-beta.
|
|
43
|
-
"@apibara/protocol": "2.1.0-beta.
|
|
48
|
+
"@apibara/indexer": "2.1.0-beta.21",
|
|
49
|
+
"@apibara/protocol": "2.1.0-beta.21",
|
|
44
50
|
"postgres-range": "^1.1.4"
|
|
45
51
|
}
|
|
46
52
|
}
|
package/src/constants.ts
ADDED