@apibara/indexer 2.0.0-beta.0 → 2.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/package.json +30 -19
  2. package/src/context.ts +8 -3
  3. package/src/hooks/index.ts +1 -0
  4. package/src/hooks/useSink.ts +13 -0
  5. package/src/index.ts +1 -0
  6. package/src/indexer.test.ts +70 -41
  7. package/src/indexer.ts +168 -168
  8. package/src/plugins/config.ts +4 -4
  9. package/src/plugins/kv.ts +2 -2
  10. package/src/plugins/persistence.test.ts +10 -6
  11. package/src/plugins/persistence.ts +3 -3
  12. package/src/sink.ts +21 -24
  13. package/src/sinks/csv.test.ts +15 -3
  14. package/src/sinks/csv.ts +68 -7
  15. package/src/sinks/drizzle/Int8Range.ts +52 -0
  16. package/src/sinks/drizzle/delete.ts +42 -0
  17. package/src/sinks/drizzle/drizzle.test.ts +239 -0
  18. package/src/sinks/drizzle/drizzle.ts +115 -0
  19. package/src/sinks/drizzle/index.ts +6 -0
  20. package/src/sinks/drizzle/insert.ts +39 -0
  21. package/src/sinks/drizzle/select.ts +44 -0
  22. package/src/sinks/drizzle/transaction.ts +49 -0
  23. package/src/sinks/drizzle/update.ts +47 -0
  24. package/src/sinks/drizzle/utils.ts +36 -0
  25. package/src/sinks/sqlite.test.ts +13 -1
  26. package/src/sinks/sqlite.ts +65 -5
  27. package/src/testing/indexer.ts +15 -8
  28. package/src/testing/setup.ts +5 -5
  29. package/src/testing/vcr.ts +42 -4
  30. package/src/vcr/record.ts +2 -2
  31. package/src/vcr/replay.ts +3 -3
  32. package/.turbo/turbo-build.log +0 -37
  33. package/CHANGELOG.md +0 -83
  34. package/LICENSE.txt +0 -202
  35. package/build.config.ts +0 -16
  36. package/dist/index.cjs +0 -34
  37. package/dist/index.d.cts +0 -21
  38. package/dist/index.d.mts +0 -21
  39. package/dist/index.d.ts +0 -21
  40. package/dist/index.mjs +0 -19
  41. package/dist/shared/indexer.371c0482.mjs +0 -15
  42. package/dist/shared/indexer.3852a4d3.d.ts +0 -91
  43. package/dist/shared/indexer.50aa7ab0.mjs +0 -268
  44. package/dist/shared/indexer.7c118fb5.d.cts +0 -28
  45. package/dist/shared/indexer.7c118fb5.d.mts +0 -28
  46. package/dist/shared/indexer.7c118fb5.d.ts +0 -28
  47. package/dist/shared/indexer.a27bcb35.d.cts +0 -91
  48. package/dist/shared/indexer.c8ef02ea.cjs +0 -289
  49. package/dist/shared/indexer.e05aedca.cjs +0 -19
  50. package/dist/shared/indexer.f7dd57e5.d.mts +0 -91
  51. package/dist/sinks/csv.cjs +0 -66
  52. package/dist/sinks/csv.d.cts +0 -34
  53. package/dist/sinks/csv.d.mts +0 -34
  54. package/dist/sinks/csv.d.ts +0 -34
  55. package/dist/sinks/csv.mjs +0 -59
  56. package/dist/sinks/sqlite.cjs +0 -71
  57. package/dist/sinks/sqlite.d.cts +0 -41
  58. package/dist/sinks/sqlite.d.mts +0 -41
  59. package/dist/sinks/sqlite.d.ts +0 -41
  60. package/dist/sinks/sqlite.mjs +0 -68
  61. package/dist/testing/index.cjs +0 -63
  62. package/dist/testing/index.d.cts +0 -29
  63. package/dist/testing/index.d.mts +0 -29
  64. package/dist/testing/index.d.ts +0 -29
  65. package/dist/testing/index.mjs +0 -59
  66. package/tsconfig.json +0 -11
@@ -1,71 +0,0 @@
1
- 'use strict';
2
-
3
- const sink = require('../shared/indexer.e05aedca.cjs');
4
- require('hookable');
5
-
6
- var __defProp = Object.defineProperty;
7
- var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
8
- var __publicField = (obj, key, value) => {
9
- __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
10
- return value;
11
- };
12
- class SqliteSink extends sink.Sink {
13
- constructor(options) {
14
- super();
15
- __publicField(this, "_config");
16
- __publicField(this, "_db");
17
- const { database, ...config } = options;
18
- this._config = config;
19
- this._db = database;
20
- }
21
- async write({ data, endCursor, finality }) {
22
- await this.callHook("write", { data });
23
- data = this.processCursorColumn(data, endCursor);
24
- await this.insertJsonArray(data);
25
- await this.callHook("flush", { endCursor, finality });
26
- }
27
- async insertJsonArray(data) {
28
- if (data.length === 0)
29
- return;
30
- const columns = Object.keys(data[0]);
31
- const columnNames = columns.join(", ");
32
- const placeholders = columns.map(() => "?").join(", ");
33
- const conflictClause = this.buildConflictClause();
34
- const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
35
- const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
36
- const statement = insertSQL + valuePlaceholders + conflictClause;
37
- const values = data.flatMap((row) => columns.map((col) => row[col]));
38
- this._db.prepare(statement).run(values);
39
- }
40
- processCursorColumn(data, endCursor) {
41
- const { cursorColumn } = this._config;
42
- if (cursorColumn && data.some(
43
- (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
44
- )) {
45
- throw new Error(
46
- `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
47
- );
48
- }
49
- if (cursorColumn) {
50
- return data;
51
- }
52
- return data.map((row) => ({
53
- ...row,
54
- _cursor: Number(endCursor?.orderKey)
55
- }));
56
- }
57
- buildConflictClause() {
58
- const { on, update } = this._config.onConflict || {};
59
- if (on && update && update.length > 0) {
60
- const updateColumns = update.map((col) => `${col}=excluded.${col}`).join(", ");
61
- return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
62
- }
63
- return "";
64
- }
65
- }
66
- const sqlite = (args) => {
67
- return new SqliteSink(args);
68
- };
69
-
70
- exports.SqliteSink = SqliteSink;
71
- exports.sqlite = sqlite;
@@ -1,41 +0,0 @@
1
- import { Database } from 'better-sqlite3';
2
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.cjs';
3
- import '@apibara/protocol';
4
- import 'hookable';
5
-
6
- type SqliteSinkOptions = {
7
- /**
8
- * Database instance of better-sqlite3
9
- */
10
- database: Database;
11
- /**
12
- * The name of the table where data will be inserted.
13
- */
14
- tableName: string;
15
- /**
16
- * An optional column name used to store the cursor value. If specified,
17
- * the value of this column must match the `endCursor.orderKey` for each row.
18
- */
19
- cursorColumn?: string;
20
- /**
21
- * An optional configuration to handle conflicts during data insertion.
22
- * - `on`: The column name on which conflicts are detected.
23
- * - `update`: An array of column names to be updated if a conflict occurs.
24
- */
25
- onConflict?: {
26
- on: string;
27
- update: string[];
28
- };
29
- };
30
- declare class SqliteSink extends Sink {
31
- private _config;
32
- private _db;
33
- constructor(options: SqliteSinkOptions);
34
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
35
- private insertJsonArray;
36
- private processCursorColumn;
37
- private buildConflictClause;
38
- }
39
- declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
40
-
41
- export { SqliteSink, type SqliteSinkOptions, sqlite };
@@ -1,41 +0,0 @@
1
- import { Database } from 'better-sqlite3';
2
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.mjs';
3
- import '@apibara/protocol';
4
- import 'hookable';
5
-
6
- type SqliteSinkOptions = {
7
- /**
8
- * Database instance of better-sqlite3
9
- */
10
- database: Database;
11
- /**
12
- * The name of the table where data will be inserted.
13
- */
14
- tableName: string;
15
- /**
16
- * An optional column name used to store the cursor value. If specified,
17
- * the value of this column must match the `endCursor.orderKey` for each row.
18
- */
19
- cursorColumn?: string;
20
- /**
21
- * An optional configuration to handle conflicts during data insertion.
22
- * - `on`: The column name on which conflicts are detected.
23
- * - `update`: An array of column names to be updated if a conflict occurs.
24
- */
25
- onConflict?: {
26
- on: string;
27
- update: string[];
28
- };
29
- };
30
- declare class SqliteSink extends Sink {
31
- private _config;
32
- private _db;
33
- constructor(options: SqliteSinkOptions);
34
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
35
- private insertJsonArray;
36
- private processCursorColumn;
37
- private buildConflictClause;
38
- }
39
- declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
40
-
41
- export { SqliteSink, type SqliteSinkOptions, sqlite };
@@ -1,41 +0,0 @@
1
- import { Database } from 'better-sqlite3';
2
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.js';
3
- import '@apibara/protocol';
4
- import 'hookable';
5
-
6
- type SqliteSinkOptions = {
7
- /**
8
- * Database instance of better-sqlite3
9
- */
10
- database: Database;
11
- /**
12
- * The name of the table where data will be inserted.
13
- */
14
- tableName: string;
15
- /**
16
- * An optional column name used to store the cursor value. If specified,
17
- * the value of this column must match the `endCursor.orderKey` for each row.
18
- */
19
- cursorColumn?: string;
20
- /**
21
- * An optional configuration to handle conflicts during data insertion.
22
- * - `on`: The column name on which conflicts are detected.
23
- * - `update`: An array of column names to be updated if a conflict occurs.
24
- */
25
- onConflict?: {
26
- on: string;
27
- update: string[];
28
- };
29
- };
30
- declare class SqliteSink extends Sink {
31
- private _config;
32
- private _db;
33
- constructor(options: SqliteSinkOptions);
34
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
35
- private insertJsonArray;
36
- private processCursorColumn;
37
- private buildConflictClause;
38
- }
39
- declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
40
-
41
- export { SqliteSink, type SqliteSinkOptions, sqlite };
@@ -1,68 +0,0 @@
1
- import { S as Sink } from '../shared/indexer.371c0482.mjs';
2
- import 'hookable';
3
-
4
- var __defProp = Object.defineProperty;
5
- var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
6
- var __publicField = (obj, key, value) => {
7
- __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
8
- return value;
9
- };
10
- class SqliteSink extends Sink {
11
- constructor(options) {
12
- super();
13
- __publicField(this, "_config");
14
- __publicField(this, "_db");
15
- const { database, ...config } = options;
16
- this._config = config;
17
- this._db = database;
18
- }
19
- async write({ data, endCursor, finality }) {
20
- await this.callHook("write", { data });
21
- data = this.processCursorColumn(data, endCursor);
22
- await this.insertJsonArray(data);
23
- await this.callHook("flush", { endCursor, finality });
24
- }
25
- async insertJsonArray(data) {
26
- if (data.length === 0)
27
- return;
28
- const columns = Object.keys(data[0]);
29
- const columnNames = columns.join(", ");
30
- const placeholders = columns.map(() => "?").join(", ");
31
- const conflictClause = this.buildConflictClause();
32
- const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
33
- const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
34
- const statement = insertSQL + valuePlaceholders + conflictClause;
35
- const values = data.flatMap((row) => columns.map((col) => row[col]));
36
- this._db.prepare(statement).run(values);
37
- }
38
- processCursorColumn(data, endCursor) {
39
- const { cursorColumn } = this._config;
40
- if (cursorColumn && data.some(
41
- (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
42
- )) {
43
- throw new Error(
44
- `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
45
- );
46
- }
47
- if (cursorColumn) {
48
- return data;
49
- }
50
- return data.map((row) => ({
51
- ...row,
52
- _cursor: Number(endCursor?.orderKey)
53
- }));
54
- }
55
- buildConflictClause() {
56
- const { on, update } = this._config.onConflict || {};
57
- if (on && update && update.length > 0) {
58
- const updateColumns = update.map((col) => `${col}=excluded.${col}`).join(", ");
59
- return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
60
- }
61
- return "";
62
- }
63
- }
64
- const sqlite = (args) => {
65
- return new SqliteSink(args);
66
- };
67
-
68
- export { SqliteSink, sqlite };
@@ -1,63 +0,0 @@
1
- 'use strict';
2
-
3
- const vitest = require('vitest');
4
- const replay = require('../shared/indexer.c8ef02ea.cjs');
5
- require('node:fs/promises');
6
- require('node:path');
7
- require('hookable');
8
- require('klona/full');
9
- require('node:assert');
10
- require('node:fs');
11
- require('@apibara/protocol/testing');
12
- require('consola');
13
- require('node:async_hooks');
14
- require('unctx');
15
- require('@opentelemetry/api');
16
- require('../shared/indexer.e05aedca.cjs');
17
-
18
- const test = vitest.test.extend({
19
- vcr: {
20
- withClient
21
- }
22
- });
23
- async function withClient(cassetteName, range, callback) {
24
- const vcrConfig = {
25
- cassetteDir: "cassettes"
26
- };
27
- const cassetteOptions = {
28
- name: cassetteName,
29
- startingCursor: {
30
- orderKey: range.fromBlock
31
- },
32
- endingCursor: {
33
- orderKey: range.toBlock
34
- }
35
- };
36
- const context = {
37
- async run(indexer) {
38
- const client = replay.loadCassette(vcrConfig, cassetteName);
39
- if (!replay.isCassetteAvailable(vcrConfig, cassetteName)) {
40
- await replay.record(vcrConfig, client, indexer, cassetteOptions);
41
- }
42
- return await replay.replay(vcrConfig, indexer, cassetteName);
43
- }
44
- };
45
- await callback(context);
46
- }
47
-
48
- function generateMockMessages(count = 10) {
49
- return [...Array(count)].map((_, i) => ({
50
- _tag: "data",
51
- data: {
52
- cursor: { orderKey: BigInt(5e6 - 1) },
53
- finality: "accepted",
54
- data: [{ data: `${5e6 + i}` }],
55
- endCursor: { orderKey: BigInt(5e6 + i) }
56
- }
57
- }));
58
- }
59
-
60
- exports.VcrSink = replay.VcrSink;
61
- exports.vcr = replay.vcr;
62
- exports.generateMockMessages = generateMockMessages;
63
- exports.test = test;
@@ -1,29 +0,0 @@
1
- import * as vitest from 'vitest';
2
- import { I as Indexer, i as VcrReplayResult } from '../shared/indexer.a27bcb35.cjs';
3
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.cjs';
4
- import { MockStreamResponse } from '@apibara/protocol/testing';
5
- import '@apibara/protocol';
6
- import 'hookable';
7
-
8
- declare const test: vitest.TestAPI<{
9
- vcr: {
10
- withClient: typeof withClient;
11
- };
12
- }>;
13
- type WithClientContext<TFilter, TBlock, TRet> = {
14
- run: (indexerArgs: Indexer<TFilter, TBlock, TRet>) => Promise<VcrReplayResult>;
15
- };
16
- declare function withClient<TFilter, TBlock, TRet>(cassetteName: string, range: {
17
- fromBlock: bigint;
18
- toBlock: bigint;
19
- }, callback: (context: WithClientContext<TFilter, TBlock, TRet>) => Promise<void>): Promise<void>;
20
-
21
- declare class VcrSink extends Sink {
22
- result: VcrReplayResult["outputs"];
23
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
24
- }
25
- declare function vcr(): VcrSink;
26
-
27
- declare function generateMockMessages(count?: number): MockStreamResponse[];
28
-
29
- export { VcrSink, generateMockMessages, test, vcr };
@@ -1,29 +0,0 @@
1
- import * as vitest from 'vitest';
2
- import { I as Indexer, i as VcrReplayResult } from '../shared/indexer.f7dd57e5.mjs';
3
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.mjs';
4
- import { MockStreamResponse } from '@apibara/protocol/testing';
5
- import '@apibara/protocol';
6
- import 'hookable';
7
-
8
- declare const test: vitest.TestAPI<{
9
- vcr: {
10
- withClient: typeof withClient;
11
- };
12
- }>;
13
- type WithClientContext<TFilter, TBlock, TRet> = {
14
- run: (indexerArgs: Indexer<TFilter, TBlock, TRet>) => Promise<VcrReplayResult>;
15
- };
16
- declare function withClient<TFilter, TBlock, TRet>(cassetteName: string, range: {
17
- fromBlock: bigint;
18
- toBlock: bigint;
19
- }, callback: (context: WithClientContext<TFilter, TBlock, TRet>) => Promise<void>): Promise<void>;
20
-
21
- declare class VcrSink extends Sink {
22
- result: VcrReplayResult["outputs"];
23
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
24
- }
25
- declare function vcr(): VcrSink;
26
-
27
- declare function generateMockMessages(count?: number): MockStreamResponse[];
28
-
29
- export { VcrSink, generateMockMessages, test, vcr };
@@ -1,29 +0,0 @@
1
- import * as vitest from 'vitest';
2
- import { I as Indexer, i as VcrReplayResult } from '../shared/indexer.3852a4d3.js';
3
- import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.js';
4
- import { MockStreamResponse } from '@apibara/protocol/testing';
5
- import '@apibara/protocol';
6
- import 'hookable';
7
-
8
- declare const test: vitest.TestAPI<{
9
- vcr: {
10
- withClient: typeof withClient;
11
- };
12
- }>;
13
- type WithClientContext<TFilter, TBlock, TRet> = {
14
- run: (indexerArgs: Indexer<TFilter, TBlock, TRet>) => Promise<VcrReplayResult>;
15
- };
16
- declare function withClient<TFilter, TBlock, TRet>(cassetteName: string, range: {
17
- fromBlock: bigint;
18
- toBlock: bigint;
19
- }, callback: (context: WithClientContext<TFilter, TBlock, TRet>) => Promise<void>): Promise<void>;
20
-
21
- declare class VcrSink extends Sink {
22
- result: VcrReplayResult["outputs"];
23
- write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
24
- }
25
- declare function vcr(): VcrSink;
26
-
27
- declare function generateMockMessages(count?: number): MockStreamResponse[];
28
-
29
- export { VcrSink, generateMockMessages, test, vcr };
@@ -1,59 +0,0 @@
1
- import { test as test$1 } from 'vitest';
2
- import { l as loadCassette, i as isCassetteAvailable, b as record, e as replay } from '../shared/indexer.50aa7ab0.mjs';
3
- export { V as VcrSink, v as vcr } from '../shared/indexer.50aa7ab0.mjs';
4
- import 'node:fs/promises';
5
- import 'node:path';
6
- import 'hookable';
7
- import 'klona/full';
8
- import 'node:assert';
9
- import 'node:fs';
10
- import '@apibara/protocol/testing';
11
- import 'consola';
12
- import 'node:async_hooks';
13
- import 'unctx';
14
- import '@opentelemetry/api';
15
- import '../shared/indexer.371c0482.mjs';
16
-
17
- const test = test$1.extend({
18
- vcr: {
19
- withClient
20
- }
21
- });
22
- async function withClient(cassetteName, range, callback) {
23
- const vcrConfig = {
24
- cassetteDir: "cassettes"
25
- };
26
- const cassetteOptions = {
27
- name: cassetteName,
28
- startingCursor: {
29
- orderKey: range.fromBlock
30
- },
31
- endingCursor: {
32
- orderKey: range.toBlock
33
- }
34
- };
35
- const context = {
36
- async run(indexer) {
37
- const client = loadCassette(vcrConfig, cassetteName);
38
- if (!isCassetteAvailable(vcrConfig, cassetteName)) {
39
- await record(vcrConfig, client, indexer, cassetteOptions);
40
- }
41
- return await replay(vcrConfig, indexer, cassetteName);
42
- }
43
- };
44
- await callback(context);
45
- }
46
-
47
- function generateMockMessages(count = 10) {
48
- return [...Array(count)].map((_, i) => ({
49
- _tag: "data",
50
- data: {
51
- cursor: { orderKey: BigInt(5e6 - 1) },
52
- finality: "accepted",
53
- data: [{ data: `${5e6 + i}` }],
54
- endCursor: { orderKey: BigInt(5e6 + i) }
55
- }
56
- }));
57
- }
58
-
59
- export { generateMockMessages, test };
package/tsconfig.json DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "extends": "../../tsconfig.json",
3
- "compilerOptions": {
4
- "outDir": "dist",
5
- "declarationDir": "dist",
6
- "noEmit": false,
7
- "rootDir": "src",
8
- "types": ["node"]
9
- },
10
- "include": ["src/"]
11
- }