@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.10

package/dist/index.cjs

@@ -1,50 +1,282 @@
 'use strict';
 
-const vcr_index = require('./shared/indexer.541d43eb.cjs');
-const sink = require('./shared/indexer.a8b7ab1f.cjs');
-const helper = require('./shared/indexer.318d3617.cjs');
-const plugins_index = require('./plugins/index.cjs');
-require('node:fs/promises');
-require('node:path');
-require('klona/full');
-require('consola');
-require('hookable');
-require('node:assert');
-require('node:fs');
-require('@apibara/protocol/testing');
+const protocol = require('@apibara/protocol');
+const consola = require('consola');
+const hookable = require('hookable');
+const assert = require('node:assert');
+const context = require('./shared/indexer.077335f3.cjs');
+const api = require('@opentelemetry/api');
 require('node:async_hooks');
 require('unctx');
-require('@opentelemetry/api');
 
-function useKVStore() {
-  const ctx = helper.useIndexerContext();
-  if (!ctx?.kv)
-    throw new Error("KV Plugin is not available in context!");
-  return ctx.kv;
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
+const assert__default = /*#__PURE__*/_interopDefaultCompat(assert);
+
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
 }
 
-function useSink({
-  context
+const tracer = api.trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
 }) {
-  if (!context.sinkTransaction) {
-    throw new Error("Transaction context doesn't exist!");
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: hookable.createHooks()
+  };
+  if (indexer.options.debug) {
+    hookable.createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
   }
-  return context.sinkTransaction;
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof protocol.ClientError) {
+        if (error.code === protocol.Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola__default.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await context.indexerAsyncContext.callAsync({}, async () => {
+    const context$1 = context.useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    let startingCursor;
+    if (indexer.options.startingCursor) {
+      startingCursor = indexer.options.startingCursor;
+    } else if (indexer.options.startingBlock !== void 0) {
+      if (indexer.options.startingBlock === 0n) {
+        startingCursor = void 0;
+      } else if (indexer.options.startingBlock > 0n) {
+        startingCursor = {
+          orderKey: indexer.options.startingBlock - 1n
+        };
+      }
+    }
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context$1.cursor = cursor;
+            context$1.endCursor = endCursor;
+            context$1.finality = finality;
+            await middleware(context$1, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert__default(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context: context$1
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert__default(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context: context$1
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context$1.cursor = void 0;
+          context$1.endCursor = void 0;
+          context$1.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola__default.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola__default.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola__default.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
 }
 
-exports.createIndexer = vcr_index.createIndexer;
-exports.defineIndexer = vcr_index.defineIndexer;
-exports.loadCassette = vcr_index.loadCassette;
-exports.record = vcr_index.record;
-exports.replay = vcr_index.replay;
-exports.run = vcr_index.run;
-exports.DefaultSink = sink.DefaultSink;
-exports.Sink = sink.Sink;
-exports.defaultSink = sink.defaultSink;
-exports.deserialize = helper.deserialize;
-exports.isCassetteAvailable = helper.isCassetteAvailable;
-exports.serialize = helper.serialize;
-exports.useIndexerContext = helper.useIndexerContext;
-exports.defineIndexerPlugin = plugins_index.defineIndexerPlugin;
-exports.useKVStore = useKVStore;
-exports.useSink = useSink;
+exports.useIndexerContext = context.useIndexerContext;
+exports.createIndexer = createIndexer;
+exports.defineIndexer = defineIndexer;
+exports.run = run;
+exports.runWithReconnect = runWithReconnect;

package/dist/index.d.cts

@@ -1,18 +1,3 @@
-import { a as IndexerContext } from './shared/indexer.c7ed6b83.cjs';
-export { f as Indexer, c as IndexerConfig, b as IndexerHooks, I as IndexerPlugin, d as IndexerWithStreamConfig, g as createIndexer, e as defineIndexer, h as defineIndexerPlugin, r as run, u as useIndexerContext } from './shared/indexer.c7ed6b83.cjs';
-export { D as DefaultSink, S as Sink, a as SinkCursorParams, b as SinkData, d as defaultSink } from './shared/indexer.b9c8f0d8.cjs';
-export { C as CassetteOptions, V as VcrConfig, a as VcrReplayResult, l as loadCassette, r as replay } from './shared/indexer.500fd281.cjs';
-export { CassetteDataType, deserialize, isCassetteAvailable, record, serialize } from './vcr/index.cjs';
-import { KVStore } from './plugins/kv.cjs';
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.cjs';
 import '@apibara/protocol';
 import 'hookable';
-import 'better-sqlite3';
-
-type UseKVStoreResult = InstanceType<typeof KVStore>;
-declare function useKVStore(): UseKVStoreResult;
-
-declare function useSink<TTxnParams>({ context, }: {
-    context: IndexerContext<TTxnParams>;
-}): NonNullable<TTxnParams>;
-
-export { type UseKVStoreResult, useKVStore, useSink };

package/dist/index.d.mts

@@ -1,18 +1,3 @@
-import { a as IndexerContext } from './shared/indexer.e8bd138d.mjs';
-export { f as Indexer, c as IndexerConfig, b as IndexerHooks, I as IndexerPlugin, d as IndexerWithStreamConfig, g as createIndexer, e as defineIndexer, h as defineIndexerPlugin, r as run, u as useIndexerContext } from './shared/indexer.e8bd138d.mjs';
-export { D as DefaultSink, S as Sink, a as SinkCursorParams, b as SinkData, d as defaultSink } from './shared/indexer.b9c8f0d8.mjs';
-export { C as CassetteOptions, V as VcrConfig, a as VcrReplayResult, l as loadCassette, r as replay } from './shared/indexer.e1856641.mjs';
-export { CassetteDataType, deserialize, isCassetteAvailable, record, serialize } from './vcr/index.mjs';
-import { KVStore } from './plugins/kv.mjs';
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.mjs';
 import '@apibara/protocol';
 import 'hookable';
-import 'better-sqlite3';
-
-type UseKVStoreResult = InstanceType<typeof KVStore>;
-declare function useKVStore(): UseKVStoreResult;
-
-declare function useSink<TTxnParams>({ context, }: {
-    context: IndexerContext<TTxnParams>;
-}): NonNullable<TTxnParams>;
-
-export { type UseKVStoreResult, useKVStore, useSink };

package/dist/index.d.ts

@@ -1,18 +1,3 @@
-import { a as IndexerContext } from './shared/indexer.f761abcd.js';
-export { f as Indexer, c as IndexerConfig, b as IndexerHooks, I as IndexerPlugin, d as IndexerWithStreamConfig, g as createIndexer, e as defineIndexer, h as defineIndexerPlugin, r as run, u as useIndexerContext } from './shared/indexer.f761abcd.js';
-export { D as DefaultSink, S as Sink, a as SinkCursorParams, b as SinkData, d as defaultSink } from './shared/indexer.b9c8f0d8.js';
-export { C as CassetteOptions, V as VcrConfig, a as VcrReplayResult, l as loadCassette, r as replay } from './shared/indexer.e4f2430f.js';
-export { CassetteDataType, deserialize, isCassetteAvailable, record, serialize } from './vcr/index.js';
-import { KVStore } from './plugins/kv.js';
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.js';
 import '@apibara/protocol';
 import 'hookable';
-import 'better-sqlite3';
-
-type UseKVStoreResult = InstanceType<typeof KVStore>;
-declare function useKVStore(): UseKVStoreResult;
-
-declare function useSink<TTxnParams>({ context, }: {
-    context: IndexerContext<TTxnParams>;
-}): NonNullable<TTxnParams>;
-
-export { type UseKVStoreResult, useKVStore, useSink };

package/dist/index.mjs

@@ -1,34 +1,271 @@
-export { c as createIndexer, d as defineIndexer, l as loadCassette, a as record, b as replay, r as run } from './shared/indexer.36530330.mjs';
-export { D as DefaultSink, S as Sink, d as defaultSink } from './shared/indexer.93d6b2eb.mjs';
-import { u as useIndexerContext } from './shared/indexer.2c23c9cd.mjs';
-export { d as deserialize, i as isCassetteAvailable, s as serialize } from './shared/indexer.2c23c9cd.mjs';
-export { defineIndexerPlugin } from './plugins/index.mjs';
-import 'node:fs/promises';
-import 'node:path';
-import 'klona/full';
-import 'consola';
-import 'hookable';
-import 'node:assert';
-import 'node:fs';
-import '@apibara/protocol/testing';
+import { ClientError, Status } from '@apibara/protocol';
+import consola from 'consola';
+import { createHooks, createDebugger } from 'hookable';
+import assert from 'node:assert';
+import { i as indexerAsyncContext, u as useIndexerContext } from './shared/indexer.a55ad619.mjs';
+import { trace } from '@opentelemetry/api';
 import 'node:async_hooks';
 import 'unctx';
-import '@opentelemetry/api';
 
-function useKVStore() {
-  const ctx = useIndexerContext();
-  if (!ctx?.kv)
-    throw new Error("KV Plugin is not available in context!");
-  return ctx.kv;
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
 }
 
-function useSink({
-  context
+const tracer = trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
 }) {
-  if (!context.sinkTransaction) {
-    throw new Error("Transaction context doesn't exist!");
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: createHooks()
+  };
+  if (indexer.options.debug) {
+    createDebugger(indexer.hooks, { tag: "indexer" });
   }
-  return context.sinkTransaction;
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof ClientError) {
+        if (error.code === Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    let startingCursor;
+    if (indexer.options.startingCursor) {
+      startingCursor = indexer.options.startingCursor;
+    } else if (indexer.options.startingBlock !== void 0) {
+      if (indexer.options.startingBlock === 0n) {
+        startingCursor = void 0;
+      } else if (indexer.options.startingBlock > 0n) {
+        startingCursor = {
+          orderKey: indexer.options.startingBlock - 1n
+        };
+      }
+    }
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context.cursor = cursor;
+            context.endCursor = endCursor;
+            context.finality = finality;
+            await middleware(context, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context.cursor = void 0;
+          context.endCursor = void 0;
+          context.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
 }
 
-export { useIndexerContext, useKVStore, useSink };
+export { createIndexer, defineIndexer, run, runWithReconnect, useIndexerContext };

package/dist/internal/index.cjs

@@ -0,0 +1,10 @@
+'use strict';
+
+function generateIndexerId(fileBasedName, identifier) {
+  return `indexer_${fileBasedName}_${identifier || "default"}`.replace(
+    /[^a-zA-Z0-9_]/g,
+    "_"
+  );
+}
+
+exports.generateIndexerId = generateIndexerId;