@apibara/indexer 2.0.0-beta.3 → 2.0.0-beta.30

package/dist/index.cjs

@@ -0,0 +1,270 @@
+'use strict';
+
+const protocol = require('@apibara/protocol');
+const consola = require('consola');
+const hookable = require('hookable');
+const assert = require('node:assert');
+const context = require('./shared/indexer.077335f3.cjs');
+const api = require('@opentelemetry/api');
+require('node:async_hooks');
+require('unctx');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
+const assert__default = /*#__PURE__*/_interopDefaultCompat(assert);
+
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
+}
+
+const tracer = api.trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
+}) {
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: hookable.createHooks()
+  };
+  if (indexer.options.debug) {
+    hookable.createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof protocol.ClientError) {
+        if (error.code === protocol.Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola__default.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await context.indexerAsyncContext.callAsync({}, async () => {
+    const context$1 = context.useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor: indexer.options.startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context$1.cursor = cursor;
+            context$1.endCursor = endCursor;
+            context$1.finality = finality;
+            await middleware(context$1, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert__default(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context: context$1
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert__default(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context: context$1
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context$1.cursor = void 0;
+          context$1.endCursor = void 0;
+          context$1.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola__default.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola__default.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola__default.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
+}
+
+exports.useIndexerContext = context.useIndexerContext;
+exports.createIndexer = createIndexer;
+exports.defineIndexer = defineIndexer;
+exports.run = run;
+exports.runWithReconnect = runWithReconnect;

package/dist/index.d.cts

@@ -0,0 +1,3 @@
+export { c as Indexer, b as IndexerConfig, e as IndexerHooks, I as IndexerWithStreamConfig, R as ReconnectOptions, h as RunOptions, U as UseMiddlewareFunction, g as createIndexer, f as defineIndexer, i as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.8939ecc8.cjs';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.d.mts

@@ -0,0 +1,3 @@
+export { c as Indexer, b as IndexerConfig, e as IndexerHooks, I as IndexerWithStreamConfig, R as ReconnectOptions, h as RunOptions, U as UseMiddlewareFunction, g as createIndexer, f as defineIndexer, i as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.8939ecc8.mjs';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { c as Indexer, b as IndexerConfig, e as IndexerHooks, I as IndexerWithStreamConfig, R as ReconnectOptions, h as RunOptions, U as UseMiddlewareFunction, g as createIndexer, f as defineIndexer, i as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.8939ecc8.js';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.mjs

@@ -0,0 +1,259 @@
+import { ClientError, Status } from '@apibara/protocol';
+import consola from 'consola';
+import { createHooks, createDebugger } from 'hookable';
+import assert from 'node:assert';
+import { i as indexerAsyncContext, u as useIndexerContext } from './shared/indexer.a55ad619.mjs';
+import { trace } from '@opentelemetry/api';
+import 'node:async_hooks';
+import 'unctx';
+
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
+}
+
+const tracer = trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
+}) {
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: createHooks()
+  };
+  if (indexer.options.debug) {
+    createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof ClientError) {
+        if (error.code === Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor: indexer.options.startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context.cursor = cursor;
+            context.endCursor = endCursor;
+            context.finality = finality;
+            await middleware(context, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context.cursor = void 0;
+          context.endCursor = void 0;
+          context.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
+}
+
+export { createIndexer, defineIndexer, run, runWithReconnect, useIndexerContext };

package/dist/internal/testing.cjs

@@ -0,0 +1,109 @@
+'use strict';
+
+const protocol = require('@apibara/protocol');
+const testing = require('@apibara/protocol/testing');
+const context = require('../shared/indexer.077335f3.cjs');
+const index = require('../index.cjs');
+const config = require('../shared/indexer.601ceab0.cjs');
+require('consola');
+require('node:async_hooks');
+require('unctx');
+require('hookable');
+require('node:assert');
+require('@opentelemetry/api');
+
+function generateMockMessages(count = 10, options) {
+  const invalidateAt = options?.invalidate;
+  const finalizeAt = options?.finalize;
+  const messages = [];
+  for (let i = 0; i < count; i++) {
+    if (invalidateAt && i === invalidateAt.invalidateTriggerIndex) {
+      messages.push({
+        _tag: "invalidate",
+        invalidate: {
+          cursor: {
+            orderKey: BigInt(5e6 + invalidateAt.invalidateFromIndex)
+          }
+        }
+      });
+    } else if (finalizeAt && i === finalizeAt.finalizeTriggerIndex) {
+      messages.push({
+        _tag: "finalize",
+        finalize: {
+          cursor: {
+            orderKey: BigInt(5e6 + finalizeAt.finalizeToIndex)
+          }
+        }
+      });
+    } else {
+      messages.push({
+        _tag: "data",
+        data: {
+          cursor: { orderKey: BigInt(5e6 + i - 1) },
+          finality: "accepted",
+          data: [{ data: `${5e6 + i}` }],
+          endCursor: { orderKey: BigInt(5e6 + i) }
+        }
+      });
+    }
+  }
+  return messages;
+}
+function getMockIndexer({
+  plugins,
+  override
+} = {}) {
+  return index.createIndexer(
+    index.defineIndexer(testing.MockStream)({
+      streamUrl: "https://sepolia.ethereum.a5a.ch",
+      finality: "accepted",
+      filter: {},
+      async transform({ block: { data }, context }) {
+      },
+      plugins,
+      ...override
+    })
+  );
+}
+function mockSink({
+  output,
+  metadata
+}) {
+  return config.defineIndexerPlugin((indexer) => {
+    indexer.hooks.hook("connect:before", ({ request }) => {
+      if (metadata?.lastCursor && protocol.isCursor(metadata.lastCursor)) {
+        request.startingCursor = metadata.lastCursor;
+      }
+      if (metadata?.lastFilter) {
+        request.filter[1] = metadata.lastFilter;
+      }
+    });
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (request.filter[1]) {
+        if (metadata) {
+          metadata.lastCursor = endCursor;
+          metadata.lastFilter = request.filter[1];
+        }
+      }
+    });
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (context, next) => {
+        context.output = output;
+        await next();
+        context.output = null;
+        if (metadata) {
+          metadata.lastCursor = context.endCursor;
+        }
+      });
+    });
+  });
+}
+function useMockSink() {
+  const context$1 = context.useIndexerContext();
+  return { output: context$1.output };
+}
+
+exports.generateMockMessages = generateMockMessages;
+exports.getMockIndexer = getMockIndexer;
+exports.mockSink = mockSink;
+exports.useMockSink = useMockSink;

package/dist/internal/testing.d.cts

@@ -0,0 +1,40 @@
+import { a as IndexerPlugin, b as IndexerConfig, c as Indexer } from '../shared/indexer.8939ecc8.cjs';
+import { MockStreamResponse, MockFilter, MockBlock } from '@apibara/protocol/testing';
+import '@apibara/protocol';
+import 'hookable';
+
+type MockMessagesOptions = {
+    invalidate?: {
+        invalidateFromIndex: number;
+        invalidateTriggerIndex: number;
+    };
+    finalize?: {
+        finalizeToIndex: number;
+        finalizeTriggerIndex: number;
+    };
+};
+declare function generateMockMessages(count?: number, options?: MockMessagesOptions): MockStreamResponse[];
+declare function getMockIndexer({ plugins, override, }?: {
+    plugins?: ReadonlyArray<IndexerPlugin<MockFilter, MockBlock>>;
+    override?: Partial<IndexerConfig<MockFilter, MockBlock>>;
+}): Indexer<{
+    readonly filter?: string | undefined;
+}, {
+    readonly data?: string | undefined;
+}>;
+type MockRet = {
+    data: string;
+};
+/**
+ * A mock sink used for testing. The indexer function can write to the output array.
+ * The indexer context is optionally written to the metadata object.
+ */
+declare function mockSink<TFilter, TBlock>({ output, metadata, }: {
+    output: unknown[];
+    metadata?: Record<string, unknown>;
+}): IndexerPlugin<TFilter, TBlock>;
+declare function useMockSink(): {
+    output: unknown[];
+};
+
+export { type MockMessagesOptions, type MockRet, generateMockMessages, getMockIndexer, mockSink, useMockSink };