@apibara/indexer 2.0.0-beta.4 → 2.0.0-beta.41

package/dist/index.cjs

@@ -0,0 +1,278 @@
+'use strict';
+
+const protocol = require('@apibara/protocol');
+const consola = require('consola');
+const hookable = require('hookable');
+const assert = require('node:assert');
+const context = require('./shared/indexer.077335f3.cjs');
+const api = require('@opentelemetry/api');
+require('node:async_hooks');
+require('unctx');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
+const assert__default = /*#__PURE__*/_interopDefaultCompat(assert);
+
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
+}
+
+const tracer = api.trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
+}) {
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: hookable.createHooks()
+  };
+  if (indexer.options.debug) {
+    hookable.createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof protocol.ClientError) {
+        if (error.code === protocol.Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola__default.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await context.indexerAsyncContext.callAsync({}, async () => {
+    const context$1 = context.useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    let startingCursor;
+    if (indexer.options.startingCursor) {
+      startingCursor = indexer.options.startingCursor;
+    } else if (indexer.options.startingBlock !== void 0) {
+      startingCursor = {
+        orderKey: indexer.options.startingBlock
+      };
+    }
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context$1.cursor = cursor;
+            context$1.endCursor = endCursor;
+            context$1.finality = finality;
+            await middleware(context$1, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert__default(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context: context$1
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert__default(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context: context$1
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context$1.cursor = void 0;
+          context$1.endCursor = void 0;
+          context$1.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola__default.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola__default.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola__default.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
+}
+
+exports.useIndexerContext = context.useIndexerContext;
+exports.createIndexer = createIndexer;
+exports.defineIndexer = defineIndexer;
+exports.run = run;
+exports.runWithReconnect = runWithReconnect;

package/dist/index.d.cts

@@ -0,0 +1,3 @@
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.cjs';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.d.mts

@@ -0,0 +1,3 @@
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.mjs';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { a as Indexer, c as IndexerConfig, e as IndexerHooks, f as IndexerStartingCursor, I as IndexerWithStreamConfig, R as ReconnectOptions, i as RunOptions, U as UseMiddlewareFunction, h as createIndexer, g as defineIndexer, j as run, r as runWithReconnect, u as useIndexerContext } from './shared/indexer.fedcd831.js';
+import '@apibara/protocol';
+import 'hookable';

package/dist/index.mjs

@@ -0,0 +1,267 @@
+import { ClientError, Status } from '@apibara/protocol';
+import consola from 'consola';
+import { createHooks, createDebugger } from 'hookable';
+import assert from 'node:assert';
+import { i as indexerAsyncContext, u as useIndexerContext } from './shared/indexer.a55ad619.mjs';
+import { trace } from '@opentelemetry/api';
+import 'node:async_hooks';
+import 'unctx';
+
+function compose(middleware) {
+  return (context, next) => {
+    let index = -1;
+    return dispatch(0);
+    async function dispatch(i) {
+      if (i <= index) {
+        throw new Error("next() called multiple times");
+      }
+      index = i;
+      let handler;
+      if (i >= middleware.length) {
+        if (next) {
+          await next();
+        }
+        return;
+      }
+      if (middleware[i]) {
+        handler = middleware[i];
+      } else {
+        handler = i === middleware.length ? next : void 0;
+      }
+      if (!handler) {
+        throw new Error("Handler not found");
+      }
+      await handler(context, () => dispatch(i + 1));
+    }
+  };
+}
+
+const tracer = trace.getTracer("@apibara/indexer");
+
+function defineIndexer(streamConfig) {
+  return (config) => ({
+    streamConfig,
+    ...config
+  });
+}
+function createIndexer({
+  streamConfig,
+  ...options
+}) {
+  const indexer = {
+    options,
+    streamConfig,
+    hooks: createHooks()
+  };
+  if (indexer.options.debug) {
+    createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+  return indexer;
+}
+async function runWithReconnect(client, indexer, options = {}) {
+  let retryCount = 0;
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1e3;
+  const maxWait = options.maxWait ?? 3e4;
+  const runOptions = {
+    onConnect() {
+      retryCount = 0;
+    }
+  };
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      retryCount++;
+      if (error instanceof ClientError) {
+        if (error.code === Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola.error(
+              "Internal server error, reconnecting...",
+              error.message
+            );
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise(
+              (resolve) => setTimeout(resolve, Math.min(retryCount * delay, maxWait))
+            );
+            continue;
+          }
+        }
+      }
+      throw error;
+    }
+  }
+}
+async function run(client, indexer, runOptions = {}) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
+    await indexer.hooks.callHook("run:before");
+    const isFactoryMode = indexer.options.factory !== void 0;
+    let startingCursor;
+    if (indexer.options.startingCursor) {
+      startingCursor = indexer.options.startingCursor;
+    } else if (indexer.options.startingBlock !== void 0) {
+      startingCursor = {
+        orderKey: indexer.options.startingBlock
+      };
+    }
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor
+    });
+    const options = {};
+    await indexer.hooks.callHook("connect:before", { request, options });
+    let mainFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+    let stream = client.streamData(request, options)[Symbol.asyncIterator]();
+    await indexer.hooks.callHook("connect:after", { request });
+    let onConnectCalled = false;
+    while (true) {
+      const { value: message, done } = await stream.next();
+      if (done) {
+        break;
+      }
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+      await indexer.hooks.callHook("message", { message });
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+            context.cursor = cursor;
+            context.endCursor = endCursor;
+            context.finality = finality;
+            await middleware(context, async () => {
+              let block;
+              if (isFactoryMode) {
+                assert(indexer.options.factory !== void 0);
+                const [factoryBlock, mainBlock] = blocks;
+                block = mainBlock;
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context
+                  });
+                  if (filter) {
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter
+                    );
+                    const request2 = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor
+                    });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request: request2,
+                      endCursor
+                    });
+                    stream = client.streamData(request2, options)[Symbol.asyncIterator]();
+                    const { value: message2 } = await stream.next();
+                    assert(message2._tag === "data");
+                    const [_factoryBlock, _block] = message2.data.data;
+                    block = _block;
+                  }
+                }
+              } else {
+                block = blocks[0];
+              }
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span2) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context
+                  });
+                  span2.end();
+                });
+              }
+            });
+            span.end();
+          });
+          context.cursor = void 0;
+          context.endCursor = void 0;
+          context.finality = void 0;
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
+          });
+          break;
+        }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola.info(message.systemMessage.output.stdout);
+                  break;
+                }
+              }
+              await indexer.hooks.callHook("message:systemMessage", {
+                message
+              });
+              span.end();
+            }
+          );
+          break;
+        }
+        default: {
+          consola.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}
+async function registerMiddleware(indexer) {
+  const middleware = [];
+  const use = (fn) => {
+    middleware.push(fn);
+  };
+  await indexer.hooks.callHook("handler:middleware", { use });
+  const composed = compose(middleware);
+  return async function _composedIndexerMiddleware(context, next) {
+    await composed(context, next);
+  };
+}
+
+export { createIndexer, defineIndexer, run, runWithReconnect, useIndexerContext };

package/dist/internal/index.cjs

@@ -0,0 +1,10 @@
+'use strict';
+
+function generateIndexerId(fileBasedName, identifier) {
+  return `indexer_${fileBasedName}_${identifier || "default"}`.replace(
+    /[^a-zA-Z0-9_]/g,
+    "_"
+  );
+}
+
+exports.generateIndexerId = generateIndexerId;

package/dist/internal/index.d.cts

@@ -0,0 +1,3 @@
+declare function generateIndexerId(fileBasedName: string, identifier?: string): string;
+
+export { generateIndexerId };

package/dist/internal/index.d.mts

@@ -0,0 +1,3 @@
+declare function generateIndexerId(fileBasedName: string, identifier?: string): string;
+
+export { generateIndexerId };

package/dist/internal/index.d.ts

@@ -0,0 +1,3 @@
+declare function generateIndexerId(fileBasedName: string, identifier?: string): string;
+
+export { generateIndexerId };

package/dist/internal/index.mjs

@@ -0,0 +1,8 @@
+function generateIndexerId(fileBasedName, identifier) {
+  return `indexer_${fileBasedName}_${identifier || "default"}`.replace(
+    /[^a-zA-Z0-9_]/g,
+    "_"
+  );
+}
+
+export { generateIndexerId };

package/dist/internal/plugins.cjs

@@ -0,0 +1,38 @@
+'use strict';
+
+const context = require('../shared/indexer.077335f3.cjs');
+const config = require('../shared/indexer.601ceab0.cjs');
+require('node:async_hooks');
+require('unctx');
+
+const INTERNAL_CONTEXT_PROPERTY = "_internal";
+function internalContext(values) {
+  return config.defineIndexerPlugin((indexer) => {
+    indexer.hooks.hook("run:before", () => {
+      try {
+        const ctx = context.useIndexerContext();
+        ctx[INTERNAL_CONTEXT_PROPERTY] = {
+          ...ctx[INTERNAL_CONTEXT_PROPERTY] || {},
+          ...values
+        };
+      } catch (error) {
+        throw new Error("Failed to set internal context", {
+          cause: error
+        });
+      }
+    });
+  });
+}
+function useInternalContext() {
+  const ctx = context.useIndexerContext();
+  if (ctx[INTERNAL_CONTEXT_PROPERTY] === void 0) {
+    throw new Error(
+      "Internal context is not available, possibly 'internalContext' plugin is missing!"
+    );
+  }
+  return ctx[INTERNAL_CONTEXT_PROPERTY];
+}
+
+exports.INTERNAL_CONTEXT_PROPERTY = INTERNAL_CONTEXT_PROPERTY;
+exports.internalContext = internalContext;
+exports.useInternalContext = useInternalContext;

package/dist/internal/plugins.d.cts

@@ -0,0 +1,13 @@
+import { b as IndexerPlugin } from '../shared/indexer.fedcd831.cjs';
+import '@apibara/protocol';
+import 'hookable';
+
+declare const INTERNAL_CONTEXT_PROPERTY = "_internal";
+declare function internalContext<TFilter, TBlock, TTxnParams>(values: Record<string, unknown>): IndexerPlugin<TFilter, TBlock>;
+type InternalContext = {
+    indexerName: string;
+    availableIndexers: string[];
+};
+declare function useInternalContext(): InternalContext;
+
+export { INTERNAL_CONTEXT_PROPERTY, type InternalContext, internalContext, useInternalContext };

package/dist/internal/plugins.d.mts

@@ -0,0 +1,13 @@
+import { b as IndexerPlugin } from '../shared/indexer.fedcd831.mjs';
+import '@apibara/protocol';
+import 'hookable';
+
+declare const INTERNAL_CONTEXT_PROPERTY = "_internal";
+declare function internalContext<TFilter, TBlock, TTxnParams>(values: Record<string, unknown>): IndexerPlugin<TFilter, TBlock>;
+type InternalContext = {
+    indexerName: string;
+    availableIndexers: string[];
+};
+declare function useInternalContext(): InternalContext;
+
+export { INTERNAL_CONTEXT_PROPERTY, type InternalContext, internalContext, useInternalContext };

package/dist/internal/plugins.d.ts

@@ -0,0 +1,13 @@
+import { b as IndexerPlugin } from '../shared/indexer.fedcd831.js';
+import '@apibara/protocol';
+import 'hookable';
+
+declare const INTERNAL_CONTEXT_PROPERTY = "_internal";
+declare function internalContext<TFilter, TBlock, TTxnParams>(values: Record<string, unknown>): IndexerPlugin<TFilter, TBlock>;
+type InternalContext = {
+    indexerName: string;
+    availableIndexers: string[];
+};
+declare function useInternalContext(): InternalContext;
+
+export { INTERNAL_CONTEXT_PROPERTY, type InternalContext, internalContext, useInternalContext };