apibara 2.0.0-beta.9 → 2.1.0-beta.3

package/src/rollup/plugins/config.ts

@@ -0,0 +1,12 @@
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+
+export function appConfig(apibara: Apibara) {
+  return virtual({
+    "#apibara-internal-virtual/config": `
+    import * as projectConfig from '${apibara.options._c12.configFile}';
+
+    export const config = projectConfig.default;
+    `,
+  });
+}

package/src/rollup/plugins/esm-shim.ts

@@ -0,0 +1,69 @@
+import MagicString from "magic-string";
+import type { Plugin } from "rollup";
+
+/**
+ * An alternative to @rollup/plugin-esm-shim
+ */
+export function esmShim(): Plugin {
+  const ESMShim = `
+// -- Shims --
+import cjsUrl from 'node:url';
+import cjsPath from 'node:path';
+const __filename = cjsUrl.fileURLToPath(import.meta.url);
+const __dirname = cjsPath.dirname(__filename);
+// -- End Shims --
+`;
+
+  const CJSyntaxRegex = /__filename|__dirname/;
+
+  return {
+    name: "esm-shim",
+
+    renderChunk(code, _chunk, opts) {
+      if (opts.format === "es") {
+        if (code.includes(ESMShim) || !CJSyntaxRegex.test(code)) {
+          return null;
+        }
+
+        let endIndexOfLastImport = -1;
+
+        // Find the last import statement and its ending index
+        for (const match of code.matchAll(/^import\s.*';$/gm)) {
+          if (match.length === 0 || typeof match.index !== "number") {
+            continue;
+          }
+
+          endIndexOfLastImport = match.index + match[0].length;
+        }
+
+        const s = new MagicString(code);
+        s.appendRight(endIndexOfLastImport, ESMShim);
+
+        const sourceMap = s.generateMap({
+          includeContent: true,
+        });
+
+        let sourcesContent: string[] | undefined;
+        if (Array.isArray(sourceMap.sourcesContent)) {
+          sourcesContent = [];
+          for (let i = 0; i < sourceMap.sourcesContent.length; i++) {
+            const content = sourceMap.sourcesContent[i];
+            if (typeof content === "string") {
+              sourcesContent.push(content);
+            }
+          }
+        }
+
+        return {
+          code: s.toString(),
+          map: {
+            ...sourceMap,
+            sourcesContent,
+          },
+        };
+      }
+
+      return null;
+    },
+  };
+}

package/src/rollup/plugins/indexers.ts

@@ -0,0 +1,17 @@
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+import { hash } from "ohash";
+
+export function indexers(apibara: Apibara) {
+  const indexers = [...new Set(apibara.indexers)];
+
+  return virtual({
+    "#apibara-internal-virtual/indexers": `
+    ${indexers.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
+
+    export const indexers = [
+      ${indexers.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
+    ];
+    `,
+  });
+}

package/src/runtime/dev.ts

@@ -0,0 +1,64 @@
+import { runWithReconnect } from "@apibara/indexer";
+import { createClient } from "@apibara/protocol";
+import { defineCommand, runMain } from "citty";
+import { availableIndexers, createIndexer } from "./internal/app";
+
+const startCommand = defineCommand({
+  meta: {
+    name: "start",
+    description: "Start the indexer",
+  },
+  args: {
+    indexers: {
+      type: "string",
+      description: "Which indexers to run",
+    },
+    preset: {
+      type: "string",
+      description: "Preset to use",
+    },
+  },
+  async run({ args }) {
+    const { indexers: indexersArgs, preset } = args;
+
+    let selectedIndexers = availableIndexers;
+    if (indexersArgs) {
+      selectedIndexers = indexersArgs.split(",");
+    }
+
+    for (const indexer of selectedIndexers) {
+      if (!availableIndexers.includes(indexer)) {
+        throw new Error(
+          `Specified indexer "${indexer}" but it was not defined`,
+        );
+      }
+    }
+
+    await Promise.all(
+      selectedIndexers.map(async (indexer) => {
+        const indexerInstance = createIndexer(indexer, preset);
+
+        const client = createClient(
+          indexerInstance.streamConfig,
+          indexerInstance.options.streamUrl,
+        );
+
+        await runWithReconnect(client, indexerInstance);
+      }),
+    );
+  },
+});
+
+export const mainCli = defineCommand({
+  meta: {
+    name: "indexer-dev-runner",
+    description: "Run indexer in dev mode",
+  },
+  subCommands: {
+    start: () => startCommand,
+  },
+});
+
+runMain(mainCli);
+
+export default {};

package/src/runtime/index.ts

@@ -0,0 +1,2 @@
+export { createIndexer } from "./internal/app";
+export { createLogger } from "./internal/logger";

package/src/runtime/internal/app.ts

@@ -0,0 +1,78 @@
+import { createIndexer as _createIndexer } from "@apibara/indexer";
+import {
+  type InternalContext,
+  internalContext,
+} from "@apibara/indexer/internal/plugins";
+import {
+  type ConsolaReporter,
+  inMemoryPersistence,
+  logger,
+} from "@apibara/indexer/plugins";
+import { config } from "#apibara-internal-virtual/config";
+import { indexers } from "#apibara-internal-virtual/indexers";
+import { createLogger } from "./logger";
+
+export const availableIndexers = indexers.map((i) => i.name);
+
+export function createIndexer(indexerName: string, preset?: string) {
+  let runtimeConfig: Record<string, unknown> = { ...config.runtimeConfig };
+
+  if (preset) {
+    if (config.presets === undefined) {
+      throw new Error(
+        `Specified preset "${preset}" but no presets were defined`,
+      );
+    }
+
+    if (config.presets[preset] === undefined) {
+      throw new Error(`Specified preset "${preset}" but it was not defined`);
+    }
+
+    const presetValue = config.presets[preset] as {
+      runtimeConfig: Record<string, unknown>;
+    };
+    runtimeConfig = { ...runtimeConfig, ...presetValue.runtimeConfig };
+  }
+
+  const indexerDefinition = indexers.find((i) => i.name === indexerName);
+
+  if (indexerDefinition === undefined) {
+    throw new Error(
+      `Specified indexer "${indexerName}" but it was not defined`,
+    );
+  }
+
+  const definition =
+    typeof indexerDefinition.indexer === "function"
+      ? indexerDefinition.indexer(runtimeConfig)
+      : indexerDefinition.indexer;
+
+  let reporter: ConsolaReporter = createLogger({
+    indexer: indexerName,
+    preset,
+    indexers: availableIndexers,
+  });
+
+  if (config.logger) {
+    reporter = config.logger({
+      indexer: indexerName,
+      preset,
+      indexers: availableIndexers,
+    });
+  }
+
+  // Put the in-memory persistence plugin first so that it can be overridden by any user-defined
+  // persistence plugin.
+  // Put the logger last since we want to override any user-defined logger.
+  definition.plugins = [
+    internalContext({
+      indexerName,
+      availableIndexers,
+    } as InternalContext),
+    inMemoryPersistence(),
+    ...(definition.plugins ?? []),
+    logger({ logger: reporter }),
+  ];
+
+  return _createIndexer(definition);
+}

package/src/runtime/internal/logger.ts

@@ -0,0 +1,70 @@
+import type {
+  ConsolaOptions,
+  ConsolaReporter,
+  LogLevel,
+  LogObject,
+  LogType,
+} from "consola";
+import { type ColorName, colors, getColor } from "consola/utils";
+import { murmurHash } from "ohash";
+
+const INDEXER_COLOR_MAP = [
+  colors.red,
+  colors.green,
+  colors.yellow,
+  colors.blue,
+  colors.magenta,
+  colors.cyan,
+];
+
+const TYPE_COLOR_MAP: { [k in LogType]?: string } = {
+  info: "cyan",
+  fail: "red",
+  success: "green",
+  ready: "green",
+  start: "magenta",
+};
+
+const LEVEL_COLOR_MAP: { [k in LogLevel]?: string } = {
+  0: "red",
+  1: "yellow",
+};
+
+const MAX_INDEXER_NAME_LENGTH = 20;
+
+class DefaultReporter implements ConsolaReporter {
+  private tag: string;
+
+  constructor(indexer: string, indexers: string[], preset?: string) {
+    const color =
+      INDEXER_COLOR_MAP[murmurHash(indexer) % INDEXER_COLOR_MAP.length];
+
+    const presetLength = preset ? preset.length : 0;
+
+    const longestIndexerName =
+      Math.max(...indexers.map((i) => i.length), indexer.length) + presetLength;
+
+    const paddedIndexer = `${indexer}${preset ? `:${preset} ` : ""}`
+      .padEnd(longestIndexerName, " ")
+      .slice(0, Math.min(longestIndexerName, MAX_INDEXER_NAME_LENGTH));
+
+    this.tag = color(`${paddedIndexer} |`);
+  }
+
+  log(logObj: LogObject, ctx: { options: ConsolaOptions }) {
+    const { args } = logObj;
+    const typeColor =
+      TYPE_COLOR_MAP[logObj.type] || LEVEL_COLOR_MAP[logObj.level] || "gray";
+
+    const type = getColor(typeColor as ColorName, "white")(logObj.type);
+    console.log(`${this.tag} ${type}`, ...args);
+  }
+}
+
+export function createLogger({
+  indexer,
+  indexers,
+  preset,
+}: { indexer: string; indexers: string[]; preset?: string }) {
+  return new DefaultReporter(indexer, indexers, preset);
+}

package/src/runtime/start.ts

@@ -0,0 +1,48 @@
+import { runWithReconnect } from "@apibara/indexer";
+import { createClient } from "@apibara/protocol";
+import { defineCommand, runMain } from "citty";
+import { createIndexer } from "./internal/app";
+
+const startCommand = defineCommand({
+  meta: {
+    name: "start",
+    description: "Start the indexer",
+  },
+  args: {
+    indexer: {
+      type: "string",
+      description: "Indexer name",
+      required: true,
+    },
+    preset: {
+      type: "string",
+      description: "Preset to use",
+    },
+  },
+  async run({ args }) {
+    const { indexer, preset } = args;
+
+    const indexerInstance = createIndexer(indexer, preset);
+
+    const client = createClient(
+      indexerInstance.streamConfig,
+      indexerInstance.options.streamUrl,
+    );
+
+    await runWithReconnect(client, indexerInstance);
+  },
+});
+
+export const mainCli = defineCommand({
+  meta: {
+    name: "indexer-runner",
+    description: "Run an indexer",
+  },
+  subCommands: {
+    start: () => startCommand,
+  },
+});
+
+runMain(mainCli);
+
+export default {};

package/src/types/apibara.ts

@@ -3,9 +3,17 @@ import type { Hookable } from "hookable";
 import type { ApibaraDynamicConfig, ApibaraOptions } from "./config";
 import type { ApibaraHooks } from "./hooks";
 
+export type IndexerDefinition = {
+  // Name of the indexer.
+  name: string;
+  // Path to the indexer file.
+  indexer: string;
+};
+
 export interface Apibara {
   options: ApibaraOptions;
   hooks: Hookable<ApibaraHooks>;
+  indexers: IndexerDefinition[];
   logger: ConsolaInstance;
   close: () => Promise<void>;
   updateConfig: (config: ApibaraDynamicConfig) => void | Promise<void>;

package/src/types/config.ts

@@ -1,4 +1,5 @@
-import type { Sink } from "@apibara/indexer";
+import type { ConsolaReporter } from "@apibara/indexer/plugins";
+import type { RollupCommonJSOptions } from "@rollup/plugin-commonjs";
 import type {
   C12InputConfig,
   ConfigWatcher,
@@ -7,28 +8,30 @@ import type {
 } from "c12";
 import type { WatchOptions } from "chokidar";
 import type { NestedHooks } from "hookable";
-import type { TSConfig } from "pkg-types";
 import type { DeepPartial } from "./_utils";
 import type { ApibaraHooks } from "./hooks";
 import type { RollupConfig } from "./rollup";
 
+export type LoggerFactory = ({
+  indexer,
+  preset,
+}: { indexer: string; indexers: string[]; preset?: string }) => ConsolaReporter;
+
 /**
  * Apibara Config type (apibara.config)
  */
 export interface ApibaraConfig<
-  // biome-ignore lint/complexity/noBannedTypes: <explanation>
-  T extends Record<string, DeepPartial<ApibaraConfig<T, R>>> = {},
-  // biome-ignore lint/complexity/noBannedTypes: <explanation>
-  R extends Record<string, unknown> = {},
-> extends DeepPartial<Omit<ApibaraOptions<T, R>, "preset" | "presets" | "dev">>,
+  T extends Record<
+    string,
+    DeepPartial<Pick<ApibaraConfig<T, R>, "runtimeConfig">>
+  > = Record<string, never>,
+  R extends Record<string, unknown> = Record<string, never>,
+> extends Partial<Omit<ApibaraOptions<T, R>, "preset" | "presets" | "dev">>,
     C12InputConfig<ApibaraConfig<T, R>> {
-  sink?: {
-    default: () => Sink;
-    [key: string]: () => Sink;
-  };
   runtimeConfig?: R;
   presets?: T;
   preset?: keyof T;
+  logger?: LoggerFactory;
 }
 
 export type ApibaraDynamicConfig = Pick<ApibaraConfig, "runtimeConfig">;
@@ -42,10 +45,11 @@ export interface LoadConfigOptions {
 }
 
 export interface ApibaraOptions<
-  // biome-ignore lint/complexity/noBannedTypes: <explanation>
-  T extends Record<string, DeepPartial<ApibaraConfig<T, R>>> = {},
-  // biome-ignore lint/complexity/noBannedTypes: <explanation>
-  R extends Record<string, unknown> = {},
+  T extends Record<
+    string,
+    DeepPartial<Pick<ApibaraConfig<T, R>, "runtimeConfig">>
+  > = Record<string, never>,
+  R extends Record<string, unknown> = Record<string, never>,
 > {
   // Internal
   _config: ApibaraConfig<T, R>;
@@ -53,12 +57,6 @@ export interface ApibaraOptions<
     | ResolvedConfig<ApibaraConfig<T, R>>
     | ConfigWatcher<ApibaraConfig<T, R>>;
 
-  // Sink
-  sink: {
-    default: () => Sink;
-    [key: string]: () => Sink;
-  };
-
   // Presets
   presets?: T;
   preset?: keyof T;
@@ -69,25 +67,28 @@ export interface ApibaraOptions<
   rootDir: string;
   buildDir: string;
   outputDir: string;
+  indexersDir: string;
+
   // Dev
   dev: boolean;
   watchOptions: WatchOptions;
 
   // Hooks
   hooks: NestedHooks<ApibaraHooks>;
+
+  // Logging
+  logger?: LoggerFactory;
+
   // Rollup
-  rollupConfig?: RollupConfig;
+  rollupConfig?: Partial<RollupConfig>;
+  sourceMap?: boolean;
   entry: string;
-  minify: boolean;
+  commonJS?: RollupCommonJSOptions;
 
   // Advanced
   typescript: {
     strict?: boolean;
     internalPaths?: boolean;
     generateRuntimeConfigTypes?: boolean;
-    generateTsConfig?: boolean;
-    /** the path of the generated `tsconfig.json`, relative to buildDir */
-    tsconfigPath: string;
-    tsConfig?: Partial<TSConfig>;
   };
 }

package/src/types/hooks.ts

@@ -4,6 +4,7 @@ import type { RollupConfig } from "./rollup";
 export interface ApibaraHooks {
   "rollup:before": (apibara: Apibara, rollupConfig: RollupConfig) => void;
   compiled: (apibara: Apibara) => void;
+  "dev:restart": () => void;
   "dev:reload": () => void;
   "rollup:reload": () => void;
   restart: () => void;

package/src/types/virtual/config.d.ts

@@ -0,0 +1,3 @@
+import type { ApibaraConfig } from "apibara/types";
+
+export const config: ApibaraConfig = {};

package/src/types/virtual/indexers.d.ts

@@ -0,0 +1,10 @@
+import type { IndexerWithStreamConfig } from "@apibara/indexer";
+import type { ApibaraRuntimeConfig } from "apibara/types";
+
+export type IndexerConstructor =
+  | ((
+      runtimeConfig: ApibaraRuntimeConfig,
+    ) => IndexerWithStreamConfig<unknown, unknown, unknown>)
+  | IndexerWithStreamConfig<unknown, unknown, unknown>;
+
+export const indexers: { name: string; indexer: IndexerConstructor }[] = [];

package/dist/internal/citty/index.d.mts

@@ -1 +0,0 @@
-export * from 'citty';

package/dist/internal/citty/index.d.ts

@@ -1 +0,0 @@
-export * from 'citty';

package/dist/internal/citty/index.mjs

@@ -1 +0,0 @@
-export * from 'citty';

package/dist/internal/consola/index.d.mts

@@ -1,2 +0,0 @@
-export * from 'consola';
-import 'citty';

package/dist/internal/consola/index.d.ts

@@ -1,2 +0,0 @@
-export * from 'consola';
-import 'citty';

package/dist/internal/consola/index.mjs

@@ -1 +0,0 @@
-export * from 'consola';

package/src/internal/citty/index.ts

@@ -1 +0,0 @@
-export * from "citty";

package/src/internal/consola/index.ts

@@ -1 +0,0 @@
-export * from "consola";