apibara 2.1.0-beta.5 → 2.1.0-beta.50

package/src/rolldown/plugins/instrumentation.ts

@@ -0,0 +1,68 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function instrumentation(apibara: Apibara) {
+  const instrumentationPath = join(
+    apibara.options._c12.cwd!,
+    `instrumentation.${apibara.options._c12.configFile?.endsWith(".ts") ? "ts" : apibara.options._c12.configFile?.endsWith(".mjs") ? "mjs" : "js"}`,
+  );
+
+  if (!existsSync(instrumentationPath)) {
+    return virtual({
+      "#apibara-internal-virtual/instrumentation": `
+      let register = undefined;
+      let logger = undefined;
+  
+      export { register, logger };
+      `,
+    }) as RolldownPluginOption;
+  }
+
+  /**
+   * We are importing the instrumentation file inline with "require" instead of "import" at the top of the file to avoid warnings from rolldown
+   * when some methods are not defined in the instrumentation file.
+   *
+   * Example warning:
+   * 
+   * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+   *     ╭─[virtual:#apibara-internal-virtual/instrumentation:11:35]
+   *     │
+   *  11 │     if (instrumentation && typeof instrumentation.logger === "function") {
+   *     │                                   ───────────┬──────────  
+   *     │                                              ╰──────────── 
+   * ────╯
+
+  * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+  *     ╭─[virtual:#apibara-internal-virtual/instrumentation:12:16]
+  *     │
+  *  12 │       logger = instrumentation.logger;
+  *     │                ───────────┬──────────  
+  *     │                           ╰──────────── 
+  * ────╯ 
+  */
+  return virtual({
+    "#apibara-internal-virtual/instrumentation": `
+    let register = undefined;
+    let logger = undefined;
+
+    try {
+      const instrumentation = require('${instrumentationPath}');
+      
+      if (instrumentation?.register && typeof instrumentation.register === "function") {
+        register = instrumentation.register;
+      }
+      
+      if (instrumentation?.logger && typeof instrumentation.logger === "function") {
+        logger = instrumentation.logger;
+      }
+    } catch {
+      // Silently handle any require errors
+    }
+
+    export { register, logger };
+    `,
+  }) as RolldownPluginOption;
+}

package/src/rolldown/plugins/static-config.ts

@@ -0,0 +1,21 @@
+import virtual from "@rollup/plugin-virtual";
+import { USER_ENV_APIBARA_RUNTIME_CONFIG } from "apibara/common";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function staticConfig(apibara: Apibara) {
+  const presetString = apibara.options.preset ?? "";
+  const presetsStringified = JSON.stringify(apibara.options.presets ?? {});
+  const runtimeConfigStringified = JSON.stringify(
+    apibara.options.runtimeConfig ?? {},
+  );
+
+  return virtual({
+    "#apibara-internal-virtual/static-config": `
+    export const preset = ${presetString ? `"${presetString}"` : "undefined"};
+    export const presets = ${presetsStringified};
+    export const runtimeConfig = ${runtimeConfigStringified};
+    export const userEnvRuntimeConfig = JSON.parse(process.env.${USER_ENV_APIBARA_RUNTIME_CONFIG} ?? "{}");
+    `,
+  }) as RolldownPluginOption;
+}

package/src/runtime/dev.ts

@@ -1,8 +1,53 @@
-import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { ReloadIndexerRequest, runWithReconnect } from "@apibara/indexer";
+import { createAuthenticatedClient } from "@apibara/protocol";
+import { getRuntimeDataFromEnv } from "apibara/common";
 import { defineCommand, runMain } from "citty";
+import type { ConsolaInstance } from "consola";
+import { blueBright } from "picocolors";
 import { availableIndexers, createIndexer } from "./internal/app";
 
+async function startIndexer(indexer: string) {
+  let _logger: ConsolaInstance | undefined;
+  while (true) {
+    try {
+      const { processedRuntimeConfig, preset } = getRuntimeDataFromEnv();
+
+      const { indexer: indexerInstance, logger } =
+        (await createIndexer({
+          indexerName: indexer,
+          processedRuntimeConfig,
+          preset,
+        })) ?? {};
+
+      _logger = logger;
+
+      if (!indexerInstance) {
+        return;
+      }
+
+      const client = createAuthenticatedClient(
+        indexerInstance.streamConfig,
+        indexerInstance.options.streamUrl,
+        indexerInstance.options.clientOptions,
+      );
+
+      if (logger) {
+        logger.info(`Indexer ${blueBright(indexer)} started`);
+      }
+
+      await runWithReconnect(client, indexerInstance);
+
+      return;
+    } catch (error) {
+      if (error instanceof ReloadIndexerRequest) {
+        _logger?.info(`Indexer ${blueBright(indexer)} reloaded`);
+        continue;
+      }
+      throw error;
+    }
+  }
+}
+
 const startCommand = defineCommand({
   meta: {
     name: "start",
@@ -13,13 +58,9 @@ const startCommand = defineCommand({
       type: "string",
       description: "Which indexers to run",
     },
-    preset: {
-      type: "string",
-      description: "Preset to use",
-    },
   },
   async run({ args }) {
-    const { indexers: indexersArgs, preset } = args;
+    const { indexers: indexersArgs } = args;
 
     let selectedIndexers = availableIndexers;
     if (indexersArgs) {
@@ -34,18 +75,7 @@ const startCommand = defineCommand({
       }
     }
 
-    await Promise.all(
-      selectedIndexers.map(async (indexer) => {
-        const indexerInstance = createIndexer(indexer, preset);
-
-        const client = createClient(
-          indexerInstance.streamConfig,
-          indexerInstance.options.streamUrl,
-        );
-
-        await runWithReconnect(client, indexerInstance);
-      }),
-    );
+    await Promise.all(selectedIndexers.map((indexer) => startIndexer(indexer)));
   },
 });
 

package/src/runtime/internal/app.ts

@@ -8,32 +8,28 @@ import {
   inMemoryPersistence,
   logger,
 } from "@apibara/indexer/plugins";
-import { config } from "#apibara-internal-virtual/config";
+import consola from "consola";
 import { indexers } from "#apibara-internal-virtual/indexers";
+import { logger as instrumentationLogger } from "#apibara-internal-virtual/instrumentation";
 import { createLogger } from "./logger";
 
 export const availableIndexers = indexers.map((i) => i.name);
 
-export function createIndexer(indexerName: string, preset?: string) {
-  let runtimeConfig: Record<string, unknown> = { ...config.runtimeConfig };
-
-  if (preset) {
-    if (config.presets === undefined) {
-      throw new Error(
-        `Specified preset "${preset}" but no presets were defined`,
-      );
-    }
-
-    if (config.presets[preset] === undefined) {
-      throw new Error(`Specified preset "${preset}" but it was not defined`);
-    }
-
-    const presetValue = config.presets[preset] as {
-      runtimeConfig: Record<string, unknown>;
-    };
-    runtimeConfig = { ...runtimeConfig, ...presetValue.runtimeConfig };
-  }
-
+export async function createIndexer({
+  indexerName,
+  processedRuntimeConfig,
+  preset,
+}: {
+  indexerName: string;
+  /**
+   * Final processed runtime config to be used by the indexer
+   */
+  processedRuntimeConfig: Record<string, unknown>;
+  /**
+   * Preset name which was used to generate the runtime config
+   */
+  preset?: string;
+}) {
   const indexerDefinition = indexers.find((i) => i.name === indexerName);
 
   if (indexerDefinition === undefined) {
@@ -42,10 +38,18 @@ export function createIndexer(indexerName: string, preset?: string) {
     );
   }
 
+  const indexerModule = indexerDefinition.indexer?.default;
+  if (indexerModule === undefined) {
+    consola.warn(
+      `Specified indexer "${indexerName}" but it does not export a default. Ignoring.`,
+    );
+    return;
+  }
+
   const definition =
-    typeof indexerDefinition.indexer === "function"
-      ? indexerDefinition.indexer(runtimeConfig)
-      : indexerDefinition.indexer;
+    typeof indexerModule === "function"
+      ? await indexerModule(processedRuntimeConfig)
+      : indexerModule;
 
   let reporter: ConsolaReporter = createLogger({
     indexer: indexerName,
@@ -53,26 +57,35 @@ export function createIndexer(indexerName: string, preset?: string) {
     indexers: availableIndexers,
   });
 
-  if (config.logger) {
-    reporter = config.logger({
+  // Check if a custom logger is provided through instrumentation
+  if (instrumentationLogger) {
+    // Create a reporter using the custom logger function
+    const _reporter = instrumentationLogger({
       indexer: indexerName,
       preset,
       indexers: availableIndexers,
     });
+
+    // If the reporter is valid (has a log method), use it instead of the default
+    if (_reporter && "log" in _reporter) {
+      reporter = _reporter;
+    }
   }
 
   // Put the in-memory persistence plugin first so that it can be overridden by any user-defined
   // persistence plugin.
-  // Put the logger last since we want to override any user-defined logger.
   definition.plugins = [
     internalContext({
       indexerName,
       availableIndexers,
     } as InternalContext),
+    logger({ logger: reporter }),
     inMemoryPersistence(),
     ...(definition.plugins ?? []),
-    logger({ logger: reporter }),
   ];
 
-  return _createIndexer(definition);
+  return {
+    indexer: _createIndexer(definition),
+    logger: consola.create({ reporters: [reporter] }),
+  };
 }

package/src/runtime/project-info.ts

@@ -0,0 +1,90 @@
+import { writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import {
+  getProcessedRuntimeConfig,
+  getRuntimeDataFromEnv,
+} from "apibara/common";
+import { defineCommand, runMain } from "citty";
+import { availableIndexers, createIndexer } from "./internal/app";
+
+type ProjectInfo = {
+  indexers: {
+    [indexerName: string]: {
+      [presetName: string]: {
+        type: string;
+        isFactory: boolean;
+      };
+    };
+  };
+};
+
+const startCommand = defineCommand({
+  meta: {
+    name: "write-project-info",
+    description: "Write json-encoded information about the project.",
+  },
+  args: {
+    "build-dir": {
+      type: "string",
+      description: "project build directory",
+    },
+  },
+  async run({ args }) {
+    const projectInfo: ProjectInfo = {
+      indexers: {},
+    };
+
+    // get all original runtime data from env
+    const { presets, runtimeConfig, userEnvRuntimeConfig } =
+      getRuntimeDataFromEnv();
+
+    for (const preset of Object.keys(presets ?? {})) {
+      // process runtime config for each preset
+      const processedRuntimeConfig = getProcessedRuntimeConfig({
+        preset,
+        presets,
+        runtimeConfig,
+        userEnvRuntimeConfig,
+      });
+      for (const indexer of availableIndexers) {
+        const { indexer: indexerInstance } =
+          (await createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          })) ?? {};
+        if (!indexerInstance) {
+          continue;
+        }
+        projectInfo.indexers[indexer] = {
+          ...(projectInfo.indexers[indexer] ?? {}),
+          [preset]: {
+            type: indexerInstance.streamConfig.name,
+            isFactory: indexerInstance.options.factory !== undefined,
+          },
+        };
+      }
+    }
+
+    const projectInfoPath = resolve(
+      args["build-dir"] ?? ".apibara",
+      "project-info.json",
+    );
+
+    writeFileSync(projectInfoPath, JSON.stringify(projectInfo, null, 2));
+  },
+});
+
+export const mainCli = defineCommand({
+  meta: {
+    name: "write-project-info-runner",
+    description: "Write json-encoded information about the project.",
+  },
+  subCommands: {
+    start: () => startCommand,
+  },
+});
+
+runMain(mainCli);
+
+export default {};

package/src/runtime/start.ts

@@ -1,6 +1,21 @@
-import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { ReloadIndexerRequest, runWithReconnect } from "@apibara/indexer";
+import { createAuthenticatedClient } from "@apibara/protocol";
+import {
+  checkForUnknownArgs,
+  getProcessedRuntimeConfig,
+  getRuntimeDataFromEnv,
+} from "apibara/common";
 import { defineCommand, runMain } from "citty";
+import consola, { type ConsolaInstance } from "consola";
+import { blueBright } from "picocolors";
+import { register } from "#apibara-internal-virtual/instrumentation";
+// used when running with node .apibara/build/start.mjs as these values are made static during build time (except userEnvRuntimeConfig)
+import {
+  preset as originalPreset,
+  presets,
+  runtimeConfig,
+  userEnvRuntimeConfig,
+} from "#apibara-internal-virtual/static-config";
 import { createIndexer } from "./internal/app";
 
 const startCommand = defineCommand({
@@ -16,20 +31,85 @@ const startCommand = defineCommand({
     },
     preset: {
       type: "string",
-      description: "Preset to use",
+      description: "Preset name",
+    },
+    standalone: {
+      type: "boolean",
+      default: true,
+      description:
+        "--standalone: can run the indexer just with node and without apibara cli \n\t\t\t  --no-standalone: must run the indexer with apibara cli",
     },
   },
-  async run({ args }) {
-    const { indexer, preset } = args;
+  async run({ args, cmd }) {
+    const { indexer, preset: argPreset, standalone } = args;
+    await checkForUnknownArgs(args, cmd);
+
+    let _logger: ConsolaInstance | undefined;
+
+    while (true) {
+      try {
+        let preset: string | undefined;
+        let processedRuntimeConfig: Record<string, unknown> | undefined;
+
+        if (standalone) {
+          // when user does node .apibara/build/start.mjs
+          preset = argPreset ?? originalPreset;
+          processedRuntimeConfig = getProcessedRuntimeConfig({
+            preset,
+            presets,
+            runtimeConfig,
+            userEnvRuntimeConfig,
+          });
+        } else {
+          // When user does apibara start
+          const envResult = getRuntimeDataFromEnv();
+          preset = envResult.preset;
+          processedRuntimeConfig = envResult.processedRuntimeConfig;
+        }
+
+        const { indexer: indexerInstance, logger } =
+          (await createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          })) ?? {};
+
+        _logger = logger;
+
+        if (!indexerInstance) {
+          consola.error(
+            `Specified indexer "${indexer}" but it was not defined`,
+          );
+          process.exit(1);
+        }
+
+        const client = createAuthenticatedClient(
+          indexerInstance.streamConfig,
+          indexerInstance.options.streamUrl,
+          indexerInstance.options.clientOptions,
+        );
+
+        if (register) {
+          consola.start("Registering from instrumentation");
+          await register();
+          consola.success("Registered from instrumentation");
+        }
 
-    const indexerInstance = createIndexer(indexer, preset);
+        if (logger) {
+          logger.info(`Indexer ${blueBright(indexer)} started`);
+        }
 
-    const client = createClient(
-      indexerInstance.streamConfig,
-      indexerInstance.options.streamUrl,
-    );
+        await runWithReconnect(client, indexerInstance);
 
-    await runWithReconnect(client, indexerInstance);
+        return;
+      } catch (error) {
+        if (error instanceof ReloadIndexerRequest) {
+          _logger?.info(`Indexer ${blueBright(indexer)} reloaded`);
+          continue;
+        }
+        throw error;
+      }
+    }
   },
 });
 

package/src/types/config.ts

@@ -1,21 +1,29 @@
 import type { ConsolaReporter } from "@apibara/indexer/plugins";
-import type { RollupCommonJSOptions } from "@rollup/plugin-commonjs";
 import type {
   C12InputConfig,
   ConfigWatcher,
   ResolvedConfig,
   WatchConfigOptions,
 } from "c12";
-import type { WatchOptions } from "chokidar";
 import type { NestedHooks } from "hookable";
+import type { WatchOptions } from "rolldown";
+import type { RolldownOptions } from "rolldown";
 import type { DeepPartial } from "./_utils";
 import type { ApibaraHooks } from "./hooks";
-import type { RollupConfig } from "./rollup";
 
-export type LoggerFactory = ({
+export type RegisterFn = () => Promise<void>;
+
+export type LoggerFactoryFn = ({
   indexer,
+  indexers,
   preset,
-}: { indexer: string; indexers: string[]; preset?: string }) => ConsolaReporter;
+}: LoggerFactoryArgs) => ConsolaReporter;
+
+export type LoggerFactoryArgs = {
+  indexer: string;
+  indexers: string[];
+  preset?: string;
+};
 
 /**
  * Apibara Config type (apibara.config)
@@ -31,10 +39,12 @@ export interface ApibaraConfig<
   runtimeConfig?: R;
   presets?: T;
   preset?: keyof T;
-  logger?: LoggerFactory;
 }
 
-export type ApibaraDynamicConfig = Pick<ApibaraConfig, "runtimeConfig">;
+export type ApibaraDynamicConfig = Pick<
+  ApibaraConfig,
+  "runtimeConfig" | "preset" | "presets"
+>;
 
 /**
  * Config loader options
@@ -68,22 +78,26 @@ export interface ApibaraOptions<
   buildDir: string;
   outputDir: string;
   indexersDir: string;
+  disableLogs?: boolean;
 
   // Dev
   dev: boolean;
-  watchOptions: WatchOptions;
+  watchOptions: WatchOptions["watch"];
 
   // Hooks
   hooks: NestedHooks<ApibaraHooks>;
 
-  // Logging
-  logger?: LoggerFactory;
+  // Rolldown
+  rolldownConfig?: Partial<RolldownOptions>;
 
-  // Rollup
-  rollupConfig?: Partial<RollupConfig>;
+  /**
+   * @deprecated Use rolldownConfig instead. This option will be removed in future releases.
+   */
+  rollupConfig?: unknown;
   sourceMap?: boolean;
   entry: string;
-  commonJS?: RollupCommonJSOptions;
+  node: boolean;
+  exportConditions?: string[];
 
   // Advanced
   typescript: {

package/src/types/hooks.ts

@@ -1,12 +1,15 @@
+import type { RolldownOptions } from "rolldown";
 import type { Apibara } from "./apibara";
-import type { RollupConfig } from "./rollup";
 
 export interface ApibaraHooks {
-  "rollup:before": (apibara: Apibara, rollupConfig: RollupConfig) => void;
+  "rolldown:before": (
+    apibara: Apibara,
+    rolldownConfig: RolldownOptions,
+  ) => void;
   compiled: (apibara: Apibara) => void;
-  "dev:restart": () => void;
-  "dev:reload": () => void;
-  "rollup:reload": () => void;
+  "dev:restart": () => Promise<void>;
+  "dev:reload": () => Promise<void>;
+  "rolldown:reload": () => Promise<void>;
   restart: () => void;
   close: () => void;
 }

package/src/types/index.ts

@@ -2,5 +2,5 @@ export * from "./apibara";
 export * from "./_utils";
 export * from "./config";
 export * from "./hooks";
-export * from "./rollup";
+export * from "./rolldown";
 export * from "./runtime";

package/src/types/rolldown.ts

@@ -0,0 +1,5 @@
+import type { InputOptions, OutputOptions } from "rolldown";
+
+export type RolldownConfig = InputOptions & {
+  output: OutputOptions;
+};

package/src/types/virtual/indexers.d.ts

@@ -7,4 +7,7 @@ export type IndexerConstructor =
     ) => IndexerWithStreamConfig<unknown, unknown, unknown>)
   | IndexerWithStreamConfig<unknown, unknown, unknown>;
 
-export const indexers: { name: string; indexer: IndexerConstructor }[] = [];
+export const indexers: {
+  name: string;
+  indexer: { default?: IndexerConstructor | undefined };
+}[] = [];

package/src/types/virtual/instrumentation.d.ts

@@ -0,0 +1,4 @@
+import type { LoggerFactoryFn, RegisterFn } from "apibara/types";
+
+export const register: RegisterFn | undefined = undefined;
+export const logger: LoggerFactoryFn | undefined = undefined;

package/src/types/virtual/static-config.d.ts

@@ -0,0 +1,4 @@
+export const preset: string | undefined = undefined;
+export const presets: Record<string, unknown> = {};
+export const runtimeConfig: Record<string, unknown> = {};
+export const userEnvRuntimeConfig: Record<string, unknown> = {};

package/dist/rollup/index.d.mts

@@ -1,6 +0,0 @@
-import { Apibara, RollupConfig } from 'apibara/types';
-export { Plugin } from 'rollup';
-
-declare function getRollupConfig(apibara: Apibara): RollupConfig;
-
-export { getRollupConfig };

package/dist/rollup/index.d.ts

@@ -1,6 +0,0 @@
-import { Apibara, RollupConfig } from 'apibara/types';
-export { Plugin } from 'rollup';
-
-declare function getRollupConfig(apibara: Apibara): RollupConfig;
-
-export { getRollupConfig };