apibara 2.1.0-beta.4 → 2.1.0-beta.40

package/src/create/utils.ts

@@ -4,7 +4,7 @@ import * as prettier from "prettier";
 import prompts from "prompts";
 import { blue, cyan, red, yellow } from "./colors";
 import { dnaUrls, networks } from "./constants";
-import type { Chain, Language, Network, PkgInfo } from "./types";
+import type { Chain, FileExtension, Language, Network, PkgInfo } from "./types";
 
 export function isEmpty(path: string) {
   const files = fs.readdirSync(path);
@@ -31,7 +31,8 @@ export function validateLanguage(language?: string, throwError = false) {
     language === "typescript" ||
     language === "ts" ||
     language === "javascript" ||
-    language === "js"
+    language === "js" ||
+    language === "mjs"
   ) {
     return true;
   }
@@ -49,7 +50,7 @@ export function getLanguageFromAlias(alias: string): Language {
   if (alias === "ts" || alias === "typescript") {
     return "typescript";
   }
-  if (alias === "js" || alias === "javascript") {
+  if (alias === "js" || alias === "javascript" || alias === "mjs") {
     return "javascript";
   }
 
@@ -184,19 +185,31 @@ export function validateDnaUrl(dnaUrl?: string, throwError = false) {
 export function hasApibaraConfig(cwd: string): boolean {
   const configPathJS = path.join(cwd, "apibara.config.js");
   const configPathTS = path.join(cwd, "apibara.config.ts");
+  const configPathMJS = path.join(cwd, "apibara.config.mjs");
 
-  return fs.existsSync(configPathJS) || fs.existsSync(configPathTS);
+  return (
+    fs.existsSync(configPathJS) ||
+    fs.existsSync(configPathTS) ||
+    fs.existsSync(configPathMJS)
+  );
 }
 
-export function getApibaraConfigLanguage(cwd: string): Language {
+export function getApibaraConfigLanguage(cwd: string): {
+  language: Language;
+  extension: FileExtension;
+} {
   const configPathJS = path.join(cwd, "apibara.config.js");
   const configPathTS = path.join(cwd, "apibara.config.ts");
+  const configPathMJS = path.join(cwd, "apibara.config.mjs");
 
+  if (fs.existsSync(configPathMJS)) {
+    return { language: "javascript", extension: "mjs" };
+  }
   if (fs.existsSync(configPathJS)) {
-    return "javascript";
+    return { language: "javascript", extension: "js" };
   }
   if (fs.existsSync(configPathTS)) {
-    return "typescript";
+    return { language: "typescript", extension: "ts" };
   }
 
   throw new Error(red("✖") + " No apibara.config found");

package/src/hooks/useRuntimeConfig.ts

@@ -1,5 +1,8 @@
+import { ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME } from "apibara/common";
 import type { ApibaraRuntimeConfig } from "apibara/types";
 
 export function useRuntimeConfig(): ApibaraRuntimeConfig {
-  return JSON.parse(process.env.APIBARA_RUNTIME_CONFIG || "{}");
+  return JSON.parse(
+    process.env[ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME] || "{}",
+  );
 }

package/src/indexer/index.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer";

package/src/indexer/plugins.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/plugins";

package/src/indexer/testing.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/testing";

package/src/indexer/vcr.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/vcr";

package/src/rolldown/config.ts

@@ -0,0 +1,85 @@
+import { existsSync } from "node:fs";
+import { builtinModules } from "node:module";
+import type { Apibara } from "apibara/types";
+import defu from "defu";
+import { join } from "pathe";
+import type {
+  ConfigExport,
+  RolldownOptions,
+  RolldownPluginOption,
+} from "rolldown";
+import { indexers } from "./plugins/indexers";
+import { instrumentation } from "./plugins/instrumentation";
+import { staticConfig } from "./plugins/static-config";
+
+const runtimeDependencies = [
+  "better-sqlite3",
+  "@electric-sql/pglite",
+  "pg",
+  // https://socket.io/docs/v4/server-installation/#additional-packages
+  "utf-8-validate",
+  "bufferutil",
+  // was giving unresolved import warnings from `node-fetch` library.
+  "encoding",
+];
+
+export function getRolldownConfig(apibara: Apibara): RolldownOptions {
+  const extensions: string[] = [
+    ".ts",
+    ".mjs",
+    ".js",
+    ".json",
+    ".node",
+    ".tsx",
+    ".jsx",
+  ];
+
+  const tsConfigExists = existsSync(
+    join(apibara.options.rootDir, "tsconfig.json"),
+  );
+
+  const rolldownConfig: RolldownOptions & {
+    plugins: RolldownPluginOption[];
+  } = defu(
+    // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rolldownConfig is typed
+    apibara.options.rolldownConfig as any,
+    <ConfigExport>{
+      platform: "node",
+      input: apibara.options.entry,
+      output: {
+        dir: join(apibara.options.outputDir || "./.apibara/build"),
+        format: "esm",
+        entryFileNames: "[name].mjs",
+        chunkFileNames: "chunks/[name]-[hash].mjs",
+        sourcemap: true,
+      },
+      plugins: [],
+      onwarn(warning, rolldownWarn) {
+        if (
+          !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
+            warning.code || "",
+          ) &&
+          !warning.message.includes("Unsupported source map comment") &&
+          !warning.message.includes("@__PURE__") &&
+          !warning.message.includes("/*#__PURE__*/")
+        ) {
+          rolldownWarn(warning);
+        }
+      },
+      resolve: {
+        extensions,
+        preferBuiltins: !!apibara.options.node,
+        mainFields: ["main"],
+        exportConditions: apibara.options.exportConditions,
+        tsconfigFilename: tsConfigExists ? "tsconfig.json" : undefined,
+      },
+      treeshake: true,
+      external: [...builtinModules, ...runtimeDependencies],
+    },
+  );
+
+  rolldownConfig.plugins?.push(staticConfig(apibara));
+  rolldownConfig.plugins?.push(instrumentation(apibara));
+  rolldownConfig.plugins?.push(indexers(apibara));
+  return rolldownConfig;
+}

package/src/rolldown/index.ts

@@ -0,0 +1,2 @@
+export * from "./config";
+export type { Plugin } from "rolldown";

package/src/{rollup → rolldown}/plugins/indexers.ts

@@ -1,17 +1,17 @@
 import virtual from "@rollup/plugin-virtual";
 import type { Apibara } from "apibara/types";
 import { hash } from "ohash";
+import type { RolldownPluginOption } from "rolldown";
 
 export function indexers(apibara: Apibara) {
   const indexers = [...new Set(apibara.indexers)];
-
   return virtual({
     "#apibara-internal-virtual/indexers": `
-    ${indexers.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
+    ${indexers.map((i) => `import * as _${hash(i)} from '${i.indexer}';`).join("\n")}
 
     export const indexers = [
       ${indexers.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
     ];
     `,
-  });
+  }) as RolldownPluginOption;
 }

package/src/rolldown/plugins/instrumentation.ts

@@ -0,0 +1,68 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function instrumentation(apibara: Apibara) {
+  const instrumentationPath = join(
+    apibara.options._c12.cwd!,
+    `instrumentation.${apibara.options._c12.configFile?.endsWith(".ts") ? "ts" : apibara.options._c12.configFile?.endsWith(".mjs") ? "mjs" : "js"}`,
+  );
+
+  if (!existsSync(instrumentationPath)) {
+    return virtual({
+      "#apibara-internal-virtual/instrumentation": `
+      let register = undefined;
+      let logger = undefined;
+  
+      export { register, logger };
+      `,
+    }) as RolldownPluginOption;
+  }
+
+  /**
+   * We are importing the instrumentation file inline with "require" instead of "import" at the top of the file to avoid warnings from rolldown
+   * when some methods are not defined in the instrumentation file.
+   *
+   * Example warning:
+   * 
+   * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+   *     ╭─[virtual:#apibara-internal-virtual/instrumentation:11:35]
+   *     │
+   *  11 │     if (instrumentation && typeof instrumentation.logger === "function") {
+   *     │                                   ───────────┬──────────  
+   *     │                                              ╰──────────── 
+   * ────╯
+
+  * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+  *     ╭─[virtual:#apibara-internal-virtual/instrumentation:12:16]
+  *     │
+  *  12 │       logger = instrumentation.logger;
+  *     │                ───────────┬──────────  
+  *     │                           ╰──────────── 
+  * ────╯ 
+  */
+  return virtual({
+    "#apibara-internal-virtual/instrumentation": `
+    let register = undefined;
+    let logger = undefined;
+
+    try {
+      const instrumentation = require('${instrumentationPath}');
+      
+      if (instrumentation?.register && typeof instrumentation.register === "function") {
+        register = instrumentation.register;
+      }
+      
+      if (instrumentation?.logger && typeof instrumentation.logger === "function") {
+        logger = instrumentation.logger;
+      }
+    } catch {
+      // Silently handle any require errors
+    }
+
+    export { register, logger };
+    `,
+  }) as RolldownPluginOption;
+}

package/src/rolldown/plugins/static-config.ts

@@ -0,0 +1,21 @@
+import virtual from "@rollup/plugin-virtual";
+import { USER_ENV_APIBARA_RUNTIME_CONFIG } from "apibara/common";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function staticConfig(apibara: Apibara) {
+  const presetString = apibara.options.preset ?? "";
+  const presetsStringified = JSON.stringify(apibara.options.presets ?? {});
+  const runtimeConfigStringified = JSON.stringify(
+    apibara.options.runtimeConfig ?? {},
+  );
+
+  return virtual({
+    "#apibara-internal-virtual/static-config": `
+    export const preset = ${presetString ? `"${presetString}"` : "undefined"};
+    export const presets = ${presetsStringified};
+    export const runtimeConfig = ${runtimeConfigStringified};
+    export const userEnvRuntimeConfig = JSON.parse(process.env.${USER_ENV_APIBARA_RUNTIME_CONFIG} ?? "{}");
+    `,
+  }) as RolldownPluginOption;
+}

package/src/runtime/dev.ts

@@ -1,6 +1,8 @@
 import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { createAuthenticatedClient } from "@apibara/protocol";
+import { getRuntimeDataFromEnv } from "apibara/common";
 import { defineCommand, runMain } from "citty";
+import { blueBright } from "picocolors";
 import { availableIndexers, createIndexer } from "./internal/app";
 
 const startCommand = defineCommand({
@@ -13,13 +15,11 @@ const startCommand = defineCommand({
       type: "string",
       description: "Which indexers to run",
     },
-    preset: {
-      type: "string",
-      description: "Preset to use",
-    },
   },
   async run({ args }) {
-    const { indexers: indexersArgs, preset } = args;
+    const { indexers: indexersArgs } = args;
+
+    const { processedRuntimeConfig, preset } = getRuntimeDataFromEnv();
 
     let selectedIndexers = availableIndexers;
     if (indexersArgs) {
@@ -36,13 +36,26 @@ const startCommand = defineCommand({
 
     await Promise.all(
       selectedIndexers.map(async (indexer) => {
-        const indexerInstance = createIndexer(indexer, preset);
+        const { indexer: indexerInstance, logger } =
+          createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          }) ?? {};
+        if (!indexerInstance) {
+          return;
+        }
 
-        const client = createClient(
+        const client = createAuthenticatedClient(
           indexerInstance.streamConfig,
           indexerInstance.options.streamUrl,
+          indexerInstance.options.clientOptions,
         );
 
+        if (logger) {
+          logger.info(`Indexer ${blueBright(indexer)} started`);
+        }
+
         await runWithReconnect(client, indexerInstance);
       }),
     );

package/src/runtime/internal/app.ts

@@ -8,32 +8,28 @@ import {
   inMemoryPersistence,
   logger,
 } from "@apibara/indexer/plugins";
-import { config } from "#apibara-internal-virtual/config";
+import consola from "consola";
 import { indexers } from "#apibara-internal-virtual/indexers";
+import { logger as instrumentationLogger } from "#apibara-internal-virtual/instrumentation";
 import { createLogger } from "./logger";
 
 export const availableIndexers = indexers.map((i) => i.name);
 
-export function createIndexer(indexerName: string, preset?: string) {
-  let runtimeConfig: Record<string, unknown> = { ...config.runtimeConfig };
-
-  if (preset) {
-    if (config.presets === undefined) {
-      throw new Error(
-        `Specified preset "${preset}" but no presets were defined`,
-      );
-    }
-
-    if (config.presets[preset] === undefined) {
-      throw new Error(`Specified preset "${preset}" but it was not defined`);
-    }
-
-    const presetValue = config.presets[preset] as {
-      runtimeConfig: Record<string, unknown>;
-    };
-    runtimeConfig = { ...runtimeConfig, ...presetValue.runtimeConfig };
-  }
-
+export function createIndexer({
+  indexerName,
+  processedRuntimeConfig,
+  preset,
+}: {
+  indexerName: string;
+  /**
+   * Final processed runtime config to be used by the indexer
+   */
+  processedRuntimeConfig: Record<string, unknown>;
+  /**
+   * Preset name which was used to generate the runtime config
+   */
+  preset?: string;
+}) {
   const indexerDefinition = indexers.find((i) => i.name === indexerName);
 
   if (indexerDefinition === undefined) {
@@ -42,10 +38,18 @@ export function createIndexer(indexerName: string, preset?: string) {
     );
   }
 
+  const indexerModule = indexerDefinition.indexer?.default;
+  if (indexerModule === undefined) {
+    consola.warn(
+      `Specified indexer "${indexerName}" but it does not export a default. Ignoring.`,
+    );
+    return;
+  }
+
   const definition =
-    typeof indexerDefinition.indexer === "function"
-      ? indexerDefinition.indexer(runtimeConfig)
-      : indexerDefinition.indexer;
+    typeof indexerModule === "function"
+      ? indexerModule(processedRuntimeConfig)
+      : indexerModule;
 
   let reporter: ConsolaReporter = createLogger({
     indexer: indexerName,
@@ -53,26 +57,35 @@ export function createIndexer(indexerName: string, preset?: string) {
     indexers: availableIndexers,
   });
 
-  if (config.logger) {
-    reporter = config.logger({
+  // Check if a custom logger is provided through instrumentation
+  if (instrumentationLogger) {
+    // Create a reporter using the custom logger function
+    const _reporter = instrumentationLogger({
       indexer: indexerName,
       preset,
       indexers: availableIndexers,
     });
+
+    // If the reporter is valid (has a log method), use it instead of the default
+    if (_reporter && "log" in _reporter) {
+      reporter = _reporter;
+    }
   }
 
   // Put the in-memory persistence plugin first so that it can be overridden by any user-defined
   // persistence plugin.
-  // Put the logger last since we want to override any user-defined logger.
   definition.plugins = [
     internalContext({
       indexerName,
       availableIndexers,
     } as InternalContext),
+    logger({ logger: reporter }),
     inMemoryPersistence(),
     ...(definition.plugins ?? []),
-    logger({ logger: reporter }),
   ];
 
-  return _createIndexer(definition);
+  return {
+    indexer: _createIndexer(definition),
+    logger: consola.create({ reporters: [reporter] }),
+  };
 }

package/src/runtime/project-info.ts

@@ -0,0 +1,90 @@
+import { writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import {
+  getProcessedRuntimeConfig,
+  getRuntimeDataFromEnv,
+} from "apibara/common";
+import { defineCommand, runMain } from "citty";
+import { availableIndexers, createIndexer } from "./internal/app";
+
+type ProjectInfo = {
+  indexers: {
+    [indexerName: string]: {
+      [presetName: string]: {
+        type: string;
+        isFactory: boolean;
+      };
+    };
+  };
+};
+
+const startCommand = defineCommand({
+  meta: {
+    name: "write-project-info",
+    description: "Write json-encoded information about the project.",
+  },
+  args: {
+    "build-dir": {
+      type: "string",
+      description: "project build directory",
+    },
+  },
+  async run({ args }) {
+    const projectInfo: ProjectInfo = {
+      indexers: {},
+    };
+
+    // get all original runtime data from env
+    const { presets, runtimeConfig, userEnvRuntimeConfig } =
+      getRuntimeDataFromEnv();
+
+    for (const preset of Object.keys(presets ?? {})) {
+      // process runtime config for each preset
+      const processedRuntimeConfig = getProcessedRuntimeConfig({
+        preset,
+        presets,
+        runtimeConfig,
+        userEnvRuntimeConfig,
+      });
+      for (const indexer of availableIndexers) {
+        const { indexer: indexerInstance } =
+          createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          }) ?? {};
+        if (!indexerInstance) {
+          continue;
+        }
+        projectInfo.indexers[indexer] = {
+          ...(projectInfo.indexers[indexer] ?? {}),
+          [preset]: {
+            type: indexerInstance.streamConfig.name,
+            isFactory: indexerInstance.options.factory !== undefined,
+          },
+        };
+      }
+    }
+
+    const projectInfoPath = resolve(
+      args["build-dir"] ?? ".apibara",
+      "project-info.json",
+    );
+
+    writeFileSync(projectInfoPath, JSON.stringify(projectInfo, null, 2));
+  },
+});
+
+export const mainCli = defineCommand({
+  meta: {
+    name: "write-project-info-runner",
+    description: "Write json-encoded information about the project.",
+  },
+  subCommands: {
+    start: () => startCommand,
+  },
+});
+
+runMain(mainCli);
+
+export default {};

package/src/runtime/start.ts

@@ -1,6 +1,21 @@
 import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { createAuthenticatedClient } from "@apibara/protocol";
+import {
+  checkForUnknownArgs,
+  getProcessedRuntimeConfig,
+  getRuntimeDataFromEnv,
+} from "apibara/common";
 import { defineCommand, runMain } from "citty";
+import consola from "consola";
+import { blueBright } from "picocolors";
+import { register } from "#apibara-internal-virtual/instrumentation";
+// used when running with node .apibara/build/start.mjs as these values are made static during build time (except userEnvRuntimeConfig)
+import {
+  preset as originalPreset,
+  presets,
+  runtimeConfig,
+  userEnvRuntimeConfig,
+} from "#apibara-internal-virtual/static-config";
 import { createIndexer } from "./internal/app";
 
 const startCommand = defineCommand({
@@ -16,19 +31,66 @@ const startCommand = defineCommand({
     },
     preset: {
       type: "string",
-      description: "Preset to use",
+      description: "Preset name",
+    },
+    standalone: {
+      type: "boolean",
+      default: true,
+      description:
+        "--standalone: can run the indexer just with node and without apibara cli \n\t\t\t  --no-standalone: must run the indexer with apibara cli",
     },
   },
-  async run({ args }) {
-    const { indexer, preset } = args;
+  async run({ args, cmd }) {
+    const { indexer, preset: argPreset, standalone } = args;
+    await checkForUnknownArgs(args, cmd);
+
+    let preset: string | undefined;
+    let processedRuntimeConfig: Record<string, unknown> | undefined;
+
+    if (standalone) {
+      // when user does node .apibara/build/start.mjs
+      preset = argPreset ?? originalPreset;
+      processedRuntimeConfig = getProcessedRuntimeConfig({
+        preset,
+        presets,
+        runtimeConfig,
+        userEnvRuntimeConfig,
+      });
+    } else {
+      // When user does apibara start
+      const envResult = getRuntimeDataFromEnv();
+      preset = envResult.preset;
+      processedRuntimeConfig = envResult.processedRuntimeConfig;
+    }
 
-    const indexerInstance = createIndexer(indexer, preset);
+    const { indexer: indexerInstance, logger } =
+      createIndexer({
+        indexerName: indexer,
+        processedRuntimeConfig,
+        preset,
+      }) ?? {};
 
-    const client = createClient(
+    if (!indexerInstance) {
+      consola.error(`Specified indexer "${indexer}" but it was not defined`);
+      process.exit(1);
+    }
+
+    const client = createAuthenticatedClient(
       indexerInstance.streamConfig,
       indexerInstance.options.streamUrl,
+      indexerInstance.options.clientOptions,
     );
 
+    if (register) {
+      consola.start("Registering from instrumentation");
+      await register();
+      consola.success("Registered from instrumentation");
+    }
+
+    if (logger) {
+      logger.info(`Indexer ${blueBright(indexer)} started`);
+    }
+
     await runWithReconnect(client, indexerInstance);
   },
 });