apibara 2.1.0-beta.3 → 2.1.0-beta.31

package/src/runtime/internal/app.ts

@@ -8,32 +8,34 @@ import {
   inMemoryPersistence,
   logger,
 } from "@apibara/indexer/plugins";
-import { config } from "#apibara-internal-virtual/config";
+import {
+  type CreateClientOptions,
+  Metadata,
+  type StreamConfig,
+  createClient,
+} from "@apibara/protocol";
+import consola from "consola";
 import { indexers } from "#apibara-internal-virtual/indexers";
+import { logger as instrumentationLogger } from "#apibara-internal-virtual/instrumentation";
 import { createLogger } from "./logger";
 
 export const availableIndexers = indexers.map((i) => i.name);
 
-export function createIndexer(indexerName: string, preset?: string) {
-  let runtimeConfig: Record<string, unknown> = { ...config.runtimeConfig };
-
-  if (preset) {
-    if (config.presets === undefined) {
-      throw new Error(
-        `Specified preset "${preset}" but no presets were defined`,
-      );
-    }
-
-    if (config.presets[preset] === undefined) {
-      throw new Error(`Specified preset "${preset}" but it was not defined`);
-    }
-
-    const presetValue = config.presets[preset] as {
-      runtimeConfig: Record<string, unknown>;
-    };
-    runtimeConfig = { ...runtimeConfig, ...presetValue.runtimeConfig };
-  }
-
+export function createIndexer({
+  indexerName,
+  processedRuntimeConfig,
+  preset,
+}: {
+  indexerName: string;
+  /**
+   * Final processed runtime config to be used by the indexer
+   */
+  processedRuntimeConfig: Record<string, unknown>;
+  /**
+   * Preset name which was used to generate the runtime config
+   */
+  preset?: string;
+}) {
   const indexerDefinition = indexers.find((i) => i.name === indexerName);
 
   if (indexerDefinition === undefined) {
@@ -42,10 +44,18 @@ export function createIndexer(indexerName: string, preset?: string) {
     );
   }
 
+  const indexerModule = indexerDefinition.indexer?.default;
+  if (indexerModule === undefined) {
+    consola.warn(
+      `Specified indexer "${indexerName}" but it does not export a default. Ignoring.`,
+    );
+    return;
+  }
+
   const definition =
-    typeof indexerDefinition.indexer === "function"
-      ? indexerDefinition.indexer(runtimeConfig)
-      : indexerDefinition.indexer;
+    typeof indexerModule === "function"
+      ? indexerModule(processedRuntimeConfig)
+      : indexerModule;
 
   let reporter: ConsolaReporter = createLogger({
     indexer: indexerName,
@@ -53,26 +63,62 @@ export function createIndexer(indexerName: string, preset?: string) {
     indexers: availableIndexers,
   });
 
-  if (config.logger) {
-    reporter = config.logger({
+  // Check if a custom logger is provided through instrumentation
+  if (instrumentationLogger) {
+    // Create a reporter using the custom logger function
+    const _reporter = instrumentationLogger({
       indexer: indexerName,
       preset,
       indexers: availableIndexers,
     });
+
+    // If the reporter is valid (has a log method), use it instead of the default
+    if (_reporter && "log" in _reporter) {
+      reporter = _reporter;
+    }
   }
 
   // Put the in-memory persistence plugin first so that it can be overridden by any user-defined
   // persistence plugin.
-  // Put the logger last since we want to override any user-defined logger.
   definition.plugins = [
     internalContext({
       indexerName,
       availableIndexers,
     } as InternalContext),
+    logger({ logger: reporter }),
     inMemoryPersistence(),
     ...(definition.plugins ?? []),
-    logger({ logger: reporter }),
   ];
 
-  return _createIndexer(definition);
+  return {
+    indexer: _createIndexer(definition),
+    logger: consola.create({ reporters: [reporter] }),
+  };
+}
+
+export function createAuthenticatedClient(
+  config: StreamConfig<unknown, unknown>,
+  streamUrl: string,
+  options?: CreateClientOptions,
+) {
+  const dnaToken = process.env.DNA_TOKEN;
+  if (!dnaToken) {
+    consola.warn(
+      "DNA_TOKEN environment variable is not set. Trying to connect without authentication.",
+    );
+  }
+
+  return createClient(config, streamUrl, {
+    ...options,
+    defaultCallOptions: {
+      ...(options?.defaultCallOptions ?? {}),
+      "*": {
+        metadata: Metadata({
+          Authorization: `Bearer ${dnaToken}`,
+        }),
+        // metadata cant be overrided with spread as its a class so we override it fully if user provided it.
+        ...(options?.defaultCallOptions?.["*"] ?? {}),
+      },
+    },
+  });
 }

package/src/runtime/project-info.ts

@@ -0,0 +1,90 @@
+import { writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import {
+  getProcessedRuntimeConfig,
+  getRuntimeDataFromEnv,
+} from "apibara/common";
+import { defineCommand, runMain } from "citty";
+import { availableIndexers, createIndexer } from "./internal/app";
+
+type ProjectInfo = {
+  indexers: {
+    [indexerName: string]: {
+      [presetName: string]: {
+        type: string;
+        isFactory: boolean;
+      };
+    };
+  };
+};
+
+const startCommand = defineCommand({
+  meta: {
+    name: "write-project-info",
+    description: "Write json-encoded information about the project.",
+  },
+  args: {
+    "build-dir": {
+      type: "string",
+      description: "project build directory",
+    },
+  },
+  async run({ args }) {
+    const projectInfo: ProjectInfo = {
+      indexers: {},
+    };
+
+    // get all original runtime data from env
+    const { presets, runtimeConfig, userEnvRuntimeConfig } =
+      getRuntimeDataFromEnv();
+
+    for (const preset of Object.keys(presets ?? {})) {
+      // process runtime config for each preset
+      const processedRuntimeConfig = getProcessedRuntimeConfig({
+        preset,
+        presets,
+        runtimeConfig,
+        userEnvRuntimeConfig,
+      });
+      for (const indexer of availableIndexers) {
+        const { indexer: indexerInstance } =
+          createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          }) ?? {};
+        if (!indexerInstance) {
+          continue;
+        }
+        projectInfo.indexers[indexer] = {
+          ...(projectInfo.indexers[indexer] ?? {}),
+          [preset]: {
+            type: indexerInstance.streamConfig.name,
+            isFactory: indexerInstance.options.factory !== undefined,
+          },
+        };
+      }
+    }
+
+    const projectInfoPath = resolve(
+      args["build-dir"] ?? ".apibara",
+      "project-info.json",
+    );
+
+    writeFileSync(projectInfoPath, JSON.stringify(projectInfo, null, 2));
+  },
+});
+
+export const mainCli = defineCommand({
+  meta: {
+    name: "write-project-info-runner",
+    description: "Write json-encoded information about the project.",
+  },
+  subCommands: {
+    start: () => startCommand,
+  },
+});
+
+runMain(mainCli);
+
+export default {};

package/src/runtime/start.ts

@@ -1,7 +1,10 @@
 import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { getRuntimeDataFromEnv } from "apibara/common";
 import { defineCommand, runMain } from "citty";
-import { createIndexer } from "./internal/app";
+import consola from "consola";
+import { blueBright } from "picocolors";
+import { register } from "#apibara-internal-virtual/instrumentation";
+import { createAuthenticatedClient, createIndexer } from "./internal/app";
 
 const startCommand = defineCommand({
   meta: {
@@ -14,21 +17,39 @@ const startCommand = defineCommand({
       description: "Indexer name",
       required: true,
     },
-    preset: {
-      type: "string",
-      description: "Preset to use",
-    },
   },
   async run({ args }) {
-    const { indexer, preset } = args;
+    const { indexer } = args;
+
+    const { processedRuntimeConfig, preset } = getRuntimeDataFromEnv();
 
-    const indexerInstance = createIndexer(indexer, preset);
+    const { indexer: indexerInstance, logger } =
+      createIndexer({
+        indexerName: indexer,
+        processedRuntimeConfig,
+        preset,
+      }) ?? {};
+    if (!indexerInstance) {
+      consola.error(`Specified indexer "${indexer}" but it was not defined`);
+      process.exit(1);
+    }
 
-    const client = createClient(
+    const client = createAuthenticatedClient(
       indexerInstance.streamConfig,
       indexerInstance.options.streamUrl,
+      indexerInstance.options.clientOptions,
     );
 
+    if (register) {
+      consola.start("Registering from instrumentation");
+      await register();
+      consola.success("Registered from instrumentation");
+    }
+
+    if (logger) {
+      logger.info(`Indexer ${blueBright(indexer)} started`);
+    }
+
     await runWithReconnect(client, indexerInstance);
   },
 });

package/src/types/config.ts

@@ -1,21 +1,29 @@
 import type { ConsolaReporter } from "@apibara/indexer/plugins";
-import type { RollupCommonJSOptions } from "@rollup/plugin-commonjs";
 import type {
   C12InputConfig,
   ConfigWatcher,
   ResolvedConfig,
   WatchConfigOptions,
 } from "c12";
-import type { WatchOptions } from "chokidar";
 import type { NestedHooks } from "hookable";
+import type { WatchOptions } from "rolldown";
+import type { RolldownOptions } from "rolldown";
 import type { DeepPartial } from "./_utils";
 import type { ApibaraHooks } from "./hooks";
-import type { RollupConfig } from "./rollup";
 
-export type LoggerFactory = ({
+export type RegisterFn = () => Promise<void>;
+
+export type LoggerFactoryFn = ({
   indexer,
+  indexers,
   preset,
-}: { indexer: string; indexers: string[]; preset?: string }) => ConsolaReporter;
+}: LoggerFactoryArgs) => ConsolaReporter;
+
+export type LoggerFactoryArgs = {
+  indexer: string;
+  indexers: string[];
+  preset?: string;
+};
 
 /**
  * Apibara Config type (apibara.config)
@@ -31,10 +39,12 @@ export interface ApibaraConfig<
   runtimeConfig?: R;
   presets?: T;
   preset?: keyof T;
-  logger?: LoggerFactory;
 }
 
-export type ApibaraDynamicConfig = Pick<ApibaraConfig, "runtimeConfig">;
+export type ApibaraDynamicConfig = Pick<
+  ApibaraConfig,
+  "runtimeConfig" | "preset" | "presets"
+>;
 
 /**
  * Config loader options
@@ -68,22 +78,26 @@ export interface ApibaraOptions<
   buildDir: string;
   outputDir: string;
   indexersDir: string;
+  disableLogs?: boolean;
 
   // Dev
   dev: boolean;
-  watchOptions: WatchOptions;
+  watchOptions: WatchOptions["watch"];
 
   // Hooks
   hooks: NestedHooks<ApibaraHooks>;
 
-  // Logging
-  logger?: LoggerFactory;
+  // Rolldown
+  rolldownConfig?: Partial<RolldownOptions>;
 
-  // Rollup
-  rollupConfig?: Partial<RollupConfig>;
+  /**
+   * @deprecated Use rolldownConfig instead. This option will be removed in future releases.
+   */
+  rollupConfig?: unknown;
   sourceMap?: boolean;
   entry: string;
-  commonJS?: RollupCommonJSOptions;
+  node: boolean;
+  exportConditions?: string[];
 
   // Advanced
   typescript: {

package/src/types/hooks.ts

@@ -1,12 +1,15 @@
+import type { RolldownOptions } from "rolldown";
 import type { Apibara } from "./apibara";
-import type { RollupConfig } from "./rollup";
 
 export interface ApibaraHooks {
-  "rollup:before": (apibara: Apibara, rollupConfig: RollupConfig) => void;
+  "rolldown:before": (
+    apibara: Apibara,
+    rolldownConfig: RolldownOptions,
+  ) => void;
   compiled: (apibara: Apibara) => void;
-  "dev:restart": () => void;
-  "dev:reload": () => void;
-  "rollup:reload": () => void;
+  "dev:restart": () => Promise<void>;
+  "dev:reload": () => Promise<void>;
+  "rolldown:reload": () => Promise<void>;
   restart: () => void;
   close: () => void;
 }

package/src/types/index.ts

@@ -2,5 +2,5 @@ export * from "./apibara";
 export * from "./_utils";
 export * from "./config";
 export * from "./hooks";
-export * from "./rollup";
+export * from "./rolldown";
 export * from "./runtime";

package/src/types/rolldown.ts

@@ -0,0 +1,5 @@
+import type { InputOptions, OutputOptions } from "rolldown";
+
+export type RolldownConfig = InputOptions & {
+  output: OutputOptions;
+};

package/src/types/virtual/indexers.d.ts

@@ -7,4 +7,7 @@ export type IndexerConstructor =
     ) => IndexerWithStreamConfig<unknown, unknown, unknown>)
   | IndexerWithStreamConfig<unknown, unknown, unknown>;
 
-export const indexers: { name: string; indexer: IndexerConstructor }[] = [];
+export const indexers: {
+  name: string;
+  indexer: { default?: IndexerConstructor | undefined };
+}[] = [];

package/src/types/virtual/instrumentation.d.ts

@@ -0,0 +1,4 @@
+import type { LoggerFactoryFn, RegisterFn } from "apibara/types";
+
+export const register: RegisterFn | undefined = undefined;
+export const logger: LoggerFactoryFn | undefined = undefined;

package/dist/rollup/index.d.mts

@@ -1,6 +0,0 @@
-import { Apibara, RollupConfig } from 'apibara/types';
-export { Plugin } from 'rollup';
-
-declare function getRollupConfig(apibara: Apibara): RollupConfig;
-
-export { getRollupConfig };

package/dist/rollup/index.d.ts

@@ -1,6 +0,0 @@
-import { Apibara, RollupConfig } from 'apibara/types';
-export { Plugin } from 'rollup';
-
-declare function getRollupConfig(apibara: Apibara): RollupConfig;
-
-export { getRollupConfig };

package/dist/rollup/index.mjs

@@ -1,150 +0,0 @@
-import { builtinModules } from 'node:module';
-import commonjs from '@rollup/plugin-commonjs';
-import json from '@rollup/plugin-json';
-import { nodeResolve } from '@rollup/plugin-node-resolve';
-import { join } from 'pathe';
-import defu from 'defu';
-import virtual from '@rollup/plugin-virtual';
-import MagicString from 'magic-string';
-import { hash } from 'ohash';
-
-function appConfig(apibara) {
-  return virtual({
-    "#apibara-internal-virtual/config": `
-    import * as projectConfig from '${apibara.options._c12.configFile}';
-
-    export const config = projectConfig.default;
-    `
-  });
-}
-
-function esmShim() {
-  const ESMShim = `
-// -- Shims --
-import cjsUrl from 'node:url';
-import cjsPath from 'node:path';
-const __filename = cjsUrl.fileURLToPath(import.meta.url);
-const __dirname = cjsPath.dirname(__filename);
-// -- End Shims --
-`;
-  const CJSyntaxRegex = /__filename|__dirname/;
-  return {
-    name: "esm-shim",
-    renderChunk(code, _chunk, opts) {
-      if (opts.format === "es") {
-        if (code.includes(ESMShim) || !CJSyntaxRegex.test(code)) {
-          return null;
-        }
-        let endIndexOfLastImport = -1;
-        for (const match of code.matchAll(/^import\s.*';$/gm)) {
-          if (match.length === 0 || typeof match.index !== "number") {
-            continue;
-          }
-          endIndexOfLastImport = match.index + match[0].length;
-        }
-        const s = new MagicString(code);
-        s.appendRight(endIndexOfLastImport, ESMShim);
-        const sourceMap = s.generateMap({
-          includeContent: true
-        });
-        let sourcesContent;
-        if (Array.isArray(sourceMap.sourcesContent)) {
-          sourcesContent = [];
-          for (let i = 0; i < sourceMap.sourcesContent.length; i++) {
-            const content = sourceMap.sourcesContent[i];
-            if (typeof content === "string") {
-              sourcesContent.push(content);
-            }
-          }
-        }
-        return {
-          code: s.toString(),
-          map: {
-            ...sourceMap,
-            sourcesContent
-          }
-        };
-      }
-      return null;
-    }
-  };
-}
-
-function indexers(apibara) {
-  const indexers2 = [...new Set(apibara.indexers)];
-  return virtual({
-    "#apibara-internal-virtual/indexers": `
-    ${indexers2.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
-
-    export const indexers = [
-      ${indexers2.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
-    ];
-    `
-  });
-}
-
-const runtimeDependencies = ["better-sqlite3", "@electric-sql/pglite"];
-function getRollupConfig(apibara) {
-  const extensions = [
-    ".ts",
-    ".mjs",
-    ".js",
-    ".json",
-    ".node",
-    ".tsx",
-    ".jsx"
-  ];
-  const rollupConfig = defu(
-    // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rollupConfig is typed
-    apibara.options.rollupConfig,
-    {
-      input: apibara.options.entry,
-      output: {
-        dir: join(apibara.options.outputDir || "./.apibara/build"),
-        format: "esm",
-        exports: "auto",
-        entryFileNames: "[name].mjs",
-        chunkFileNames: "chunks/[name]-[hash].mjs",
-        generatedCode: {
-          constBindings: true
-        },
-        sourcemap: true,
-        sourcemapExcludeSources: true,
-        sourcemapIgnoreList(relativePath, sourcemapPath) {
-          return relativePath.includes("node_modules");
-        }
-      },
-      plugins: [],
-      onwarn(warning, rollupWarn) {
-        if (!["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
-          warning.code || ""
-        ) && !warning.message.includes("Unsupported source map comment") && !warning.message.includes("@__PURE__") && !warning.message.includes("/*#__PURE__*/")) {
-          rollupWarn(warning);
-        }
-      },
-      treeshake: true,
-      external: [...builtinModules, ...runtimeDependencies]
-    }
-  );
-  rollupConfig.plugins.push(esmShim());
-  rollupConfig.plugins.push(json());
-  rollupConfig.plugins.push(
-    commonjs({
-      strictRequires: true,
-      requireReturnsDefault: "auto",
-      ...apibara.options.commonJS
-    })
-  );
-  rollupConfig.plugins.push(
-    nodeResolve({
-      extensions,
-      preferBuiltins: true,
-      mainFields: ["main"]
-    })
-  );
-  rollupConfig.plugins.push(indexers(apibara));
-  rollupConfig.plugins.push(appConfig(apibara));
-  return rollupConfig;
-}
-
-export { getRollupConfig };

package/dist/shared/apibara.1b515d04.mjs

@@ -1,8 +0,0 @@
-const commonArgs = {
-  dir: {
-    type: "string",
-    description: "project root directory"
-  }
-};
-
-export { commonArgs as c };

package/src/core/config/resolvers/preset.resolver.ts

@@ -1,9 +0,0 @@
-import type { ApibaraOptions } from "apibara/types";
-import defu from "defu";
-
-export async function presetResolver(options: ApibaraOptions) {
-  if (options.preset && options.presets?.[options.preset]) {
-    const new_options = defu(options.presets[options.preset], options);
-    Object.assign(options, new_options);
-  }
-}

package/src/core/config/resolvers/runtime-config.resolver.ts

@@ -1,6 +0,0 @@
-import type { ApibaraOptions } from "apibara/types";
-
-export async function resolveRuntimeConfigOptions(options: ApibaraOptions) {
-  options.runtimeConfig = { ...options.runtimeConfig };
-  process.env.APIBARA_RUNTIME_CONFIG = JSON.stringify(options.runtimeConfig);
-}