apibara 2.1.0-beta.3 → 2.1.0-beta.31

package/src/create/templates.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import { consola } from "consola";
+import prompts from "prompts";
 import { type ObjectLiteralExpression, Project, SyntaxKind } from "ts-morph";
 import { cyan, green, magenta, yellow } from "./colors";
 import { packageVersions } from "./constants";
@@ -14,21 +15,18 @@ export function generatePackageJson(isTypeScript: boolean) {
     private: true,
     type: "module",
     scripts: {
-      prepare: "apibara prepare",
+      ...(isTypeScript && { prepare: "apibara prepare" }),
       dev: "apibara dev",
       start: "apibara start",
       build: "apibara build",
       ...(isTypeScript && { typecheck: "tsc --noEmit" }),
     },
     dependencies: {
-      "@apibara/indexer": packageVersions["@apibara/indexer"],
       "@apibara/protocol": packageVersions["@apibara/protocol"],
       apibara: packageVersions.apibara,
     },
     devDependencies: {
       ...(isTypeScript && {
-        "@rollup/plugin-typescript":
-          packageVersions["@rollup/plugin-typescript"],
         "@types/node": packageVersions["@types/node"],
         typescript: packageVersions.typescript,
       }),
@@ -58,17 +56,10 @@ export function generateTsConfig() {
 }
 
 export function generateApibaraConfig(isTypeScript: boolean) {
-  return `${isTypeScript ? 'import typescript from "@rollup/plugin-typescript";\nimport type { Plugin } from "apibara/rollup";\n' : ""}import { defineConfig } from "apibara/config";
+  return `import { defineConfig } from "apibara/config";
 
 export default defineConfig({
-  runtimeConfig: {},${
-    isTypeScript
-      ? `
-  rollupConfig: {
-    plugins: [typescript()${isTypeScript ? " as Plugin" : ""}],
-  },`
-      : ""
-  }
+  runtimeConfig: {},
 });\n`;
 }
 
@@ -78,9 +69,10 @@ export function generateIndexer({
   chain,
   language,
 }: IndexerOptions) {
-  return `import { defineIndexer } from "@apibara/indexer";
-import { useLogger } from "@apibara/indexer/plugins";
+  return `import { defineIndexer } from "apibara/indexer";
+import { useLogger } from "apibara/plugins";
 ${storage === "postgres" ? `import { drizzleStorage } from "@apibara/plugin-drizzle";` : ""}
+${storage === "postgres" ? `import { drizzle } from "@apibara/plugin-drizzle";` : ""}
 ${
   chain === "ethereum"
     ? `import { EvmStream } from "@apibara/evm";`
@@ -91,19 +83,17 @@ ${
         : ""
 }
 ${language === "typescript" ? `import type { ApibaraRuntimeConfig } from "apibara/types";` : ""}
-${
-  storage === "postgres"
-    ? `import { getDrizzlePgDatabase } from "../lib/db";`
-    : ""
-}
+${storage === "postgres" ? `import * as schema from "../lib/schema";` : ""}
 
 
 export default function (runtimeConfig${language === "typescript" ? ": ApibaraRuntimeConfig" : ""}) {
   const indexerId = "${indexerId}";
-  const { startingBlock, streamUrl${storage === "postgres" ? ", postgresConnectionString" : ""} } = runtimeConfig[indexerId];
+  const { startingBlock, streamUrl } = runtimeConfig[indexerId];
   ${
     storage === "postgres"
-      ? "const { db } = getDrizzlePgDatabase(postgresConnectionString);"
+      ? `const db = drizzle({
+    schema,
+  });`
       : ""
   }
 
@@ -122,7 +112,7 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
     filter: {
       header: "always",
     },
-    plugins: [${storage === "postgres" ? "drizzleStorage({ db, persistState: true })" : ""}],
+    plugins: [${storage === "postgres" ? "drizzleStorage({ db, migrate: { migrationsFolder: './drizzle' } })" : ""}],
     async transform({ endCursor, finality }) {
       const logger = useLogger();
 
@@ -136,14 +126,12 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
       ${
         storage === "postgres"
           ? `// Example snippet to insert data into db using drizzle with postgres
-      //   const { db } = useDrizzleStorage();
-      //   const { logs } = block;
-      //   for (const log of logs) {
-      //     await db.insert(exampleTable).values({
-      //       number: Number(endCursor?.orderKey),
-      //       hash: log.transactionHash,
-      //     });
-      //   }`
+      // const { db: database } = useDrizzleStorage();
+
+      // await database.insert(schema.cursorTable).values({
+      //   endCursor: Number(endCursor?.orderKey),
+      //   uniqueKey: \`\${endCursor?.uniqueKey}\`,
+      // });`
           : ""
       }
     },
@@ -235,12 +223,8 @@ export async function updateApibaraConfigFile({
 
   const runtimeConfigString = `{
   startingBlock: 0,
-  streamUrl: "${dnaUrl ?? getDnaUrl(chain, network)}"${
-    storage === "postgres"
-      ? `,
-  postgresConnectionString: process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://${indexerId}"`
-      : ""
-  }}`;
+  streamUrl: "${dnaUrl ?? getDnaUrl(chain, network)}"  
+}`;
 
   const project = new Project();
   const sourceFile = project.addSourceFileAtPath(pathToConfig);
@@ -284,7 +268,7 @@ export async function updateApibaraConfigFile({
 }
 
 export async function createDrizzleStorageFiles(options: IndexerOptions) {
-  const { cwd, language, storage } = options;
+  const { cwd, language, storage, indexerId } = options;
 
   if (storage !== "postgres") return;
 
@@ -312,11 +296,11 @@ export async function createDrizzleStorageFiles(options: IndexerOptions) {
     const drizzleConfigContent = `${language === "typescript" ? 'import type { Config } from "drizzle-kit";' : ""}
 
 export default {
-  schema: "./lib/schema.ts",
+  schema: "./lib/schema.${fileExtension}",
   out: "./drizzle",
   dialect: "postgresql",
   dbCredentials: {
-    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "",
+    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://${indexerId}",
   },
 }${language === "typescript" ? " satisfies Config" : ""};`;
 
@@ -347,14 +331,14 @@ export default {
     });
 
   if (!schemaExists || schemaOverwrite) {
-    const schemaContent = `//  --- Add your pg table schemas here ---- 
+    const schemaContent = `//  --- Add your pg table schemas here ----
 
 // import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-// export const exampleTable = pgTable("example_table", {
+// export const cursorTable = pgTable("cursor_table", {
 //   id: uuid("id").primaryKey().defaultRandom(),
-//   number: bigint("number", { mode: "number" }),
-//   hash: text("hash"),
+//   endCursor: bigint("end_cursor", { mode: "number" }),
+//   uniqueKey: text("unique_key"),
 // });
 
 export {};
@@ -369,63 +353,6 @@ export {};
     consola.success(`Created ${cyan("lib/schema.ts")}`);
   }
 
-  /**
-   *
-   *
-   * DB File
-   *
-   *
-   */
-  const dbFileName = `db.${fileExtension}`;
-
-  const dbPath = path.join(cwd, "lib", dbFileName);
-
-  const { exists: dbExists, overwrite: dbOverwrite } = await checkFileExists(
-    dbPath,
-    {
-      askPrompt: true,
-      fileName: `lib/${dbFileName}`,
-      allowIgnore: true,
-    },
-  );
-
-  if (!dbExists || dbOverwrite) {
-    const dbContent = `import * as schema from "./schema";
-import { drizzle as nodePgDrizzle } from "drizzle-orm/node-postgres";
-import { drizzle as pgLiteDrizzle } from "drizzle-orm/pglite";
-import pg from "pg";
-
-
-export function getDrizzlePgDatabase(connectionString${language === "typescript" ? ": string" : ""}) {
-  // Create pglite instance
-  if (connectionString.includes("memory")) {
-    return {
-      db: pgLiteDrizzle({
-        schema,
-        connection: {
-          dataDir: connectionString,
-        },
-      }),
-    };
-  }
-
-  // Create node-postgres instance
-  const pool = new pg.Pool({
-    connectionString,
-  });
-
-  return { db: nodePgDrizzle(pool, { schema }) };
-}`;
-
-    // create directory if it doesn't exist
-    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
-    fs.writeFileSync(dbPath, dbContent);
-
-    await formatFile(dbPath);
-
-    consola.success(`Created ${cyan(`lib/${dbFileName}`)}`);
-  }
-
   console.log("\n");
 
   // If schema file is created, show the example
@@ -445,17 +372,17 @@ ${yellow(`
 
 import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-export const exampleTable = pgTable("example_table", {
+export const cursorTable = pgTable("cursor_table", {
   id: uuid("id").primaryKey().defaultRandom(),
-  number: bigint("number", { mode: "number" }),
-  hash: text("hash"),
+  endCursor: bigint("end_cursor", { mode: "number" }),
+  uniqueKey: text("unique_key"),
 });`)}`);
 
     console.log("\n");
   }
 
   consola.info(
-    `Run ${green(`${options.packageManager} run drizzle:generate`)} & ${green(`${options.packageManager} run drizzle:migrate`)} to generate and apply migrations.`,
+    `Run ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:generate`)} & ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:migrate`)} to generate and apply migrations.`,
   );
 }
 
@@ -466,3 +393,103 @@ export async function createStorageRelatedFiles(options: IndexerOptions) {
     await createDrizzleStorageFiles(options);
   }
 }
+
+const gitIgnoreItems: {
+  isRecommended: boolean;
+  description?: string;
+  value: string;
+}[] = [
+  {
+    isRecommended: false,
+    value: "node_modules",
+  },
+  {
+    isRecommended: false,
+    value: "dist",
+  },
+  {
+    isRecommended: true,
+    description: "build and dev files of apibara",
+    value: ".apibara",
+  },
+  {
+    isRecommended: false,
+    value: ".env",
+  },
+  {
+    isRecommended: false,
+    description: "for mac users",
+    value: ".DS_Store",
+  },
+];
+
+export async function createGitIgnoreFile(cwd: string) {
+  const gitIgnorePath = path.join(cwd, ".gitignore");
+
+  if (fs.existsSync(gitIgnorePath)) {
+    const result = await prompts([
+      {
+        type: "select",
+        name: "overwrite",
+        message: `${cyan(".gitignore")} already exists. Please choose how to proceed:`,
+        initial: 0,
+        choices: [
+          {
+            title: "Choose items to append in your .gitignore",
+            value: "append",
+          },
+          {
+            title: "Keep original",
+            value: "ignore",
+          },
+          {
+            title: "Overwrite",
+            value: "overwrite",
+          },
+        ],
+      },
+      {
+        type: (overwrite: "append" | "ignore" | "overwrite") =>
+          overwrite === "append" ? "multiselect" : null,
+        name: "ignoreItems",
+        message: "Choose items to append in your .gitignore",
+        choices: gitIgnoreItems.map((item) => ({
+          title: `${yellow(item.value)}${
+            item.description ? ` - ${item.description}` : ""
+          }${item.isRecommended ? ` ${green("(recommended)")}` : ""}`,
+          value: item.value,
+        })),
+      },
+    ]);
+
+    const { overwrite, ignoreItems } = result as {
+      overwrite: "append" | "ignore" | "overwrite";
+      ignoreItems: string[];
+    };
+
+    if (overwrite === "append" && ignoreItems.length > 0) {
+      const gitIgnoreContent = fs.readFileSync(gitIgnorePath, "utf8");
+      fs.writeFileSync(
+        gitIgnorePath,
+        `${gitIgnoreContent}\n${result.ignoreItems.join("\n")}`,
+      );
+      consola.success(`Updated ${cyan(".gitignore")}`);
+      return;
+    }
+
+    if (overwrite === "overwrite") {
+      fs.writeFileSync(
+        gitIgnorePath,
+        gitIgnoreItems.map((item) => item.value).join("\n"),
+      );
+      consola.success(`Updated ${cyan(".gitignore")}`);
+      return;
+    }
+  }
+
+  fs.writeFileSync(
+    gitIgnorePath,
+    gitIgnoreItems.map((item) => item.value).join("\n"),
+  );
+  consola.success(`Created ${cyan(".gitignore")}`);
+}

package/src/hooks/useRuntimeConfig.ts

@@ -1,5 +1,8 @@
+import { ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME } from "apibara/common";
 import type { ApibaraRuntimeConfig } from "apibara/types";
 
 export function useRuntimeConfig(): ApibaraRuntimeConfig {
-  return JSON.parse(process.env.APIBARA_RUNTIME_CONFIG || "{}");
+  return JSON.parse(
+    process.env[ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME] || "{}",
+  );
 }

package/src/indexer/index.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer";

package/src/indexer/plugins.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/plugins";

package/src/indexer/testing.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/testing";

package/src/indexer/vcr.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/vcr";

package/src/rolldown/config.ts

@@ -0,0 +1,84 @@
+import { existsSync } from "node:fs";
+import { builtinModules } from "node:module";
+import type { Apibara } from "apibara/types";
+import defu from "defu";
+import { join } from "pathe";
+import type {
+  ConfigExport,
+  RolldownOptions,
+  RolldownPluginOption,
+} from "rolldown";
+import { indexers } from "./plugins/indexers";
+import { instrumentation } from "./plugins/instrumentation";
+
+const runtimeDependencies = [
+  "better-sqlite3",
+  "@electric-sql/pglite",
+  "pg",
+  // https://socket.io/docs/v4/server-installation/#additional-packages
+  "utf-8-validate",
+  "bufferutil",
+  // was giving unresolved import warnings from `node-fetch` library.
+  "encoding",
+];
+
+export function getRolldownConfig(apibara: Apibara): RolldownOptions {
+  const extensions: string[] = [
+    ".ts",
+    ".mjs",
+    ".js",
+    ".json",
+    ".node",
+    ".tsx",
+    ".jsx",
+  ];
+
+  const tsConfigExists = existsSync(
+    join(apibara.options.rootDir, "tsconfig.json"),
+  );
+
+  const rolldownConfig: RolldownOptions & {
+    plugins: RolldownPluginOption[];
+  } = defu(
+    // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rolldownConfig is typed
+    apibara.options.rolldownConfig as any,
+    <ConfigExport>{
+      platform: "node",
+      input: apibara.options.entry,
+      output: {
+        dir: join(apibara.options.outputDir || "./.apibara/build"),
+        format: "esm",
+        entryFileNames: "[name].mjs",
+        chunkFileNames: "chunks/[name]-[hash].mjs",
+        sourcemap: true,
+      },
+      plugins: [],
+      onwarn(warning, rolldownWarn) {
+        if (
+          !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
+            warning.code || "",
+          ) &&
+          !warning.message.includes("Unsupported source map comment") &&
+          !warning.message.includes("@__PURE__") &&
+          !warning.message.includes("/*#__PURE__*/")
+        ) {
+          rolldownWarn(warning);
+        }
+      },
+      resolve: {
+        extensions,
+        preferBuiltins: !!apibara.options.node,
+        mainFields: ["main"],
+        exportConditions: apibara.options.exportConditions,
+        tsconfigFilename: tsConfigExists ? "tsconfig.json" : undefined,
+      },
+      treeshake: true,
+      external: [...builtinModules, ...runtimeDependencies],
+    },
+  );
+
+  rolldownConfig.plugins?.push(instrumentation(apibara));
+  rolldownConfig.plugins?.push(indexers(apibara));
+
+  return rolldownConfig;
+}

package/src/rolldown/index.ts

@@ -0,0 +1,2 @@
+export * from "./config";
+export type { Plugin } from "rolldown";

package/src/{rollup → rolldown}/plugins/indexers.ts

@@ -1,17 +1,17 @@
 import virtual from "@rollup/plugin-virtual";
 import type { Apibara } from "apibara/types";
 import { hash } from "ohash";
+import type { RolldownPluginOption } from "rolldown";
 
 export function indexers(apibara: Apibara) {
   const indexers = [...new Set(apibara.indexers)];
-
   return virtual({
     "#apibara-internal-virtual/indexers": `
-    ${indexers.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
+    ${indexers.map((i) => `import * as _${hash(i)} from '${i.indexer}';`).join("\n")}
 
     export const indexers = [
       ${indexers.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
     ];
     `,
-  });
+  }) as RolldownPluginOption;
 }

package/src/rolldown/plugins/instrumentation.ts

@@ -0,0 +1,68 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function instrumentation(apibara: Apibara) {
+  const instrumentationPath = join(
+    apibara.options._c12.cwd!,
+    `instrumentation.${apibara.options._c12.configFile?.endsWith(".ts") ? "ts" : "js"}`,
+  );
+
+  if (!existsSync(instrumentationPath)) {
+    return virtual({
+      "#apibara-internal-virtual/instrumentation": `
+      let register = undefined;
+      let logger = undefined;
+  
+      export { register, logger };
+      `,
+    }) as RolldownPluginOption;
+  }
+
+  /**
+   * We are importing the instrumentation file inline with "require" instead of "import" at the top of the file to avoid warnings from rolldown
+   * when some methods are not defined in the instrumentation file.
+   *
+   * Example warning:
+   * 
+   * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+   *     ╭─[virtual:#apibara-internal-virtual/instrumentation:11:35]
+   *     │
+   *  11 │     if (instrumentation && typeof instrumentation.logger === "function") {
+   *     │                                   ───────────┬──────────  
+   *     │                                              ╰──────────── 
+   * ────╯
+
+  * [IMPORT_IS_UNDEFINED] Warning: Import `logger` will always be undefined because there is no matching export in 'instrumentation.ts'
+  *     ╭─[virtual:#apibara-internal-virtual/instrumentation:12:16]
+  *     │
+  *  12 │       logger = instrumentation.logger;
+  *     │                ───────────┬──────────  
+  *     │                           ╰──────────── 
+  * ────╯ 
+  */
+  return virtual({
+    "#apibara-internal-virtual/instrumentation": `
+    let register = undefined;
+    let logger = undefined;
+
+    try {
+      const instrumentation = require('${instrumentationPath}');
+      
+      if (instrumentation?.register && typeof instrumentation.register === "function") {
+        register = instrumentation.register;
+      }
+      
+      if (instrumentation?.logger && typeof instrumentation.logger === "function") {
+        logger = instrumentation.logger;
+      }
+    } catch {
+      // Silently handle any require errors
+    }
+
+    export { register, logger };
+    `,
+  }) as RolldownPluginOption;
+}

package/src/runtime/dev.ts

@@ -1,7 +1,12 @@
 import { runWithReconnect } from "@apibara/indexer";
-import { createClient } from "@apibara/protocol";
+import { getRuntimeDataFromEnv } from "apibara/common";
 import { defineCommand, runMain } from "citty";
-import { availableIndexers, createIndexer } from "./internal/app";
+import { blueBright } from "picocolors";
+import {
+  availableIndexers,
+  createAuthenticatedClient,
+  createIndexer,
+} from "./internal/app";
 
 const startCommand = defineCommand({
   meta: {
@@ -13,13 +18,11 @@ const startCommand = defineCommand({
       type: "string",
       description: "Which indexers to run",
     },
-    preset: {
-      type: "string",
-      description: "Preset to use",
-    },
   },
   async run({ args }) {
-    const { indexers: indexersArgs, preset } = args;
+    const { indexers: indexersArgs } = args;
+
+    const { processedRuntimeConfig, preset } = getRuntimeDataFromEnv();
 
     let selectedIndexers = availableIndexers;
     if (indexersArgs) {
@@ -36,13 +39,26 @@ const startCommand = defineCommand({
 
     await Promise.all(
       selectedIndexers.map(async (indexer) => {
-        const indexerInstance = createIndexer(indexer, preset);
+        const { indexer: indexerInstance, logger } =
+          createIndexer({
+            indexerName: indexer,
+            processedRuntimeConfig,
+            preset,
+          }) ?? {};
+        if (!indexerInstance) {
+          return;
+        }
 
-        const client = createClient(
+        const client = createAuthenticatedClient(
           indexerInstance.streamConfig,
           indexerInstance.options.streamUrl,
+          indexerInstance.options.clientOptions,
         );
 
+        if (logger) {
+          logger.info(`Indexer ${blueBright(indexer)} started`);
+        }
+
         await runWithReconnect(client, indexerInstance);
       }),
     );