apibara 2.1.0-beta.1 → 2.1.0-beta.11

package/src/create/templates.ts

@@ -1,11 +1,12 @@
 import fs from "node:fs";
 import path from "node:path";
 import { consola } from "consola";
+import prompts from "prompts";
 import { type ObjectLiteralExpression, Project, SyntaxKind } from "ts-morph";
 import { cyan, green, magenta, yellow } from "./colors";
 import { packageVersions } from "./constants";
 import type { IndexerOptions } from "./types";
-import { checkFileExists, getDnaUrl } from "./utils";
+import { checkFileExists, formatFile, getDnaUrl } from "./utils";
 
 export function generatePackageJson(isTypeScript: boolean) {
   return {
@@ -14,7 +15,7 @@ export function generatePackageJson(isTypeScript: boolean) {
     private: true,
     type: "module",
     scripts: {
-      prepare: "apibara prepare",
+      ...(isTypeScript && { prepare: "apibara prepare" }),
       dev: "apibara dev",
       start: "apibara start",
       build: "apibara build",
@@ -27,8 +28,6 @@ export function generatePackageJson(isTypeScript: boolean) {
     },
     devDependencies: {
       ...(isTypeScript && {
-        "@rollup/plugin-typescript":
-          packageVersions["@rollup/plugin-typescript"],
         "@types/node": packageVersions["@types/node"],
         typescript: packageVersions.typescript,
       }),
@@ -58,17 +57,10 @@ export function generateTsConfig() {
 }
 
 export function generateApibaraConfig(isTypeScript: boolean) {
-  return `${isTypeScript ? 'import typescript from "@rollup/plugin-typescript";\nimport type { Plugin } from "apibara/rollup";\n' : ""}import { defineConfig } from "apibara/config";
+  return `import { defineConfig } from "apibara/config";
 
 export default defineConfig({
-  runtimeConfig: {},${
-    isTypeScript
-      ? `
-  rollupConfig: {
-    plugins: [typescript()${isTypeScript ? " as Plugin" : ""}],
-  },`
-      : ""
-  }
+  runtimeConfig: {},
 });\n`;
 }
 
@@ -78,24 +70,21 @@ export function generateIndexer({
   chain,
   language,
 }: IndexerOptions) {
-  return `${
-    chain === "ethereum"
-      ? `import { EvmStream } from "@apibara/evm";`
-      : chain === "beaconchain"
-        ? `import { BeaconChainStream } from "@apibara/beaconchain";`
-        : chain === "starknet"
-          ? `import { StarknetStream } from "@apibara/starknet";`
-          : ""
-  }
-import { defineIndexer } from "@apibara/indexer";
-${storage === "postgres" ? `import { drizzleStorage } from "@apibara/plugin-drizzle";` : ""}
-${language === "typescript" ? `import type { ApibaraRuntimeConfig } from "apibara/types";` : ""}
+  return `import { defineIndexer } from "@apibara/indexer";
 import { useLogger } from "@apibara/indexer/plugins";
+${storage === "postgres" ? `import { drizzleStorage } from "@apibara/plugin-drizzle";` : ""}
+${storage === "postgres" ? `import { drizzle } from "@apibara/plugin-drizzle";` : ""}
 ${
-  storage === "postgres"
-    ? `import { getDrizzlePgDatabase } from "../lib/db";`
-    : ""
+  chain === "ethereum"
+    ? `import { EvmStream } from "@apibara/evm";`
+    : chain === "beaconchain"
+      ? `import { BeaconChainStream } from "@apibara/beaconchain";`
+      : chain === "starknet"
+        ? `import { StarknetStream } from "@apibara/starknet";`
+        : ""
 }
+${language === "typescript" ? `import type { ApibaraRuntimeConfig } from "apibara/types";` : ""}
+${storage === "postgres" ? `import * as schema from "../lib/schema";` : ""}
 
 
 export default function (runtimeConfig${language === "typescript" ? ": ApibaraRuntimeConfig" : ""}) {
@@ -103,7 +92,10 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
   const { startingBlock, streamUrl${storage === "postgres" ? ", postgresConnectionString" : ""} } = runtimeConfig[indexerId];
   ${
     storage === "postgres"
-      ? "const { db } = getDrizzlePgDatabase(postgresConnectionString);"
+      ? `const db = drizzle({
+    schema,
+    connectionString: postgresConnectionString,
+  });`
       : ""
   }
 
@@ -122,7 +114,7 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
     filter: {
       header: "always",
     },
-    plugins: [${storage === "postgres" ? "drizzleStorage({ db, persistState: true })" : ""}],
+    plugins: [${storage === "postgres" ? "drizzleStorage({ db, migrate: { migrationsFolder: './drizzle' } })" : ""}],
     async transform({ endCursor, finality }) {
       const logger = useLogger();
 
@@ -136,14 +128,12 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
       ${
         storage === "postgres"
           ? `// Example snippet to insert data into db using drizzle with postgres
-      //   const { db } = useDrizzleStorage();
-      //   const { logs } = block;
-      //   for (const log of logs) {
-      //     await db.insert(exampleTable).values({
-      //       number: Number(endCursor?.orderKey),
-      //       hash: log.transactionHash,
-      //     });
-      //   }`
+      // const { db: database } = useDrizzleStorage();
+
+      // await database.insert(schema.cursorTable).values({
+      //   endCursor: Number(endCursor?.orderKey),
+      //   uniqueKey: \`\${endCursor?.uniqueKey}\`,
+      // });`
           : ""
       }
     },
@@ -169,17 +159,19 @@ export async function createIndexerFile(options: IndexerOptions) {
 
   fs.mkdirSync(path.dirname(indexerFilePath), { recursive: true });
   fs.writeFileSync(indexerFilePath, indexerContent);
+
+  await formatFile(indexerFilePath);
 }
 
-export function updatePackageJson({
+export async function updatePackageJson({
   cwd,
   chain,
   storage,
   language,
 }: IndexerOptions) {
-  const packageJson = JSON.parse(
-    fs.readFileSync(path.join(cwd, "package.json"), "utf8"),
-  );
+  const packageJsonPath = path.join(cwd, "package.json");
+
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
 
   if (chain === "ethereum") {
     packageJson.dependencies["@apibara/evm"] = packageVersions["@apibara/evm"];
@@ -212,13 +204,12 @@ export function updatePackageJson({
     }
   }
 
-  fs.writeFileSync(
-    path.join(cwd, "package.json"),
-    JSON.stringify(packageJson, null, 2),
-  );
+  fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
+
+  await formatFile(packageJsonPath);
 }
 
-export function updateApibaraConfigFile({
+export async function updateApibaraConfigFile({
   indexerId,
   cwd,
   chain,
@@ -279,15 +270,11 @@ export function updateApibaraConfigFile({
   // Save the changes
   sourceFile.saveSync();
 
-  sourceFile.formatText({
-    tabSize: 2,
-    insertSpaceAfterOpeningAndBeforeClosingEmptyBraces: true,
-    insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces: true,
-  });
+  await formatFile(pathToConfig);
 }
 
 export async function createDrizzleStorageFiles(options: IndexerOptions) {
-  const { cwd, language, storage } = options;
+  const { cwd, language, storage, indexerId } = options;
 
   if (storage !== "postgres") return;
 
@@ -315,16 +302,18 @@ export async function createDrizzleStorageFiles(options: IndexerOptions) {
     const drizzleConfigContent = `${language === "typescript" ? 'import type { Config } from "drizzle-kit";' : ""}
 
 export default {
-  schema: "./lib/schema.ts",
+  schema: "./lib/schema.${fileExtension}",
   out: "./drizzle",
   dialect: "postgresql",
   dbCredentials: {
-    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "",
+    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://${indexerId}",
   },
 }${language === "typescript" ? " satisfies Config" : ""};`;
 
     fs.writeFileSync(drizzleConfigPath, drizzleConfigContent);
 
+    await formatFile(drizzleConfigPath);
+
     consola.success(`Created ${cyan(drizzleConfigFileName)}`);
   }
 
@@ -348,14 +337,14 @@ export default {
     });
 
   if (!schemaExists || schemaOverwrite) {
-    const schemaContent = `//  --- Add your pg table schemas here ---- 
+    const schemaContent = `//  --- Add your pg table schemas here ----
 
 // import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-// export const exampleTable = pgTable("example_table", {
+// export const cursorTable = pgTable("cursor_table", {
 //   id: uuid("id").primaryKey().defaultRandom(),
-//   number: bigint("number", { mode: "number" }),
-//   hash: text("hash"),
+//   endCursor: bigint("end_cursor", { mode: "number" }),
+//   uniqueKey: text("unique_key"),
 // });
 
 export {};
@@ -365,62 +354,9 @@ export {};
     fs.mkdirSync(path.dirname(schemaPath), { recursive: true });
     fs.writeFileSync(schemaPath, schemaContent);
 
-    consola.success(`Created ${cyan("lib/schema.ts")}`);
-  }
-
-  /**
-   *
-   *
-   * DB File
-   *
-   *
-   */
-  const dbFileName = `db.${fileExtension}`;
-
-  const dbPath = path.join(cwd, "lib", dbFileName);
+    await formatFile(schemaPath);
 
-  const { exists: dbExists, overwrite: dbOverwrite } = await checkFileExists(
-    dbPath,
-    {
-      askPrompt: true,
-      fileName: `lib/${dbFileName}`,
-      allowIgnore: true,
-    },
-  );
-
-  if (!dbExists || dbOverwrite) {
-    const dbContent = `import * as schema from "./schema";
-import { drizzle as nodePgDrizzle } from "drizzle-orm/node-postgres";
-import { drizzle as pgLiteDrizzle } from "drizzle-orm/pglite";
-import pg from "pg";
-
-
-export function getDrizzlePgDatabase(connectionString${language === "typescript" ? ": string" : ""}) {
-  // Create pglite instance
-  if (connectionString.includes("memory")) {
-    return {
-      db: pgLiteDrizzle({
-        schema,
-        connection: {
-          dataDir: connectionString,
-        },
-      }),
-    };
-  }
-
-  // Create node-postgres instance
-  const pool = new pg.Pool({
-    connectionString,
-  });
-
-  return { db: nodePgDrizzle(pool, { schema }) };
-}`;
-
-    // create directory if it doesn't exist
-    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
-    fs.writeFileSync(dbPath, dbContent);
-
-    consola.success(`Created ${cyan(`lib/${dbFileName}`)}`);
+    consola.success(`Created ${cyan("lib/schema.ts")}`);
   }
 
   console.log("\n");
@@ -442,17 +378,17 @@ ${yellow(`
 
 import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-export const exampleTable = pgTable("example_table", {
+export const cursorTable = pgTable("cursor_table", {
   id: uuid("id").primaryKey().defaultRandom(),
-  number: bigint("number", { mode: "number" }),
-  hash: text("hash"),
+  endCursor: bigint("end_cursor", { mode: "number" }),
+  uniqueKey: text("unique_key"),
 });`)}`);
 
     console.log("\n");
   }
 
   consola.info(
-    `Run ${green(`${options.packageManager} run drizzle:generate`)} & ${green(`${options.packageManager} run drizzle:migrate`)} to generate and apply migrations.`,
+    `Run ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:generate`)} & ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:migrate`)} to generate and apply migrations.`,
   );
 }
 
@@ -463,3 +399,103 @@ export async function createStorageRelatedFiles(options: IndexerOptions) {
     await createDrizzleStorageFiles(options);
   }
 }
+
+const gitIgnoreItems: {
+  isRecommended: boolean;
+  description?: string;
+  value: string;
+}[] = [
+  {
+    isRecommended: false,
+    value: "node_modules",
+  },
+  {
+    isRecommended: false,
+    value: "dist",
+  },
+  {
+    isRecommended: true,
+    description: "build and dev files of apibara",
+    value: ".apibara",
+  },
+  {
+    isRecommended: false,
+    value: ".env",
+  },
+  {
+    isRecommended: false,
+    description: "for mac users",
+    value: ".DS_Store",
+  },
+];
+
+export async function createGitIgnoreFile(cwd: string) {
+  const gitIgnorePath = path.join(cwd, ".gitignore");
+
+  if (fs.existsSync(gitIgnorePath)) {
+    const result = await prompts([
+      {
+        type: "select",
+        name: "overwrite",
+        message: `${cyan(".gitignore")} already exists. Please choose how to proceed:`,
+        initial: 0,
+        choices: [
+          {
+            title: "Choose items to append in your .gitignore",
+            value: "append",
+          },
+          {
+            title: "Keep original",
+            value: "ignore",
+          },
+          {
+            title: "Overwrite",
+            value: "overwrite",
+          },
+        ],
+      },
+      {
+        type: (overwrite: "append" | "ignore" | "overwrite") =>
+          overwrite === "append" ? "multiselect" : null,
+        name: "ignoreItems",
+        message: "Choose items to append in your .gitignore",
+        choices: gitIgnoreItems.map((item) => ({
+          title: `${yellow(item.value)}${
+            item.description ? ` - ${item.description}` : ""
+          }${item.isRecommended ? ` ${green("(recommended)")}` : ""}`,
+          value: item.value,
+        })),
+      },
+    ]);
+
+    const { overwrite, ignoreItems } = result as {
+      overwrite: "append" | "ignore" | "overwrite";
+      ignoreItems: string[];
+    };
+
+    if (overwrite === "append" && ignoreItems.length > 0) {
+      const gitIgnoreContent = fs.readFileSync(gitIgnorePath, "utf8");
+      fs.writeFileSync(
+        gitIgnorePath,
+        `${gitIgnoreContent}\n${result.ignoreItems.join("\n")}`,
+      );
+      consola.success(`Updated ${cyan(".gitignore")}`);
+      return;
+    }
+
+    if (overwrite === "overwrite") {
+      fs.writeFileSync(
+        gitIgnorePath,
+        gitIgnoreItems.map((item) => item.value).join("\n"),
+      );
+      consola.success(`Updated ${cyan(".gitignore")}`);
+      return;
+    }
+  }
+
+  fs.writeFileSync(
+    gitIgnorePath,
+    gitIgnoreItems.map((item) => item.value).join("\n"),
+  );
+  consola.success(`Created ${cyan(".gitignore")}`);
+}

package/src/create/utils.ts

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path, { basename } from "node:path";
+import * as prettier from "prettier";
 import prompts from "prompts";
 import { blue, cyan, red, yellow } from "./colors";
 import { dnaUrls, networks } from "./constants";
@@ -410,3 +411,12 @@ function pkgFromUserAgent(userAgent: string | undefined): PkgInfo | undefined {
     version: pkgSpecArr[1],
   };
 }
+
+export async function formatFile(path: string) {
+  const file = fs.readFileSync(path, "utf8");
+  const formatted = await prettier.format(file, {
+    filepath: path,
+    tabWidth: 2,
+  });
+  fs.writeFileSync(path, formatted);
+}

package/src/hooks/useRuntimeConfig.ts

@@ -1,5 +1,6 @@
 import type { ApibaraRuntimeConfig } from "apibara/types";
+import { deserialize } from "../utils/helper";
 
 export function useRuntimeConfig(): ApibaraRuntimeConfig {
-  return JSON.parse(process.env.APIBARA_RUNTIME_CONFIG || "{}");
+  return deserialize(process.env.APIBARA_RUNTIME_CONFIG || "{}");
 }

package/src/rolldown/config.ts

@@ -0,0 +1,108 @@
+import { existsSync } from "node:fs";
+import { builtinModules } from "node:module";
+import replace from "@rollup/plugin-replace";
+import type { Apibara } from "apibara/types";
+import defu from "defu";
+import { join } from "pathe";
+import type {
+  ConfigExport,
+  RolldownOptions,
+  RolldownPluginOption,
+} from "rolldown";
+import { serialize } from "../utils/helper";
+import { appConfig } from "./plugins/config";
+import { indexers } from "./plugins/indexers";
+
+const runtimeDependencies = [
+  "better-sqlite3",
+  "@electric-sql/pglite",
+  "pg",
+  // https://socket.io/docs/v4/server-installation/#additional-packages
+  "utf-8-validate",
+  "bufferutil",
+  "node-fetch",
+];
+
+export function getRolldownConfig(apibara: Apibara): RolldownOptions {
+  const extensions: string[] = [
+    ".ts",
+    ".mjs",
+    ".js",
+    ".json",
+    ".node",
+    ".tsx",
+    ".jsx",
+  ];
+
+  const tsConfigExists = existsSync(
+    join(apibara.options.rootDir, "tsconfig.json"),
+  );
+
+  const rolldownConfig: RolldownOptions & {
+    plugins: RolldownPluginOption[];
+  } = defu(
+    // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rolldownConfig is typed
+    apibara.options.rolldownConfig as any,
+    <ConfigExport>{
+      platform: "node",
+      input: apibara.options.entry,
+      output: {
+        dir: join(apibara.options.outputDir || "./.apibara/build"),
+        format: "esm",
+        entryFileNames: "[name].mjs",
+        chunkFileNames: "chunks/[name]-[hash].mjs",
+        sourcemap: true,
+      },
+      plugins: [],
+      onwarn(warning, rolldownWarn) {
+        if (
+          !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
+            warning.code || "",
+          ) &&
+          !warning.message.includes("Unsupported source map comment") &&
+          !warning.message.includes("@__PURE__") &&
+          !warning.message.includes("/*#__PURE__*/")
+        ) {
+          rolldownWarn(warning);
+        }
+      },
+      resolve: {
+        extensions,
+        preferBuiltins: !!apibara.options.node,
+        mainFields: ["main"],
+        exportConditions: apibara.options.exportConditions,
+        tsconfigFilename: tsConfigExists ? "tsconfig.json" : undefined,
+      },
+      treeshake: true,
+      external: [...builtinModules, ...runtimeDependencies],
+    },
+  );
+
+  rolldownConfig.plugins?.push(
+    replace({
+      preventAssignment: true,
+      values: {
+        "process.env.APIBARA_CONFIG": getSerializedRuntimeConfig(apibara),
+      },
+    }) as RolldownPluginOption,
+  );
+  rolldownConfig.plugins?.push(indexers(apibara));
+  rolldownConfig.plugins?.push(appConfig(apibara));
+
+  return rolldownConfig;
+}
+
+function getSerializedRuntimeConfig(apibara: Apibara) {
+  try {
+    return serialize({
+      runtimeConfig: apibara.options.runtimeConfig,
+      preset: apibara.options.preset,
+      presets: apibara.options.presets,
+    });
+  } catch (error) {
+    throw new Error(
+      "Failed to serialize runtime config. Please ensure all values in your runtime configuration are JSON serializable. BigInt values are supported, but functions, symbols, etc. are not. Check your configuration for non-serializable values.",
+      { cause: error },
+    );
+  }
+}

package/src/rolldown/index.ts

@@ -0,0 +1,2 @@
+export * from "./config";
+export type { Plugin } from "rolldown";

package/src/rolldown/plugins/config.ts

@@ -0,0 +1,25 @@
+import virtual from "@rollup/plugin-virtual";
+import type { Apibara } from "apibara/types";
+import type { RolldownPluginOption } from "rolldown";
+
+export function appConfig(apibara: Apibara) {
+  return virtual({
+    "#apibara-internal-virtual/config": `
+    const serializedConfig = \`process.env.APIBARA_CONFIG\`;
+
+    if (serializedConfig === undefined || serializedConfig === "") {
+      throw new Error("APIBARA_CONFIG is not defined");
+    }
+
+    function deserialize(str) {
+      return JSON.parse(str, (_, value) =>
+        typeof value === "string" && value.match(/^\\d+n$/)
+          ? BigInt(value.slice(0, -1))
+          : value,
+      );
+    }
+
+    export const config = deserialize(serializedConfig);
+    `,
+  }) as RolldownPluginOption;
+}

package/src/{rollup → rolldown}/plugins/indexers.ts

@@ -1,17 +1,17 @@
 import virtual from "@rollup/plugin-virtual";
 import type { Apibara } from "apibara/types";
 import { hash } from "ohash";
+import type { RolldownPluginOption } from "rolldown";
 
 export function indexers(apibara: Apibara) {
   const indexers = [...new Set(apibara.indexers)];
-
   return virtual({
     "#apibara-internal-virtual/indexers": `
-    ${indexers.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
+    ${indexers.map((i) => `import * as _${hash(i)} from '${i.indexer}';`).join("\n")}
 
     export const indexers = [
       ${indexers.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
     ];
     `,
-  });
+  }) as RolldownPluginOption;
 }

package/src/runtime/dev.ts

@@ -37,6 +37,9 @@ const startCommand = defineCommand({
     await Promise.all(
       selectedIndexers.map(async (indexer) => {
         const indexerInstance = createIndexer(indexer, preset);
+        if (!indexerInstance) {
+          return;
+        }
 
         const client = createClient(
           indexerInstance.streamConfig,