apibara 2.1.0-beta.5 → 2.1.0-beta.51

package/src/create/add.ts

@@ -37,6 +37,7 @@ type Options = {
   argNetwork?: string;
   argStorage?: string;
   argDnaUrl?: string;
+  argRootDir?: string;
 };
 
 export async function addIndexer({
@@ -45,8 +46,10 @@ export async function addIndexer({
   argNetwork,
   argStorage,
   argDnaUrl,
+  argRootDir,
 }: Options) {
-  const configExists = hasApibaraConfig(process.cwd());
+  const cwd = path.join(process.cwd(), argRootDir ?? ".");
+  const configExists = hasApibaraConfig(cwd);
 
   if (!configExists) {
     consola.error("No apibara.config found in the current directory.");
@@ -74,7 +77,7 @@ export async function addIndexer({
     }
   }
 
-  const language = getApibaraConfigLanguage(process.cwd());
+  const { language, extension } = getApibaraConfigLanguage(cwd);
 
   validateIndexerId(argIndexerId, true);
   validateChain(argChain, true);
@@ -92,14 +95,10 @@ export async function addIndexer({
         validate: (id) =>
           validateIndexerId(id)
             ? checkFileExists(
-                path.join(
-                  process.cwd(),
-                  "indexers",
-                  `${id}.indexer.${language === "typescript" ? "ts" : "js"}`,
-                ),
+                path.join(cwd, "indexers", `${id}.indexer.${extension}`),
               ).then(({ exists }) =>
                 exists
-                  ? `Indexer ${cyan(`${id}.indexer.${language === "typescript" ? "ts" : "js"}`)} already exists`
+                  ? `Indexer ${cyan(`${id}.indexer.${extension}`)} already exists`
                   : true,
               )
             : "Invalid indexer ID, it cannot be empty and must be in kebab-case format",
@@ -197,7 +196,7 @@ export async function addIndexer({
   const pkgManager = getPackageManager();
 
   const options: IndexerOptions = {
-    cwd: process.cwd(),
+    cwd: cwd,
     indexerFileId,
     indexerId: convertKebabToCamelCase(indexerFileId),
     chain: (argChain as Chain) ?? prompt_chain?.name!,
@@ -206,13 +205,12 @@ export async function addIndexer({
     dnaUrl: argDnaUrl ?? prompt_dnaUrl,
     language,
     packageManager: pkgManager.name,
+    extension,
   };
 
   await updateApibaraConfigFile(options);
 
-  consola.success(
-    `Updated ${cyan("apibara.config." + (language === "typescript" ? "ts" : "js"))}`,
-  );
+  consola.success(`Updated ${cyan(`apibara.config.${extension}`)}`);
 
   await updatePackageJson(options);
 
@@ -220,19 +218,15 @@ export async function addIndexer({
 
   await createIndexerFile(options);
 
-  consola.success(
-    `Created ${cyan(`${indexerFileId}.indexer.${language === "typescript" ? "ts" : "js"}`)}`,
-  );
+  consola.success(`Created ${cyan(`${indexerFileId}.indexer.${extension}`)}`);
 
   await createStorageRelatedFiles(options);
 
   console.log();
 
+  const baseCommand = `${options.packageManager} install`;
+  const tsCommand = `${baseCommand} && ${options.packageManager} run prepare`;
   consola.info(
-    `Before running the indexer, run ${cyan(`${options.packageManager} run install`)}${
-      language === "typescript"
-        ? " & " + cyan(`${options.packageManager} run prepare`)
-        : ""
-    }`,
+    `Before running the indexer, run ${cyan(language === "typescript" ? tsCommand : baseCommand)}`,
   );
 }

package/src/create/constants.ts

@@ -67,7 +67,6 @@ export const storages: StorageDataType[] = [
 export const packageVersions = {
   // Required Dependencies
   apibara: "next",
-  "@apibara/indexer": "next",
   "@apibara/protocol": "next",
   // Chain Dependencies
   "@apibara/evm": "next",
@@ -79,20 +78,19 @@ export const packageVersions = {
   "@apibara/plugin-sqlite": "next",
   // Postgres Dependencies
   "@electric-sql/pglite": "^0.2.17",
-  "drizzle-orm": "^0.37.0",
+  "drizzle-orm": "^0.40.1",
   pg: "^8.13.1",
   "@types/pg": "^8.11.10",
   "drizzle-kit": "^0.29.0",
   // Typescript Dependencies
   typescript: "^5.6.2",
-  "@rollup/plugin-typescript": "^11.1.6",
   "@types/node": "^20.5.2",
 };
 
 export const dnaUrls = {
-  ethereum: "https://ethereum.preview.apibara.org",
-  ethereumSepolia: "https://ethereum-sepolia.preview.apibara.org",
+  ethereum: "https://mainnet.ethereum.a5a.ch",
+  ethereumSepolia: "https://sepolia.ethereum.a5a.ch",
   beaconchain: "https://beaconchain.preview.apibara.org",
-  starknet: "https://starknet.preview.apibara.org",
-  starknetSepolia: "https://starknet-sepolia.preview.apibara.org",
+  starknet: "https://mainnet.starknet.a5a.ch",
+  starknetSepolia: "https://sepolia.starknet.a5a.ch",
 };

package/src/create/init.ts

@@ -166,12 +166,12 @@ export async function initializeProject({
   if (!argNoCreateIndexer) {
     consola.info("Let's create an indexer\n");
 
-    await addIndexer({});
+    await addIndexer({ argRootDir: argTargetDir });
   } else {
     const pkgManager = getPackageManager();
     consola.info(
       "Run ",
-      green(`${pkgManager.name} run install`),
+      green(`${pkgManager.name} install`),
       " to install all dependencies",
     );
   }

package/src/create/templates.ts

@@ -15,21 +15,18 @@ export function generatePackageJson(isTypeScript: boolean) {
     private: true,
     type: "module",
     scripts: {
-      prepare: "apibara prepare",
+      ...(isTypeScript && { prepare: "apibara prepare" }),
       dev: "apibara dev",
       start: "apibara start",
       build: "apibara build",
       ...(isTypeScript && { typecheck: "tsc --noEmit" }),
     },
     dependencies: {
-      "@apibara/indexer": packageVersions["@apibara/indexer"],
       "@apibara/protocol": packageVersions["@apibara/protocol"],
       apibara: packageVersions.apibara,
     },
     devDependencies: {
       ...(isTypeScript && {
-        "@rollup/plugin-typescript":
-          packageVersions["@rollup/plugin-typescript"],
         "@types/node": packageVersions["@types/node"],
         typescript: packageVersions.typescript,
       }),
@@ -59,17 +56,10 @@ export function generateTsConfig() {
 }
 
 export function generateApibaraConfig(isTypeScript: boolean) {
-  return `${isTypeScript ? 'import typescript from "@rollup/plugin-typescript";\nimport type { Plugin } from "apibara/rollup";\n' : ""}import { defineConfig } from "apibara/config";
+  return `import { defineConfig } from "apibara/config";
 
 export default defineConfig({
-  runtimeConfig: {},${
-    isTypeScript
-      ? `
-  rollupConfig: {
-    plugins: [typescript()${isTypeScript ? " as Plugin" : ""}],
-  },`
-      : ""
-  }
+  runtimeConfig: {},
 });\n`;
 }
 
@@ -79,9 +69,10 @@ export function generateIndexer({
   chain,
   language,
 }: IndexerOptions) {
-  return `import { defineIndexer } from "@apibara/indexer";
-import { useLogger } from "@apibara/indexer/plugins";
+  return `import { defineIndexer } from "apibara/indexer";
+import { useLogger } from "apibara/plugins";
 ${storage === "postgres" ? `import { drizzleStorage } from "@apibara/plugin-drizzle";` : ""}
+${storage === "postgres" ? `import { drizzle } from "@apibara/plugin-drizzle";` : ""}
 ${
   chain === "ethereum"
     ? `import { EvmStream } from "@apibara/evm";`
@@ -92,19 +83,16 @@ ${
         : ""
 }
 ${language === "typescript" ? `import type { ApibaraRuntimeConfig } from "apibara/types";` : ""}
-${
-  storage === "postgres"
-    ? `import { getDrizzlePgDatabase } from "../lib/db";`
-    : ""
-}
+${storage === "postgres" ? `import * as schema from "../lib/schema";` : ""}
 
 
 export default function (runtimeConfig${language === "typescript" ? ": ApibaraRuntimeConfig" : ""}) {
-  const indexerId = "${indexerId}";
-  const { startingBlock, streamUrl${storage === "postgres" ? ", postgresConnectionString" : ""} } = runtimeConfig[indexerId];
+  const { startingBlock, streamUrl } = runtimeConfig["${indexerId}"];
   ${
     storage === "postgres"
-      ? "const { db } = getDrizzlePgDatabase(postgresConnectionString);"
+      ? `const db = drizzle({
+    schema,
+  });`
       : ""
   }
 
@@ -121,9 +109,9 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
     finality: "accepted",
     startingBlock: BigInt(startingBlock),
     filter: {
-      header: "always",
+      ${chain === "ethereum" ? "logs: []," : chain === "starknet" ? "events:  []," : ""}
     },
-    plugins: [${storage === "postgres" ? "drizzleStorage({ db, persistState: true })" : ""}],
+    plugins: [${storage === "postgres" ? "drizzleStorage({ db, migrate: { migrationsFolder: './drizzle' } })" : ""}],
     async transform({ endCursor, finality }) {
       const logger = useLogger();
 
@@ -137,14 +125,12 @@ export default function (runtimeConfig${language === "typescript" ? ": ApibaraRu
       ${
         storage === "postgres"
           ? `// Example snippet to insert data into db using drizzle with postgres
-      //   const { db } = useDrizzleStorage();
-      //   const { logs } = block;
-      //   for (const log of logs) {
-      //     await db.insert(exampleTable).values({
-      //       number: Number(endCursor?.orderKey),
-      //       hash: log.transactionHash,
-      //     });
-      //   }`
+      // const { db: database } = useDrizzleStorage();
+
+      // await database.insert(schema.cursorTable).values({
+      //   endCursor: Number(endCursor?.orderKey),
+      //   uniqueKey: \`\${endCursor?.uniqueKey}\`,
+      // });`
           : ""
       }
     },
@@ -157,7 +143,7 @@ export async function createIndexerFile(options: IndexerOptions) {
   const indexerFilePath = path.join(
     options.cwd,
     "indexers",
-    `${options.indexerFileId}.indexer.${options.language === "typescript" ? "ts" : "js"}`,
+    `${options.indexerFileId}.indexer.${options.extension}`,
   );
 
   const { exists, overwrite } = await checkFileExists(indexerFilePath, {
@@ -228,20 +214,14 @@ export async function updateApibaraConfigFile({
   language,
   network,
   dnaUrl,
+  extension,
 }: IndexerOptions) {
-  const pathToConfig = path.join(
-    cwd,
-    `apibara.config.${language === "typescript" ? "ts" : "js"}`,
-  );
+  const pathToConfig = path.join(cwd, `apibara.config.${extension}`);
 
   const runtimeConfigString = `{
   startingBlock: 0,
-  streamUrl: "${dnaUrl ?? getDnaUrl(chain, network)}"${
-    storage === "postgres"
-      ? `,
-  postgresConnectionString: process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://${indexerId}"`
-      : ""
-  }}`;
+  streamUrl: "${dnaUrl ?? getDnaUrl(chain, network)}"  
+}`;
 
   const project = new Project();
   const sourceFile = project.addSourceFileAtPath(pathToConfig);
@@ -285,12 +265,16 @@ export async function updateApibaraConfigFile({
 }
 
 export async function createDrizzleStorageFiles(options: IndexerOptions) {
-  const { cwd, language, storage } = options;
+  const {
+    cwd,
+    language,
+    storage,
+    indexerId,
+    extension: fileExtension,
+  } = options;
 
   if (storage !== "postgres") return;
 
-  const fileExtension = language === "typescript" ? "ts" : "js";
-
   /**
    *
    *
@@ -313,11 +297,11 @@ export async function createDrizzleStorageFiles(options: IndexerOptions) {
     const drizzleConfigContent = `${language === "typescript" ? 'import type { Config } from "drizzle-kit";' : ""}
 
 export default {
-  schema: "./lib/schema.ts",
+  schema: "./lib/schema.${fileExtension}",
   out: "./drizzle",
   dialect: "postgresql",
   dbCredentials: {
-    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "",
+    url: process.env["POSTGRES_CONNECTION_STRING"] ?? "memory://${indexerId}",
   },
 }${language === "typescript" ? " satisfies Config" : ""};`;
 
@@ -348,14 +332,14 @@ export default {
     });
 
   if (!schemaExists || schemaOverwrite) {
-    const schemaContent = `//  --- Add your pg table schemas here ---- 
+    const schemaContent = `//  --- Add your pg table schemas here ----
 
 // import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-// export const exampleTable = pgTable("example_table", {
+// export const cursorTable = pgTable("cursor_table", {
 //   id: uuid("id").primaryKey().defaultRandom(),
-//   number: bigint("number", { mode: "number" }),
-//   hash: text("hash"),
+//   endCursor: bigint("end_cursor", { mode: "number" }),
+//   uniqueKey: text("unique_key"),
 // });
 
 export {};
@@ -367,64 +351,7 @@ export {};
 
     await formatFile(schemaPath);
 
-    consola.success(`Created ${cyan("lib/schema.ts")}`);
-  }
-
-  /**
-   *
-   *
-   * DB File
-   *
-   *
-   */
-  const dbFileName = `db.${fileExtension}`;
-
-  const dbPath = path.join(cwd, "lib", dbFileName);
-
-  const { exists: dbExists, overwrite: dbOverwrite } = await checkFileExists(
-    dbPath,
-    {
-      askPrompt: true,
-      fileName: `lib/${dbFileName}`,
-      allowIgnore: true,
-    },
-  );
-
-  if (!dbExists || dbOverwrite) {
-    const dbContent = `import * as schema from "./schema";
-import { drizzle as nodePgDrizzle } from "drizzle-orm/node-postgres";
-import { drizzle as pgLiteDrizzle } from "drizzle-orm/pglite";
-import pg from "pg";
-
-
-export function getDrizzlePgDatabase(connectionString${language === "typescript" ? ": string" : ""}) {
-  // Create pglite instance
-  if (connectionString.includes("memory")) {
-    return {
-      db: pgLiteDrizzle({
-        schema,
-        connection: {
-          dataDir: connectionString,
-        },
-      }),
-    };
-  }
-
-  // Create node-postgres instance
-  const pool = new pg.Pool({
-    connectionString,
-  });
-
-  return { db: nodePgDrizzle(pool, { schema }) };
-}`;
-
-    // create directory if it doesn't exist
-    fs.mkdirSync(path.dirname(dbPath), { recursive: true });
-    fs.writeFileSync(dbPath, dbContent);
-
-    await formatFile(dbPath);
-
-    consola.success(`Created ${cyan(`lib/${dbFileName}`)}`);
+    consola.success(`Created ${cyan(`lib/${schemaFileName}`)}`);
   }
 
   console.log("\n");
@@ -441,22 +368,22 @@ export function getDrizzlePgDatabase(connectionString${language === "typescript"
     
 ${yellow(`
 ┌──────────────────────────────────────────┐
-│               lib/schema.ts              │
+│               lib/schema                 │
 └──────────────────────────────────────────┘
 
 import { bigint, pgTable, text, uuid } from "drizzle-orm/pg-core";
 
-export const exampleTable = pgTable("example_table", {
+export const cursorTable = pgTable("cursor_table", {
   id: uuid("id").primaryKey().defaultRandom(),
-  number: bigint("number", { mode: "number" }),
-  hash: text("hash"),
+  endCursor: bigint("end_cursor", { mode: "number" }),
+  uniqueKey: text("unique_key"),
 });`)}`);
 
     console.log("\n");
   }
 
   consola.info(
-    `Run ${green(`${options.packageManager} run drizzle:generate`)} & ${green(`${options.packageManager} run drizzle:migrate`)} to generate and apply migrations.`,
+    `Run ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:generate`)} & ${green(`${options.packageManager}${options.packageManager === "npm" ? " run" : ""} drizzle:migrate`)} to generate and apply migrations.`,
   );
 }
 

package/src/create/types.ts

@@ -16,6 +16,8 @@ export type Network =
 
 export type Storage = "postgres" | "none";
 
+export type FileExtension = "js" | "ts" | "mjs";
+
 export type IndexerOptions = {
   cwd: string;
   indexerFileId: string;
@@ -26,6 +28,7 @@ export type IndexerOptions = {
   dnaUrl?: string;
   packageManager: string;
   language: Language;
+  extension: FileExtension;
 };
 
 export type PkgInfo = {

package/src/create/utils.ts

@@ -4,7 +4,7 @@ import * as prettier from "prettier";
 import prompts from "prompts";
 import { blue, cyan, red, yellow } from "./colors";
 import { dnaUrls, networks } from "./constants";
-import type { Chain, Language, Network, PkgInfo } from "./types";
+import type { Chain, FileExtension, Language, Network, PkgInfo } from "./types";
 
 export function isEmpty(path: string) {
   const files = fs.readdirSync(path);
@@ -31,7 +31,8 @@ export function validateLanguage(language?: string, throwError = false) {
     language === "typescript" ||
     language === "ts" ||
     language === "javascript" ||
-    language === "js"
+    language === "js" ||
+    language === "mjs"
   ) {
     return true;
   }
@@ -49,7 +50,7 @@ export function getLanguageFromAlias(alias: string): Language {
   if (alias === "ts" || alias === "typescript") {
     return "typescript";
   }
-  if (alias === "js" || alias === "javascript") {
+  if (alias === "js" || alias === "javascript" || alias === "mjs") {
     return "javascript";
   }
 
@@ -184,19 +185,31 @@ export function validateDnaUrl(dnaUrl?: string, throwError = false) {
 export function hasApibaraConfig(cwd: string): boolean {
   const configPathJS = path.join(cwd, "apibara.config.js");
   const configPathTS = path.join(cwd, "apibara.config.ts");
+  const configPathMJS = path.join(cwd, "apibara.config.mjs");
 
-  return fs.existsSync(configPathJS) || fs.existsSync(configPathTS);
+  return (
+    fs.existsSync(configPathJS) ||
+    fs.existsSync(configPathTS) ||
+    fs.existsSync(configPathMJS)
+  );
 }
 
-export function getApibaraConfigLanguage(cwd: string): Language {
+export function getApibaraConfigLanguage(cwd: string): {
+  language: Language;
+  extension: FileExtension;
+} {
   const configPathJS = path.join(cwd, "apibara.config.js");
   const configPathTS = path.join(cwd, "apibara.config.ts");
+  const configPathMJS = path.join(cwd, "apibara.config.mjs");
 
+  if (fs.existsSync(configPathMJS)) {
+    return { language: "javascript", extension: "mjs" };
+  }
   if (fs.existsSync(configPathJS)) {
-    return "javascript";
+    return { language: "javascript", extension: "js" };
   }
   if (fs.existsSync(configPathTS)) {
-    return "typescript";
+    return { language: "typescript", extension: "ts" };
   }
 
   throw new Error(red("✖") + " No apibara.config found");

package/src/hooks/useRuntimeConfig.ts

@@ -1,5 +1,8 @@
+import { ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME } from "apibara/common";
 import type { ApibaraRuntimeConfig } from "apibara/types";
 
 export function useRuntimeConfig(): ApibaraRuntimeConfig {
-  return JSON.parse(process.env.APIBARA_RUNTIME_CONFIG || "{}");
+  return JSON.parse(
+    process.env[ENV_INTERNAL_APIBARA_PROCESSED_RUNTIME] || "{}",
+  );
 }

package/src/indexer/index.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer";

package/src/indexer/plugins.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/plugins";

package/src/indexer/testing.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/testing";

package/src/indexer/vcr.ts

@@ -0,0 +1 @@
+export * from "@apibara/indexer/vcr";

package/src/rolldown/config.ts

@@ -0,0 +1,86 @@
+import { existsSync } from "node:fs";
+import { builtinModules } from "node:module";
+import type { Apibara } from "apibara/types";
+import defu from "defu";
+import { join } from "pathe";
+import type {
+  ConfigExport,
+  RolldownOptions,
+  RolldownPluginOption,
+} from "rolldown";
+import { indexers } from "./plugins/indexers";
+import { instrumentation } from "./plugins/instrumentation";
+import { staticConfig } from "./plugins/static-config";
+
+const runtimeDependencies = [
+  "better-sqlite3",
+  "@electric-sql/pglite",
+  "pg",
+  // https://socket.io/docs/v4/server-installation/#additional-packages
+  "utf-8-validate",
+  "bufferutil",
+  "fsevents",
+  // was giving unresolved import warnings from `node-fetch` library.
+  "encoding",
+];
+
+export function getRolldownConfig(apibara: Apibara): RolldownOptions {
+  const extensions: string[] = [
+    ".ts",
+    ".mjs",
+    ".js",
+    ".json",
+    ".node",
+    ".tsx",
+    ".jsx",
+  ];
+
+  const tsConfigExists = existsSync(
+    join(apibara.options.rootDir, "tsconfig.json"),
+  );
+
+  const rolldownConfig: RolldownOptions & {
+    plugins: RolldownPluginOption[];
+  } = defu(
+    // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rolldownConfig is typed
+    apibara.options.rolldownConfig as any,
+    <ConfigExport>{
+      platform: "node",
+      input: apibara.options.entry,
+      output: {
+        dir: join(apibara.options.outputDir || "./.apibara/build"),
+        format: "esm",
+        entryFileNames: "[name].mjs",
+        chunkFileNames: "chunks/[name]-[hash].mjs",
+        sourcemap: true,
+      },
+      plugins: [],
+      onwarn(warning, rolldownWarn) {
+        if (
+          !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
+            warning.code || "",
+          ) &&
+          !warning.message.includes("Unsupported source map comment") &&
+          !warning.message.includes("@__PURE__") &&
+          !warning.message.includes("/*#__PURE__*/")
+        ) {
+          rolldownWarn(warning);
+        }
+      },
+      resolve: {
+        extensions,
+        preferBuiltins: !!apibara.options.node,
+        mainFields: ["main"],
+        exportConditions: apibara.options.exportConditions,
+        tsconfigFilename: tsConfigExists ? "tsconfig.json" : undefined,
+      },
+      treeshake: true,
+      external: [...builtinModules, ...runtimeDependencies],
+    },
+  );
+
+  rolldownConfig.plugins?.push(staticConfig(apibara));
+  rolldownConfig.plugins?.push(instrumentation(apibara));
+  rolldownConfig.plugins?.push(indexers(apibara));
+  return rolldownConfig;
+}

package/src/rolldown/index.ts

@@ -0,0 +1,2 @@
+export * from "./config";
+export type { Plugin } from "rolldown";

package/src/{rollup → rolldown}/plugins/indexers.ts

@@ -1,17 +1,17 @@
 import virtual from "@rollup/plugin-virtual";
 import type { Apibara } from "apibara/types";
 import { hash } from "ohash";
+import type { RolldownPluginOption } from "rolldown";
 
 export function indexers(apibara: Apibara) {
   const indexers = [...new Set(apibara.indexers)];
-
   return virtual({
     "#apibara-internal-virtual/indexers": `
-    ${indexers.map((i) => `import _${hash(i)} from '${i.indexer}';`).join("\n")}
+    ${indexers.map((i) => `import * as _${hash(i)} from '${i.indexer}';`).join("\n")}
 
     export const indexers = [
       ${indexers.map((i) => `{ name: "${i.name}", indexer: _${hash(i)} }`).join(",\n")}
     ];
     `,
-  });
+  }) as RolldownPluginOption;
 }