@mikeatlast/ponder 0.16.1-fork.1

package/src/bin/isolatedWorker.ts

@@ -0,0 +1,192 @@
+import { isMainThread, parentPort, workerData } from "node:worker_threads";
+import { createBuild } from "@/build/index.js";
+import { createDatabase } from "@/database/index.js";
+import type { Common } from "@/internal/common.js";
+import { createLogger } from "@/internal/logger.js";
+import { IsolatedMetricsService } from "@/internal/metrics.js";
+import { createShutdown } from "@/internal/shutdown.js";
+import { createTelemetry } from "@/internal/telemetry.js";
+import type {
+  CrashRecoveryCheckpoint,
+  NamespaceBuild,
+} from "@/internal/types.js";
+import { runIsolated } from "@/runtime/isolated.js";
+
+if (isMainThread) {
+  throw new Error("'isolatedWorker.ts' must be run in a worker thread");
+}
+
+try {
+  await isolatedWorker(workerData);
+
+  parentPort!.postMessage({ type: "done" });
+} catch (err) {
+  const error = err as Error;
+
+  parentPort!.postMessage({
+    type: "error",
+    error: {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    },
+  });
+}
+
+export async function isolatedWorker({
+  options,
+  namespaceBuild,
+  crashRecoveryCheckpoint,
+  chainIds,
+}: {
+  options: Common["options"];
+  namespaceBuild: NamespaceBuild;
+  crashRecoveryCheckpoint: CrashRecoveryCheckpoint;
+  chainIds: number[];
+}) {
+  // Note: telemetry is disabled because the main thread will report telemetry
+  options.telemetryDisabled = true;
+
+  const logger = createLogger({
+    level: options.logLevel,
+    mode: options.logFormat,
+  });
+
+  const metrics = new IsolatedMetricsService();
+  const shutdown = createShutdown();
+  const telemetry = createTelemetry({ options, logger, shutdown });
+  const common = {
+    options,
+    logger,
+    metrics,
+    telemetry,
+    shutdown,
+    buildShutdown: shutdown,
+    apiShutdown: shutdown,
+  };
+
+  let isKilled = false;
+  parentPort!.on("message", async (msg) => {
+    if (msg.type === "kill") {
+      if (isKilled) return;
+      isKilled = true;
+      await shutdown.kill();
+    }
+  });
+
+  const build = await createBuild({
+    common,
+    cliOptions: workerData.cliOptions,
+  });
+
+  // Note: build is guaranteed to be successful because the main thread
+  // has already run the build.
+
+  // Note: `namespaceCompile`
+  globalThis.PONDER_NAMESPACE_BUILD = namespaceBuild;
+
+  const configResult = await build.executeConfig();
+  if (configResult.status === "error") {
+    throw configResult.error;
+  }
+
+  const schemaResult = await build.executeSchema();
+  if (schemaResult.status === "error") {
+    throw schemaResult.error;
+  }
+  const preBuildResult = build.preCompile(configResult.result);
+  if (preBuildResult.status === "error") {
+    throw preBuildResult.error;
+  }
+
+  const schemaBuildResult = build.compileSchema({
+    ...schemaResult.result,
+    preBuild: preBuildResult.result,
+  });
+  if (schemaBuildResult.status === "error") {
+    throw schemaBuildResult.error;
+  }
+
+  const configBuildResult = build.compileConfig({
+    configResult: configResult.result,
+  });
+  if (configBuildResult.status === "error") {
+    throw configBuildResult.error;
+  }
+
+  const indexingResult = await build.executeIndexingFunctions();
+  if (indexingResult.status === "error") {
+    throw indexingResult.error;
+  }
+
+  const indexingBuildResult = await build.compileIndexing({
+    configResult: configResult.result,
+    schemaResult: schemaResult.result,
+    indexingResult: indexingResult.result,
+    configBuild: configBuildResult.result,
+  });
+  if (indexingBuildResult.status === "error") {
+    throw indexingBuildResult.error;
+  }
+
+  options.indexingCacheMaxBytes = Math.floor(
+    options.indexingCacheMaxBytes / indexingBuildResult.result.chains.length,
+  );
+  options.rpcMaxConcurrency = Math.floor(
+    options.rpcMaxConcurrency / indexingBuildResult.result.chains.length,
+  );
+  options.syncEventsQuerySize = Math.floor(
+    options.syncEventsQuerySize / indexingBuildResult.result.chains.length,
+  );
+
+  const database = createDatabase({
+    common,
+    namespace: namespaceBuild,
+    preBuild: preBuildResult.result,
+    schemaBuild: schemaBuildResult.result,
+  });
+
+  await Promise.all(
+    chainIds.map(async (chainId) => {
+      const chainIndex = indexingBuildResult.result.chains.findIndex(
+        (c) => c.id === chainId,
+      );
+
+      const indexingBuild = {
+        ...indexingBuildResult.result,
+        chains: [indexingBuildResult.result.chains[chainIndex]!],
+        rpcs: [indexingBuildResult.result.rpcs[chainIndex]!],
+        finalizedBlocks: [
+          indexingBuildResult.result.finalizedBlocks[chainIndex]!,
+        ],
+        eventCallbacks: [
+          indexingBuildResult.result.eventCallbacks[chainIndex]!,
+        ],
+        setupCallbacks: [
+          indexingBuildResult.result.setupCallbacks[chainIndex]!,
+        ],
+        contracts: [indexingBuildResult.result.contracts[chainIndex]!],
+      };
+
+      common.metrics.initializeIndexingMetrics({
+        indexingBuild,
+        schemaBuild: schemaBuildResult.result,
+      });
+
+      await runIsolated({
+        common,
+        preBuild: preBuildResult.result,
+        namespaceBuild,
+        schemaBuild: schemaBuildResult.result,
+        indexingBuild,
+        crashRecoveryCheckpoint,
+        database,
+        onReady: () => {
+          parentPort!.postMessage({ type: "ready", chainId });
+        },
+      });
+
+      parentPort!.postMessage({ type: "done", chainId });
+    }),
+  );
+}

package/src/bin/ponder.ts

@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { Prettify } from "@/types/utils.js";
+import { Command } from "@commander-js/extra-typings";
+import dotenv from "dotenv";
+import { codegen } from "./commands/codegen.js";
+import { createViews } from "./commands/createViews.js";
+import { dev } from "./commands/dev.js";
+import { list } from "./commands/list.js";
+import { prune } from "./commands/prune.js";
+import { serve } from "./commands/serve.js";
+import { start } from "./commands/start.js";
+
+dotenv.config({ path: ".env.local" });
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const packageJsonPath = resolve(__dirname, "../../../package.json");
+const packageJson = JSON.parse(
+  readFileSync(packageJsonPath, { encoding: "utf8" }),
+);
+
+const ponder = new Command("ponder")
+  .usage("<command> [OPTIONS]")
+  .helpOption("-h, --help", "Show this help message")
+  .helpCommand(false)
+  .option(
+    "--root <PATH>",
+    "Path to the project root directory (default: working directory)",
+  )
+  .option(
+    "--config <PATH>",
+    "Path to the project config file",
+    "ponder.config.ts",
+  )
+  .option(
+    "-v, --debug",
+    "Enable debug logs, e.g. realtime blocks, internal events",
+  )
+  .option(
+    "-vv, --trace",
+    "Enable trace logs, e.g. db queries, indexing checkpoints",
+  )
+  .option(
+    "--log-level <LEVEL>",
+    'Minimum log level ("error", "warn", "info", "debug", or "trace", default: "info")',
+  )
+  .option(
+    "--log-format <FORMAT>",
+    'The log format ("pretty" or "json")',
+    "pretty",
+  )
+  .version(packageJson.version, "-V, --version", "Show the version number")
+  .configureHelp({ showGlobalOptions: true })
+  .allowExcessArguments(false)
+  .showHelpAfterError()
+  .enablePositionalOptions(false);
+
+type GlobalOptions = {
+  command: "dev" | "start" | "serve" | "codegen";
+  version: string;
+} & ReturnType<typeof ponder.opts>;
+
+const devCommand = new Command("dev")
+  .description("Start the development server with hot reloading")
+  .option("--schema <SCHEMA>", "Database schema (max: 45 characters)", String)
+  .option("-p, --port <PORT>", "Port for the web server", Number, 42069)
+  // NOTE: Do not set a default for hostname. We currently rely on the Node.js
+  // default behavior when passing undefined to http.Server.listen(), which
+  // detects the available interfaces (IPv4 and/or IPv6) and uses them.
+  // Documentation: https://arc.net/l/quote/dnjmtumq
+  .option(
+    "-H, --hostname <HOSTNAME>",
+    'Hostname for the web server (default: "0.0.0.0" or "::")',
+  )
+  .option("--disable-ui", "Disable the terminal UI")
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await dev({ cliOptions });
+  });
+
+const startCommand = new Command("start")
+  .description("Start the production server")
+  .option("--schema <SCHEMA>", "Database schema (max: 45 characters)", String)
+  .option(
+    "--views-schema <SCHEMA>",
+    "Views database schema (max: 45 characters)",
+    String,
+  )
+  .option("-p, --port <PORT>", "Port for the web server", Number, 42069)
+  .option(
+    "-H, --hostname <HOSTNAME>",
+    'Hostname for the web server (default: "0.0.0.0" or "::")',
+  )
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await start({ cliOptions });
+  });
+
+const serveCommand = new Command("serve")
+  .description("Start the production HTTP server without the indexer")
+  .option("--schema <SCHEMA>", "Database schema (max: 45 characters)", String)
+  .option("-p, --port <PORT>", "Port for the web server", Number, 42069)
+  .option(
+    "-H, --hostname <HOSTNAME>",
+    'Hostname for the web server (default: "0.0.0.0" or "::")',
+  )
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await serve({ cliOptions });
+  });
+
+const createViewsCommand = new Command("create-views")
+  .description("Create database views for the views pattern")
+  .option("--schema <SCHEMA>", "Database schema (max: 45 characters)", String)
+  .option(
+    "--views-schema <SCHEMA>",
+    "Views database schema (max: 45 characters)",
+    String,
+  )
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await createViews({ cliOptions });
+  });
+
+const dbCommand = new Command("db").description("Database management commands");
+
+const listCommand = new Command("list")
+  .description("List all Ponder deployments")
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await list({ cliOptions });
+  });
+
+const pruneCommand = new Command("prune")
+  .description(
+    "Drop all database tables, functions, and schemas created by Ponder deployments that are not active",
+  )
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await prune({ cliOptions });
+  });
+
+const codegenCommand = new Command("codegen")
+  .description("Generate the ponder-env.d.ts file, then exit")
+  .showHelpAfterError()
+  .action(async (_, command) => {
+    const cliOptions = {
+      ...command.optsWithGlobals(),
+      command: command.name(),
+      version: packageJson.version,
+    } as GlobalOptions & ReturnType<typeof command.opts>;
+    await codegen({ cliOptions });
+  });
+
+dbCommand.addCommand(listCommand);
+dbCommand.addCommand(pruneCommand);
+dbCommand.addCommand(createViewsCommand);
+
+ponder.addCommand(devCommand);
+ponder.addCommand(startCommand);
+ponder.addCommand(serveCommand);
+ponder.addCommand(dbCommand);
+ponder.addCommand(codegenCommand);
+
+export type CliOptions = Prettify<
+  GlobalOptions &
+    Partial<
+      ReturnType<typeof devCommand.opts> &
+        ReturnType<typeof startCommand.opts> &
+        ReturnType<typeof serveCommand.opts> &
+        ReturnType<typeof dbCommand.opts> &
+        ReturnType<typeof codegenCommand.opts>
+    >
+>;
+
+await ponder.parseAsync();

package/src/bin/utils/codegen.ts

@@ -0,0 +1,32 @@
+import { writeFileSync } from "node:fs";
+import path from "node:path";
+import type { Common } from "@/internal/common.js";
+
+export const ponderEnv = `/// <reference types="ponder/virtual" />
+
+declare module "ponder:internal" {
+  const config: typeof import("./ponder.config.ts");
+  const schema: typeof import("./ponder.schema.ts");
+}
+
+declare module "ponder:schema" {
+  export * from "./ponder.schema.ts";
+}
+
+// This file enables type checking and editor autocomplete for this Ponder project.
+// After upgrading, you may find that changes have been made to this file.
+// If this happens, please commit the changes. Do not manually edit this file.
+// See https://ponder.sh/docs/requirements#typescript for more information.
+`;
+
+export function runCodegen({ common }: { common: Common }) {
+  writeFileSync(
+    path.join(common.options.rootDir, "ponder-env.d.ts"),
+    ponderEnv,
+    "utf8",
+  );
+
+  common.logger.debug({
+    msg: `Wrote file "ponder-env.d.ts"`,
+  });
+}

package/src/bin/utils/exit.ts

@@ -0,0 +1,112 @@
+import os from "node:os";
+import readline from "node:readline";
+import type { Common } from "@/internal/common.js";
+import { NonRetryableUserError, ShutdownError } from "@/internal/errors.js";
+import type { Options } from "@/internal/options.js";
+
+const SHUTDOWN_GRACE_PERIOD_MS = 5_000;
+
+/** Sets up shutdown handlers for the process. Accepts additional cleanup logic to run. */
+export const createExit = ({
+  common,
+  options,
+}: {
+  common: Pick<
+    Common,
+    "logger" | "telemetry" | "shutdown" | "buildShutdown" | "apiShutdown"
+  >;
+  options: Options;
+}) => {
+  let isShuttingDown = false;
+
+  const exit = async ({ code }: { code: 0 | 1 | 75 }) => {
+    if (isShuttingDown) return;
+    isShuttingDown = true;
+    const timeout = setTimeout(async () => {
+      common.logger.error({
+        msg: "Failed to shutdown within 5 seconds",
+        code,
+      });
+      process.exit(code);
+    }, SHUTDOWN_GRACE_PERIOD_MS);
+
+    common.logger.warn({
+      msg: "Started shutdown sequence",
+    });
+
+    common.telemetry.record({
+      name: "lifecycle:session_end",
+      properties: { duration_seconds: process.uptime() },
+    });
+
+    await Promise.all([
+      common.shutdown.kill(),
+      common.apiShutdown.kill(),
+      common.buildShutdown.kill(),
+    ]);
+    clearTimeout(timeout);
+
+    if (process.stdin.isTTY) {
+      process.stdin.setRawMode(false);
+      process.stdin.pause();
+    }
+
+    process.exit(code);
+  };
+
+  if (os.platform() === "win32") {
+    const readlineInterface = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+    readlineInterface.on("SIGINT", () => {
+      if (isShuttingDown) return;
+      common.logger.warn({ msg: "Received SIGINT" });
+      exit({ code: 0 });
+    });
+  }
+
+  process.on("SIGINT", () => {
+    if (isShuttingDown) return;
+    common.logger.warn({ msg: "Received SIGINT" });
+    exit({ code: 0 });
+  });
+  process.on("SIGTERM", () => {
+    if (isShuttingDown) return;
+    common.logger.warn({ msg: "Received SIGTERM" });
+    exit({ code: 0 });
+  });
+  process.on("SIGQUIT", () => {
+    if (isShuttingDown) return;
+    common.logger.warn({ msg: "Received SIGQUIT" });
+    exit({ code: 0 });
+  });
+  if (options.command !== "dev") {
+    process.on("uncaughtException", (error: Error) => {
+      if (error instanceof ShutdownError) return;
+      common.logger.error({
+        msg: "uncaughtException",
+        error,
+      });
+      if (error instanceof NonRetryableUserError) {
+        exit({ code: 1 });
+      } else {
+        exit({ code: 75 });
+      }
+    });
+    process.on("unhandledRejection", (error: Error) => {
+      if (error instanceof ShutdownError) return;
+      common.logger.error({
+        msg: "unhandledRejection",
+        error,
+      });
+      if (error instanceof NonRetryableUserError) {
+        exit({ code: 1 });
+      } else {
+        exit({ code: 75 });
+      }
+    });
+  }
+
+  return exit;
+};