@mikeatlast/ponder 0.16.1-fork.1

package/src/utils/pg.ts

@@ -0,0 +1,230 @@
+import type { Logger } from "@/internal/logger.js";
+import pg, { type PoolConfig } from "pg";
+import parse from "pg-connection-string";
+
+// The default parser for numeric[] (1231) seems to parse values as Number
+// or perhaps through JSON.parse(). Use the int8[] (1016) parser instead,
+// which properly returns an array of strings.
+const bigIntArrayParser = pg.types.getTypeParser(1016);
+pg.types.setTypeParser(1231, bigIntArrayParser);
+
+export const PG_BIGINT_MAX = 9223372036854775807n;
+export const PG_INTEGER_MAX = 2147483647;
+
+export function getDatabaseName(conf: PoolConfig) {
+  const config = parseBaseConfig(conf);
+  // https://github.com/brianc/node-postgres/blob/ecff60dc8aa0bd1ad5ea8f4623af0756a86dc110/packages/pg/lib/defaults.js#L3-L73
+  const defaults = {
+    hostname: "localhost",
+    port: "5432",
+    database: undefined,
+  };
+  const envVars = {
+    hostname: process.env.PGHOST,
+    database: process.env.PGDATABASE,
+    port: process.env.PGPORT,
+  };
+
+  // https://github.com/brianc/node-postgres/blob/ecff60dc8aa0bd1ad5ea8f4623af0756a86dc110/packages/pg/lib/connection-parameters.js#L18
+  // precedence is config > env vars > default
+  const hostname = config.hostname || envVars.hostname || defaults.hostname;
+  const database = config.database || envVars.database || defaults.database;
+  const port = config.port || envVars.port || defaults.port;
+
+  return `${hostname}:${port}/${database ?? ""}`;
+}
+
+const parseBaseConfig = (
+  config: PoolConfig,
+): {
+  hostname?: string;
+  port?: string;
+  database?: string;
+} => {
+  const conf = {
+    hostname: config.host,
+    port: config.port?.toString(),
+    database: config.database,
+  };
+  if (!config.connectionString) return conf;
+  // https://github.com/brianc/node-postgres/blob/ecff60dc8aa0bd1ad5ea8f4623af0756a86dc110/packages/pg/lib/connection-parameters.js#L53-L57
+  // connString values override other values, even if a value from connstring is missing
+  const parsed = (parse as unknown as typeof parse.parse)(
+    config.connectionString,
+  );
+  return {
+    hostname: parsed.host ?? undefined,
+    database: parsed.database ?? undefined,
+    port: parsed.port ?? undefined,
+  };
+};
+
+export function createPool(config: PoolConfig, logger: Logger) {
+  class Client extends pg.Client {
+    // @ts-expect-error
+    override connect(
+      callback: (err: Error) => void | undefined,
+    ): void | Promise<void> {
+      if (callback) {
+        super.connect(() => {
+          this.query(
+            `
+            SET synchronous_commit = off;
+            SET idle_in_transaction_session_timeout = 3600000;`,
+            callback,
+          );
+        });
+      } else {
+        return super.connect().then(() =>
+          this.query(`
+            SET synchronous_commit = off;
+            SET idle_in_transaction_session_timeout = 3600000;`).then(() => {}),
+        );
+      }
+    }
+  }
+
+  const pool = new pg.Pool({
+    // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout
+    statement_timeout: 2 * 60 * 1000, // 2 minutes
+    // @ts-expect-error: The custom Client is an undocumented option.
+    Client: Client,
+    ...config,
+  });
+
+  function onPoolError(error: Error) {
+    const client = (error as any).client as any | undefined;
+    const pid = (client?.processID as number | undefined) ?? "unknown";
+    const applicationName =
+      (client?.connectionParameters?.application_name as string | undefined) ??
+      "unknown";
+
+    logger.warn({
+      msg: "Postgres pool error",
+      application_name: applicationName,
+      pid,
+      error,
+    });
+
+    // NOTE: Errors thrown here cause an uncaughtException. It's better to just log and ignore -
+    // if the underlying problem persists, the process will crash due to downstream effects.
+  }
+
+  function onClientError(error: Error) {
+    logger.warn({ msg: "Postgres client error", error });
+
+    // NOTE: Errors thrown here cause an uncaughtException. It's better to just log and ignore -
+    // if the underlying problem persists, the process will crash due to downstream effects.
+  }
+
+  function onNotice(notice: { message?: string; code?: string }) {
+    const level =
+      typeof notice.code === "string" &&
+      ["42P06", "42P07"].includes(notice.code)
+        ? "trace"
+        : "debug";
+    logger[level]({
+      msg: "Postgres notice",
+      message: notice.message,
+      code: notice.code,
+    });
+  }
+
+  pool.on("error", onPoolError);
+  pool.on("connect", (client) => {
+    client.on("notice", onNotice);
+    client.on("error", onClientError);
+  });
+
+  return pool;
+}
+
+export function createReadonlyPool(
+  config: PoolConfig,
+  logger: Logger,
+  namespace: string,
+) {
+  class ReadonlyClient extends pg.Client {
+    // @ts-expect-error
+    override connect(
+      callback: (err: Error) => void | undefined,
+    ): void | Promise<void> {
+      if (callback) {
+        super.connect(() => {
+          this.query(
+            `
+          SET search_path = "${namespace}";
+          SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;
+          SET work_mem = '512MB';
+          SET lock_timeout = '500ms';`,
+            callback,
+          );
+        });
+      } else {
+        return super.connect().then(() =>
+          this.query(
+            `
+          SET search_path = "${namespace}";
+          SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;
+          SET work_mem = '512MB';
+          SET lock_timeout = '500ms';`,
+          ).then(() => {}),
+        );
+      }
+    }
+  }
+
+  const pool = new pg.Pool({
+    // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout
+    statement_timeout: 30 * 1000, // 30s
+    // @ts-expect-error: The custom Client is an undocumented option.
+    Client: ReadonlyClient,
+    ...config,
+  });
+
+  function onPoolError(error: Error) {
+    const client = (error as any).client as any | undefined;
+    const pid = (client?.processID as number | undefined) ?? "unknown";
+    const applicationName =
+      (client?.connectionParameters?.application_name as string | undefined) ??
+      "unknown";
+
+    logger.warn({
+      msg: "Postgres pool error",
+      application_name: applicationName,
+      pid,
+      error,
+    });
+
+    // NOTE: Errors thrown here cause an uncaughtException. It's better to just log and ignore -
+    // if the underlying problem persists, the process will crash due to downstream effects.
+  }
+
+  function onClientError(error: Error) {
+    logger.warn({ msg: "Postgres client error", error });
+
+    // NOTE: Errors thrown here cause an uncaughtException. It's better to just log and ignore -
+    // if the underlying problem persists, the process will crash due to downstream effects.
+  }
+
+  function onNotice(notice: { message?: string; code?: string }) {
+    const level =
+      typeof notice.code === "string" &&
+      ["42P06", "42P07"].includes(notice.code)
+        ? "trace"
+        : "debug";
+    logger[level]({
+      msg: "Postgres notice",
+      message: notice.message,
+      code: notice.code,
+    });
+  }
+
+  pool.on("error", onPoolError);
+  pool.on("connect", (client) => {
+    client.on("notice", onNotice);
+    client.on("error", onClientError);
+  });
+
+  return pool;
+}

package/src/utils/pglite.ts

@@ -0,0 +1,97 @@
+import { mkdirSync } from "node:fs";
+import type { Prettify } from "@/types/utils.js";
+import { type PGliteOptions as Options, PGlite } from "@electric-sql/pglite";
+import {
+  CompiledQuery,
+  type DatabaseConnection,
+  type Dialect,
+  type Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+  type QueryResult,
+  type TransactionSettings,
+} from "kysely";
+
+export type PGliteOptions = Prettify<Options & { dataDir: string }>;
+
+export function createPglite(options: PGliteOptions) {
+  // PGlite uses the memory FS by default, and Windows doesn't like the
+  // "memory://" path, so it's better to pass `undefined` here.
+  if (options.dataDir === "memory://") {
+    // @ts-expect-error
+    options.dataDir = undefined;
+  } else {
+    mkdirSync(options.dataDir, { recursive: true });
+  }
+
+  return new PGlite(options);
+}
+
+// Adapted from dnlsandiego/kysely-pglite
+// https://github.com/dnlsandiego/kysely-pglite/blob/3891a0c4d9327a21bff26addf371784f0109260b/src/kysely-pglite.ts
+export function createPgliteKyselyDialect(instance: PGlite) {
+  return {
+    createAdapter: () => new PostgresAdapter(),
+    createDriver: () => new PGliteDriver(instance),
+    createIntrospector: (db: Kysely<any>) => new PostgresIntrospector(db),
+    createQueryCompiler: () => new PostgresQueryCompiler(),
+  } satisfies Dialect;
+}
+
+// Adapted from dnlsandiego/kysely-pglite
+// https://github.com/dnlsandiego/kysely-pglite/blob/3891a0c4d9327a21bff26addf371784f0109260b/src/pglite-driver.ts
+export class PGliteDriver {
+  #client: PGlite;
+
+  constructor(client: PGlite) {
+    this.#client = client;
+  }
+
+  async acquireConnection(): Promise<DatabaseConnection> {
+    return new PGliteConnection(this.#client);
+  }
+
+  async beginTransaction(
+    connection: DatabaseConnection,
+    _settings: TransactionSettings,
+  ): Promise<void> {
+    await connection.executeQuery(CompiledQuery.raw("BEGIN"));
+  }
+
+  async commitTransaction(connection: DatabaseConnection): Promise<void> {
+    await connection.executeQuery(CompiledQuery.raw("COMMIT"));
+  }
+
+  async rollbackTransaction(connection: DatabaseConnection): Promise<void> {
+    await connection.executeQuery(CompiledQuery.raw("ROLLBACK"));
+  }
+
+  async destroy(): Promise<void> {
+    await this.#client.close();
+  }
+
+  async init(): Promise<void> {}
+  async releaseConnection(_connection: DatabaseConnection): Promise<void> {}
+}
+
+class PGliteConnection implements DatabaseConnection {
+  #client: PGlite;
+
+  constructor(client: PGlite) {
+    this.#client = client;
+  }
+
+  async executeQuery<R>(
+    compiledQuery: CompiledQuery<any>,
+  ): Promise<QueryResult<R>> {
+    return await this.#client.query<R>(compiledQuery.sql, [
+      ...compiledQuery.parameters,
+    ]);
+  }
+
+  // biome-ignore lint/correctness/useYield: <explanation>
+  async *streamQuery(): AsyncGenerator<never, void, unknown> {
+    throw new Error("PGlite does not support streaming.");
+  }
+}

package/src/utils/port.ts

@@ -0,0 +1,34 @@
+import http from "node:http";
+import type { Common } from "@/internal/common.js";
+
+export const getNextAvailablePort = async ({ common }: { common: Common }) => {
+  const server = http.createServer();
+
+  let port = common.options.port;
+
+  return new Promise<number>((resolve, reject) => {
+    server.once("error", (error: Error & { code: string }) => {
+      if (error.code === "EADDRINUSE") {
+        common.logger.warn({
+          msg: "Port in use",
+          port,
+        });
+        port += 1;
+        setTimeout(() => {
+          server.listen(port, common.options.hostname);
+        }, 5);
+      } else {
+        reject(error);
+      }
+    });
+
+    server.once("listening", () => {
+      // Port is available
+      resolve(port);
+    });
+
+    server.listen(port, common.options.hostname);
+  }).finally(() => {
+    server.close();
+  });
+};

package/src/utils/print.ts

@@ -0,0 +1,31 @@
+// Adapted from viem.
+// https://github.com/wagmi-dev/viem/blob/021ce8e5a3fb02db6139564345a91fc77cba08a6/src/errors/transaction.ts#L6-L19
+export function prettyPrint(
+  args?: Record<string, bigint | number | string | undefined | false | unknown>,
+) {
+  if (args === undefined) return "(undefined)";
+
+  const entries = Object.entries(args)
+    .map(([key, value]) => {
+      if (value === undefined) return null;
+
+      const trimmedValue =
+        typeof value === "string" && value.length > 80
+          ? value.slice(0, 80).concat("...")
+          : value;
+
+      return [key, trimmedValue];
+    })
+    .filter(Boolean) as [string, string][];
+
+  if (entries.length === 0) return "  (empty object)";
+
+  const maxLength = entries.reduce(
+    (acc, [key]) => Math.max(acc, key.length),
+    0,
+  );
+
+  return entries
+    .map(([key, value]) => `  ${`${key}`.padEnd(maxLength + 1)}  ${value}`)
+    .join("\n");
+}

package/src/utils/promiseAllSettledWithThrow.ts

@@ -0,0 +1,27 @@
+/**
+ * Like `Promise.allSettled` but throws if any of the promises reject.
+ *
+ * @dev This is very useful when dealing with multiple concurrent promises
+ * in a database transaction.
+ */
+export async function promiseAllSettledWithThrow<T>(
+  promises: Promise<T>[],
+): Promise<T[]> {
+  let firstError: Error | undefined;
+
+  const result = await Promise.all(
+    promises.map((promise) =>
+      promise.catch((error) => {
+        if (firstError === undefined) {
+          firstError = error;
+        }
+      }),
+    ),
+  );
+
+  if (firstError === undefined) {
+    return result as T[];
+  }
+
+  throw firstError;
+}

package/src/utils/promiseWithResolvers.ts

@@ -0,0 +1,20 @@
+export type PromiseWithResolvers<TPromise> = {
+  resolve: (arg: TPromise) => void;
+  reject: (error: Error) => void;
+  promise: Promise<TPromise>;
+};
+
+/**
+ * @description Application level polyfill.
+ */
+export const promiseWithResolvers = <
+  TPromise,
+>(): PromiseWithResolvers<TPromise> => {
+  let resolve: (arg: TPromise) => void;
+  let reject: (error: Error) => void;
+  const promise = new Promise<TPromise>((_resolve, _reject) => {
+    resolve = _resolve;
+    reject = _reject;
+  });
+  return { resolve: resolve!, reject: reject!, promise };
+};

package/src/utils/queue.ts

@@ -0,0 +1,258 @@
+import {
+  type PromiseWithResolvers,
+  promiseWithResolvers,
+} from "./promiseWithResolvers.js";
+
+export type InnerQueue<returnType, taskType> = {
+  task: taskType;
+  resolve: (arg: returnType) => void;
+  reject: (error: Error) => void;
+}[];
+
+export type Queue<returnType, taskType> = {
+  size: () => number;
+  pending: () => Promise<number>;
+  add: (task: taskType) => Promise<returnType>;
+  clear: (
+    callback?: (e: InnerQueue<returnType, taskType>[number]) => void,
+  ) => void;
+  isStarted: () => boolean;
+  start: () => Promise<void>;
+  pause: () => void;
+  onIdle: () => Promise<void>;
+  onEmpty: () => Promise<void>;
+  setParameters: (
+    parameters: Pick<
+      CreateQueueParameters<unknown, unknown>,
+      "frequency" | "concurrency"
+    >,
+  ) => void;
+};
+
+export type CreateQueueParameters<returnType, taskType> = {
+  worker: (task: taskType) => Promise<returnType>;
+  initialStart?: boolean;
+  browser?: boolean;
+} & (
+  | {
+      concurrency: number;
+      frequency: number;
+    }
+  | { concurrency: number; frequency?: undefined }
+  | { concurrency?: undefined; frequency: number }
+);
+
+const validateParameters = ({
+  concurrency,
+  frequency,
+}: Pick<
+  CreateQueueParameters<unknown, unknown>,
+  "frequency" | "concurrency"
+>) => {
+  if (concurrency === undefined && frequency === undefined) {
+    throw new Error(
+      "Invalid queue configuration, must specify either 'concurrency' or 'frequency'.",
+    );
+  }
+
+  if (concurrency !== undefined && concurrency <= 0) {
+    throw new Error(
+      `Invalid value for queue 'concurrency' option. Got ${concurrency}, expected a number greater than zero.`,
+    );
+  }
+
+  if (frequency !== undefined && frequency <= 0) {
+    throw new Error(
+      `Invalid value for queue 'frequency' option. Got ${frequency}, expected a number greater than zero.`,
+    );
+  }
+};
+
+export const createQueue = <returnType, taskType = void>({
+  worker,
+  initialStart = false,
+  browser = true,
+  ..._parameters
+}: CreateQueueParameters<returnType, taskType>): Queue<
+  returnType,
+  taskType
+> => {
+  validateParameters(_parameters);
+
+  const parameters: Pick<
+    CreateQueueParameters<unknown, unknown>,
+    "frequency" | "concurrency"
+  > = _parameters;
+  let queue = new Array<InnerQueue<returnType, taskType>[number]>();
+  let pending = 0;
+  let timestamp = 0;
+  let requests = 0;
+  let isStarted = initialStart;
+
+  let timer: NodeJS.Timeout | undefined;
+
+  let emptyPromiseWithResolvers:
+    | (PromiseWithResolvers<void> & { completed: boolean })
+    | undefined = undefined;
+  let idlePromiseWithResolvers:
+    | (PromiseWithResolvers<void> & { completed: boolean })
+    | undefined = undefined;
+
+  const next = () => {
+    if (!isStarted) return;
+
+    const _timestamp = Date.now();
+
+    if (Math.floor(_timestamp / 1_000) !== timestamp) {
+      requests = 0;
+      timestamp = Math.floor(_timestamp / 1_000);
+    }
+
+    if (timer) return;
+
+    while (
+      (parameters.frequency !== undefined
+        ? requests < parameters.frequency
+        : true) &&
+      (parameters.concurrency !== undefined
+        ? pending < parameters.concurrency
+        : true) &&
+      queue.length > 0
+    ) {
+      const { task, resolve, reject } = queue.shift()!;
+
+      requests++;
+      pending++;
+
+      worker(task)
+        .then(resolve)
+        .catch(reject)
+        .finally(() => {
+          pending--;
+
+          if (
+            idlePromiseWithResolvers !== undefined &&
+            queue.length === 0 &&
+            pending === 0
+          ) {
+            idlePromiseWithResolvers.resolve();
+            idlePromiseWithResolvers.completed = true;
+          }
+
+          browser ? next() : process.nextTick(next);
+        });
+
+      if (emptyPromiseWithResolvers !== undefined && queue.length === 0) {
+        emptyPromiseWithResolvers.resolve();
+        emptyPromiseWithResolvers.completed = true;
+      }
+    }
+
+    if (
+      parameters.frequency !== undefined &&
+      requests >= parameters.frequency
+    ) {
+      timer = setTimeout(
+        () => {
+          timer = undefined;
+          next();
+        },
+        1_000 - (_timestamp % 1_000),
+      );
+      return;
+    }
+  };
+
+  return {
+    size: () => queue.length,
+    pending: () => {
+      if (browser) {
+        return new Promise<number>((resolve) =>
+          setTimeout(() => resolve(pending)),
+        );
+      } else {
+        return new Promise<number>((resolve) =>
+          setImmediate(() => resolve(pending)),
+        );
+      }
+    },
+    add: (task: taskType) => {
+      const { promise, resolve, reject } = promiseWithResolvers<returnType>();
+      queue.push({ task, resolve, reject });
+
+      next();
+
+      return promise;
+    },
+    clear: (callback) => {
+      if (callback) {
+        for (const e of queue) {
+          callback(e);
+        }
+      }
+
+      queue = new Array<InnerQueue<returnType, taskType>[number]>();
+      clearTimeout(timer);
+      timer = undefined;
+    },
+    isStarted: () => isStarted,
+    start: () => {
+      if (browser) {
+        return new Promise<number>((resolve) =>
+          setTimeout(() => resolve(pending)),
+        ).then(() => {
+          isStarted = true;
+          next();
+        });
+      } else {
+        return new Promise<number>((resolve) =>
+          process.nextTick(() => resolve(pending)),
+        ).then(() => {
+          isStarted = true;
+          next();
+        });
+      }
+    },
+    pause: () => {
+      isStarted = false;
+    },
+    onIdle: () => {
+      if (
+        idlePromiseWithResolvers === undefined ||
+        idlePromiseWithResolvers.completed
+      ) {
+        if (queue.length === 0 && pending === 0) return Promise.resolve();
+
+        idlePromiseWithResolvers = {
+          ...promiseWithResolvers<void>(),
+          completed: false,
+        };
+      }
+      return idlePromiseWithResolvers.promise;
+    },
+    onEmpty: () => {
+      if (
+        emptyPromiseWithResolvers === undefined ||
+        emptyPromiseWithResolvers.completed
+      ) {
+        if (queue.length === 0) return Promise.resolve();
+
+        emptyPromiseWithResolvers = {
+          ...promiseWithResolvers<void>(),
+          completed: false,
+        };
+      }
+      return emptyPromiseWithResolvers.promise;
+    },
+    setParameters: (_parameters) => {
+      validateParameters(_parameters);
+
+      if ("frequency" in _parameters) {
+        parameters.frequency = _parameters.frequency;
+      }
+      if ("concurrency" in _parameters) {
+        parameters.concurrency = _parameters.concurrency;
+      }
+    },
+  } as Queue<returnType, taskType>;
+};

package/src/utils/range.ts

@@ -0,0 +1,8 @@
+/**
+ * Generates an array of integers between two bounds. Exclusive on the right.
+ *
+ * @param start Integer to start at.
+ * @param stop Integer to stop at (exclusive).
+ */
+export const range = (start: number, stop: number) =>
+  Array.from({ length: stop - start }, (_, i) => start + i);