@effectstream/db 0.100.12 → 0.100.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/db",
-  "version": "0.100.12",
+  "version": "0.100.14",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -21,11 +21,11 @@
     "./version": "./migrations/system-version.ts"
   },
   "dependencies": {
-    "@effectstream/coroutine": "0.100.12",
-    "@effectstream/log": "0.100.12",
-    "@effectstream/runtime": "0.100.12",
-    "@effectstream/sync": "0.100.12",
-    "@effectstream/utils": "0.100.12",
+    "@effectstream/coroutine": "0.100.14",
+    "@effectstream/log": "0.100.14",
+    "@effectstream/runtime": "0.100.14",
+    "@effectstream/sync": "0.100.14",
+    "@effectstream/utils": "0.100.14",
     "@electric-sql/pglite": "^0.3.14",
     "@pgtyped/runtime": "2.4.2",
     "@pgtyped/parser": "2.4.2",

package/src/mod.ts

@@ -16,21 +16,27 @@ export type {
 // https://github.com/adelsz/pgtyped/issues/565
 export {
   blockHeightDone,
+  deleteEmptyBlocks,
   getBlockByHash,
   getBlockHeights,
   getBlockSeeds,
+  getLastNonEmptyBlockHash,
   getLatestProcessedBlockHeight,
   saveLastBlock,
 } from "./sql/block-heights.queries.ts";
 export type {
   IBlockHeightDoneParams,
   IBlockHeightDoneResult,
+  IDeleteEmptyBlocksParams,
+  IDeleteEmptyBlocksResult,
   IGetBlockByHashParams,
   IGetBlockByHashResult,
   IGetBlockHeightsParams,
   IGetBlockHeightsResult,
   IGetBlockSeedsParams,
   IGetBlockSeedsResult,
+  IGetLastNonEmptyBlockHashParams,
+  IGetLastNonEmptyBlockHashResult,
   IGetLatestProcessedBlockHeightParams,
   IGetLatestProcessedBlockHeightResult,
   ISaveLastBlockParams,

package/src/snapshot-handler.test.ts

@@ -0,0 +1,187 @@
+import { equal as assertEquals, ok as assert } from "node:assert/strict";
+import { test } from "@effectstream/utils/runtime";
+import {
+  makeTempDir,
+  mkdirRecursive,
+  readDir,
+  remove,
+  setFileTimes,
+  writeTextFile,
+} from "@effectstream/utils/fs";
+import { applyRetentionPolicy } from "./snapshot-handler.ts";
+
+const MS_1H = 3_600_000;
+const MS_24H = 24 * MS_1H;
+
+const DEFAULT_RETENTION = {
+  lastDayHourly: true,
+  last3DaysSixHourly: true,
+  lastNDaysDaily: 7,
+};
+
+async function makeSnapshotDir(): Promise<string> {
+  return await makeTempDir({ prefix: "snapshot-retention-test-" });
+}
+
+/**
+ * Seeds `dir` with one file per hour for `hours` hours, mtimes descending
+ * from `now` (offset 0h, 1h, ..., hours-1h).
+ */
+async function seedHourlySnapshots(
+  dir: string,
+  hours: number,
+  now: number,
+): Promise<void> {
+  for (let i = 0; i < hours; i++) {
+    const mtimeMs = now - i * MS_1H;
+    const iso = new Date(mtimeMs).toISOString().replace(/:/g, "-").replace(/\.\d{3}/, "");
+    const path = `${dir}/snapshot-${iso}.dump`;
+    await writeTextFile(path, "x");
+    await setFileTimes(path, mtimeMs, mtimeMs);
+  }
+}
+
+async function listSnapshots(dir: string): Promise<string[]> {
+  const out: string[] = [];
+  for await (const e of readDir(dir)) {
+    if (e.isFile && e.name.startsWith("snapshot-") && e.name.endsWith(".dump")) out.push(e.name);
+  }
+  return out;
+}
+
+test("retention: single pass over 8 days populates all tiers", async () => {
+  const dir = await makeSnapshotDir();
+  // Align `now` to a 24h UTC boundary so bucket math is deterministic.
+  const now = Math.floor(Date.now() / MS_24H) * MS_24H;
+  await seedHourlySnapshots(dir, 8 * 24, now);
+
+  await applyRetentionPolicy(dir, DEFAULT_RETENTION, now);
+
+  const survivors = await listSnapshots(dir);
+  // Expected (boundary-inclusive math, `now` aligned to 24h):
+  //   hourly tier i=0..24 (age ≤ 24h) → 25 distinct hour buckets → 25 files
+  //   6h tier    i=25..72 (24h<age≤72h), mtimes span 48h → 8 distinct 6h bins → 8 files
+  //   daily tier i=73..168 (72h<age≤7d), mtimes span 96h → 4 distinct day bins → 4 files
+  //   deleted    i=169..191 (age > 7d) → 0 files
+  assertEquals(survivors.length, 25 + 8 + 4);
+});
+
+test("retention: bucket keys are stable across successive passes (regression)", async () => {
+  // The bug: age-relative bucket keys slide forward every call, so the file
+  // kept in "bucket 6h:4" gets replaced each tick by a newer file that just
+  // crossed 24h. After many iterations only ~hourly files survive.
+  //
+  // With mtime-absolute bucket keys a file's bucket is fixed at write time.
+  // Simulate 8 days of hourly snapshots with retention running each hour and
+  // confirm the survivor count stays consistent with multi-tier retention.
+  const dir = await makeSnapshotDir();
+  const baseNow = Math.floor(Date.now() / MS_24H) * MS_24H;
+
+  // Simulate 8 days * 24 ticks: each tick drops a new snapshot and runs retention.
+  const totalTicks = 8 * 24;
+  for (let tick = 0; tick < totalTicks; tick++) {
+    const now = baseNow + tick * MS_1H;
+    const iso = new Date(now).toISOString().replace(/:/g, "-").replace(/\.\d{3}/, "");
+    const path = `${dir}/snapshot-${iso}.dump`;
+    await writeTextFile(path, "x");
+    await setFileTimes(path, now, now);
+    await applyRetentionPolicy(dir, DEFAULT_RETENTION, now);
+  }
+
+  const survivors = await listSnapshots(dir);
+  // With the old (buggy) bucketing this collapses to ~26 (hourly tier plus
+  // a handful of in-flight files). With the fix we expect ~37.
+  assert(
+    survivors.length >= 32,
+    `expected multi-tier retention to keep ≥32 files after 8 days of ticks, got ${survivors.length}`,
+  );
+  assert(
+    survivors.length <= 42,
+    `expected ≤42 files after retention, got ${survivors.length}`,
+  );
+});
+
+test("retention: lastDayHourly=false keeps all files in hourly tier", async () => {
+  const dir = await makeSnapshotDir();
+  const now = Math.floor(Date.now() / MS_24H) * MS_24H;
+  await seedHourlySnapshots(dir, 24, now);
+
+  await applyRetentionPolicy(
+    dir,
+    { ...DEFAULT_RETENTION, lastDayHourly: false },
+    now,
+  );
+
+  const survivors = await listSnapshots(dir);
+  assertEquals(survivors.length, 24);
+});
+
+test("retention: last3DaysSixHourly=false keeps all files in 24h–72h tier", async () => {
+  const dir = await makeSnapshotDir();
+  const now = Math.floor(Date.now() / MS_24H) * MS_24H;
+  // Seed 72 hourly files spanning 3 days.
+  await seedHourlySnapshots(dir, 72, now);
+
+  await applyRetentionPolicy(
+    dir,
+    { ...DEFAULT_RETENTION, last3DaysSixHourly: false },
+    now,
+  );
+
+  const survivors = await listSnapshots(dir);
+  // Hourly tier (i=0..24) → 25 distinct-hour keepers
+  // 6h tier with short-circuit → all i=25..71 kept (47 files)
+  assertEquals(survivors.length, 25 + 47);
+});
+
+test("retention: lastNDaysDaily=1 deletes everything past 24h", async () => {
+  const dir = await makeSnapshotDir();
+  const now = Math.floor(Date.now() / MS_24H) * MS_24H;
+  await seedHourlySnapshots(dir, 48, now);
+
+  await applyRetentionPolicy(
+    dir,
+    { ...DEFAULT_RETENTION, lastNDaysDaily: 1 },
+    now,
+  );
+
+  const survivors = await listSnapshots(dir);
+  // Only i=0..24 survive (age ≤ 24h = MS_ND). 25 distinct hour buckets.
+  assertEquals(survivors.length, 25);
+});
+
+test("retention: empty directory is a no-op", async () => {
+  const dir = await makeSnapshotDir();
+  await applyRetentionPolicy(dir, DEFAULT_RETENTION, Date.now());
+  const survivors = await listSnapshots(dir);
+  assertEquals(survivors.length, 0);
+});
+
+test("retention: missing directory is a no-op (warns, does not throw)", async () => {
+  const dir = await makeTempDir({ prefix: "snapshot-retention-missing-" });
+  await remove(dir, { recursive: true });
+  // Should not throw.
+  await applyRetentionPolicy(dir, DEFAULT_RETENTION, Date.now());
+});
+
+test("retention: leaves non-snapshot files untouched", async () => {
+  const dir = await makeSnapshotDir();
+  await mkdirRecursive(dir);
+  const now = Math.floor(Date.now() / MS_24H) * MS_24H;
+  // Seed an ancient snapshot that would be deleted by policy (age > 7d).
+  await seedHourlySnapshots(dir, 1, now - 30 * MS_24H);
+  // Seed unrelated files that must be left alone.
+  await writeTextFile(`${dir}/README.md`, "keep me");
+  await writeTextFile(`${dir}/snapshot-bad-name.txt`, "keep me");
+  await writeTextFile(`${dir}/other.dump`, "keep me");
+
+  await applyRetentionPolicy(dir, DEFAULT_RETENTION, now);
+
+  const remaining: string[] = [];
+  for await (const e of readDir(dir)) remaining.push(e.name);
+  assert(remaining.includes("README.md"));
+  assert(remaining.includes("snapshot-bad-name.txt"));
+  assert(remaining.includes("other.dump"));
+  // Ancient snapshot-*.dump should have been deleted.
+  assertEquals(remaining.filter((n) => n.startsWith("snapshot-") && n.endsWith(".dump")).length, 0);
+});

package/src/snapshot-handler.ts

@@ -1,6 +1,17 @@
 import { ENV } from "@effectstream/utils/node-env";
+import { cwd } from "@effectstream/utils/runtime";
+import {
+  mkdirRecursive,
+  readDir,
+  remove,
+  statMtime,
+} from "@effectstream/utils/fs";
+import { spawnOutput } from "@effectstream/utils/runtime-spawn";
 import { sleep, until, type Operation } from "effection";
 
+/** `process.env` works under Node, Bun, and Deno (via node: compat). */
+declare const process: { env: Record<string, string | undefined> };
+
 export interface SnapshotRetentionConfig {
   /** One per hour for last 24 h. Default: `true` */
   lastDayHourly?: boolean;
@@ -19,6 +30,39 @@ export interface SnapshotConfig {
   retention?: SnapshotRetentionConfig;
 }
 
+interface ResolvedSnapshotConfig {
+  intervalSeconds: number;
+  path: string;
+  retention: Required<SnapshotRetentionConfig>;
+}
+
+/**
+ * Precedence: explicit config field > env var > hardcoded default.
+ * Env vars other than INTERVAL_SECONDS carry their hardcoded defaults inside
+ * `ENV.getConfig`, so `??` here only needs to cover INTERVAL_SECONDS (whose
+ * env `defaultValue` is intentionally `undefined`).
+ */
+function resolveSnapshotConfig(config?: SnapshotConfig): ResolvedSnapshotConfig {
+  return {
+    intervalSeconds:
+      config?.intervalSeconds
+      ?? ENV.EFFECTSTREAM_SNAPSHOT_INTERVAL_SECONDS
+      ?? 3600,
+    path: config?.path ?? ENV.EFFECTSTREAM_SNAPSHOT_PATH,
+    retention: {
+      lastDayHourly:
+        config?.retention?.lastDayHourly
+        ?? ENV.EFFECTSTREAM_SNAPSHOT_LAST_DAY_HOURLY,
+      last3DaysSixHourly:
+        config?.retention?.last3DaysSixHourly
+        ?? ENV.EFFECTSTREAM_SNAPSHOT_LAST_3_DAYS_SIX_HOURLY,
+      lastNDaysDaily:
+        config?.retention?.lastNDaysDaily
+        ?? ENV.EFFECTSTREAM_SNAPSHOT_LAST_N_DAYS,
+    },
+  };
+}
+
 /**
  * Creates a database snapshot using `pg_dump` in custom format (`-F c`).
  * Must be called AFTER releasing the DB mutex (pg_dump opens its own connection).
@@ -35,14 +79,22 @@ export async function createSnapshot(
     return;
   }
 
-  const snapshotDir = config?.path ?? "./snapshots";
+  const resolved = resolveSnapshotConfig(config);
+  const snapshotDir = resolved.path;
   const timestamp = new Date().toISOString().replace(/:/g, "-").replace(/\.\d{3}/, "");
   const snapshotPath = `${snapshotDir}/snapshot-${timestamp}.dump`;
 
-  await Deno.mkdir(snapshotDir, { recursive: true });
+  await mkdirRecursive(snapshotDir);
   console.log(`[Snapshot] Creating snapshot at ${snapshotPath}...`);
 
-  const cmd = new Deno.Command("pg_dump", {
+  // Inherit the ambient env and add PGPASSWORD. `process.env` is populated on
+  // Node/Bun natively and on Deno via node:compat when --allow-env is set.
+  const ambientEnv: Record<string, string> = {};
+  for (const [k, v] of Object.entries(process.env)) {
+    if (v !== undefined) ambientEnv[k] = v;
+  }
+
+  const result = await spawnOutput("pg_dump", {
     args: [
       "-h", ENV.DB_HOST,
       "-p", String(ENV.DB_PORT),
@@ -51,18 +103,16 @@ export async function createSnapshot(
       "-F", "c",
       "-f", snapshotPath,
     ],
-    env: { ...Deno.env.toObject(), PGPASSWORD: ENV.DB_PW ?? "" },
+    env: { ...ambientEnv, PGPASSWORD: ENV.DB_PW ?? "" },
     stdout: "inherit",
     stderr: "inherit",
   });
-
-  const status = await cmd.output();
-  if (!status.success) {
-    throw new Error(`[Snapshot] pg_dump failed with exit code ${status.code}`);
+  if (!result.success) {
+    throw new Error(`[Snapshot] pg_dump failed with exit code ${result.code}`);
   }
 
   console.log(`[Snapshot] Snapshot created: ${snapshotPath}`);
-  await applyRetentionPolicy(snapshotDir, config?.retention);
+  await applyRetentionPolicy(snapshotDir, resolved.retention);
 }
 
 /**
@@ -73,33 +123,38 @@ export async function createSnapshot(
 export function* runSnapshotLoop(
   snapshotConfig: SnapshotConfig,
 ): Operation<void> {
-  const intervalMs = (snapshotConfig.intervalSeconds ?? 3600) * 1000;
-  console.log(`[Snapshot] Loop started (interval ${intervalMs / 1000}s, path ${snapshotConfig.path ?? "./snapshots"}, cwd ${Deno.cwd()})`);
+  const resolved = resolveSnapshotConfig(snapshotConfig);
+  const intervalMs = resolved.intervalSeconds * 1000;
+  console.log(`[Snapshot] Loop started (interval ${intervalMs / 1000}s, path ${resolved.path}, cwd ${cwd()})`);
   while (true) {
+    yield* sleep(intervalMs);
     console.log(`[Snapshot] Firing`);
     try {
-      yield* until(createSnapshot(snapshotConfig));
+      yield* until(createSnapshot(resolved));
       console.log(`[Snapshot] Done; sleeping ${intervalMs / 1000}s`);
     } catch (e) {
       console.error("[Snapshot] Failed:", e);
     }
-    yield* sleep(intervalMs);
   }
 }
 
 /**
  * Time-based tiered retention: keeps one file per bucket (newest wins),
  * deletes everything beyond `lastNDaysDaily` days.
+ *
+ * Bucket keys are derived from the file's wall-clock mtime, not from its
+ * age relative to `now`. A file's bucket is therefore fixed at write time
+ * and does not slide forward every tick, which is what lets files actually
+ * age into the 6-hourly and daily tiers.
  */
-async function applyRetentionPolicy(
+export async function applyRetentionPolicy(
   snapshotDir: string,
-  retention?: SnapshotRetentionConfig,
+  retention: Required<SnapshotRetentionConfig>,
+  nowMs: number = Date.now(),
 ): Promise<void> {
-  const lastDayHourly      = retention?.lastDayHourly      ?? true;
-  const last3DaysSixHourly = retention?.last3DaysSixHourly ?? true;
-  const lastNDaysDaily     = retention?.lastNDaysDaily      ?? 7;
+  const { lastDayHourly, last3DaysSixHourly, lastNDaysDaily } = retention;
 
-  const now    = Date.now();
+  const now    = nowMs;
   const MS_1H  = 3_600_000;
   const MS_24H = 24 * MS_1H;
   const MS_3D  = 3  * MS_24H;
@@ -108,12 +163,12 @@ async function applyRetentionPolicy(
   const entries: { path: string; mtime: number }[] = [];
 
   try {
-    for await (const entry of Deno.readDir(snapshotDir)) {
+    for await (const entry of readDir(snapshotDir)) {
       if (!entry.isFile || !entry.name.startsWith("snapshot-") || !entry.name.endsWith(".dump")) continue;
       const filePath = `${snapshotDir}/${entry.name}`;
       try {
-        const stat = await Deno.stat(filePath);
-        entries.push({ path: filePath, mtime: stat.mtime?.getTime() ?? 0 });
+        const mtime = await statMtime(filePath);
+        entries.push({ path: filePath, mtime });
       } catch { /* file vanished between readDir and stat */ }
     }
   } catch (e) {
@@ -137,12 +192,12 @@ async function applyRetentionPolicy(
 
     if (age <= MS_24H) {
       if (!lastDayHourly) { keepers.add(entry.path); continue; }
-      bucketKey = `h:${Math.floor(age / MS_1H)}`;
+      bucketKey = `h:${Math.floor(entry.mtime / MS_1H)}`;
     } else if (age <= MS_3D) {
       if (!last3DaysSixHourly) { keepers.add(entry.path); continue; }
-      bucketKey = `6h:${Math.floor(age / (6 * MS_1H))}`;
+      bucketKey = `6h:${Math.floor(entry.mtime / (6 * MS_1H))}`;
     } else {
-      bucketKey = `d:${Math.floor(age / MS_24H)}`;
+      bucketKey = `d:${Math.floor(entry.mtime / MS_24H)}`;
     }
 
     if (!seenBuckets.has(bucketKey)) {
@@ -154,7 +209,7 @@ async function applyRetentionPolicy(
   for (const entry of entries) {
     if (!keepers.has(entry.path)) {
       try {
-        await Deno.remove(entry.path);
+        await remove(entry.path);
         console.log(`[Snapshot] Retention: deleted ${entry.path}`);
       } catch (e) {
         console.warn(`[Snapshot] Failed to delete ${entry.path}:`, e);

package/src/sql/block-heights.queries.ts

@@ -174,6 +174,34 @@ const saveLastBlockIR: any = {"usedParamSet":{"block_height":true,"ver":true,"ma
 export const saveLastBlock = new PreparedQuery<ISaveLastBlockParams,ISaveLastBlockResult>(saveLastBlockIR);
 
 
+/** 'GetLastNonEmptyBlockHash' parameters type */
+export type IGetLastNonEmptyBlockHashParams = void;
+
+/** 'GetLastNonEmptyBlockHash' return type */
+export interface IGetLastNonEmptyBlockHashResult {
+  effectstream_block_hash: Buffer | null;
+}
+
+/** 'GetLastNonEmptyBlockHash' query type */
+export interface IGetLastNonEmptyBlockHashQuery {
+  params: IGetLastNonEmptyBlockHashParams;
+  result: IGetLastNonEmptyBlockHashResult;
+}
+
+const getLastNonEmptyBlockHashIR: any = {"usedParamSet":{},"params":[],"statement":"SELECT effectstream_block_hash FROM effectstream.effectstream_blocks\nWHERE effectstream_block_hash IS NOT NULL\n  AND effectstream_block_hash != '\\x307830'::bytea\nORDER BY block_height DESC\nLIMIT 1"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT effectstream_block_hash FROM effectstream.effectstream_blocks
+ * WHERE effectstream_block_hash IS NOT NULL
+ *   AND effectstream_block_hash != '\x307830'::bytea
+ * ORDER BY block_height DESC
+ * LIMIT 1
+ * ```
+ */
+export const getLastNonEmptyBlockHash = new PreparedQuery<IGetLastNonEmptyBlockHashParams,IGetLastNonEmptyBlockHashResult>(getLastNonEmptyBlockHashIR);
+
 /** 'BlockHeightDone' parameters type */
 export interface IBlockHeightDoneParams {
   block_hash: Buffer;
@@ -189,6 +217,29 @@ export interface IBlockHeightDoneQuery {
   result: IBlockHeightDoneResult;
 }
 
+/** 'DeleteEmptyBlocks' parameters type */
+export type IDeleteEmptyBlocksParams = void;
+
+/** 'DeleteEmptyBlocks' return type */
+export type IDeleteEmptyBlocksResult = void;
+
+/** 'DeleteEmptyBlocks' query type */
+export interface IDeleteEmptyBlocksQuery {
+  params: IDeleteEmptyBlocksParams;
+  result: IDeleteEmptyBlocksResult;
+}
+
+const deleteEmptyBlocksIR: any = {"usedParamSet":{},"params":[],"statement":"DELETE FROM effectstream.effectstream_blocks\nWHERE effectstream_block_hash = '\\x307830'::bytea"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * DELETE FROM effectstream.effectstream_blocks
+ * WHERE effectstream_block_hash = '\x307830'::bytea
+ * ```
+ */
+export const deleteEmptyBlocks = new PreparedQuery<IDeleteEmptyBlocksParams,IDeleteEmptyBlocksResult>(deleteEmptyBlocksIR);
+
 const blockHeightDoneIR: any = {"usedParamSet":{"block_hash":true,"block_height":true},"params":[{"name":"block_hash","required":true,"transform":{"type":"scalar"},"locs":[{"a":70,"b":81}]},{"name":"block_height","required":true,"transform":{"type":"scalar"},"locs":[{"a":104,"b":117}]}],"statement":"UPDATE effectstream.effectstream_blocks\nSET\neffectstream_block_hash = :block_hash!\nWHERE block_height = :block_height!"};
 
 /**

package/src/sql/block-heights.sql

@@ -38,6 +38,17 @@ seed = EXCLUDED.seed,
 ms_timestamp = EXCLUDED.ms_timestamp,
 effectstream_block_hash = EXCLUDED.effectstream_block_hash;
 
+/* @name getLastNonEmptyBlockHash */
+SELECT effectstream_block_hash FROM effectstream.effectstream_blocks
+WHERE effectstream_block_hash IS NOT NULL
+  AND effectstream_block_hash != '\x307830'::bytea
+ORDER BY block_height DESC
+LIMIT 1;
+
+/* @name deleteEmptyBlocks */
+DELETE FROM effectstream.effectstream_blocks
+WHERE effectstream_block_hash = '\x307830'::bytea;
+
 /* @name blockHeightDone */
 UPDATE effectstream.effectstream_blocks
 SET