@delali/sirannon-db 0.2.3-next.38 → 0.3.0

package/dist/backup/index.d.ts

@@ -1,8 +1,8 @@
-import { B as BackupDestination, f as BackupRunReport } from '../types-CopTgHG3.js';
-export { g as BackupChain, h as BackupChainBase, i as BackupChainChange, j as BackupChainPosition, k as BackupChainRecord, l as BackupCycle, m as BackupCycleError, n as BackupCycleOptions, o as BackupCycleStatus, p as BackupGroupMembership, q as BackupGroupSource, r as BackupNodePreference, s as BackupPiece, t as BackupProgress, u as BackupSkip, v as BackupSkipReason, w as BackupToDestinationOptions, x as BackupVerifyResult, y as DEFAULT_CHAIN_NAME, z as chainLogName, C as readBackupChains, E as verifyBackupRecord } from '../types-CopTgHG3.js';
-export { B as BackupCapabilities, a as BackupChainLocation, b as BackupRestorePlan, c as BackupSafeToDeleteOptions, d as backupPiecesSafeToDelete, p as planBackupRestore } from '../database-backup-CliAhaEm.js';
-import { B as BackupRestoreOptions, a as BackupRestoreReport } from '../restore-options-O5gYlr3n.js';
-export { b as BackupRestoreProgress } from '../restore-options-O5gYlr3n.js';
+import { B as BackupDestination, f as BackupRunReport } from '../types-DApb7dvw.js';
+export { g as BackupChain, h as BackupChainBase, i as BackupChainChange, j as BackupChainPosition, k as BackupChainRecord, l as BackupCycle, m as BackupCycleError, n as BackupCycleOptions, o as BackupCycleStatus, p as BackupFileReport, q as BackupGroupMembership, r as BackupGroupSource, s as BackupLogPosition, t as BackupNodePreference, u as BackupPiece, v as BackupProgress, w as BackupSkip, x as BackupSkipReason, y as BackupToDestinationOptions, z as BackupVerifyResult, C as DEFAULT_CHAIN_NAME, E as chainLogName, F as readBackupChains, G as verifyBackupRecord } from '../types-DApb7dvw.js';
+export { B as BackupCapabilities, a as BackupChainLocation, b as BackupRestorePlan, c as BackupSafeToDeleteOptions, d as backupPiecesSafeToDelete, p as planBackupRestore } from '../database-backup-C090lMHc.js';
+import { B as BackupRestoreOptions, a as BackupRestoreReport } from '../restore-options-C9MW9gpm.js';
+export { b as BackupRestoreProgress } from '../restore-options-C9MW9gpm.js';
 import '../query-types-Cv8N7pXj.js';
 
 /** What one assembled file took to build.

package/dist/backup/index.mjs

@@ -1,8 +1,8 @@
-export { assembleFromDestination, restoreBackup } from '../chunk-P53I2G6J.mjs';
+export { assembleFromDestination, restoreBackup } from '../chunk-63VBFSCS.mjs';
 export { backupPiecesSafeToDelete, planBackupRestore } from '../chunk-YNTDOFTN.mjs';
-export { verifyBackupRecord } from '../chunk-CR6HTXSC.mjs';
-import '../chunk-KSYFFYAP.mjs';
-import '../chunk-GBM4AU24.mjs';
+export { verifyBackupRecord } from '../chunk-7SC2TZAS.mjs';
+import '../chunk-U445A3C4.mjs';
+import '../chunk-DVS4QSIZ.mjs';
 export { DEFAULT_CHAIN_NAME, chainLogName, readBackupChains } from '../chunk-YVF2ZJTE.mjs';
 import '../chunk-V5FUJEYY.mjs';
 import '../chunk-PBRXXISQ.mjs';

package/dist/backup-scheduler/index.d.ts

@@ -1,4 +1,5 @@
-import { G as SQLiteConnection, H as BackupRunRequest, f as BackupRunReport, I as BackupScheduleOptions } from '../types-CopTgHG3.js';
+import { I as SQLiteConnection, J as BackupFileCopy, K as BackupRunRequest, f as BackupRunReport, L as BackupScheduleRequest } from '../types-DApb7dvw.js';
+export { p as BackupFileReport, O as BackupScheduleOptions, R as RunExclusive } from '../types-DApb7dvw.js';
 import '../query-types-Cv8N7pXj.js';
 
 /** What one runtime needs before a copy can reach a destination without a local file.
@@ -14,15 +15,44 @@ interface BackupStreamingSupport {
 declare class BackupManager {
     private readonly streaming?;
     constructor(streaming?: BackupStreamingSupport | undefined);
-    backup(conn: SQLiteConnection, destPath: string, onFirstStep?: () => void): Promise<void>;
+    /**
+     * Copies the database behind a connection to a file, while that database
+     * stays open for reads and writes.
+     *
+     * @param conn - Connection the copy runs on, which must be the connection that writes.
+     * @param destPath - Path to write the copy to. A file already there stops the copy.
+     * @param onFirstStep - Called once the copy's first step is done, so the caller can hand the writer back.
+     * @returns What the copy moved, how long it took, and how often it restarted.
+     */
+    backup(conn: SQLiteConnection, destPath: string, onFirstStep?: () => void): Promise<BackupFileCopy>;
+    /**
+     * Reads the size of the file a copy wrote.
+     *
+     * @param resolved - Absolute path of that file.
+     * @param destPath - Path the caller named, which the error states.
+     * @returns Bytes it holds.
+     */
+    private fileBytes;
+    /**
+     * Runs the copy, and removes the file it was writing where that copy fails, so
+     * that a database missing its later pages never stays on disk as though the
+     * copy had finished. Sirannon would otherwise count that half-written file
+     * among the copies it keeps, and it would evict a whole copy to make room for
+     * one nothing can restore.
+     *
+     * Sirannon stops waiting on a copy once the stall deadline passes, and SQLite
+     * keeps writing that copy to the file. A removal that raced a live copy would
+     * leave a truncated file in place, and on Windows it would fail outright.
+     * Sirannon therefore removes the file only once the copy stops, while the
+     * caller receives the failure at once and waits for none of that.
+     */
+    private copyOrClearUp;
     copyToDestination(conn: SQLiteConnection, request: BackupRunRequest): Promise<BackupRunReport>;
     streamsToDestination(): boolean;
     generateFilename(): string;
     rotate(dir: string, maxFiles: number): void;
 }
 
-/** Runs a backup while the caller's writer lock is held, so it never shares the writer connection with another write. */
-type RunExclusive = (op: () => Promise<void>) => Promise<void>;
 /**
  * @public
  *
@@ -35,12 +65,32 @@ declare class BackupScheduler {
      * Starts repeating backups and returns a function that stops them.
      *
      * @param conn - Connection to the database being copied.
-     * @param options - Cron expression, destination directory, retention, time zone, and failure callback.
-     * @param runExclusive - Runs each copy with the database's writer held so that no write commits mid-copy.
+     * @param request - Cron expression, destination directory, retention, and time zone, along with the callbacks and the database the copies come from.
      * @returns A function that stops the schedule.
      */
-    schedule(conn: SQLiteConnection, options: BackupScheduleOptions, runExclusive?: RunExclusive): () => void;
+    schedule(conn: SQLiteConnection, request: BackupScheduleRequest): () => void;
+    /**
+     * Works out the database and the file every report names. A caller may supply
+     * either of them, and Sirannon reads the file SQLite has open where the caller
+     * named none. Sirannon puts that question to the connection that writes, so it
+     * asks with nothing else holding the writer.
+     *
+     * @param conn - Connection the copies come from.
+     * @param run - The schedule, which holds whichever of the two the caller named.
+     * @param runExclusive - Runs the question with nothing else holding the writer.
+     * @returns The database identifier and the path of the file the copies come from.
+     * @throws A `BACKUP_ERROR` where the caller named no source file and SQLite has none open.
+     */
+    private namesFor;
+    /**
+     * Passes one finished copy to the caller's completion callback, and stops
+     * waiting on that callback once its deadline passes.
+     *
+     * @param report - What the copy produced.
+     * @param run - The schedule, which holds the callback and its deadline.
+     */
+    private handToCaller;
     private run;
 }
 
-export { BackupScheduleOptions, BackupScheduler };
+export { BackupScheduleRequest, BackupScheduler };

package/dist/backup-scheduler/index.mjs

@@ -1,5 +1,6 @@
-export { BackupScheduler } from '../chunk-IPDHHIG2.mjs';
+export { BackupScheduler } from '../chunk-DXXCLTFF.mjs';
 import '../chunk-W67A4H2E.mjs';
-import '../chunk-GBM4AU24.mjs';
+import '../chunk-2QCPVE67.mjs';
+import '../chunk-DVS4QSIZ.mjs';
 import '../chunk-V5FUJEYY.mjs';
 import '../chunk-PBRXXISQ.mjs';

package/dist/{change-tracker-BLKRX8ic.d.ts → change-tracker-uG9gWvsY.d.ts}

@@ -1,4 +1,4 @@
-import { G as SQLiteConnection } from './types-CopTgHG3.js';
+import { I as SQLiteConnection } from './types-DApb7dvw.js';
 import { C as ChangeEvent } from './query-types-Cv8N7pXj.js';
 
 /**

package/dist/chunk-2QCPVE67.mjs

@@ -0,0 +1,143 @@
+import { LOG_HEADER_BYTES, readLogHeader, logFrameOffset, readValidLogFrame } from './chunk-DVS4QSIZ.mjs';
+import { SirannonError } from './chunk-PBRXXISQ.mjs';
+import { open } from 'fs/promises';
+
+var READ_CHUNK_BYTES = 4 * 1024 * 1024;
+async function openForReading(path) {
+  try {
+    return await open(path, "r");
+  } catch (err) {
+    if (err.code === "ENOENT") return void 0;
+    throw err;
+  }
+}
+async function readLogFileHeader(logPath) {
+  const file = await openForReading(logPath);
+  if (!file) return void 0;
+  try {
+    const buffer = Buffer.allocUnsafe(LOG_HEADER_BYTES);
+    const { bytesRead } = await file.read(buffer, 0, LOG_HEADER_BYTES, 0);
+    if (bytesRead < LOG_HEADER_BYTES) return void 0;
+    return readLogHeader(buffer);
+  } finally {
+    await file.close();
+  }
+}
+async function scanLogFrames(logPath, header, from) {
+  const stopped = {
+    lastCommitFrame: from.frame,
+    endOffset: logFrameOffset(from.frame + 1, header.frameBytes),
+    checksum: from.checksum
+  };
+  const file = await openForReading(logPath);
+  if (!file) return stopped;
+  try {
+    const { size } = await file.stat();
+    const framesInFile = Math.max(0, Math.floor((size - LOG_HEADER_BYTES) / header.frameBytes));
+    const framesPerChunk = Math.max(1, Math.floor(READ_CHUNK_BYTES / header.frameBytes));
+    const buffer = Buffer.allocUnsafe(framesPerChunk * header.frameBytes);
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+    let running = from.checksum;
+    let found = stopped;
+    let frame = from.frame + 1;
+    while (frame <= framesInFile) {
+      const frames = Math.min(framesPerChunk, framesInFile - frame + 1);
+      const wanted = frames * header.frameBytes;
+      let filled = 0;
+      while (filled < wanted) {
+        const { bytesRead } = await file.read(
+          buffer,
+          filled,
+          wanted - filled,
+          logFrameOffset(frame, header.frameBytes) + filled
+        );
+        if (bytesRead === 0) break;
+        filled += bytesRead;
+      }
+      const readable = Math.floor(filled / header.frameBytes);
+      if (readable === 0) break;
+      for (let inChunk = 0; inChunk < readable; inChunk++) {
+        const read = readValidLogFrame(view, inChunk * header.frameBytes, header, running);
+        if (!read) return found;
+        running = read.checksum;
+        if (read.frame.databasePages !== 0) {
+          found = {
+            lastCommitFrame: frame + inChunk,
+            endOffset: logFrameOffset(frame + inChunk + 1, header.frameBytes),
+            checksum: running
+          };
+        }
+      }
+      frame += readable;
+    }
+    return found;
+  } finally {
+    await file.close();
+  }
+}
+async function copyLogRange(logPath, startOffset, endOffset, destPath) {
+  const source = await open(logPath, "r");
+  try {
+    const dest = await open(destPath, "w");
+    try {
+      const buffer = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, Math.max(endOffset - startOffset, 1)));
+      let at = startOffset;
+      let written = 0;
+      while (at < endOffset) {
+        const wanted = Math.min(buffer.byteLength, endOffset - at);
+        const { bytesRead } = await source.read(buffer, 0, wanted, at);
+        if (bytesRead === 0) {
+          throw new SirannonError(
+            `The write-ahead log '${logPath}' ends at byte ${at} while the frames captured from it run to byte ${endOffset}, so it lost frames while Sirannon was reading it. Another connection checkpointed the log. Route every write and every checkpoint through Sirannon, and take a fresh full copy so a new chain starts from a known state.`,
+            "BACKUP_LOG_REWOUND"
+          );
+        }
+        let put = 0;
+        while (put < bytesRead) {
+          const { bytesWritten } = await dest.write(buffer, put, bytesRead - put);
+          if (bytesWritten === 0) {
+            throw new SirannonError(
+              `The capture of '${logPath}' could write no more than ${written + put} of its ${endOffset - startOffset} bytes into '${destPath}'. Check the free space and the permissions on that directory.`,
+              "BACKUP_ERROR"
+            );
+          }
+          put += bytesWritten;
+        }
+        at += bytesRead;
+        written += bytesRead;
+      }
+      return written;
+    } finally {
+      await dest.close();
+    }
+  } finally {
+    await source.close();
+  }
+}
+function logPathFor(sourcePath) {
+  return `${sourcePath}-wal`;
+}
+async function readLogPosition(sourcePath) {
+  const logPath = logPathFor(sourcePath);
+  try {
+    const header = await readLogFileHeader(logPath);
+    if (!header) return void 0;
+    const scan = await scanLogFrames(logPath, header, { frame: 0, checksum: header.checksum });
+    return {
+      logSequence: header.logSequence,
+      salt1: header.salt1,
+      salt2: header.salt2,
+      lastFrame: scan.lastCommitFrame
+    };
+  } catch {
+    return void 0;
+  }
+}
+function cursorChecksum(cursor) {
+  return { first: cursor.checksum1, second: cursor.checksum2 };
+}
+function sameLog(header, cursor) {
+  return header.salt1 === cursor.salt1 && header.salt2 === cursor.salt2;
+}
+
+export { copyLogRange, cursorChecksum, logPathFor, readLogFileHeader, readLogPosition, sameLog, scanLogFrames };

package/dist/{chunk-P53I2G6J.mjs → chunk-63VBFSCS.mjs}

@@ -1,6 +1,6 @@
 import { planBackupRestore } from './chunk-YNTDOFTN.mjs';
-import { listStoredFilePieces, fetchStoredFile, checkpointLog, LOG_HEADER_BYTES, LOG_FRAME_HEADER_BYTES, readLogHeader, foldLogChecksum, MAGIC_WITH_LITTLE_ENDIAN_CONTENT, LOG_FORMAT_VERSION } from './chunk-KSYFFYAP.mjs';
-import { destinationWithDeadline, DEFAULT_DESTINATION_TIMEOUT_MS } from './chunk-GBM4AU24.mjs';
+import { listStoredFilePieces, fetchStoredFile, checkpointLog } from './chunk-U445A3C4.mjs';
+import { destinationWithDeadline, DEFAULT_DESTINATION_TIMEOUT_MS, LOG_HEADER_BYTES, LOG_FRAME_HEADER_BYTES, readLogHeader, foldLogChecksum, MAGIC_WITH_LITTLE_ENDIAN_CONTENT, LOG_FORMAT_VERSION } from './chunk-DVS4QSIZ.mjs';
 import { readBackupChains, DEFAULT_CHAIN_NAME } from './chunk-YVF2ZJTE.mjs';
 import { randomHex } from './chunk-V5FUJEYY.mjs';
 import { SirannonError } from './chunk-PBRXXISQ.mjs';

package/dist/{chunk-CR6HTXSC.mjs → chunk-7SC2TZAS.mjs}

@@ -1,4 +1,4 @@
-import { listStoredFilePieces, fetchStoredFile } from './chunk-KSYFFYAP.mjs';
+import { listStoredFilePieces, fetchStoredFile } from './chunk-U445A3C4.mjs';
 import { SirannonError } from './chunk-PBRXXISQ.mjs';
 
 // src/core/backup/verify.ts

package/dist/chunk-DVS4QSIZ.mjs

@@ -0,0 +1,124 @@
+import { SirannonError } from './chunk-PBRXXISQ.mjs';
+
+// src/core/backup/destination-deadline.ts
+var DEFAULT_DESTINATION_TIMEOUT_MS = 6e5;
+var LONGEST_DEADLINE_MS = 2147483647;
+function assertDeadline(timeoutMs, subject) {
+  if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
+    throw new SirannonError(
+      `${subject} must be a number of milliseconds that is zero or above, and it was ${timeoutMs}`,
+      "BACKUP_ERROR"
+    );
+  }
+  if (timeoutMs > LONGEST_DEADLINE_MS) {
+    throw new SirannonError(
+      `${subject} must be no longer than ${LONGEST_DEADLINE_MS}ms, which is the longest a timer holds, and it was ${timeoutMs}`,
+      "BACKUP_ERROR"
+    );
+  }
+}
+async function withinDeadline(operation, timeoutMs, timedOut) {
+  let timer = null;
+  const deadline = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(timedOut()), timeoutMs);
+  });
+  try {
+    return await Promise.race([operation, deadline]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function destinationWithin(operation, action, timeoutMs) {
+  return withinDeadline(
+    operation,
+    timeoutMs,
+    () => new SirannonError(
+      `The destination did not ${action} within ${timeoutMs}ms, so the run stopped`,
+      "BACKUP_DESTINATION_ERROR"
+    )
+  );
+}
+function destinationWithDeadline(destination, timeoutMs) {
+  assertDeadline(timeoutMs, "The destination deadline");
+  if (timeoutMs === 0) return destination;
+  const claim = destination.writePieceIfAbsent?.bind(destination);
+  return {
+    writePiece: (name, index, bytes) => destinationWithin(destination.writePiece(name, index, bytes), `store piece ${index} of '${name}'`, timeoutMs),
+    ...claim ? {
+      writePieceIfAbsent: (name, index, bytes) => destinationWithin(claim(name, index, bytes), `claim piece ${index} of '${name}'`, timeoutMs)
+    } : {},
+    readPiece: (name, index) => destinationWithin(destination.readPiece(name, index), `return piece ${index} of '${name}'`, timeoutMs),
+    listPieces: (name) => destinationWithin(destination.listPieces(name), `list the pieces of '${name}'`, timeoutMs)
+  };
+}
+
+// src/core/backup/wal-format.ts
+var MAGIC_WITH_LITTLE_ENDIAN_CONTENT = 931071618;
+var MAGIC_WITH_BIG_ENDIAN_CONTENT = 931071619;
+var LOG_FORMAT_VERSION = 3007e3;
+var SMALLEST_PAGE_BYTES = 512;
+var LARGEST_PAGE_BYTES = 65536;
+var LOG_HEADER_BYTES = 32;
+var LOG_FRAME_HEADER_BYTES = 24;
+function foldLogChecksum(view, offset, byteLength, bigEndianContent, seed) {
+  const littleEndian = !bigEndianContent;
+  const end = offset + byteLength;
+  let first = seed.first;
+  let second = seed.second;
+  for (let at = offset; at < end; at += 8) {
+    first = first + view.getUint32(at, littleEndian) + second >>> 0;
+    second = second + view.getUint32(at + 4, littleEndian) + first >>> 0;
+  }
+  return { first, second };
+}
+function readLogHeader(bytes) {
+  if (bytes.byteLength < LOG_HEADER_BYTES) return void 0;
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const magic = view.getUint32(0, false);
+  if (magic !== MAGIC_WITH_LITTLE_ENDIAN_CONTENT && magic !== MAGIC_WITH_BIG_ENDIAN_CONTENT) return void 0;
+  const bigEndianContent = magic === MAGIC_WITH_BIG_ENDIAN_CONTENT;
+  const pageSize = view.getUint32(8, false);
+  if (pageSize < SMALLEST_PAGE_BYTES || pageSize > LARGEST_PAGE_BYTES || (pageSize & pageSize - 1) !== 0) {
+    return void 0;
+  }
+  const stored = { first: view.getUint32(24, false), second: view.getUint32(28, false) };
+  const computed = foldLogChecksum(view, 0, 24, bigEndianContent, { first: 0, second: 0 });
+  if (computed.first !== stored.first || computed.second !== stored.second) return void 0;
+  return {
+    pageSize,
+    frameBytes: LOG_FRAME_HEADER_BYTES + pageSize,
+    logSequence: view.getUint32(12, false),
+    salt1: view.getUint32(16, false),
+    salt2: view.getUint32(20, false),
+    checksum: stored,
+    bigEndianContent
+  };
+}
+function readLogFrameHeader(view, offset) {
+  return {
+    pageNumber: view.getUint32(offset, false),
+    databasePages: view.getUint32(offset + 4, false),
+    salt1: view.getUint32(offset + 8, false),
+    salt2: view.getUint32(offset + 12, false),
+    checksum: { first: view.getUint32(offset + 16, false), second: view.getUint32(offset + 20, false) }
+  };
+}
+function readValidLogFrame(view, offset, header, seed) {
+  const frame = readLogFrameHeader(view, offset);
+  if (frame.salt1 !== header.salt1 || frame.salt2 !== header.salt2) return void 0;
+  const overHeader = foldLogChecksum(view, offset, 8, header.bigEndianContent, seed);
+  const checksum = foldLogChecksum(
+    view,
+    offset + LOG_FRAME_HEADER_BYTES,
+    header.pageSize,
+    header.bigEndianContent,
+    overHeader
+  );
+  if (checksum.first !== frame.checksum.first || checksum.second !== frame.checksum.second) return void 0;
+  return { frame, checksum };
+}
+function logFrameOffset(frameNumber, frameBytes) {
+  return LOG_HEADER_BYTES + (frameNumber - 1) * frameBytes;
+}
+
+export { DEFAULT_DESTINATION_TIMEOUT_MS, LOG_FORMAT_VERSION, LOG_FRAME_HEADER_BYTES, LOG_HEADER_BYTES, MAGIC_WITH_LITTLE_ENDIAN_CONTENT, assertDeadline, destinationWithDeadline, foldLogChecksum, logFrameOffset, readLogHeader, readValidLogFrame, withinDeadline };