@noy-db/hub 0.3.0-pre.8 → 0.3.0

package/dist/with-commit/history/ledger/store.d.ts

@@ -0,0 +1,356 @@
+/**
+ * `LedgerStore` — read/write access to a compartment's hash-chained
+ * audit log.
+ *
+ * The store is a thin wrapper around the adapter's `_ledger/` internal
+ * collection. Every append:
+ *
+ *   1. Loads the current head (or treats an empty ledger as head = -1)
+ *   2. Computes `prevHash` = sha256(canonicalJson(head))
+ *   3. Builds the new entry with `index = head.index + 1`
+ *   4. Encrypts the entry with the compartment's ledger DEK
+ *   5. Writes the encrypted envelope to `_ledger/<paddedIndex>`
+ *
+ * `verify()` walks the chain from genesis forward and returns
+ * `{ ok: true, head }` on success or `{ ok: false, divergedAt }` on the
+ * first broken link.
+ *
+ * ## Thread / concurrency model
+ *
+ * For we assume a **single writer per vault**. Two
+ * concurrent `append()` calls would race on the "read head, write
+ * head+1" cycle and could produce a broken chain. The sync engine
+ * is the primary concurrent-writer scenario, and it uses
+ * optimistic-concurrency via `expectedVersion` on the adapter — but
+ * the ledger path has no such guard today. Multi-writer hardening is a
+ * follow-up.
+ *
+ * Single-writer usage IS safe, including across process restarts:
+ * `head()` reads the adapter fresh each call, so a crash between the
+ * adapter.put of a data record and the ledger append just means the
+ * ledger is missing an entry for that record. `verify()` still
+ * succeeds; a future `verifyIntegrity()` helper can cross-check the
+ * ledger against the data collections to catch the gap.
+ *
+ * ## Why hide the ledger from `vault.collection()`?
+ *
+ * The `_ledger` name starts with `_`, matching the existing prefix
+ * convention for internal collections (`_keyring`, `_sync`,
+ * `_history`). The Vault's public `collection()` method already
+ * returns entries for any name, but `loadAll()` filters out
+ * underscore-prefixed collections so backups and exports don't leak
+ * ledger metadata. We keep the ledger accessible ONLY via
+ * `vault.ledger()` to enforce the hash-chain invariants — direct
+ * puts via `collection('_ledger')` would bypass the `append()` logic.
+ */
+import type { NoydbStore } from '../../../kernel/types.js';
+import { type EnclaveKey } from '../../../kernel/enclave/index.js';
+import { type LedgerEntry } from './entry.js';
+import type { JsonPatch } from './patch.js';
+import { LEDGER_COLLECTION, LEDGER_DELTAS_COLLECTION } from './constants.js';
+import { envelopePayloadHash } from './hash.js';
+export { LEDGER_COLLECTION, LEDGER_DELTAS_COLLECTION, envelopePayloadHash };
+/**
+ * Input shape for `LedgerStore.append()`. The caller supplies the
+ * operation metadata; the store fills in `index` and `prevHash`.
+ */
+export interface AppendInput {
+    op: LedgerEntry['op'];
+    collection: string;
+    id: string;
+    version: number;
+    actor: string;
+    payloadHash: string;
+    /**
+     * Optional JSON Patch representing the delta from the previous
+     * version to the new version. Present only for `put` operations
+     * that had a previous version; omitted for genesis puts and for
+     * deletes. When present, `LedgerStore.append` persists the patch
+     * in `_ledger_deltas/<paddedIndex>` and records its sha256 hash
+     * as the entry's `deltaHash` field.
+     */
+    delta?: JsonPatch;
+    /**
+     * Present only for `op === 'amendment'` — structured audit
+     * payload for multi-record repair operations performed via
+     * `withTransactions(...)`. Carried through verbatim to the
+     * resulting ledger entry.
+     */
+    amendment?: LedgerEntry['amendment'];
+    /**
+     * Optional human-readable tag describing why this mutation happened.
+     * Threaded from `collection.put(_, _, { reason })`.
+     * Carried verbatim onto the resulting ledger entry's `reason` field;
+     * omitted from canonical JSON when undefined.
+     */
+    reason?: string;
+}
+/**
+ * Result of `LedgerStore.verify()`. On success, `head` is the hash of
+ * the last entry — the same value that should be published to any
+ * external anchoring service (blockchain, OpenTimestamps, etc.). On
+ * failure, `divergedAt` is the 0-based index of the first entry whose
+ * recorded `prevHash` does not match the recomputed hash of its
+ * predecessor. Entries at `divergedAt` and later are untrustworthy;
+ * entries before that index are still valid.
+ */
+export type VerifyResult = {
+    readonly ok: true;
+    readonly head: string;
+    readonly length: number;
+} | {
+    readonly ok: false;
+    readonly divergedAt: number;
+    readonly expected: string;
+    readonly actual: string;
+};
+/**
+ * A LedgerStore is bound to a single vault. Callers obtain one
+ * via `vault.ledger()` — there is no public constructor to keep
+ * the hash-chain invariants in one place.
+ *
+ * The class holds no mutable state beyond its dependencies (adapter,
+ * vault name, DEK resolver, actor id). Every method reads the
+ * adapter fresh so multiple instances against the same vault
+ * see each other's writes immediately (at the cost of re-parsing the
+ * ledger on every head() / verify() call; acceptable at scale).
+ */
+export declare class LedgerStore {
+    private readonly adapter;
+    private readonly vault;
+    private readonly encrypted;
+    private readonly getDEK;
+    private readonly actor;
+    /**
+     * In-memory cache of the chain head — the most recently appended
+     * entry along with its precomputed hash. Without this, every
+     * `append()` would re-load every prior entry to recompute the
+     * prevHash, making N puts O(N²) — a 1K-record stress test goes from
+     * < 100ms to a multi-second timeout.
+     *
+     * The cache is populated on first read (`append`, `head`, `verify`)
+     * and updated in-place on every successful `append`. Single-writer
+     * usage (the assumption) keeps it consistent. A second
+     * LedgerStore instance writing to the same vault would not
+     * see the first instance's appends in its cached state — that's the
+     * concurrency caveat documented at the class level.
+     *
+     * Sentinel `undefined` means "not yet loaded"; an explicit `null`
+     * value means "loaded and confirmed empty" — distinguishing these
+     * matters because an empty ledger is a valid state (genesis prevHash
+     * is the empty string), and we don't want to re-scan the adapter
+     * just because the chain is freshly initialized.
+     */
+    private headCache;
+    constructor(opts: {
+        adapter: NoydbStore;
+        vault: string;
+        encrypted: boolean;
+        getDEK: (collectionName: string) => Promise<EnclaveKey>;
+        actor: string;
+    });
+    /**
+     * Lazily load (or return cached) the current chain head. The cache
+     * sentinel is `undefined` until first access; after the first call,
+     * the cache holds either a `{ entry, hash }` for non-empty ledgers
+     * or `null` for empty ones.
+     */
+    private getCachedHead;
+    /**
+     * Append a new entry to the ledger. Returns the full entry that was
+     * written (with its assigned index and computed prevHash) so the
+     * caller can use the hash for downstream purposes (e.g., embedding
+     * in a verifiable backup).
+     *
+     * This is the **only** way to add entries. Direct adapter writes to
+     * `_ledger/` would bypass the chain math and would be caught by the
+     * next `verify()` call as a divergence.
+     *
+     * ## Multi-writer correctness
+     *
+     * Append is implemented as an optimistic-CAS retry loop. On every
+     * attempt:
+     *
+     *   1. Read fresh head (cache invalidated on retry).
+     *   2. Compute `nextIndex = head.index + 1`, `prevHash = hash(head)`.
+     *   3. Encrypt delta payload IN MEMORY (no adapter write yet) so we
+     *      can compute `deltaHash` before claiming the chain slot.
+     *   4. Build + encrypt the entry envelope.
+     *   5. `adapter.put(_ledger, paddedIndex, envelope, expectedVersion: 0)`
+     *      — the `expectedVersion: 0` asserts "this slot must not exist."
+     *      Stores with `casAtomic: true` honor the CAS check; under
+     *      contention the second writer's put throws `ConflictError`.
+     *   6. On `ConflictError`: invalidate the head cache, sleep with
+     *      bounded backoff + jitter, retry. After `MAX_APPEND_ATTEMPTS`
+     *      retries throw {@link LedgerContentionError}.
+     *   7. On success: write the delta envelope (if any) at the same
+     *      index. Update the head cache.
+     *
+     * Entry-first ordering matters: writing the delta first under
+     * contention would orphan delta records at indices the writer never
+     * actually claimed. The deltaHash is computed off the encrypted
+     * envelope's `_data` field, which doesn't require the envelope to
+     * be persisted.
+     *
+     * Stores with `casAtomic: false` (file, s3, r2 by default) silently
+     * accept the `expectedVersion: 0` argument and proceed without a
+     * CAS check. Concurrent appends against those stores remain
+     * best-effort — pair them with an advisory lock or with sync
+     * single-writer discipline.
+     */
+    append(input: AppendInput): Promise<LedgerEntry>;
+    /**
+     * One attempt at the append cycle. Throws `ConflictError` when the
+     * CAS check on the entry put fails — `append()` catches that and
+     * retries. Any other error propagates to the caller.
+     */
+    private appendOnce;
+    /**
+     * Load a delta payload by its entry index. Returns `null` if the
+     * entry at that index doesn't reference a delta (genesis puts and
+     * deletes leave the slot empty) or if the delta row is missing
+     * (possible after a `pruneHistory` fold).
+     *
+     * The caller is responsible for deciding what to do with a missing
+     * delta — `ledger.reconstruct()` uses it as a "stop walking
+     * backward" signal and falls back to the on-disk current value.
+     */
+    loadDelta(index: number): Promise<JsonPatch | null>;
+    /**
+     * Purge a record's tier-0-era plaintext deltas (#729). Deletes each
+     * `_ledger_deltas/<paddedIndex>` row whose entry matches
+     * `(collection, id)` and carries a delta, then returns the count
+     * deleted.
+     *
+     * Chain-safe by construction: `verify()` recomputes the tamper-chain
+     * from each `_ledger` entry's own canonical fields — including the
+     * stored `deltaHash` field, which lives on the entry, not on the
+     * `_ledger_deltas` row — and never re-reads `_ledger_deltas`.
+     * Deleting a delta row therefore cannot break `verify()`.
+     * `reconstruct()` already treats a missing delta (`loadDelta` →
+     * `null`) as a pruned stop, so a purge just makes the pre-purge
+     * plaintext unreachable rather than corrupting anything.
+     *
+     * No `_ledger` entry is touched and nothing is re-encrypted — this
+     * is irreversible (the deleted plaintext cannot be recovered) but
+     * leaves the entry metadata (that the record was mutated, at which
+     * version/timestamp/actor) fully intact for audit purposes.
+     */
+    purgeRecordDeltas(collection: string, id: string): Promise<number>;
+    /** Encrypt a JSON Patch into an envelope for storage. Mirrors encryptEntry. */
+    private encryptDelta;
+    /**
+     * Read all entries in ascending-index order. Used internally by
+     * `append()`, `head()`, `verify()`, and `entries()`. Decryption is
+     * serial because the entries are tiny and the overhead of a Promise
+     * pool would dominate at realistic chain lengths (< 100K entries).
+     */
+    loadAllEntries(): Promise<LedgerEntry[]>;
+    /**
+     * Return the current head of the ledger: the last entry, its hash,
+     * and the total chain length. `null` on an empty ledger so callers
+     * can distinguish "no history yet" from "empty history".
+     */
+    head(): Promise<{
+        readonly entry: LedgerEntry;
+        readonly hash: string;
+        readonly length: number;
+    } | null>;
+    /**
+     * Return entries in the requested half-open range `[from, to)`.
+     * Defaults: `from = 0`, `to = length`. The indices are clipped to
+     * the valid range; no error is thrown for out-of-range queries.
+     */
+    entries(opts?: {
+        from?: number;
+        to?: number;
+    }): Promise<LedgerEntry[]>;
+    /**
+     * Reconstruct a record's state at a given historical version by
+     * walking the ledger's delta chain backward from the current state.
+     *
+     * ## Algorithm
+     *
+     * Ledger deltas are stored in **reverse** form — each entry's
+     * patch describes how to undo that put, transforming the new
+     * record back into the previous one. `reconstruct` exploits this
+     * by:
+     *
+     *   1. Finding every ledger entry for `(collection, id)` in the
+     *      chain, sorted by index ascending.
+     *   2. Starting from `current` (the present value of the record,
+     *      as held by the caller — typically fetched via
+     *      `Collection.get()`).
+     *   3. Walking entries in **descending** index order and applying
+     *      each entry's reverse patch, stopping when we reach the
+     *      entry whose version equals `atVersion`.
+     *
+     * The result is the record as it existed immediately AFTER the
+     * put at `atVersion`. To get the state at the genesis put
+     * (version 1), the walk runs all the way back through every put
+     * after the first.
+     *
+     * ## Caveats
+     *
+     * - **Delete entries** break the walk: once we see a delete, the
+     *   record didn't exist before that point, so there's nothing to
+     *   reconstruct. We return `null` in that case.
+     * - **Missing deltas** (e.g., after `pruneHistory` folds old
+     *   entries into a base snapshot) also stop the walk. does
+     *   not ship pruneHistory, so today this only happens if an entry
+     *   was deleted out-of-band.
+     * - The caller MUST pass the correct current value. Passing a
+     *   mutated object would corrupt the reconstruction — the patch
+     *   chain is only valid against the exact state that was in
+     *   effect when the most recent put happened.
+     *
+     * For, `reconstruct` is the only way to read a historical
+     * version via deltas. The legacy `_history` collection still
+     * holds full snapshots and `Collection.getVersion()` still reads
+     * from there — the two paths coexist until pruneHistory lands in
+     * a follow-up and delta becomes the default.
+     */
+    reconstruct<T>(collection: string, id: string, current: T, atVersion: number): Promise<T | null>;
+    /**
+     * Walk the chain from genesis forward and verify every link.
+     *
+     * Returns `{ ok: true, head, length }` if every entry's `prevHash`
+     * matches the recomputed hash of its predecessor (and the genesis
+     * entry's `prevHash` is the empty string).
+     *
+     * Returns `{ ok: false, divergedAt, expected, actual }` on the first
+     * mismatch. `divergedAt` is the 0-based index of the BROKEN entry
+     * — entries before that index still verify cleanly; entries at and
+     * after `divergedAt` are untrustworthy.
+     *
+     * This method detects:
+     *   - Mutated entry content (fields changed)
+     *   - Reordered entries (if any adjacent pair swaps, the prevHash
+     *     of the second no longer matches)
+     *   - Inserted entries (the inserted entry's prevHash likely fails,
+     *     and the following entry's prevHash definitely fails)
+     *   - Deleted entries (the entry after the deletion sees a wrong
+     *     prevHash)
+     *
+     * It does NOT detect:
+     *   - Tampering with the DATA collections that bypassed the ledger
+     *     entirely (e.g., an attacker who modifies records without
+     *     appending matching ledger entries — this is why we also
+     *     plan a `verifyIntegrity()` helper in a follow-up)
+     *   - Truncation of the chain at the tail (dropping the last N
+     *     entries leaves a shorter but still consistent chain). External
+     *     anchoring of `head.hash` to a trusted service is the defense
+     *     against this.
+     */
+    verify(): Promise<VerifyResult>;
+    /**
+     * Serialize + encrypt a ledger entry into an EncryptedEnvelope. The
+     * envelope's `_v` field is set to `entry.index + 1` so the usual
+     * optimistic-concurrency machinery has a reasonable version number
+     * to compare against (the ledger is append-only, so concurrent
+     * writes should always bump the index).
+     */
+    private encryptEntry;
+    /** Decrypt an envelope into a LedgerEntry. Throws on bad key / tamper. */
+    private decryptEntry;
+}

package/dist/with-commit/history/strategy.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Strategy seam for the optional history + ledger + time-machine
+ * service. Core imports `HistoryStrategy` type-only + `NO_HISTORY`
+ * stub; real implementations of `saveHistory`, `LedgerStore`,
+ * `VaultInstant`, `computePatch`, `diff` etc. are only reachable via
+ * `withHistory()` in `./active.ts`.
+ *
+ * Applications that don't track per-record versioning, don't need the
+ * hash-chained audit ledger, and don't restore to past instants ship
+ * none of the ~1,880 LOC behind this seam.
+ *
+ * Strategy contract:
+ *
+ * - **saveHistory / pruneHistory / clearHistory** — no-ops under
+ *   NO_HISTORY. Writes still succeed; no snapshot is captured.
+ * - **getHistoryEntries / getVersionEnvelope / diff** — throw under
+ *   NO_HISTORY. These are read APIs the consumer would only call
+ *   after explicitly asking for history; the throw guides them to
+ *   `@noy-db/hub/history`.
+ * - **envelopePayloadHash / computePatch** — return empty / `[]`
+ *   under NO_HISTORY. These are only used inside the
+ *   `if (this.ledger)` branch, which is itself gated by
+ *   `buildLedger()` returning null.
+ * - **buildLedger** — returns `null` under NO_HISTORY. The Vault's
+ *   public `vault.ledger()` accessor throws when null.
+ * - **buildVaultInstant** — throws under NO_HISTORY. `vault.at()`
+ *   propagates the throw.
+ *
+ * @internal
+ */
+import type { EncryptedEnvelope, NoydbStore, HistoryOptions, PruneOptions } from '../../kernel/types.js';
+import type { LedgerStore } from './ledger/store.js';
+import type { JsonPatch } from './ledger/patch.js';
+import type { DiffEntry } from './diff.js';
+import type { VaultInstant, VaultEngine } from './time-machine.js';
+import type { EnclaveKey } from '../../kernel/enclave/index.js';
+/**
+ * Options accepted by `HistoryStrategy.buildLedger`. Mirrors the
+ * `LedgerStore` constructor verbatim — kept in this file so `core`
+ * code never imports the LedgerStore module at runtime.
+ *
+ * @internal
+ */
+export interface BuildLedgerOptions {
+    adapter: NoydbStore;
+    vault: string;
+    encrypted: boolean;
+    getDEK: (collectionName: string) => Promise<EnclaveKey>;
+    actor: string;
+}
+/**
+ * @internal
+ */
+export interface HistoryStrategy {
+    /**
+     * Persist a full encrypted envelope snapshot of the prior version
+     * under `_history/{collection}:{id}:{paddedVersion}`. No-op under
+     * `NO_HISTORY`.
+     */
+    saveHistory(adapter: NoydbStore, vault: string, collection: string, recordId: string, envelope: EncryptedEnvelope): Promise<void>;
+    /**
+     * List history envelopes for a record, newest first. Throws under
+     * `NO_HISTORY` — callers reach this via `collection.history()` /
+     * `collection.getVersion()` / `collection.diff()`, which only work
+     * with the strategy enabled.
+     */
+    getHistoryEntries(adapter: NoydbStore, vault: string, collection: string, recordId: string, options?: HistoryOptions): Promise<EncryptedEnvelope[]>;
+    /**
+     * Fetch a specific version's envelope. Throws under `NO_HISTORY`.
+     */
+    getVersionEnvelope(adapter: NoydbStore, vault: string, collection: string, recordId: string, version: number): Promise<EncryptedEnvelope | null>;
+    /**
+     * Prune history entries by retention rule. Returns `0` under
+     * `NO_HISTORY`.
+     */
+    pruneHistory(adapter: NoydbStore, vault: string, collection: string, recordId: string | undefined, options: PruneOptions): Promise<number>;
+    /**
+     * Clear all history for vault/collection/record. Returns `0` under
+     * `NO_HISTORY`.
+     */
+    clearHistory(adapter: NoydbStore, vault: string, collection?: string, recordId?: string): Promise<number>;
+    /**
+     * Crypto-shred (overwrite-to-tombstone) every `_history` version of a
+     * record for GDPR erasure. Returns the number of versions newly
+     * tombstoned, or `0` under `NO_HISTORY` (no history = nothing to shred).
+     */
+    tombstoneHistory(adapter: NoydbStore, vault: string, collection: string, recordId: string, actor: string, encrypted: boolean): Promise<number>;
+    /**
+     * Re-key every `_history` snapshot of a record from `fromDek` to `toDek`
+     * on a tier move (elevate/demote/putAtTier), mirroring the live-body
+     * rewrap so prior versions are not left decryptable at rest under a tier
+     * the record has moved away from. Optional `tier0Dek` enables the legacy
+     * fallback for snapshots written before this rewrap existed (see
+     * `history.ts` for the full contract). No-op under `NO_HISTORY` (no
+     * history = nothing to rewrap).
+     */
+    rewrapHistory(adapter: NoydbStore, vault: string, collection: string, recordId: string, fromDek: EnclaveKey, toDek: EnclaveKey, tier0Dek?: EnclaveKey): Promise<void>;
+    /**
+     * Compute the SHA-256 hash of an envelope's encrypted payload, used
+     * by `LedgerStore.append` to track tamper-evidence. Returns the
+     * empty string under `NO_HISTORY` (the call site is gated by
+     * `if (this.ledger)`, so the value is never observed).
+     */
+    envelopePayloadHash(envelope: EncryptedEnvelope | null): Promise<string>;
+    /**
+     * Compute the JSON patch from `from` → `to`. Returns `[]` under
+     * `NO_HISTORY`.
+     */
+    computePatch(from: unknown, to: unknown): JsonPatch;
+    /**
+     * Compute the typed diff between two records. Throws under
+     * `NO_HISTORY` — `collection.diff()` is a history-read API.
+     */
+    diff(recordA: unknown, recordB: unknown): DiffEntry[];
+    /**
+     * Construct (or return null) a `LedgerStore` for the vault. Returns
+     * `null` under `NO_HISTORY`; the Vault treats null as "no ledger
+     * attached" — collection write paths skip the append branch and the
+     * public `vault.ledger()` accessor throws.
+     */
+    buildLedger(opts: BuildLedgerOptions): LedgerStore | null;
+    /**
+     * Construct a `VaultInstant` for time-machine reads. Throws under
+     * `NO_HISTORY`.
+     */
+    buildVaultInstant(engine: VaultEngine, timestamp: string): VaultInstant;
+}
+/**
+ * No-history stub. Snapshots and prune/clear are no-ops; reads and
+ * time-machine throw with an actionable message; ledger construction
+ * returns null so the write-path's `if (this.ledger)` branch is dead
+ * code in the bundle.
+ *
+ * @internal
+ */
+export declare const NO_HISTORY: HistoryStrategy;