@noy-db/hub 0.3.0-pre.8 → 0.3.0

package/dist/with-pod/pod-store.d.ts

@@ -0,0 +1,54 @@
+import type { NoydbStore, NoydbPodStore } from '../kernel/types.js';
+export interface WrapPodStoreOptions {
+    /**
+     * When `true` (default), every `put()` and `delete()` flushes the full
+     * vault snapshot to the bundle backend. Set to `false` for bulk operations
+     * and call `store.flush(vaultId)` manually.
+     */
+    autoFlush?: boolean;
+}
+export interface WrappedPodNoydbStore extends NoydbStore {
+    /** Manually flush the in-memory snapshot to the bundle backend. */
+    flush(vaultId: string): Promise<void>;
+    /**
+     * Run a batch of mutations without flushing until the callback completes.
+     * A single flush is performed at the end.
+     */
+    batch(vaultId: string, fn: () => Promise<void>): Promise<void>;
+}
+/**
+ * Convert a `NoydbPodStore` (blob-oriented read/write with OCC) into the
+ * standard six-method `NoydbStore` interface expected by `createNoydb({ store })`.
+ *
+ * Bundle stores operate on the entire vault as a single serialised unit —
+ * ideal for backends like Google Drive, WebDAV, or iCloud Drive that work
+ * best with whole-file I/O rather than per-record KV operations.
+ *
+ * ## Optimistic concurrency
+ *
+ * The wrapper tracks the `version` token from the last `readBundle` and
+ * passes it as `expectedVersion` on every flush. On
+ * `PodVersionConflictError`, it re-reads, merges the remote snapshot
+ * (last-write-wins per record key), and retries (max 3 attempts).
+ *
+ * ## Flush modes
+ *
+ * By default, flushes on every mutation (O(vault size) per write). Options:
+ * - `autoFlush: false` + explicit `store.flush(vaultId)` calls
+ * - `store.batch(vaultId, async () => { ... })` — defers flush until end
+ * - Pair with `syncPolicy: { push: { mode: 'debounce' } }` from
+ */
+export declare function wrapPodStore(bundle: NoydbPodStore, options?: WrapPodStoreOptions): WrappedPodNoydbStore;
+/**
+ * Type-safe factory helper for `NoydbPodStore` implementations,
+ * analogous to `createStore` for KV stores.
+ */
+export declare function createPodStore<TOptions>(factory: (options: TOptions) => NoydbPodStore): (options: TOptions) => NoydbPodStore;
+/** @deprecated Use wrapPodStore. */
+export declare const wrapBundleStore: typeof wrapPodStore;
+/** @deprecated Use createPodStore. */
+export declare const createBundleStore: typeof createPodStore;
+/** @deprecated Use WrappedPodNoydbStore. */
+export type WrappedBundleNoydbStore = WrappedPodNoydbStore;
+/** @deprecated Use WrapPodStoreOptions. */
+export type WrapBundleStoreOptions = WrapPodStoreOptions;

package/dist/with-pod/ulid.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Minimal ULID generator — zero dependencies, Web Crypto API only.
+ *
+ *. Used by the bundle writer to generate stable opaque
+ * handles for `.noydb` containers.
+ *
+ * **What's a ULID?** A 128-bit identifier encoded as 26 Crockford
+ * base32 characters. Layout:
+ *
+ * ```
+ *   01HYABCDEFGHJKMNPQRSTVWXYZ
+ *   |--------||---------------|
+ *    48-bit     80-bit
+ *    timestamp  randomness
+ * ```
+ *
+ * The first 10 chars encode a millisecond Unix timestamp (so ULIDs
+ * sort lexicographically by creation time), and the remaining 16
+ * chars are random. Crockford base32 omits I/L/O/U to avoid
+ * ambiguity in handwriting and URLs.
+ *
+ * **Why hand-roll instead of pulling in `ulid`?** The package adds
+ * a dep, the implementation is ~30 lines, and the bundle module
+ * is the only consumer. Adding `ulid` would also drag in its own
+ * crypto polyfill that we don't need on Node 18+ or modern
+ * browsers.
+ *
+ * **Privacy consideration:** the timestamp prefix is observable in
+ * the bundle header. This is a deliberate trade-off:
+ *   - Pro: lexicographic sortability lets bundle adapters list
+ *     newest-first without an extra index.
+ *   - Con: a casual observer can read the bundle's creation time
+ *     from the handle. They cannot read it from any OTHER field
+ *     (the header explicitly forbids `_exported_at`), and a
+ *     creation timestamp is the same kind of metadata that
+ *     filesystem mtime would already expose for a downloaded
+ *     bundle. The leak is therefore equivalent to what's already
+ *     visible from the file's mtime — not a new exposure.
+ *
+ * If a future use case needs timestamp-free handles, a v2 of the
+ * format could specify "use the random portion only" without a
+ * format break — `validateBundleHeader` only checks the regex
+ * shape, not the encoded timestamp.
+ */
+/**
+ * Generate a fresh ULID. Uses `crypto.getRandomValues` for the
+ * randomness portion — same Web Crypto API the rest of the
+ * codebase uses for IVs and salt.
+ *
+ * Returns a 26-character string. Calling twice in the same
+ * millisecond produces two distinct ULIDs (the random portion
+ * differs); ULIDs from the same millisecond are NOT guaranteed
+ * to be monotonically ordered relative to each other, only
+ * relative to ULIDs from a different millisecond. The bundle
+ * format never relies on intra-millisecond ordering.
+ */
+export declare function generateULID(): string;
+/**
+ * Validate that a string is a syntactically well-formed ULID. Used
+ * by the bundle header validator. Does NOT verify that the
+ * timestamp portion decodes to a sensible date — the format only
+ * cares about the encoding shape.
+ */
+export declare function isULID(value: string): boolean;

package/dist/with-shape/blobs/blob-compaction.d.ts

@@ -0,0 +1,170 @@
+/**
+ * Blob retention + compaction.
+ *
+ * Declarative per-collection / per-slot eviction policy. Two
+ * triggers:
+ *
+ *   - **`retainDays`** — age-based TTL. A slot uploaded more than N
+ *     days ago is evicted.
+ *   - **`evictWhen(record)`** — predicate over the **decrypted**
+ *     record. Lets consumers express "the image is safe to drop once
+ *     the structured invoice has been reviewed and confirmed."
+ *
+ * Either trigger (or both) causes the slot to evict. Eviction removes
+ * the slot entry from `_blob_slots_{collection}`, decrements the
+ * blob's refCount (so unreferenced chunks can be GC'd by the next
+ * sweep), and writes one entry to the `_blob_eviction_audit`
+ * collection for tamper-evident record-keeping.
+ *
+ * The audit entry carries the eTag of the evicted blob (opaque HMAC
+ * of plaintext under the vault's `_blob` DEK) — no plaintext leakage,
+ * per the SPEC non-correlation invariant. Consumers reconstructing
+ * "what used to be attached" can look up the audit entry by record
+ * id.
+ *
+ * Compaction is **consumer-scheduled** — noy-db never runs a
+ * background daemon. Call `vault.compact()` whenever your workflow
+ * allows (cron, manual "tidy" button, cold-storage export prep, …).
+ *
+ * @module
+ */
+import type { NoydbStore, SlotInfo } from '../../kernel/types.js';
+import { type EnclaveKey } from '../../kernel/enclave/index.js';
+export interface BlobFieldPolicy<T = unknown> {
+    /**
+     * Via port brand marker — lets a `BlobFieldPolicy` satisfy the kernel's
+     * opaque `ViaDescriptor` (#629 Task 7). Optional (the
+     * `ClassifiedFieldSpec._viaBrand` precedent): `blobFields` policies are
+     * plain object literals with no declaration factory to stamp it, so a
+     * mandatory field would break every existing declaration.
+     */
+    readonly _viaBrand?: 'blob';
+    /**
+     * Age-based TTL in days. A slot whose `uploadedAt` is older than
+     * `now - retainDays × 86400s` evicts on the next `vault.compact()`.
+     * Omit to disable age-based eviction.
+     */
+    readonly retainDays?: number;
+    /**
+     * Predicate evaluated against the decrypted record. When it returns
+     * `true`, every matching slot on that record evicts. Omit to
+     * disable predicate-based eviction.
+     */
+    readonly evictWhen?: (record: T) => boolean;
+    /**
+     * **Legal hold.** When this predicate returns `true`, the slot is
+     * never evicted — `retainDays`/`evictWhen` are overridden. Use for a
+     * litigation / audit hold on a fiscal document: the blob stays until
+     * the predicate returns `false` (the hold is released). Fail-closed:
+     * if the predicate throws, the slot is treated as held.
+     */
+    readonly legalHold?: (record: T) => boolean;
+    /**
+     * **Period-bound retention.** Returns the date (Date / ISO string /
+     * epoch ms) until which the slot must be retained — typically derived
+     * from the record's fiscal period (e.g. period end + 10 years). While
+     * `now < retainUntil`, the slot is never evicted, regardless of
+     * `retainDays`. Return `null`/`undefined` to impose no floor.
+     * Fail-closed: a throwing function holds the slot.
+     */
+    readonly retainUntil?: (record: T) => Date | string | number | null | undefined;
+    /**
+     * **External projection.** When `true`, this field's bytes are stored in the
+     * vault's `ObjectProjection` (`createNoydb({ objectStore })`) as a single raw,
+     * **unencrypted** object — servable directly from S3/CDN and processable by
+     * native tooling — instead of the encrypted-chunk path. The encrypted slot
+     * record remains the catalog (anchoring invariant). Requires an `objectStore`;
+     * **outside the zero-knowledge guarantee** — use only for assets meant to
+     * leave the vault. See the as-aws-s3 design spec.
+     */
+    readonly external?: boolean;
+    /**
+     * For an `external` field: make the object world-readable (CDN origin) rather
+     * than presigned-only. Default `false` (presigned). Ignored unless `external`.
+     */
+    readonly public?: boolean;
+    /**
+     * For an `external` field: how to stamp a **backlink** (this record's
+     * vault/collection/id/field) onto the object's metadata — the self-describing
+     * "secondary store" that powers reconcile / DR / import re-pairing.
+     * - `'opaque-token'` (default): a random id; preserves the opaque-bucket
+     *   property (no names leak); the token is also recorded on the slot.
+     * - `'encrypted'`: the reference encrypted under the blob DEK (ZK-preserving;
+     *   falls back to `'opaque-token'` on a plaintext vault).
+     * - `'plain'`: the reference in cleartext metadata — **leaks structure** to
+     *   bucket readers; only for non-sensitive deployments.
+     * - `'none'`: no backlink.
+     */
+    readonly backlink?: 'opaque-token' | 'encrypted' | 'plain' | 'none';
+}
+export type BlobFieldsConfig<T = unknown> = Record<string, BlobFieldPolicy<T>>;
+export declare const BLOB_EVICTION_AUDIT_COLLECTION = "_blob_eviction_audit";
+export interface BlobEvictionEntry {
+    readonly id: string;
+    readonly collection: string;
+    readonly recordId: string;
+    readonly slotName: string;
+    readonly blobHash: string;
+    readonly reason: 'ttl' | 'predicate' | 'both';
+    readonly evictedAt: string;
+    readonly actor: string;
+}
+export interface CompactionResult {
+    /** Number of blob slots evicted across all collections. */
+    readonly evicted: number;
+    /** Number of records touched (iterated + policy checked). */
+    readonly records: number;
+    /** Number of collections with `blobFields` configured. */
+    readonly collections: number;
+    /** Number of audit entries written. Equal to `evicted`. */
+    readonly auditEntries: number;
+    /**
+     * Number of slots that would have evicted (TTL/predicate triggered)
+     * but were retained by a `legalHold` or `retainUntil` floor.
+     */
+    readonly held: number;
+    /** Per-collection breakdown for diagnostics. */
+    readonly byCollection: Record<string, {
+        records: number;
+        evicted: number;
+    }>;
+}
+export interface CompactRunOptions {
+    /** Override "now" for deterministic testing. */
+    readonly now?: Date;
+    /**
+     * Stop after this many evictions. Useful for capped batches / cron
+     * jobs that need to fit in a time window. `undefined` = unbounded.
+     */
+    readonly maxEvictions?: number;
+    /**
+     * Dry-run — evaluate policies and return the counts, but do NOT
+     * delete slots or write audit entries. Lets a consumer preview
+     * what would happen.
+     */
+    readonly dryRun?: boolean;
+}
+export interface CompactionContext {
+    readonly adapter: NoydbStore;
+    readonly vault: string;
+    readonly actor: string;
+    readonly encrypted: boolean;
+    readonly getDEK: (collection: string) => Promise<EnclaveKey>;
+    /**
+     * Resolve a collection's declared `blobFields` config. Returns an
+     * empty map for collections without the config — the walk skips
+     * those.
+     */
+    readonly getBlobFields: <T>(collection: string) => BlobFieldsConfig<T> | null;
+    /** List collection names in the vault. */
+    readonly listCollections: () => Promise<string[]>;
+    /** List record ids in a collection. */
+    readonly listRecords: (collection: string) => Promise<string[]>;
+    /** Decrypt and return the record. Null when absent. */
+    readonly getRecord: <T>(collection: string, id: string) => Promise<T | null>;
+    /** Return the BlobSet-like handle for a record's slots. */
+    readonly listSlots: (collection: string, id: string) => Promise<SlotInfo[]>;
+    /** Delete a slot and decrement its blob's refCount. */
+    readonly deleteSlot: (collection: string, id: string, slotName: string) => Promise<void>;
+}
+export declare function runCompaction(ctx: CompactionContext, options?: CompactRunOptions): Promise<CompactionResult>;

package/dist/with-shape/blobs/blob-intent.d.ts

@@ -0,0 +1,153 @@
+/**
+ * `_blob_intent` — the blob durability journal marker (#753, spec §7).
+ *
+ * One row per in-flight multi-step blob operation (`shred` | `rehome`),
+ * written BEFORE the first destructive/movement step and deleted as the
+ * operation's LAST step. An absent marker means "no operation in flight"
+ * (the normal state, zero cost on every path that doesn't crash); a present
+ * marker means the operation MUST be resumed before any other blob work on
+ * that record proceeds.
+ *
+ * **Pattern honesty (C9):** this is the family's FIRST content-bearing
+ * ENCRYPTED reserved marker. The prior reserved-collection precedent,
+ * `_mv_stale` (`with-formula/materialized-views/stale.ts`), is
+ * content-free plaintext (`{ _iv: '', _data: '{}' }` — the row's only
+ * payload is its key). `_blob_intent` carries real operational metadata
+ * (op, tiers, eTags + counts) and is encrypted, so its codec/CAS/backup/
+ * sweep conventions below are new to the codebase, not a copy of an
+ * existing pattern.
+ *
+ * **Key management (spec §4):** encrypted under the record's OWNING
+ * collection's tier-0 DEK — `getDEK(collection)`, which resolves to
+ * `dekKey(collection, 0) === collection` (see `with-party/team/tiers.ts`).
+ * Every legitimate resumer (forget()-retry, blob write entry points,
+ * tier ops) already holds this DEK; minting an elevated DEK for the
+ * marker itself would make resume impossible for a tier-0-only resumer.
+ * Documented residue: a tier-0 DEK holder can learn that an elevated
+ * record's shred/rehome was in flight and which eTags it touched —
+ * existence-adjacent, consistent with the already-documented
+ * dedup-policy residue (#741).
+ *
+ * **Key grammar (C11):** `{collection}::{recordId}`. Blob-bearing record
+ * ids already refuse `::` at the write surface (`blob-set.ts`'s
+ * `assertKeyPartSafe`, #752) — re-validated here defensively so the
+ * sweep's prefix-free key list stays unambiguous regardless of caller.
+ *
+ * Wired by `shredAllForRecord`/`forget()` (PR-1 Task 3, shred) and by
+ * `BlobSet.syncTierMove`/`rehomeForTier`/`resolvePendingIntent` (PR-2 Task
+ * 2, rehome — mint/resume/consume for `op:'rehome'` markers).
+ */
+import type { NoydbStore } from '../../kernel/types.js';
+import { type EnclaveKey } from '../../kernel/enclave/index.js';
+/** Reserved collection holding `_blob_intent` marker rows. */
+export declare const BLOB_INTENT_COLLECTION = "_blob_intent";
+/**
+ * One captured refCount hold on an eTag, authoritative on resume (spec §2a,
+ * amended by C5). `chunkCount` lets a resumed release complete chunk
+ * cleanup even if the `_blob_index` row itself is already gone (mirrors
+ * `releaseRef`'s `chunkCountHint` parameter, Task 1).
+ */
+export interface BlobIntentHold {
+    readonly eTag: string;
+    readonly n: number;
+    readonly chunkCount: number;
+}
+/**
+ * The intent marker's decrypted payload (spec §2a, amended by C5: `holds`
+ * entries carry `{ eTag, n, chunkCount }` rather than the original bare
+ * `{ eTag, n }`).
+ */
+export interface BlobIntent {
+    readonly op: 'shred' | 'rehome';
+    /** Random nonce minted at marker creation (`crypto.getRandomValues`) — the op-stamp identity. */
+    readonly opId: string;
+    /** Captured ONCE at marker creation; authoritative on every resume. */
+    readonly holds?: readonly BlobIntentHold[];
+    readonly ownerTier?: number;
+    readonly fromTier?: number;
+    readonly toTier?: number;
+    readonly policy?: 'isolate' | 'dedup';
+    /**
+     * Rehome destination `+1` row-stamps (`${opId}:${slotName}` /
+     * `${opId}:${versionKey}`) CONFIRMED applied — #746 whole-branch review
+     * (K=8 stamp-ring blocker). `BlobObject.lastOps` is a BOUNDED ring (K=8,
+     * spec C2) — correct as the sole idempotency source for SHRED (one
+     * decrement per eTag, count captured in `holds`) but not for REHOME,
+     * whose row-scoped increments can exceed K on a single destination via
+     * either ≥9 rows of identical content on ONE record, or ≥8 CONCURRENT
+     * rehomes converging on one shared destination between a crashed op's
+     * stamp-write and its resume — either way, an evicted-but-unapplied-
+     * looking stamp gets re-applied, over-counting the destination's
+     * refCount and permanently stranding it above 0. This field is THIS
+     * op's own UNBOUNDED, per-record backstop: appended (via
+     * {@link recordAppliedStamp}) immediately after a stamped `+1` lands
+     * (whether freshly applied or found already in the ring), consulted
+     * BEFORE attempting the CAS so a resume that finds its own row-stamp
+     * here skips the increment entirely — independent of the shared
+     * destination object's bounded ring and of how many OTHER ops have
+     * touched it since. Absent/empty is the normal case (no rehome — or no
+     * increments yet — in flight).
+     */
+    readonly appliedStamps?: readonly string[];
+}
+/** Resume callback signature for {@link sweepBlobIntents} — supplied by the caller (Task 3 / PR-2). */
+export type BlobIntentResume = (collection: string, recordId: string, intent: BlobIntent) => Promise<void>;
+/**
+ * Read the `_blob_intent` marker for `{collection}::{recordId}`, or `null`
+ * if none is pending. The resume-gate check every future blob mutator
+ * (Task 3 / PR-2) runs before proceeding.
+ */
+export declare function getIntent(adapter: NoydbStore, vault: string, collection: string, recordId: string, getDEK: (collection: string) => Promise<EnclaveKey>): Promise<BlobIntent | null>;
+/**
+ * Create a `_blob_intent` marker for `{collection}::{recordId}` — CAS
+ * create-if-absent (C8). `store.put(..., expectedVersion: 0)` succeeds only
+ * when no row is present (every real adapter treats a missing record +
+ * `expectedVersion: 0` as a no-conflict write — the same convention
+ * `loadOrCreateSigner` uses in `with-audit/attestation/signer.ts`); a
+ * present row always carries `_v >= 1`, so the CAS throws `ConflictError`
+ * and is converted here into {@link BlobIntentPendingError} — the signal
+ * a caller catches to resume the PENDING operation before starting a new
+ * one, rather than silently clobbering it (which would orphan the prior
+ * op's op-stamps).
+ */
+export declare function createIntent(adapter: NoydbStore, vault: string, collection: string, recordId: string, getDEK: (collection: string) => Promise<EnclaveKey>, intent: BlobIntent): Promise<void>;
+/** Delete the `_blob_intent` marker for `{collection}::{recordId}` — the operation's LAST step. */
+export declare function deleteIntent(adapter: NoydbStore, vault: string, collection: string, recordId: string): Promise<void>;
+/**
+ * Durably record that a rehome destination row-stamp has been CONFIRMED
+ * applied — #746 whole-branch review (K=8 stamp-ring blocker). Appends
+ * `stamp` to the marker's `appliedStamps` (idempotent no-op if already
+ * present), CAS-retried against concurrent resumers touching the SAME
+ * marker (mirrors `BlobSet.casUpdateSlots`'s retry shape).
+ *
+ * A no-op — never throws — when the marker is already gone: that only
+ * happens if the whole op already completed and deleted it (this call is
+ * racing its own operation's tail) or a sibling resumer already finished
+ * it; either way there is nothing left to record against. A marker whose
+ * `op` is NOT `'rehome'` (should be structurally impossible — only
+ * `BlobSet`'s own rehome machinery calls this, against its own marker) is
+ * treated the same way: nothing to record.
+ */
+export declare function recordAppliedStamp(adapter: NoydbStore, vault: string, collection: string, recordId: string, getDEK: (collection: string) => Promise<EnclaveKey>, stamp: string): Promise<void>;
+/**
+ * List every persisted `_blob_intent` marker and resume each via the
+ * caller-supplied `resume` callback (the forget-entry / vault-open sweep,
+ * mirroring `_mv_stale`'s `hydratePersistedStaleMarkers`/orphan-sweep
+ * pattern).
+ *
+ * **Per-marker isolation (#753 Task 3, carried finding from the Task 2
+ * review):** each marker's decode + `resume` runs in its own try/catch — a
+ * corrupt/DEK-mismatched marker, or a marker whose resume genuinely fails
+ * (e.g. a store error mid-release), is reported via `onResumeError` and
+ * skipped, never aborting the loop and never silently swallowed. Earlier
+ * (Task 2) this loop had no try/catch at all; that was safe only because
+ * nothing called `resume` with real logic yet — `BlobSet.mintShredIntent`'s
+ * collection-scoped sweep (this function's first real caller, Task 3) can
+ * hit a genuinely corrupt sibling marker and must not let it block a
+ * healthy one (mirrors `stale.ts`'s orphan-sweep defensive posture, #736).
+ *
+ * `resume` is the per-marker resume action (Task 3: `BlobSet`'s
+ * `resolvePendingIntent`-equivalent for a shred marker; a rehome marker is
+ * PR-2 scope and should be skipped by the callback, not thrown into here).
+ */
+export declare function sweepBlobIntents(adapter: NoydbStore, vault: string, getDEK: (collection: string) => Promise<EnclaveKey>, resume: BlobIntentResume, onResumeError?: (collection: string, recordId: string, err: unknown) => void): Promise<void>;