@smthrs/journal 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/SqlJournal.d.ts

@@ -0,0 +1,173 @@
+/**
+ * SQLite-backed logical journal with durable and lossy channels.
+ *
+ * Lossy telemetry events remain optimistic until the single scoped writer
+ * commits them, so a process crash can lose accepted-but-unwritten telemetry.
+ * Lifecycle events use `emitDurable` and return after commit, except inside
+ * `transact`, where their savepoint completes before the outer commit.
+ *
+ * Governing design: `packages/smithers/flows/journal/docs/concepts/two-channels.md`.
+ * Prior-art decision: `packages/smithers/flows/sync/docs/concepts/replay-then-follow.md`.
+ *
+ * The replay-then-follow stream follows Effect's `EventJournal` and OpenCode's
+ * upstream event stream design. The bounded send queue deliberately deviates
+ * from their synchronous durable writes by reserving a provisional per-run
+ * sequence before admission. Both channels finalize canonical order at commit.
+ * SQLite retry and transaction behavior comes through `@smthrs/database`.
+ *
+ * @since 0.1.0
+ */
+import { DurableWriter } from "@smthrs/database/DurableWriter";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Scope from "effect/Scope";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+import { Consensus } from "./Consensus.ts";
+import { Journal, JournalError, type OverflowPolicy, type Service } from "./Journal.ts";
+import { RunId, type Seq } from "./JournalEvent.ts";
+import * as Redaction from "./Redaction.ts";
+/**
+ * Automatic checkpoint-and-compact policy for the SQL journal.
+ *
+ * Disabled unless set on {@link SqlJournalOptions}. Once a run's committed
+ * entry count reaches `entryThreshold`, the journal asks `capture` for the
+ * caller's replay state at the run's durable tail, writes it as a checkpoint
+ * at that sequence, and compacts the entries strictly below it.
+ *
+ * Both channels run the policy after their allocation permit is free. Lossy
+ * commits fork scoped maintenance so the sole queue consumer keeps draining;
+ * durable emits await their attempt. `flush` awaits registered maintenance.
+ * `capture` may read or emit without blocking unrelated allocation. A capture is interrupted
+ * after 30 seconds so caller code cannot wedge journal admission indefinitely.
+ *
+ * A failed or refused attempt (a live stream behind the boundary, a capture
+ * failure) is logged at warning, damped for `entryThreshold`
+ * further committed entries, and never surfaced to the emit that triggered
+ * it.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface CompactionPolicy {
+    readonly entryThreshold: number;
+    readonly capture: (runId: RunId, upTo: Seq) => Effect.Effect<unknown, unknown>;
+}
+/**
+ * SQL journal queue and batching options.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface SqlJournalOptions {
+    /**
+     * Bound on the number of ENTRIES held in two places at once: the lossy
+     * admission queue, where `overflow` decides what happens when it is full,
+     * and the sliding `changes` buffer, where a slow subscriber silently loses
+     * the entries that fall out of it.
+     *
+     * It bounds entries, not bytes. Set {@link SqlJournalOptions.maxEntryBytes}
+     * to bound the size of one entry as well; with it unset a handful of very
+     * large payloads is still the memory bill.
+     */
+    readonly capacity: number;
+    /** Policy applied when the lossy admission queue is full. */
+    readonly overflow: OverflowPolicy;
+    /** Entries the queued writer commits per transaction. */
+    readonly batchSize?: number | undefined;
+    /**
+     * Upper bound on the in-process source-event index, the map that answers
+     * producer idempotency from memory.
+     *
+     * The index is a cache, never the authority, and a MISS costs a receipt, not
+     * correctness. On a miss for an explicit producer sequence the entry is
+     * admitted optimistically without any read: `emitLossy` returns `Accepted`
+     * where a resident entry would have returned `Duplicate`, and the insert
+     * collapses onto the committed row through the unique index
+     * `(run_id, source_id, source_seq)` rather than doubling it. A CHANGED retry
+     * behind an evicted entry therefore surfaces `idempotency_conflict` from
+     * `flush`, not from the emit. See `admitFromIndex` for why the read is gone:
+     * `emitLossy` is called from inside other write transactions, and the
+     * pre-admission SELECT deadlocked `@smthrs/agent`'s exit flush against the
+     * engine's own writer, measured at over 120 s versus about half a second
+     * without it.
+     *
+     * Bounding the cache keeps startup decode and resident memory O(bound) rather
+     * than O(total events ever written), mirroring Temporal's refusal to hold
+     * unbounded history in a shard (`service/history`). The separate
+     * sequence-floor maps start empty and grow only with runs and producers this
+     * layer instance touches; they are lazy, not governed by this bound.
+     *
+     * Idempotency compares canonical persisted JSON after redaction. Two inputs
+     * that redact to the same value are therefore the same event, and JSON's
+     * encoding intentionally treats `NaN` and `null` as the same `null` value.
+     */
+    readonly sourceEventCache?: number | undefined;
+    /**
+     * Largest single entry the journal admits, measured in UTF-8 bytes of the
+     * encoded `payload` plus `meta`.
+     *
+     * Unset by default, which means unbounded: `capacity` bounds how MANY entries
+     * the admission queue and the `changes` buffer hold, never how large one is,
+     * so without this a handful of multi-megabyte payloads is the memory bill,
+     * and every one of them is replayed to each sync subscriber and time-travel
+     * consumer on every read. Set it where the journal is fed by untrusted or
+     * unbounded producers.
+     *
+     * The bound is checked after encoding and BEFORE any sequence is allocated or
+     * anything is queued, so a refused entry costs no sequence and leaves no gap.
+     * An entry over the bound fails `invalid_event`.
+     */
+    readonly maxEntryBytes?: number | undefined;
+    /**
+     * Scrub applied to every `payload` and `meta` before it is encoded for
+     * persistence.
+     *
+     * Journal rows are permanent and are replayed verbatim to sync subscribers
+     * and time-travel consumers, so a credential that reaches `payload_json` is
+     * a durable, broadly readable leak. Redaction therefore defaults to
+     * `Redaction.make()`; pass `Redaction.makeNoop()` to persist payloads
+     * verbatim by choice.
+     */
+    readonly redact?: Redaction.Redactor | undefined;
+    /**
+     * Automatic checkpoint-and-compact policy. Off by default: without it the
+     * journal never deletes an entry, and checkpointing stays a caller-driven
+     * `checkpoint` / `compact` call.
+     */
+    readonly compaction?: CompactionPolicy | undefined;
+}
+/**
+ * Constructs the SQL journal over the context's SQL client, durable writer,
+ * and consensus strategy.
+ *
+ * `emitLossy` validates and admits telemetry to the non-blocking queue;
+ * `emitDurable` allocates and commits inside the database transaction, fenced
+ * through `Consensus.guard` in that same transaction.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export declare const make: (options: SqlJournalOptions) => Effect.Effect<Service, JournalError, Consensus | DurableWriter | SqlClient.SqlClient | Scope.Scope>;
+/**
+ * Provides the SQL journal over an explicitly injected consensus strategy.
+ *
+ * The strategy in context arbitrates the owner fence: `emitDurable`,
+ * `checkpoint`, and `compact` join its `guard` inside their write
+ * transaction. Provide the SAME strategy instance to every service that must
+ * agree on ownership — `RunStore.layerWith` in particular — because
+ * `Consensus.layerLocal` keeps its leases per layer build.
+ *
+ * @category layers
+ * @since 1.0.0
+ */
+export declare const layerWith: (options: SqlJournalOptions) => Layer.Layer<Journal, JournalError, Consensus | DurableWriter | SqlClient.SqlClient>;
+/**
+ * Provides the SQL journal fenced by the database-backed strategy,
+ * `SqlConsensus`, over the same client. Use {@link layerWith} to inject a
+ * different strategy.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layer: (options: SqlJournalOptions) => Layer.Layer<Journal, JournalError, DurableWriter | SqlClient.SqlClient>;
+//# sourceMappingURL=SqlJournal.d.ts.map

package/dist/cjs/SqlJournal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SqlJournal.d.ts","sourceRoot":"","sources":["../../src/SqlJournal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAA8B,aAAa,EAAE,MAAM,gCAAgC,CAAA;AAI1F,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAOrC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAGrC,OAAO,KAAK,SAAS,MAAM,+BAA+B,CAAA;AAE1D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAE1C,OAAO,EASL,OAAO,EACP,YAAY,EAGZ,KAAK,cAAc,EACnB,KAAK,OAAO,EAEb,MAAM,cAAc,CAAA;AACrB,OAAO,EAKL,KAAK,EACL,KAAK,GAAG,EAGT,MAAM,mBAAmB,CAAA;AAK1B,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA;AAM3C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;CAC/E;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;;;;;;OASG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACvC;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9C;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC3C;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAA;IAChD;;;;OAIG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;CACnD;AA+WD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,IAAI,GACf,SAAS,iBAAiB,KACzB,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,GAAG,aAAa,GAAG,SAAS,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAqlEjG,CAAA;AAEJ;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,SAAS,GACpB,SAAS,iBAAiB,KACzB,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,GAAG,aAAa,GAAG,SAAS,CAAC,SAAS,CAC/C,CAAA;AAEtC;;;;;;;GAOG;AACH,eAAO,MAAM,KAAK,GAChB,SAAS,iBAAiB,KACzB,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,aAAa,GAAG,SAAS,CAAC,SAAS,CACb,CAAA"}