@bett3r-dev/pv3-types 1.0.0 → 1.4.0

package/build/outboxManager.d.ts

@@ -1,37 +1,881 @@
-import { CommittedEvent } from '@bett3r-dev/pv3-types';
-export declare const UdpConfigSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
-    port?: number | undefined;
-    multicastAddress?: string | undefined;
-    ttl?: number | undefined;
+import { CommittedEvent, CustomCriteriaRegexpType, StructureAggregateType, StructureDispatcherSubscriptionType, StructureState, StructureSubscriptionType, TypedEventEmitter } from './index';
+export declare const EventDispatcherConfigSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    maxConsumerQueueSize?: number | undefined;
+    metricsInterval?: number | undefined;
+    defaultRetryableStatuses?: number[] | undefined;
+    retryAfterSeconds?: number | undefined;
+    retriesBeforeFailing?: number | undefined;
+    breakerThreshold?: number | undefined;
+    breakerCooldownSeconds?: number | undefined;
+    breakerCooldownMaxSeconds?: number | undefined;
+    commitOrderStream?: {
+        streamKey?: string | undefined;
+    } | undefined;
+    seedDedupToTipOnColdStart?: boolean | undefined;
+    structureAckReadiness?: {
+        enabled?: boolean | undefined;
+        windowMs?: number | undefined;
+    } | undefined;
 }, true>;
-export type UdpConfigType = typeof UdpConfigSchema.type;
+export type EventDispatcherConfigType = typeof EventDispatcherConfigSchema.type;
 export declare const AdminServicesConfigSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
-    udp?: {
-        port?: number | undefined;
-        multicastAddress?: string | undefined;
-        ttl?: number | undefined;
-    } | undefined;
     structure?: {
         structureEventstoreStream?: string | undefined;
         structureAggregateCollection?: string | undefined;
         structureEventstoreCollection?: string | undefined;
+        defaultAutomaticDataSplits?: number | undefined;
     } | undefined;
     infrastructure?: {
+        adapter?: "docker" | "kubernetes" | undefined;
+        kubernetes?: {
+            namespace?: string | undefined;
+            labelSelector?: string | undefined;
+        } | undefined;
         infrastructureLeaderRandomBackoff?: number | undefined;
+        infrastructureReconcileIntervalMs?: number | undefined;
+    } | undefined;
+    eventDispatcher?: {
+        maxConsumerQueueSize?: number | undefined;
+        metricsInterval?: number | undefined;
+        defaultRetryableStatuses?: number[] | undefined;
+        retryAfterSeconds?: number | undefined;
+        retriesBeforeFailing?: number | undefined;
+        breakerThreshold?: number | undefined;
+        breakerCooldownSeconds?: number | undefined;
+        breakerCooldownMaxSeconds?: number | undefined;
+        commitOrderStream?: {
+            streamKey?: string | undefined;
+        } | undefined;
+        seedDedupToTipOnColdStart?: boolean | undefined;
+        structureAckReadiness?: {
+            enabled?: boolean | undefined;
+            windowMs?: number | undefined;
+        } | undefined;
     } | undefined;
-    metricsInterval?: number | undefined;
 }, true>;
 export type AdminServicesConfigType = typeof AdminServicesConfigSchema.type;
+export type StructureChangeQueueType = StructureAggregateType;
+export type EventDispatcherDataSplitType = StructureDispatcherSubscriptionType & {
+    id: string;
+};
+export type EventDispatcherSubscriptionState = {
+    /** The subscription Definition */
+    subscription: StructureSubscriptionType;
+    /** The total data splits among all the different outbox services */
+    splitCount: number;
+    /** The status of the subscription */
+    status: StructureSubscriptionType['status'];
+    /** The DataSplits handled by the current service */
+    dataSplits: Record<string, EventDispatcherDataSplitType>;
+};
+export type CollectServiceStructureAckHelperType = {
+    close: () => void;
+    waitForRevisionAck: (state: StructureAggregateType) => Promise<boolean>;
+};
+export type ChannelType = {
+    send: (event: CommittedEvent, host: DispatcherHostType, endpoint: string) => Promise<void>;
+};
+export type ChannelsType = {
+    [name: string]: ChannelType;
+};
+export type InstrumentChannelSend = (send: ChannelType['send']) => ChannelType['send'];
+export type DispatcherHostType = {
+    host: string;
+    service: string;
+    channel: string;
+    port: number;
+};
+export declare const EventDispatcherDataSplitErrorSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    status?: number | undefined;
+    error?: string | undefined;
+    event: {
+        id: string;
+        position: number;
+        name?: any;
+        timestamp: string;
+        data?: import("@bett3r-dev/jsonschema-definer/dist/base").Any;
+        version: number | null;
+        stream: string;
+        metadata: {
+            causationId?: string | undefined;
+            correlationId?: string | undefined;
+        };
+    };
+}, true>;
+export type EventDispatcherDataSplitErrorType = typeof EventDispatcherDataSplitErrorSchema.type;
+export declare const EventDispatcherDataSplitStatusSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    position: number;
+    status: "critical" | "ok" | "retryable";
+    lastError?: {
+        status?: number | undefined;
+        error?: string | undefined;
+        event: {
+            id: string;
+            position: number;
+            name?: any;
+            timestamp: string;
+            data?: import("@bett3r-dev/jsonschema-definer/dist/base").Any;
+            version: number | null;
+            stream: string;
+            metadata: {
+                causationId?: string | undefined;
+                correlationId?: string | undefined;
+            };
+        };
+    } | undefined;
+}, true>;
+export type EventDispatcherDataSplitStatusType = typeof EventDispatcherDataSplitStatusSchema.type;
+export declare const EventDispatcherSubscriptionEventSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    ok: number[];
+    retryable: number[];
+    critical: number[];
+    ignored: number[];
+}, true>;
+export type EventDispatcherSubscriptionEventType = typeof EventDispatcherSubscriptionEventSchema.type;
+export declare const ProcessingByDataSplitSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    events: {
+        [x: string]: string | null;
+    };
+    lastProcessed?: number | undefined;
+}, true>;
+export type ProcessingByDataSplitType = typeof ProcessingByDataSplitSchema.type;
+export type EventDispatcherHistoricDataSplitByClusterRevision = {
+    dataSplits: {
+        [index: number]: {
+            id: string;
+            index: number;
+            position: number;
+            status: 'ok' | 'retryable' | 'critical';
+            customCriteriaRegexp: CustomCriteriaRegexpType[];
+            lastError?: EventDispatcherDataSplitErrorType;
+            events?: {
+                [event: string]: EventDispatcherSubscriptionEventType;
+            };
+        };
+    };
+    clusterRevision: number;
+    dataSplitsCount: number;
+};
+export type PullEventsForDataSplitParamsType = {
+    subscriptionId: string;
+    dataSplitId: string;
+    position: number;
+    shouldReprocess?: boolean;
+    shouldIgnoreSequence?: boolean;
+    retryAttempt?: number;
+    onBeforeUnpause?: () => Promise<void>;
+};
+export type ProcessEventForDataSplitParamsType = {
+    subscriptionId: string;
+    event: CommittedEvent;
+    dataSplit: EventDispatcherDataSplitType;
+    eventstorePosition?: number;
+    shouldReprocess?: boolean;
+    shouldIgnoreSequence?: boolean;
+    retryAttempt?: number;
+    streamEntryId?: string;
+    asFirstDelivery?: boolean;
+    isBreakerProbe?: boolean;
+    asNewReplay?: boolean;
+};
+export type EventDispatcherQueueType = ProcessEventForDataSplitParamsType;
+export type ErroredStreamDetailType = {
+    stream: string;
+    position: number;
+    error: string;
+    errorDetails?: Record<string, any>;
+    stackTrace?: string;
+    event?: CommittedEvent;
+    erroredAt?: string;
+};
+export type OutboxCriticalErrorPayloadType = {
+    subscriptionId: string;
+    dataSplit: EventDispatcherDataSplitType;
+    event: CommittedEvent;
+    dataSplitPosition: number;
+    retryAttempt: number;
+    error: Error;
+};
+export type OutboxEventEmitterType = {
+    'stream-errored': [OutboxCriticalErrorPayloadType];
+    'stream-fixed': [{
+        subscriptionId: string;
+        stream: string;
+    }];
+    'split-halted': [OutboxCriticalErrorPayloadType];
+    'split-restarted': [{
+        subscriptionId: string;
+        dataSplitIndex: string;
+    }];
+};
+export type EventProcessingType = {
+    pullFromAllThisBrokerSubscriptions: (status?: StructureSubscriptionType['status'][]) => Promise<void>;
+    pullEventForErroredStream: (subscriptionId: string, stream: string) => Promise<boolean>;
+    pullBrokerSubscriptions: (subscriptionId: string) => Promise<void>;
+    pullEventsForDataSplit: (params: PullEventsForDataSplitParamsType, retryAttempt?: number) => Promise<void>;
+    /**
+     * The commit-order stream dispatch (subWm dedup). `version > wm` ⇒ send then CAS-advance the
+     * shared per-(sub,stream) watermark; `version <= wm` ⇒ skip — UNLESS `shouldReprocess` (a
+     * force-replay) bypasses the skip and re-sends an already-watermarked version, which is then
+     * stamped `event.metadata.isRedelivery = true` on a shallow copy. TV1-1950.
+     */
+    processStreamEventForDataSplit: (params: ProcessEventForDataSplitParamsType) => Promise<void>;
+    resumeDataSplit: (dataSplitId: string) => void;
+    resumeAllOwnedDataSplits: () => void;
+    /**
+     * Operator scoped replay (TV1-1950 slice 9; generalized to bulk scope — a stream list or the whole
+     * subscription — by TV1-1957). The forward stream loop is cursor-driven and won't react to a stale
+     * watermark, so a replay must POKE the recovery path: this rolls back the by-position recovery cursor
+     * of EVERY in-scope split and writes one field per in-scope split index onto the per-`(sub)` refill
+     * request HASH (`subRefill:{sub}`) that the owning instance's `pumpOnce` honors — each split index has
+     * exactly one owner, so its field is written once and cleared once (no atomic RMW / Lua needed).
+     * Selector: a non-empty `streams` list scopes to `unique(streams.map(getDataSplitIndex))`;
+     * `allStreams: true` scopes to every split `0..splitCount-1`. `stream` is DEPRECATED sugar for
+     * `streams: [stream]` (kept for the pre-TV1-1957 single-stream call sites). Neither mode mutates any
+     * dedup state (watermark/bitmap) here (TV1-1957 slice 3) — the mode alone decides how `pumpOnce`'s
+     * refill pass dispatches each replayed event: `as-new` ⇒ `asNewReplay` bypass (isRedelivery UNSET);
+     * `force-replay` ⇒ `shouldReprocess` bypass (isRedelivery = true). See `asNewReplay` on
+     * `ProcessEventForDataSplitParamsType` for the dispatch-time mechanics.
+     */
+    requestScopedRefill: (subscriptionId: string, params: {
+        fromPosition: number;
+        mode: RefillModeType;
+        stream?: string;
+        streams?: string[];
+        allStreams?: boolean;
+    }) => Promise<void>;
+    /**
+     * Cold-start dedup seed to eventstore tip (TV1-1980 — "Option C"). Seeds this subscription's dedup
+     * state (per-(sub,stream) version watermark + null-class position bitmap) to the eventstore TIP so a
+     * cold-started (sandbox) instance lifts already "caught up" and dispatches NOTHING at boot, while an
+     * operator force-replay still re-emits events stamped `isRedelivery = true`. Gated by the flag
+     * (`config.seedDedupToTipOnColdStart`, default false ⇒ dead path in production) AND the persistent
+     * `subColdSeeded:{sub}` marker (seed iff flag ON AND marker ABSENT; marker SET after the scan). NEVER
+     * sends, NEVER drops (ADR-013) — boot-silence comes from advancing monotonic/idempotent dedup state.
+     * Awaited in the subscription bootstrap before the pump; exposed for deterministic test drive.
+     */
+    seedDedupToTip: (subscriptionId: string) => Promise<void>;
+    /**
+     * Operator DLQ-drain (TV1-1954 slice 6). Re-reads each dead-lettered ref `(stream, position)` from
+     * the EVENTSTORE by position and re-dispatches it as a GENUINE FIRST delivery (`asFirstDelivery` ⇒
+     * `isRedelivery = false`) — a dead-lettered event was NEVER successfully delivered, so stamping it a
+     * redelivery would suppress the consumer's redelivery-detection signal on what is actually its first
+     * successful receipt. On a CONFIRMED successful re-dispatch the entry is REMOVED from `subDlq:{sub}`;
+     * a re-dispatch that FAILS AGAIN stays in the DLQ (not silently dropped, and NOT duplicated). The
+     * re-dispatch goes through the SAME send→commit boundary (`commitNullEvent` — SETBIT idempotent +
+     * breaker reset). Returns the counts `{ drained, remaining }`. With `only` set, retries just the
+     * entries matching that (stream, position) ref — the operator single-event retry.
+     */
+    drainDlq: (subscriptionId: string, only?: {
+        stream: string;
+        position: number;
+    }) => Promise<{
+        drained: number;
+        remaining: number;
+    }>;
+    /**
+     * Operator restart of a HALTED datasplit (TV1-1954 split-halt). A split whose packed status is
+     * `critical` was stopped by the retryable-exhausted escalation (the live dispatch gate drops its
+     * events without advancing any watermark, so nothing is lost). This clears the split back to `ok`
+     * and replays it BY POSITION from `min(halt position, lowest gate-dropped position) − 1` through the
+     * normal subWm/bitmap-deduped dispatch, then live flow resumes. If the consumer is still sick the
+     * replay burns another retry budget and re-halts. Returns false (admin maps to 404) for an unknown
+     * subscription, a split this instance does not own, or a split that is not halted.
+     */
+    restartDataSplit: (subscriptionId: string, dataSplitIndex: string) => Promise<boolean>;
+    eventEmitter: TypedEventEmitter<OutboxEventEmitterType>;
+};
+/**
+ * Consumer lag on the commit-order stream, measured on the POSITION axis (events of the global
+ * feed still ahead of the instance's read cursor). Works identically for versioned (subWm) and
+ * null-class (bitmap) subscriptions — the cursor is dedup-agnostic. Computed from EXISTING state
+ * only: `subCursor:{sub}` (per-instance read cursor), the TTL'd `subHb` heartbeat (liveness +
+ * host), and the stream itself (tail / cursor entry positions).
+ */
+export type SubscriptionLagType = {
+    /** Global position of the stream's live tail entry (undefined when the stream is empty). */
+    tailPosition?: number;
+    /** Highest lag across live instances — the subscription's headline "events behind" figure. */
+    maxLag: number;
+    /** Per LIVE instance (dead instances' cursors are excluded via the TTL'd heartbeat). */
+    instances: Record<string, {
+        host?: string;
+        cursor: string;
+        lag: number;
+    }>;
+};
+export type EventDispatcherType = EventProcessingType & {
+    start: (structureState?: StructureState) => Promise<void>;
+    stop: () => Promise<void>;
+    /**
+     * Consumer lag per subscription (see SubscriptionLagType). Optional — wired at the OutboxManager
+     * composition root (it needs the commit-order stream handle); bare EventDispatcher test stubs omit it.
+     */
+    getSubscriptionsLag?: (subscriptionId?: string) => Promise<Record<string, SubscriptionLagType>>;
+};
+export declare const EventDispatcherDataSplitStateSchema: import("@bett3r-dev/jsonschema-definer").ObjectSchema<{
+    host: string;
+    index: number;
+    isManual?: boolean | undefined;
+    customCriteriaRegexp?: {
+        value: string;
+        regexp: string;
+        initialPosition?: number | undefined;
+        isErrorDataSplit?: boolean | undefined;
+        lastErrorDataSplitIndex?: number | undefined;
+        lastErrorClusterRevision?: number | undefined;
+    }[] | undefined;
+    position: number;
+    status: "critical" | "ok" | "retryable";
+    lastError?: {
+        status?: number | undefined;
+        error?: string | undefined;
+        event: {
+            id: string;
+            position: number;
+            name?: any;
+            timestamp: string;
+            data?: import("@bett3r-dev/jsonschema-definer/dist/base").Any;
+            version: number | null;
+            stream: string;
+            metadata: {
+                causationId?: string | undefined;
+                correlationId?: string | undefined;
+            };
+        };
+    } | undefined;
+    id: string;
+    subscription_id: string;
+    cluster_revision: number;
+    data_splits_count: number;
+    events?: {
+        [x: string]: {
+            ok: number[];
+            retryable: number[];
+            critical: number[];
+            ignored: number[];
+        };
+    } | undefined;
+    last_updated?: string | undefined;
+}, true>;
+export type EventDispatcherDataSplitStateType = typeof EventDispatcherDataSplitStateSchema.type;
+export declare const ErrorHandlingStrategyMap: {
+    readonly 'custom-datasplit': 0;
+    readonly 'subscriptionStop': 1;
+};
+export type ErrorHandlingStrategy = keyof typeof ErrorHandlingStrategyMap;
+export declare const DataSplitStatusIntMap: {
+    readonly ok: 0;
+    readonly retryable: 1;
+    readonly critical: 2;
+    readonly splitHalted: 3;
+};
+export type DataSplitStatusIntMapType = keyof typeof DataSplitStatusIntMap;
+export declare const DataSplitIntStatusMap: {
+    readonly 0: 'ok';
+    readonly 1: 'retryable';
+    readonly 2: 'critical';
+};
+export type DataSplitIntStatusMapType = keyof typeof DataSplitIntStatusMap;
+export declare const SubscriptionStatusIntMap: {
+    readonly off: 0;
+    readonly ok: 1;
+    readonly withErrors: 2;
+    readonly retrying: 3;
+    readonly critical: 4;
+};
+export type SubscriptionStatuses = keyof typeof SubscriptionStatusIntMap;
+export declare const SubscriptionIntStatusMap: {
+    0: string;
+    1: string;
+    2: string;
+    3: string;
+    4: string;
+};
+export type AllSubscriptionsStatusType = {
+    status: string;
+    errorStreams: Record<string, number>;
+    dataSplits: Record<string, {
+        position: number;
+        status: number;
+    } | undefined>;
+    delivered: number;
+    streamsTracked: number;
+    dlq?: number;
+};
+/**
+ * Operator scoped-replay mode (TV1-1950 slice 9). The admin replay endpoint pokes the cursor-driven
+ * recovery path by writing a per-`(sub)` refill marker; the owning instance's `pumpOnce` honors it.
+ * Both modes roll back ONLY the recovery cursor — neither touches the watermark/bitmap up front
+ * (TV1-1957 slice 3 removed that off-queue reset). The dedup-bypass instead happens at DISPATCH TIME:
+ * - `as-new`: the refill dispatches each replayed event with the `asNewReplay` bypass flag ⇒ the
+ *   redelivery skip is bypassed and the outgoing copy is stamped `isRedelivery = false` (fresh).
+ * - `force-replay`: the refill dispatches with `shouldReprocess` ⇒ the skip is bypassed and the
+ *   outgoing copy is stamped `isRedelivery = true` (consistent with the slice-4 stamp).
+ * Either way the only dedup MUTATION is the existing post-send monotonic `advanceSubWatermark` /
+ * idempotent `setSubBit`, run on the split's concurrency:1 queue — never off-queue (ADR-007).
+ */
+export type RefillModeType = 'as-new' | 'force-replay';
+/**
+ * Per-in-scope-split payload of the bulk scoped-refill request HASH (`subRefill:{sub}`, TV1-1957 —
+ * generalizes the pre-existing single-key `RefillRequestType` marker, which it REPLACES; a single
+ * marker cleared by the first owner to pump silently skipped splits owned by other instances).
+ * `streams` (a list selector) and `allStreams` (the whole-subscription selector) are mutually
+ * exclusive; `allStreams: true` means "no stream filter within this split" (every stream routed to it).
+ */
+export type ScopedRefillEntryType = {
+    mode: RefillModeType;
+    fromPosition: number;
+    streams?: string[];
+    allStreams?: boolean;
+};
+/**
+ * Live retry progress for a split whose send loop is in the in-place retryable backoff (TV1-1954
+ * retry visibility). Written on EACH retry attempt (the loop is about to sleep `retryAfterSecs`, so
+ * one HSET per attempt is noise), cleared on every loop exit (success / split-halt / quarantine /
+ * null dead-letter) so it can never go stale. `nextRetryAt` is epoch millis — the backoff is a FIXED
+ * `retryAfterSecs` per attempt, so the admin UI renders a live "retry N of M · next in Xs" countdown.
+ */
+export type SplitRetryStateType = {
+    attempt: number;
+    maxRetries: number;
+    retryAfterSecs: number;
+    nextRetryAt: number;
+    stream: string;
+    position: number;
+    error?: {
+        status?: number;
+        message?: string;
+    };
+};
+export type StateType = {
+    getDataSplitStateForPosition: (subscriptionId: string, dataSplitId: string, position: number) => Promise<{
+        status: 'alreadyProcessed' | 'subscriptionOff' | 'subscriptionHasErrors' | 'ok';
+        dataSplitState?: {
+            position: number;
+            status: DataSplitStatusIntMapType;
+        };
+        errorStreams?: Record<string, number>;
+    }>;
+    getErrorStreamPosition: (subscription: string, stream: string) => Promise<number | undefined>;
+    getSubscriptionDataSplits: (subscription: string) => Promise<Record<string, {
+        position: number;
+        status: DataSplitStatusIntMapType;
+        delivered: number;
+        erroredStreams: number;
+        retry?: SplitRetryStateType;
+        heldSinceHalt?: number;
+    } | undefined>>;
+    getSubscriptionStatus: (subscription: string) => Promise<{
+        status?: string;
+        errorStreams?: Record<string, number>;
+        delivered?: number;
+        streamsTracked?: number;
+    }>;
+    getAllSubscriptionsStatus: () => Promise<Record<string, AllSubscriptionsStatusType>>;
+    /**
+     * Per-`(sub,stream)` version watermark (TV1-1950): the gapless-commit-order-stream dedup.
+     * `subWm:{sub}` is a single HASH (field = stream → highest dispatched version); the `{sub}` hash
+     * tag co-locates all of a subscription's streams in ONE Redis slot so the CAS advance is a
+     * single-slot Lua script. The dispatch decide is `version > storedWm ⇒ send` and the commit is a
+     * monotonic CAS advance AFTER a successful send, so an interruption between send and advance
+     * re-dispatches (at-least-once, never drop). Because the watermark is SHARED (keyed on
+     * `(sub,stream)`, not a per-instance cursor) a rebalance hands a split over cleanly regardless of
+     * cursor skew — progress travels with the stream.
+     */
+    getSubWatermark: (subscription: string, stream: string) => Promise<number | undefined>;
+    getSubWatermarks: (subscription: string) => Promise<Record<string, number>>;
+    /** Monotonic CAS advance of `subWm:{sub}[stream]` to `version` (never regresses); returns the resulting watermark. */
+    advanceSubWatermark: (subscription: string, stream: string, version: number) => Promise<number>;
+    /**
+     * Entry-id-recording monotonic CAS advance (TV1-1950 slice 8). Same as `advanceSubWatermark`, but it
+     * ALSO records — atomically, in the parallel `subWmId:{sub}` HASH (field = stream → entryId) — the
+     * stream ENTRY-ID at which the watermark last advanced. That recorded id is the bridge the bounded
+     * watermark GC needs: `subWm` stores a per-stream VERSION, but the trim horizon (`firstId()`) is a
+     * stream ENTRY-ID, so the GC compares the recorded entry-id (not the version) against the horizon.
+     * The dispatch path uses this when it has the live stream entry-id; the by-position refill falls back
+     * to `advanceSubWatermark` (no stream entry-id), leaving the field absent from `subWmId` → the GC
+     * conservatively retains it.
+     */
+    advanceSubWatermarkAt: (subscription: string, stream: string, version: number, entryId: string) => Promise<number>;
+    /**
+     * Bounded watermark GC (TV1-1950 slice 8): drop every `(sub,stream)` watermark (from BOTH `subWm`
+     * and `subWmId`) whose recorded entry-id is STRICTLY BELOW the trim horizon (the global stream's
+     * first-entry-id, `firstId()`, after the slice-8a XTRIM). An entry strictly below the horizon was
+     * already trimmed away and can never be re-read off the live stream, so dropping its watermark
+     * induces NO re-dispatch; an entry AT or ABOVE the horizon is still in the live window and is
+     * RETAINED. A `subWm` field with no recorded `subWmId` is conservatively retained. An `undefined`
+     * horizon (empty stream — nothing trimmed) is a no-op. Returns the dropped stream fields.
+     */
+    gcSubWatermarks: (subscription: string, trimHorizon: string | undefined) => Promise<string[]>;
+    /**
+     * Per-`(sub,instance)` read cursor + TTL'd heartbeat for `XREAD` resume. The cursor is
+     * instance-local and may skew across instances; correctness rests on the SHARED `subWm` dedup,
+     * not on the cursor. The heartbeat is TTL'd so a dead instance can no longer pin the stream
+     * (its cursor stops counting toward the trim floor once the heartbeat expires).
+     */
+    getStreamCursor: (subscription: string, instance: string) => Promise<string | undefined>;
+    setStreamCursor: (subscription: string, instance: string, entryId: string) => Promise<void>;
+    /**
+     * `host` is embedded in the heartbeat VALUE (the key's existence/TTL is the liveness contract and
+     * stays untouched — the WAL Reader never parses the value). It maps the per-process instance UUID
+     * back to the structure's broker host/IP, so the admin lag view can attribute a cursor to the
+     * host that owns each data split without any additional state.
+     */
+    heartbeatStreamConsumer: (subscription: string, instance: string, ttlSeconds?: number, host?: string) => Promise<void>;
+    /**
+     * Per-`(sub,split)` by-position recovery cursor (TV1-1950 slice 9). The CONSERVATIVE start point
+     * for the by-position refill (the KEPT `reduceStream`; never a by-version read) — it only BOUNDS
+     * the re-scan; correctness rests on the version watermark (design critique #3). A refill advances
+     * it to the highest scanned position; an operator replay rolls it back to re-open a range. Keyed by
+     * split INDEX (deterministic from the stream) so the admin can address it from `(sub,stream)` alone.
+     */
+    getRecoveryCursor: (subscription: string, splitIndex: string) => Promise<number | undefined>;
+    setRecoveryCursor: (subscription: string, splitIndex: string, position: number) => Promise<void>;
+    /**
+     * Persistent per-`(sub)` cold-seed marker (TV1-1980 — "Option C"). Existence of `subColdSeeded:{sub}`
+     * means this subscription's dedup state was already seeded to the eventstore tip on a prior cold start.
+     * `seedDedupToTip` seeds iff the flag is ON AND `getColdSeeded` is FALSE, then `setColdSeeded` AFTER the
+     * scan (self-healing — a failed seed leaves the marker unset and retries next cycle). The marker gates
+     * RE-seeding unconditionally: a warm restart must never re-seed, else a new event that advanced the
+     * watermark past the old tip would be masked to a silent drop (violates ADR-013). Torn down with the
+     * subscription in `deleteSubscription`.
+     */
+    getColdSeeded: (subscription: string) => Promise<boolean>;
+    setColdSeeded: (subscription: string) => Promise<void>;
+    /**
+     * Null-class position bitmap dedup (TV1-1954 slice 2 — hybrid dedup). VERSION-NULL streams
+     * (external/integration, no OCC) are delivered in the WAL Reader's commit-LSN order, NOT position
+     * order, so the ordered `subWm` high-water would skip (silently DROP) a lower-position event that
+     * commits AFTER a higher one on the same stream. Instead each null-version event dedups ORDER-FREE
+     * on its POSITION bit in `subBitmap:{sub}:{epoch}` (epoch = floor(position / EPOCH_SIZE), offset =
+     * position % EPOCH_SIZE, EPOCH_SIZE = 1_000_000 so a single key stays ≤ 2^32 bits): `getSubBit` is
+     * the DECIDE (set ⇒ redelivery ⇒ skip) and `setSubBit` is the COMMIT, run ONLY AFTER a successful
+     * send (send → commit boundary preserved for both classes → at-least-once, never drop). The `{sub}`
+     * hash tag co-locates a subscription's epoch keys in ONE Redis slot.
+     */
+    getSubBit: (subscription: string, position: number) => Promise<boolean>;
+    setSubBit: (subscription: string, position: number) => Promise<void>;
+    /**
+     * Batched, set-only null-class SETBIT (TV1-1980 slice 2 — cold-seed flush). The same set-only,
+     * idempotent semantics as `setSubBit` (SETBIT 1, never CLRBIT) but for MANY positions at once. It
+     * groups positions by epoch (each `subBitmap:{sub}:{epoch}` key) and issues one PIPELINED batch of
+     * SETBITs PER epoch — collapsing an O(positions) per-event round-trip loop to O(epochs) round trips,
+     * the cold-seed batching gate. Correctness rests, like `setSubBit`, on consumer idempotency
+     * (at-least-once, ADR-013): it only BOUNDS duplicate volume, it is not itself the never-drop mechanism.
+     */
+    setSubBits: (subscription: string, positions: number[]) => Promise<void>;
+    /**
+     * Bounded null-class bitmap GC (TV1-1954 slice 3) — the POSITION-axis analogue of `gcSubWatermarks`.
+     * The watermark GC compares ENTRY-IDs (commit order); the null-class bitmap is indexed by POSITION —
+     * a DIFFERENT axis — so the caller crosses axes FIRST: it maps the trim floor (`firstId()` ENTRY-ID)
+     * to a POSITION by reading the first live stream entry's `event.position` and passes that as
+     * `floorPosition`. This DELs every `subBitmap:{sub}:{epoch}` key whose epoch is STRICTLY BELOW
+     * `floor((floorPosition − KEEP_FLOOR_MARGIN) / EPOCH_SIZE)` (`KEEP_FLOOR_MARGIN` = one epoch = 1_000_000
+     * positions). The one-epoch margin absorbs the commit-order↔position skew: a low-position event that
+     * committed LATE is still LIVE (entry-id ≥ floor) and its bit is still needed, so its epoch (down to
+     * floorEpoch − 1) is RETAINED. If the skew ever exceeds the margin the effect is OVER-prune → a
+     * duplicate (idempotent-safe), NEVER a drop, so the margin is a dup-tuning knob, not a correctness
+     * bound. An undefined/negative/small `floorPosition` (pruneBelowEpoch <= 0) is a no-op. Returns the
+     * count of epoch keys deleted (observability).
+     */
+    pruneBitmap: (subscription: string, floorPosition: number) => Promise<number>;
+    /**
+     * Null-class dead-letter + circuit-breaker counter (TV1-1954 slice 4 — licensed by order-freeness).
+     * A version-null event has NO per-stream order to protect, so on retries-exhausted OR a critical send
+     * failure it is DEAD-LETTERED and the split keeps flowing — unlike the versioned per-stream halt. This
+     * runs ONE atomic Lua (all three keys share the `{sub}` slot) that (1) LPUSHes a durable ref onto
+     * `subDlq:{sub}`, (2) SETBITs the `position` bit (resolved-but-not-delivered — a recovery re-read sees
+     * GETBIT=1 and does NOT re-blast it; SETBIT-without-send is SAFE because the DLQ IS the durable
+     * capture, so there is no crash window where the event is neither delivered nor captured), and (3)
+     * INCRs the per-`(sub,host)` breaker count. Returns the NEW breaker count. Slice 4 only RECORDS the
+     * count; the trip/pause + success-reset is slice 5. `error` is SANITIZED to `{ status, message }` —
+     * stacks/secrets are NEVER written to the durable DLQ.
+     */
+    deadLetterNullEvent: (subscription: string, host: string, stream: string, position: number, name: string, error: {
+        status?: number;
+        message?: string;
+        name?: string;
+        details?: any;
+    }) => Promise<number>;
+    /**
+     * Read the dead-letter queue for a subscription, newest-first (matching the LPUSH in
+     * `deadLetterNullEvent`). Each ref is the `{ stream, position, name, error }` tuple captured on a
+     * null-class failure; the operator DLQ-drain (slice 6) re-reads each from the eventstore by position.
+     */
+    getDlq: (subscription: string) => Promise<Array<{
+        stream: string;
+        position: number;
+        name: string;
+        error: {
+            status?: number;
+            message?: string;
+            name?: string;
+            details?: any;
+        };
+    }>>;
+    /**
+     * The RAW dead-letter entries for a subscription — the EXACT stored strings (`LRANGE subDlq:{sub}
+     * 0 -1`), NOT re-parsed. The operator DLQ-drain (TV1-1954 slice 6) reads these so it can hand each
+     * back BYTE-FOR-BYTE to `removeDlqEntry` on a confirmed re-dispatch, avoiding re-serialization drift
+     * (`getDlq` re-parses for assertions/display and MUST NOT be used as the LREM key).
+     */
+    getDlqRaw: (subscription: string) => Promise<string[]>;
+    /**
+     * Remove ONE exact-match occurrence of a raw dead-letter entry (`LREM subDlq:{sub} 1 rawEntry`) —
+     * the DLQ-drain's confirmed-success removal (TV1-1954 slice 6). Using the RAW stored string (from
+     * `getDlqRaw`) guarantees the LREM matches byte-for-byte; only ONE occurrence is removed so a
+     * duplicate ref (same event dead-lettered twice) is not over-deleted.
+     */
+    removeDlqEntry: (subscription: string, rawEntry: string) => Promise<void>;
+    /**
+     * The per-`(sub,host)` consecutive-failure count `deadLetterNullEvent` INCRs. SHARED across outbox
+     * instances (one Redis counter) so the effective breaker threshold does not multiply by instance
+     * count. Absent key ⇒ 0 (no failures recorded yet).
+     */
+    getBreakerCount: (subscription: string, host: string) => Promise<number>;
+    /**
+     * Null-class HAPPY-PATH commit (TV1-1954 slice 5) — REPLACES the bare `setSubBit` at the null commit
+     * site so a SUCCESS resets the circuit breaker ATOMICALLY with the position-bit commit (no extra
+     * hot-path round-trip). ONE Lua (all keys share the `{sub}` slot) that (1) SETBITs the `position` bit
+     * (order-free dedup commit, run ONLY after a successful send — send→commit boundary preserved →
+     * at-least-once, never drop), (2) DELs `breaker:{sub}:{host}` (any success resets the consecutive
+     * failure count to 0), (3) DELs `breakerCooldown:{sub}:{host}` (reset the half-open backoff to base),
+     * and (4) DELs `breakerOpenedAt:{sub}:{host}`. Returns whether the breaker WAS open — i.e. this
+     * success CLOSED it — so the caller can resume the host's paused splits + emit `onBreakerClose`.
+     */
+    /**
+     * Happy-path null commit: SETBIT the position bit, bump the per-split delivered counter
+     * (`subNullSplit:{sub}` HASH, incremented only when the bit flips 0→1 so redelivered duplicates
+     * don't inflate it — the bitmap is streamless, so split attribution only exists at commit time),
+     * and reset the (sub,host) circuit breaker — all in ONE atomic Lua. Returns whether the breaker
+     * WAS open (this success closed it).
+     */
+    commitNullEvent: (subscription: string, host: string, position: number, splitIndex: string | number) => Promise<boolean>;
+    /**
+     * OPEN the per-`(sub,host)` circuit breaker (TV1-1954 slice 5). ONE Lua on the shared breaker clock:
+     * if `breakerOpenedAt:{sub}:{host}` is ABSENT → stamp it with redis `TIME` seconds (the SHARED clock,
+     * not a local one) and init `breakerCooldown` to `baseCooldownSeconds`, returning `true` (freshly
+     * opened). If ALREADY set → return `false` WITHOUT resetting the clock. The cooldown is DOUBLED to the
+     * `maxCooldownSeconds` cap ONLY when `probeFailed` is true (a FAILED half-open probe → exponential
+     * backoff, measured from the probe the tryProbeBreaker reset stamped); an already-open in-flight-batch
+     * failure (`probeFailed` false) is a NO-OP on the clock/cooldown, so a post-trip burst can neither
+     * inflate the cooldown to the cap nor storm. Idempotent under concurrent trips.
+     */
+    openBreaker: (subscription: string, host: string, baseCooldownSeconds: number, maxCooldownSeconds: number, probeFailed: boolean) => Promise<boolean>;
+    /**
+     * Half-open PROBE decision (TV1-1954 slice 5) — the Lua is the AUTHORITY, a local wake-up timer is
+     * only a HINT it can reject (kills local/shared clock drift). ONE Lua that reads
+     * `breakerOpenedAt:{sub}:{host}` + redis `TIME`: if `now - openedAt >= cooldownSeconds` it ALLOWS the
+     * probe (returns `true`) AND resets `openedAt = now` so the NEXT cooldown is measured from this probe;
+     * else it REJECTS (`false`, stay paused). Absent `openedAt` (breaker not open) → `false`.
+     */
+    tryProbeBreaker: (subscription: string, host: string, cooldownSeconds: number) => Promise<boolean>;
+    /**
+     * The current half-open cooldown (seconds) for `(sub,host)` — `breakerCooldown:{sub}:{host}`. Set to
+     * the base on a fresh open, DOUBLED on each probe failure (cap max), reset (deleted) on close. Absent
+     * ⇒ `undefined` (no breaker open / no failures recorded); callers default to the configured base.
+     */
+    getBreakerCooldown: (subscription: string, host: string) => Promise<number | undefined>;
+    /**
+     * Per-`(sub)` pending scoped-refill request HASH (TV1-1950 slice 9; generalized to bulk scope by
+     * TV1-1957) — field = in-scope SPLIT INDEX → payload — the operator poke the cursor-driven forward
+     * loop honors as a THIRD refill trigger (besides cold-start / eviction). REPLACES the earlier
+     * single-key `RefillRequestType` marker (first-clearer-wins, so a multi-split request would let the
+     * first owning instance to pump clear it before other owners ever saw it). A split index has EXACTLY
+     * ONE owner, so `setRefillRequest`/`clearRefillRequest` are single-field HSET/HDEL — no atomic
+     * read-modify-write or Lua needed. `getRefillRequests` reads the whole hash (one round-trip); the
+     * owning instance's `pumpOnce` filters to the fields it owns, batches them into ONE `refillByPosition`
+     * pass, then clears ONLY those fields — and only AFTER that pass's covered splits drain and their
+     * recovery cursors checkpoint (HDEL-after-drain).
+     */
+    getRefillRequests: (subscription: string) => Promise<Record<string, ScopedRefillEntryType>>;
+    setRefillRequest: (subscription: string, splitIndex: string, request: ScopedRefillEntryType) => Promise<void>;
+    clearRefillRequest: (subscription: string, splitIndex: string) => Promise<void>;
+    deleteSubscription: (subscription: string) => Promise<void>;
+    storeDataSplitPosition: (subscription: string, dataSplitId: string, stream: string, position: number, markAsProcessed?: boolean, dataSplitStatus?: DataSplitStatusIntMapType, errorHandlingStrategy?: ErrorHandlingStrategy, forceSet?: boolean) => Promise<void>;
+    storeErroredStream: (subscription: string, stream: string, position: number, isLastEvent?: boolean) => Promise<void>;
+    removeErrorStream: (subscription: string, stream: string, dataSplitId?: string) => Promise<void>;
+    /**
+     * Rich error detail for a quarantined (sub,stream) — the operator-facing companion of the
+     * `subErrorStreams` ZSET (which only holds stream → position). One HASH `subErrorDetails:{sub}`
+     * (field = stream → JSON detail) written at quarantine time, removed with the quarantine
+     * (removeErrorStream HDELs it; deleteSubscription drops the key). Best-effort bookkeeping: the
+     * quarantine itself never depends on it.
+     */
+    storeErroredStreamDetail: (subscription: string, stream: string, detail: ErroredStreamDetailType) => Promise<void>;
+    getErroredStreamDetails: (subscription: string) => Promise<Record<string, ErroredStreamDetailType>>;
+    setPositionForAllDataSplits: (subscription: string, splitCount: number, clusterVersion: number) => Promise<number>;
+    /**
+     * The packed per-split status (one ZSCORE on `subSplit:{sub}` + unpack) — the live dispatch gate's
+     * read. `critical` here means the split is HALTED (only the splitHalted write packs critical bits
+     * since TV1-1954 split-halt; the per-stream quarantine no longer paints the split).
+     */
+    getDataSplitStatus: (subscription: string, dataSplitId: string) => Promise<'ok' | 'retryable' | 'critical' | undefined>;
+    /**
+     * Halt-floor bookkeeping (TV1-1954 split-halt). While a split is halted the dispatch gate DROPS its
+     * events; a null-version event can arrive with a position BELOW the halt position (commit-LSN order
+     * ≠ position order), so the restart replay must start from the LOWEST dropped position, not just the
+     * halt position. `recordHaltDrop` is an atomic min (`ZADD LT` on `subHaltFloor:{sub}`, member =
+     * split index); restart reads it to bound the replay and clears it.
+     */
+    recordHaltDrop: (subscription: string, splitIndex: string, position: number) => Promise<void>;
+    getHaltFloor: (subscription: string, splitIndex: string) => Promise<number | undefined>;
+    clearHaltFloor: (subscription: string, splitIndex: string) => Promise<void>;
+    /**
+     * Rich failure detail for a HALTED split — the split-level analogue of `storeErroredStreamDetail`
+     * (`subHaltDetails:{sub}` HASH, field = split index → JSON). Written at halt time, removed by
+     * restart-datasplit; best-effort operator bookkeeping, never part of the halt correctness contract.
+     */
+    storeSplitHaltDetail: (subscription: string, splitIndex: string, detail: ErroredStreamDetailType) => Promise<void>;
+    getSplitHaltDetails: (subscription: string) => Promise<Record<string, ErroredStreamDetailType>>;
+    removeSplitHaltDetail: (subscription: string, splitIndex: string) => Promise<void>;
+    /**
+     * Live per-split retry progress (TV1-1954 retry visibility) — `subRetryState:{sub}` HASH, field =
+     * split index → JSON SplitRetryStateType. Written on each retryable attempt, cleared on every loop
+     * exit; surfaced through getSubscriptionDataSplits as each split's `retry` field. Best-effort
+     * operator bookkeeping, never part of the retry correctness contract.
+     */
+    setSplitRetryState: (subscription: string, splitIndex: string, retry: SplitRetryStateType) => Promise<void>;
+    clearSplitRetryState: (subscription: string, splitIndex: string) => Promise<void>;
+    getSplitRetryStates: (subscription: string) => Promise<Record<string, SplitRetryStateType>>;
+    start: () => Promise<void>;
+};
 export type OutboxManagerType = {
-    start: (activateDiscoveredSubscriptions?: boolean) => Promise<void>;
+    /** Receives an optional Structure State Variable with the result of the infrastructure change that happened when the service started */
+    start: (structureState?: StructureState) => Promise<void>;
     stop: () => Promise<void>;
+    state: StateType;
+    eventDispatcher: EventDispatcherType;
+};
+/**
+ * Callbacks for outbox event processing instrumentation.
+ * Implementations MUST NOT throw — callbacks are invoked inline during event processing
+ * and an unhandled exception would disrupt outbox behavior. Use try/catch internally.
+ * The built-in `instrumentEventProcessing()` factory wraps each callback in try/catch.
+ */
+export type EventProcessingInstrumentation = {
+    onErrorClassified: (params: {
+        type: 'retryable' | 'critical' | 'sequence';
+        subscription: string;
+        event: string;
+        statusCode?: number;
+    }) => void;
+    onRetryAttempt: (params: {
+        subscription: string;
+        event: string;
+        attempt: number;
+    }) => void;
+    onRetryExhausted: (params: {
+        subscription: string;
+        event: string;
+    }) => void;
+    onStreamErrored: (params: {
+        subscription: string;
+        stream: string;
+    }) => void;
+    onStreamFixed: (params: {
+        subscription: string;
+        stream: string;
+    }) => void;
+    onDataSplitStatusChange: (params: {
+        subscription: string;
+        datasplit: string;
+        previousStatus: string;
+        newStatus: string;
+    }) => void;
+    onBackpressure: (params: {
+        subscription: string;
+        datasplit: string;
+    }) => void;
+    onSequenceRepull: (params: {
+        subscription: string;
+        datasplit: string;
+    }) => void;
+    onCacheHit?: (params: {
+        subscription: string;
+        source: 'debounce' | 'wait' | 'pull';
+        eventsServed: number;
+    }) => void;
+    onCacheMiss?: (params: {
+        subscription: string;
+        eventsRead: number;
+        durationMs: number;
+    }) => void;
+    onHydrate?: (params: {
+        eventstore: string;
+        source: 'redis' | 'db';
+        eventsRead: number;
+        durationMs: number;
+    }) => void;
+    onLazyFetch?: (params: {
+        subscription: string;
+        event: string;
+        durationMs: number;
+    }) => void;
+    /**
+     * Global commit-order stream DEPTH (`tail − floor`): how many entry-ids separate the stream's
+     * last entry from its trim floor (`firstId()`). Emitted per pump from the stream source, which is
+     * the layer that has both the tail and the floor. The backlog of un-trimmed events on the stream.
+     */
+    onStreamDepth?: (params: {
+        subscription: string;
+        depth: number;
+    }) => void;
+    /**
+     * Per-`(sub,instance)` consumer LAG = `tail − this instance's read cursor`: how far behind the
+     * live stream tail this consumer's XREAD cursor is (in entry-id distance). Emitted per pump BEFORE
+     * the cursor advances, from the stream source (the only layer that holds the per-instance cursor).
+     */
+    onConsumerLag?: (params: {
+        subscription: string;
+        instance: string;
+        lag: number;
+    }) => void;
+    /**
+     * By-position REFILL activity (cold-start / eviction / operator-replay recovery): the count of
+     * events re-dispatched by one `refillByPosition` pass. Emitted once per refill; a rising rate
+     * signals recovery churn (evictions, replays).
+     */
+    onRefill?: (params: {
+        subscription: string;
+        refilled: number;
+    }) => void;
+    /**
+     * Per-`(sub,stream)` WATERMARK progress: the version the shared `subWm` advanced to after a
+     * successful send (dispatch progress). Emitted on every watermark advance in the stream dispatch.
+     */
+    onWatermarkAdvance?: (params: {
+        subscription: string;
+        stream: string;
+        version: number;
+    }) => void;
+    /**
+     * Null-class circuit breaker TRIPPED (TV1-1954 slice 5): `breakerThreshold` consecutive failures for
+     * `(subscription, host)` reached, the host's split queues were PAUSED. The alarm the operator acts on
+     * (a dead consumer). Emitted from the dead-letter branch when the count crosses the threshold.
+     */
+    onBreakerOpen?: (params: {
+        subscription: string;
+        host: string;
+        count: number;
+    }) => void;
+    /**
+     * Null-class circuit breaker CLOSED (TV1-1954 slice 5): a successful null send (a recovered probe)
+     * reset the count and cleared the open clock, the host's split queues resumed. The auto-recovery signal.
+     */
+    onBreakerClose?: (params: {
+        subscription: string;
+        host: string;
+    }) => void;
 };
 export type TcpEventMessageRequest = {
     endpoint: string;
     event: CommittedEvent;
 };
 export type TcpEventMessageResponse = {
-    id: number;
+    id: string;
+    position: number;
     status: number;
     error?: string;
 };