@swirls/looms 0.0.2 → 0.2.0

package/dist/s2/snapshot-store.js

@@ -1,19 +1,26 @@
 import { AppendInput, AppendRecord, S2, S2Endpoints, S2Error } from '@s2-dev/streamstore';
-import { Effect, Layer, Option, Schema } from 'effect';
-import { createKeyedSerializer, SnapshotStoreError, SnapshotStoreTag, } from '@swirls/looms/core';
+import { Effect, Layer, Option, Schema, Stream } from 'effect';
+import { createKeyedSerializer, assertSnapshotByteLimit, RunStateSchema, RunSummarySchema, SnapshotStoreError, SnapshotPayloadTooLargeError, SnapshotStoreTag, } from '@swirls/looms/core';
 import { S2ConfigSchema } from './config.js';
+import { paginateS2Stream } from './pagination.js';
 const DEFAULT_PREFIX = 'snapshots';
 const DEFAULT_MAX_CHUNK_BYTES = 900 * 1024;
 const SnapshotFrameSchema = Schema.Struct({
     v: Schema.Literal(1),
     runId: Schema.String,
-    cursor: Schema.Number,
+    cursor: Schema.Finite,
     stateHash: Schema.String,
-    takenAt: Schema.Number,
-    chunk: Schema.Number,
-    chunks: Schema.Number,
+    takenAt: Schema.Finite,
+    chunk: Schema.Finite,
+    chunks: Schema.Finite,
     data: Schema.String,
 });
+const SnapshotFrameJson = Schema.fromJsonString(SnapshotFrameSchema);
+const decodeFrame = Schema.decodeSync(SnapshotFrameJson);
+const encodeFrame = Schema.encodeSync(SnapshotFrameJson);
+const HeaderJson = Schema.fromJsonString(RunSummarySchema);
+const decodeHeader = Schema.decodeUnknownOption(HeaderJson);
+const encodeHeader = Schema.encodeSync(HeaderJson);
 export function frameSnapshot(snapshot, maxChunkBytes = DEFAULT_MAX_CHUNK_BYTES) {
     const json = JSON.stringify(snapshot.state);
     const chunks = Math.max(1, Math.ceil(json.length / maxChunkBytes));
@@ -37,6 +44,8 @@ export function assembleSnapshot(frames) {
         throw new SnapshotStoreError('Cannot assemble snapshot from empty frames');
     }
     const first = frames[0];
+    // Oxlint's immutable alternative requires ES2023; sorting a fresh copy is safe on ES2022.
+    // oxlint-disable-next-line unicorn/no-array-sort
     const ordered = [...frames].sort((a, b) => a.chunk - b.chunk);
     for (const frame of ordered) {
         if (frame.cursor !== first.cursor ||
@@ -54,8 +63,7 @@ export function assembleSnapshot(frames) {
         }
     }
     const json = ordered.map((frame) => frame.data).join('');
-    // SAFETY: frames are produced by `frameSnapshot` from a serialized RunState.
-    const state = JSON.parse(json);
+    const state = Schema.decodeSync(Schema.fromJsonString(RunStateSchema))(json);
     return {
         runId: first.runId,
         cursor: first.cursor,
@@ -84,74 +92,56 @@ function createClient(config) {
 }
 function parseFrame(body) {
     try {
-        return Schema.decodeUnknownSync(SnapshotFrameSchema)(JSON.parse(body));
+        return decodeFrame(body);
     }
     catch {
         return undefined;
     }
 }
+function toSnapshotError(cause, message) {
+    return cause instanceof SnapshotStoreError
+        ? cause
+        : new SnapshotStoreError(`${message}: ${cause instanceof Error ? cause.message : String(cause)}`, cause);
+}
 /**
  * Trailing command records (the trim written by `prune`) are skipped by
  * `ignoreCommandRecords`, so a `tailOffset: 1` read can come back empty.
  * Probe a few records from the tail instead.
  */
 const TAIL_PROBE = 4;
-async function readFromTail(stream, tailOffset) {
-    try {
-        const page = await stream.read({
+function readFromTail(stream, tailOffset) {
+    return Effect.tryPromise({
+        try: () => stream.read({
             start: { from: { tailOffset } },
             ignoreCommandRecords: true,
-        });
-        return [...page.records];
-    }
-    catch (cause) {
-        if (cause instanceof S2Error && (cause.status === 404 || cause.status === 416)) {
-            return null;
-        }
-        throw cause;
-    }
+        }),
+        catch: (cause) => toSnapshotError(cause, 'Failed to read snapshot stream tail'),
+    }).pipe(Effect.map((page) => [...page.records]), Effect.catchIf((error) => error.cause instanceof S2Error &&
+        (error.cause.status === 404 || error.cause.status === 416), () => Effect.succeed([])));
 }
 /** Latest complete data frame near the tail, with its record, or undefined. */
-async function readLatestFrame(stream) {
-    const records = (await readFromTail(stream, TAIL_PROBE)) ?? [];
-    for (let i = records.length - 1; i >= 0; i--) {
-        const record = records[i];
-        const frame = parseFrame(record.body);
-        if (frame) {
-            return { record, frame };
+function readLatestFrame(stream) {
+    return readFromTail(stream, TAIL_PROBE).pipe(Effect.map((records) => {
+        for (let i = records.length - 1; i >= 0; i--) {
+            const record = records[i];
+            const frame = parseFrame(record.body);
+            if (frame) {
+                return { record, frame };
+            }
         }
-    }
-    return undefined;
+        return undefined;
+    }));
 }
-async function readAllSnapshotRecords(stream) {
-    const records = [];
-    let cursor = 0;
-    while (true) {
-        const page = await stream
-            .read({
+function readAllSnapshotRecords(stream) {
+    return paginateS2Stream((cursor, count) => Effect.tryPromise({
+        try: () => stream.read({
             start: { from: { seqNum: cursor }, clamp: true },
-            stop: { limits: { count: 1000 } },
+            stop: { limits: { count } },
             ignoreCommandRecords: true,
-        })
-            .catch((cause) => {
-            // 404: no stream yet. 416: nothing at or past `cursor` (e.g. only a trim record).
-            if (cause instanceof S2Error && (cause.status === 404 || cause.status === 416)) {
-                return null;
-            }
-            throw cause;
-        });
-        if (!page || page.records.length === 0) {
-            break;
-        }
-        records.push(...page.records);
-        const last = page.records[page.records.length - 1];
-        const next = last.seqNum + 1;
-        if (page.tail?.seqNum !== undefined && next >= page.tail.seqNum) {
-            break;
-        }
-        cursor = next;
-    }
-    return records;
+        }),
+        catch: (cause) => toSnapshotError(cause, 'Failed to scan snapshot stream'),
+    }).pipe(Effect.catchIf((error) => error.cause instanceof S2Error &&
+        (error.cause.status === 404 || error.cause.status === 416), () => Effect.succeed(null)))).pipe(Stream.runCollect);
 }
 function groupCompleteSnapshots(records) {
     const byCursor = new Map();
@@ -173,144 +163,181 @@ function groupCompleteSnapshots(records) {
             continue;
         }
     }
+    // `assembled` is function-local, so in-place sorting cannot mutate caller-owned data.
+    // oxlint-disable-next-line unicorn/no-array-sort
     return assembled.sort((a, b) => a.cursor - b.cursor);
 }
 export function s2SnapshotStore(config) {
-    const parsed = Schema.decodeUnknownSync(S2ConfigSchema)(config);
+    const parsed = Schema.decodeSync(S2ConfigSchema)(config);
     const client = createClient(parsed);
     const basin = client.basin(parsed.basin);
     const prefix = parsed.snapshotPrefix ?? DEFAULT_PREFIX;
     const appends = createKeyedSerializer();
     const ensured = new Set();
+    const headers = new Map();
     let basinEnsured = false;
+    let headersLoaded = false;
     const streamName = (runId) => `${prefix}/${runId}`;
-    const ensureBasin = async () => {
+    const ensureBasin = Effect.suspend(() => {
         if (basinEnsured) {
-            return;
+            return Effect.void;
         }
-        try {
-            await client.basins.create({ basin: parsed.basin });
-        }
-        catch (err) {
-            if (!(err instanceof S2Error && err.status === 409)) {
-                throw err;
-            }
-        }
-        basinEnsured = true;
-    };
-    const ensureStream = async (runId) => {
-        await ensureBasin();
+        return Effect.tryPromise({
+            try: () => client.basins.create({ basin: parsed.basin }),
+            catch: (cause) => toSnapshotError(cause, `Failed to ensure basin ${parsed.basin}`),
+        }).pipe(Effect.catchIf((error) => error.cause instanceof S2Error && error.cause.status === 409, () => Effect.void), Effect.tap(() => Effect.sync(() => {
+            basinEnsured = true;
+        })), Effect.asVoid);
+    });
+    const ensureStream = (runId) => Effect.gen(function* () {
+        yield* ensureBasin;
         const name = streamName(runId);
         if (!ensured.has(name)) {
-            try {
-                await basin.streams.create({
+            yield* Effect.tryPromise({
+                try: () => basin.streams.create({
                     stream: name,
                     config: { retentionPolicy: { infinite: {} } },
-                });
-            }
-            catch (err) {
-                if (!(err instanceof S2Error &&
-                    (err.status === 409 || err.status === 404 || err.status === 405))) {
-                    throw err;
-                }
-            }
+                }),
+                catch: (cause) => toSnapshotError(cause, `Failed to ensure stream ${name}`),
+            }).pipe(Effect.catchIf((error) => error.cause instanceof S2Error &&
+                (error.cause.status === 409 ||
+                    error.cause.status === 404 ||
+                    error.cause.status === 405), () => Effect.void));
             ensured.add(name);
         }
         return basin.stream(name);
-    };
-    const toError = (cause, message) => cause instanceof SnapshotStoreError
-        ? cause
-        : new SnapshotStoreError(`${message}: ${cause instanceof Error ? cause.message : String(cause)}`, cause);
+    });
     const service = {
-        save: (snapshot) => Effect.tryPromise({
-            try: async () => {
-                const stream = await ensureStream(snapshot.runId);
-                const frames = frameSnapshot(snapshot);
-                const records = frames.map((frame) => AppendRecord.string({
-                    body: JSON.stringify(frame),
-                    headers: [
-                        ['looms-cursor', String(frame.cursor)],
-                        ['looms-chunk', `${frame.chunk}/${frame.chunks}`],
-                    ],
-                }));
-                await appends.run(snapshot.runId, async () => {
+        save: (snapshot) => Effect.gen(function* () {
+            yield* Effect.try({
+                try: () => assertSnapshotByteLimit(snapshot),
+                catch: (cause) => cause instanceof SnapshotPayloadTooLargeError
+                    ? cause
+                    : new SnapshotStoreError('snapshot size validation failed', cause),
+            });
+            const stream = yield* ensureStream(snapshot.runId);
+            const frames = frameSnapshot(snapshot);
+            const records = frames.map((frame) => AppendRecord.string({
+                body: encodeFrame(frame),
+                headers: [
+                    ['looms-cursor', String(frame.cursor)],
+                    ['looms-chunk', `${frame.chunk}/${frame.chunks}`],
+                ],
+            }));
+            yield* Effect.tryPromise({
+                try: () => appends.run(snapshot.runId, () => {
                     if (records.length === 1) {
-                        await stream.append(AppendInput.create(records));
-                        return;
+                        return stream.append(AppendInput.create(records)).then(() => undefined);
                     }
+                    let pending = Promise.resolve();
                     for (const record of records) {
-                        await stream.append(AppendInput.create([record]));
+                        pending = pending.then(() => stream.append(AppendInput.create([record])).then(() => undefined));
                     }
-                });
-            },
-            catch: (cause) => toError(cause, `Failed to save snapshot for ${snapshot.runId}`),
+                    return pending;
+                }),
+                catch: (cause) => toSnapshotError(cause, `Failed to save snapshot for ${snapshot.runId}`),
+            });
         }),
-        loadLatest: (runId) => Effect.tryPromise({
-            try: async () => {
-                const stream = await ensureStream(runId);
-                const tailFrame = await readLatestFrame(stream);
-                if (!tailFrame) {
-                    return Option.none();
-                }
-                if (tailFrame.frame.chunk === tailFrame.frame.chunks - 1) {
-                    const lastFrame = tailFrame.frame;
-                    const records = (await readFromTail(stream, lastFrame.chunks + TAIL_PROBE)) ?? [];
-                    const frames = records
-                        .map((record) => parseFrame(record.body))
-                        .filter((frame) => frame !== undefined)
-                        .filter((frame) => frame.cursor === lastFrame.cursor);
-                    if (frames.length === lastFrame.chunks) {
-                        return Option.some(assembleSnapshot(frames));
-                    }
+        loadLatest: (runId) => Effect.gen(function* () {
+            const stream = yield* ensureStream(runId);
+            const tailFrame = yield* readLatestFrame(stream);
+            if (!tailFrame) {
+                return Option.none();
+            }
+            if (tailFrame.frame.chunk === tailFrame.frame.chunks - 1) {
+                const lastFrame = tailFrame.frame;
+                const records = (yield* readFromTail(stream, lastFrame.chunks + TAIL_PROBE)) ?? [];
+                const frames = records
+                    .map((record) => parseFrame(record.body))
+                    .filter((frame) => frame !== undefined)
+                    .filter((frame) => frame.cursor === lastFrame.cursor);
+                if (frames.length === lastFrame.chunks) {
+                    return Option.some(assembleSnapshot(frames));
                 }
-                // Incomplete or interleaved tail: fall back to scanning the retained stream.
-                const records = await readAllSnapshotRecords(stream);
-                const complete = groupCompleteSnapshots(records);
-                const newest = complete[complete.length - 1];
-                return newest ? Option.some(newest) : Option.none();
-            },
-            catch: (cause) => toError(cause, `Failed to load snapshot for ${runId}`),
+            }
+            // Incomplete or interleaved tail: fall back to scanning the retained stream.
+            const records = yield* readAllSnapshotRecords(stream);
+            const complete = groupCompleteSnapshots(records);
+            const newest = complete[complete.length - 1];
+            return newest ? Option.some(newest) : Option.none();
         }),
-        listCursors: (runId) => Effect.tryPromise({
-            try: async () => {
-                const stream = await ensureStream(runId);
-                const records = await readAllSnapshotRecords(stream);
-                return groupCompleteSnapshots(records).map((snapshot) => snapshot.cursor);
-            },
-            catch: (cause) => toError(cause, `Failed to list snapshot cursors for ${runId}`),
+        listCursors: (runId) => Effect.gen(function* () {
+            const stream = yield* ensureStream(runId);
+            const records = yield* readAllSnapshotRecords(stream);
+            return groupCompleteSnapshots(records).map((snapshot) => snapshot.cursor);
         }),
-        prune: (runId, keepLatest) => Effect.tryPromise({
-            try: async () => {
-                const stream = await ensureStream(runId);
-                if (keepLatest <= 1) {
-                    // Fast path: the latest complete frame tells us where the latest snapshot starts.
-                    const latest = await readLatestFrame(stream);
-                    if (!latest || latest.frame.chunk !== latest.frame.chunks - 1) {
-                        return;
-                    }
-                    const start = latest.record.seqNum - latest.frame.chunk;
-                    if (start <= 0) {
-                        return;
-                    }
-                    await appends.run(runId, () => stream.append(AppendInput.create([AppendRecord.trim(start)])));
+        prune: (runId, keepLatest) => Effect.gen(function* () {
+            const stream = yield* ensureStream(runId);
+            if (keepLatest <= 1) {
+                // Fast path: the latest complete frame tells us where the latest snapshot starts.
+                const latest = yield* readLatestFrame(stream);
+                if (!latest || latest.frame.chunk !== latest.frame.chunks - 1) {
                     return;
                 }
-                const records = await readAllSnapshotRecords(stream);
-                const complete = groupCompleteSnapshots(records);
-                if (complete.length <= keepLatest) {
+                const start = latest.record.seqNum - latest.frame.chunk;
+                if (start <= 0) {
                     return;
                 }
-                const keep = complete[complete.length - keepLatest];
-                if (!keep) {
-                    return;
+                yield* Effect.tryPromise({
+                    try: () => appends.run(runId, () => stream.append(AppendInput.create([AppendRecord.trim(start)]))),
+                    catch: (cause) => toSnapshotError(cause, `Failed to prune snapshots for ${runId}`),
+                });
+                return;
+            }
+            const records = yield* readAllSnapshotRecords(stream);
+            const complete = groupCompleteSnapshots(records);
+            if (complete.length <= keepLatest) {
+                return;
+            }
+            const keep = complete[complete.length - keepLatest];
+            if (!keep) {
+                return;
+            }
+            const firstKept = records.find((record) => parseFrame(record.body)?.cursor === keep.cursor);
+            if (!firstKept) {
+                return;
+            }
+            yield* Effect.tryPromise({
+                try: () => appends.run(runId, () => stream.append(AppendInput.create([AppendRecord.trim(firstKept.seqNum)]))),
+                catch: (cause) => toSnapshotError(cause, `Failed to prune snapshots for ${runId}`),
+            });
+        }),
+        saveHeader: (header) => Effect.gen(function* () {
+            const stream = yield* ensureStream('__headers');
+            yield* Effect.tryPromise({
+                try: () => appends.run('__headers', () => stream.append(AppendInput.create([AppendRecord.string({ body: encodeHeader(header) })]))),
+                catch: (cause) => toSnapshotError(cause, `Failed to save run header for ${header.runId}`),
+            });
+            headers.set(header.runId, header);
+        }),
+        listHeaders: Effect.gen(function* () {
+            if (headersLoaded) {
+                return [...headers.values()];
+            }
+            const stream = yield* ensureStream('__headers');
+            const records = yield* paginateS2Stream((cursor, count) => Effect.tryPromise({
+                try: () => stream.read({
+                    start: { from: { seqNum: cursor }, clamp: true },
+                    stop: { limits: { count } },
+                    ignoreCommandRecords: true,
+                }),
+                catch: (cause) => toSnapshotError(cause, 'Failed to read run headers'),
+            }).pipe(Effect.catchIf((error) => error.cause instanceof S2Error &&
+                (error.cause.status === 404 || error.cause.status === 416), () => Effect.succeed(null)))).pipe(Stream.runCollect);
+            const loaded = new Map();
+            for (const record of records) {
+                const decoded = decodeHeader(record.body);
+                if (Option.isSome(decoded)) {
+                    loaded.set(decoded.value.runId, decoded.value);
                 }
-                const firstKept = records.find((record) => parseFrame(record.body)?.cursor === keep.cursor);
-                if (!firstKept) {
-                    return;
+            }
+            for (const [runId, header] of loaded) {
+                if (!headers.has(runId)) {
+                    headers.set(runId, header);
                 }
-                await appends.run(runId, () => stream.append(AppendInput.create([AppendRecord.trim(firstKept.seqNum)])));
-            },
-            catch: (cause) => toError(cause, `Failed to prune snapshots for ${runId}`),
+            }
+            headersLoaded = true;
+            return [...headers.values()];
         }),
     };
     return service;

package/dist/s2/store.d.ts

@@ -1,26 +1,8 @@
-import { Effect, Layer, Stream } from 'effect';
+import { Effect, Layer } from 'effect';
 import { EventStoreError, EventStoreTag, type EventStore } from '@swirls/looms/core';
 import { type S2Config } from './config';
-export interface S2ReadBatchLike<T> {
-    readonly records: ReadonlyArray<T>;
-    readonly tail?: {
-        readonly seqNum: number;
-    } | null;
-}
-export interface ReadAllPagesOptions {
-    readonly fromSeq?: number;
-    readonly limit?: number;
-    readonly maxPageSize?: number;
-}
-/**
- * Paginate an S2 stream lazily using Effect Streams.
- *
- * Pulls pages on demand until the stream tail is reached, an empty page is returned,
- * or the caller's optional limit is satisfied.
- */
-export declare function paginateS2Stream<T extends {
-    readonly seqNum: number;
-}>(fetchPage: (cursor: number, count: number) => Effect.Effect<S2ReadBatchLike<T> | null, EventStoreError>, options?: ReadAllPagesOptions): Stream.Stream<T, EventStoreError>;
+import { paginateS2Stream, type ReadAllPagesOptions, type S2ReadBatchLike } from './pagination';
+export { paginateS2Stream, type ReadAllPagesOptions, type S2ReadBatchLike };
 export declare function readAllPages<T extends {
     readonly seqNum: number;
 }>(fetchPage: (cursor: number, count: number) => Promise<S2ReadBatchLike<T> | null>, options?: ReadAllPagesOptions): Promise<T[]>;