@swirls/looms 0.4.0 → 0.5.1

package/dist/core/blob-store.js

@@ -0,0 +1,164 @@
+import { Data, Effect, Schema } from 'effect';
+const SHA256 = /^[a-f0-9]{64}$/;
+const decodeDigestList = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.String)));
+export class BlobStoreError extends Data.TaggedError('BlobStoreError') {
+    cause;
+    constructor(message, cause) {
+        super({ message, cause });
+        this.name = 'BlobStoreError';
+        this.cause = cause;
+    }
+}
+export class BlobNotFoundError extends Data.TaggedError('BlobNotFoundError') {
+    constructor(digest) {
+        super({ digest, message: `blob ${digest} was not found` });
+        this.name = 'BlobNotFoundError';
+    }
+}
+export function isSha256Digest(value) {
+    return SHA256.test(value);
+}
+const EMPTY = new Uint8Array(0);
+export function memoryBlobStore() {
+    return blobStoreFromBytes(memoryBlobBytes());
+}
+export function blobStoreFromBytes(bytes) {
+    return {
+        put: (digest, body) => Effect.gen(function* () {
+            yield* assertDigest(digest);
+            const actual = yield* sha256Hex(body);
+            if (actual !== digest) {
+                return yield* new BlobStoreError(`blob body does not match digest ${digest}`);
+            }
+            const existing = yield* bytes.read(objectKey(digest));
+            if (existing) {
+                return yield* Effect.void;
+            }
+            return yield* bytes.write(objectKey(digest), body);
+        }),
+        get: (digest) => Effect.gen(function* () {
+            yield* assertDigest(digest);
+            const body = yield* bytes.read(objectKey(digest));
+            if (!body) {
+                return yield* new BlobNotFoundError(digest);
+            }
+            const actual = yield* sha256Hex(body);
+            if (actual !== digest) {
+                return yield* new BlobStoreError(`blob ${digest} failed its integrity check`);
+            }
+            return body;
+        }),
+        // Markers go down before the existence check, and `collect` lists markers again after
+        // deleting. Whichever side runs second sees the other, so a referenced body is never lost:
+        // either `collect` restores it or this append fails and its retry uploads the body again.
+        retain: (runId, digests) => Effect.gen(function* () {
+            const current = new Set(yield* readRunDigests(bytes, runId));
+            const added = [...new Set(digests)].filter((digest) => !current.has(digest));
+            if (added.length === 0) {
+                return yield* Effect.void;
+            }
+            yield* Effect.forEach(added, assertDigest, { discard: true });
+            yield* Effect.forEach(added, (digest) => bytes.write(refKey(digest, runId), EMPTY), {
+                concurrency: REF_CONCURRENCY,
+                discard: true,
+            });
+            yield* writeRunDigests(bytes, runId, [...current, ...added]);
+            const present = yield* Effect.forEach(added, (digest) => bytes.exists(objectKey(digest)), {
+                concurrency: REF_CONCURRENCY,
+            });
+            const missing = added.filter((_, index) => !present[index]);
+            if (missing.length > 0) {
+                return yield* new BlobStoreError(`blobs ${missing.join(', ')} were deleted before run ${runId} referenced them; retry the append`);
+            }
+            return yield* Effect.void;
+        }),
+        replaceRefs: (runId, keep) => Effect.gen(function* () {
+            const previous = yield* readRunDigests(bytes, runId);
+            const dropped = previous.filter((digest) => !keep.has(digest));
+            if (dropped.length === 0) {
+                return;
+            }
+            yield* Effect.forEach(dropped, (digest) => bytes.remove(refKey(digest, runId)).pipe(Effect.andThen(collect(bytes, digest))), { concurrency: REF_CONCURRENCY, discard: true });
+            yield* writeRunDigests(bytes, runId, previous.filter((digest) => keep.has(digest)));
+        }),
+    };
+}
+const REF_CONCURRENCY = 8;
+/** Deletes a body no run references, and puts it back if a run referenced it meanwhile. */
+function collect(bytes, digest) {
+    const referenced = bytes.list(refPrefix(digest)).pipe(Effect.map((keys) => keys.length > 0));
+    return Effect.gen(function* () {
+        if (yield* referenced) {
+            return;
+        }
+        const body = yield* bytes.read(objectKey(digest));
+        if (!body) {
+            return;
+        }
+        yield* bytes.remove(objectKey(digest));
+        if (yield* referenced) {
+            yield* bytes.write(objectKey(digest), body);
+        }
+    });
+}
+function memoryBlobBytes() {
+    const objects = new Map();
+    return {
+        read: (key) => Effect.sync(() => {
+            const body = objects.get(key);
+            return body ? body.slice() : null;
+        }),
+        write: (key, body) => Effect.sync(() => {
+            objects.set(key, body.slice());
+        }),
+        remove: (key) => Effect.sync(() => {
+            objects.delete(key);
+        }),
+        exists: (key) => Effect.sync(() => objects.has(key)),
+        list: (prefix) => Effect.sync(() => [...objects.keys()].filter((key) => key.startsWith(prefix))),
+    };
+}
+function objectKey(digest) {
+    return `objects/${digest}`;
+}
+function runKey(runId) {
+    return `runs/${encodeURIComponent(runId)}`;
+}
+function refPrefix(digest) {
+    return `refs/${digest}/`;
+}
+function refKey(digest, runId) {
+    return `${refPrefix(digest)}${encodeURIComponent(runId)}`;
+}
+function assertDigest(digest) {
+    if (isSha256Digest(digest)) {
+        return Effect.void;
+    }
+    return Effect.fail(new BlobStoreError(`invalid blob digest ${digest}`));
+}
+export function sha256Hex(body) {
+    const copy = new Uint8Array(body.byteLength);
+    copy.set(body);
+    return Effect.tryPromise({
+        try: () => crypto.subtle.digest('SHA-256', copy),
+        catch: (cause) => new BlobStoreError('SHA-256 failed', cause),
+    }).pipe(Effect.map((buffer) => hex(new Uint8Array(buffer))));
+}
+function hex(bytes) {
+    let encoded = '';
+    for (const byte of bytes) {
+        encoded += byte.toString(16).padStart(2, '0');
+    }
+    return encoded;
+}
+function readRunDigests(bytes, runId) {
+    return bytes.read(runKey(runId)).pipe(Effect.flatMap((body) => {
+        if (!body) {
+            return Effect.succeed([]);
+        }
+        return decodeDigestList(new TextDecoder().decode(body)).pipe(Effect.mapError((cause) => new BlobStoreError(`blob references for ${runId} are unreadable`, cause)));
+    }));
+}
+function writeRunDigests(bytes, runId, digests) {
+    return bytes.write(runKey(runId), new TextEncoder().encode(JSON.stringify(digests)));
+}

package/dist/core/blobs.d.ts

@@ -0,0 +1,37 @@
+import { Effect } from 'effect';
+import { type BlobStore, type BlobStoreFailure } from './blob-store';
+import { type JsonValue } from './types';
+export declare const DEFAULT_BLOB_THRESHOLD_BYTES: number;
+declare const BLOB_KIND = "blob";
+declare const BLOB_MEDIA_TYPE = "application/json";
+/**
+ * Reserved stored shape. A JSON object with exactly these fields is a claim check,
+ * not caller data. `bytes` is the UTF-8 length of the JSON body named by `sha256`.
+ */
+export interface BlobRef {
+    readonly $looms: typeof BLOB_KIND;
+    readonly sha256: string;
+    readonly bytes: number;
+    readonly mediaType: typeof BLOB_MEDIA_TYPE;
+    readonly [key: string]: JsonValue;
+}
+export declare function isBlobRef(value: JsonValue): value is BlobRef;
+export interface OffloadJsonOptions {
+    readonly thresholdBytes?: number;
+    /** When false, the root value stays inline even if it is over the threshold. Children are still claimed. */
+    readonly claimRoot?: boolean;
+}
+/** Store `value` and return a reference, even when it is under the offload threshold. */
+export declare function claimJson(value: JsonValue, blobs: BlobStore): Effect.Effect<BlobRef, BlobStoreFailure>;
+/** Replace values larger than `thresholdBytes` with {@link BlobRef}s. Bodies are stored by hash. */
+export declare function offloadJson(value: JsonValue, blobs: BlobStore, options?: OffloadJsonOptions): Effect.Effect<JsonValue, BlobStoreFailure>;
+/** Replace {@link BlobRef}s with the JSON they name, including references nested inside a body. */
+export declare function resolveJson(value: JsonValue, blobs: BlobStore): Effect.Effect<JsonValue, BlobStoreFailure>;
+/**
+ * Digests named by `value`, plus digests nested inside those bodies.
+ * A parent blob that itself contains references still keeps those bodies alive.
+ */
+export declare function expandBlobDigests(blobs: BlobStore, seeds: ReadonlySet<string>): Effect.Effect<ReadonlySet<string>, BlobStoreFailure>;
+/** Record every blob digest embedded in `value`. Does not read blob bodies. */
+export declare function collectBlobDigests(value: JsonValue, into: Set<string>): void;
+export {};

package/dist/core/blobs.js

@@ -0,0 +1,160 @@
+import { Effect, Predicate, Schema } from 'effect';
+import { BlobStoreError, isSha256Digest, sha256Hex, } from './blob-store.js';
+import { utf8JsonBytes } from './limits.js';
+import { isJsonObject } from './types.js';
+export const DEFAULT_BLOB_THRESHOLD_BYTES = 64 * 1024;
+const BLOB_KIND = 'blob';
+const BLOB_MEDIA_TYPE = 'application/json';
+const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json));
+export function isBlobRef(value) {
+    if (!isJsonObject(value) || Object.keys(value).length !== 4) {
+        return false;
+    }
+    const kind = value.$looms;
+    const sha256 = value.sha256;
+    const bytes = value.bytes;
+    const mediaType = value.mediaType;
+    return (kind === BLOB_KIND &&
+        Predicate.isString(sha256) &&
+        isSha256Digest(sha256) &&
+        Predicate.isNumber(bytes) &&
+        Number.isInteger(bytes) &&
+        bytes >= 0 &&
+        mediaType === BLOB_MEDIA_TYPE);
+}
+/** Store `value` and return a reference, even when it is under the offload threshold. */
+export function claimJson(value, blobs) {
+    return putJson(value, blobs);
+}
+/** Replace values larger than `thresholdBytes` with {@link BlobRef}s. Bodies are stored by hash. */
+export function offloadJson(value, blobs, options) {
+    const thresholdBytes = options?.thresholdBytes ?? DEFAULT_BLOB_THRESHOLD_BYTES;
+    return offloadNode(value, blobs, thresholdBytes, options?.claimRoot ?? true);
+}
+/** Replace {@link BlobRef}s with the JSON they name, including references nested inside a body. */
+export function resolveJson(value, blobs) {
+    return resolveNode(value, blobs, new Set());
+}
+/**
+ * Digests named by `value`, plus digests nested inside those bodies.
+ * A parent blob that itself contains references still keeps those bodies alive.
+ */
+export function expandBlobDigests(blobs, seeds) {
+    if (seeds.size === 0) {
+        return Effect.succeed(new Set());
+    }
+    return Effect.gen(function* () {
+        const all = new Set();
+        const pending = [...seeds];
+        while (pending.length > 0) {
+            const digest = pending.pop();
+            if (digest === undefined || all.has(digest)) {
+                continue;
+            }
+            all.add(digest);
+            const body = yield* blobs.get(digest);
+            const json = yield* decodeBody(body);
+            const nested = new Set();
+            collectBlobDigests(json, nested);
+            for (const child of nested) {
+                if (!all.has(child)) {
+                    pending.push(child);
+                }
+            }
+        }
+        return all;
+    });
+}
+/** Record every blob digest embedded in `value`. Does not read blob bodies. */
+export function collectBlobDigests(value, into) {
+    if (isBlobRef(value)) {
+        into.add(value.sha256);
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            collectBlobDigests(item, into);
+        }
+        return;
+    }
+    if (isJsonObject(value)) {
+        for (const child of Object.values(value)) {
+            collectBlobDigests(child, into);
+        }
+    }
+}
+function offloadNode(value, blobs, thresholdBytes, claim) {
+    if (isBlobRef(value)) {
+        return Effect.succeed(value);
+    }
+    if (Array.isArray(value)) {
+        return Effect.forEach(value, (item) => offloadNode(item, blobs, thresholdBytes, true)).pipe(Effect.flatMap((items) => storeIfOver(items, blobs, thresholdBytes, claim)));
+    }
+    if (isJsonObject(value)) {
+        return offloadObject(value, blobs, thresholdBytes, claim);
+    }
+    return storeIfOver(value, blobs, thresholdBytes, claim);
+}
+function offloadObject(value, blobs, thresholdBytes, claim) {
+    return Effect.gen(function* () {
+        const next = {};
+        for (const key of Object.keys(value)) {
+            const child = value[key];
+            if (child === undefined) {
+                continue;
+            }
+            next[key] = yield* offloadNode(child, blobs, thresholdBytes, true);
+        }
+        return yield* storeIfOver(next, blobs, thresholdBytes, claim);
+    });
+}
+function storeIfOver(value, blobs, thresholdBytes, claim) {
+    // Strings only. Objects and arrays stay in place so schema-shaped snapshot state still decodes.
+    // An event that is still over the byte limit claims its whole payload afterward.
+    if (!claim || !Predicate.isString(value) || utf8JsonBytes(value) <= thresholdBytes) {
+        return Effect.succeed(value);
+    }
+    return putJson(value, blobs);
+}
+function putJson(value, blobs) {
+    const body = new TextEncoder().encode(JSON.stringify(value));
+    return sha256Hex(body).pipe(Effect.flatMap((sha256) => blobs.put(sha256, body).pipe(Effect.map(() => ({
+        $looms: BLOB_KIND,
+        sha256,
+        bytes: body.byteLength,
+        mediaType: BLOB_MEDIA_TYPE,
+    })))));
+}
+function resolveNode(value, blobs, seen) {
+    if (isBlobRef(value)) {
+        if (seen.has(value.sha256)) {
+            return Effect.fail(new BlobStoreError(`blob ${value.sha256} refers to itself`));
+        }
+        const next = new Set(seen);
+        next.add(value.sha256);
+        return blobs.get(value.sha256).pipe(Effect.flatMap((body) => decodeBody(body)), Effect.flatMap((decoded) => resolveNode(decoded, blobs, next)));
+    }
+    if (Array.isArray(value)) {
+        return Effect.forEach(value, (item) => resolveNode(item, blobs, seen));
+    }
+    if (isJsonObject(value)) {
+        return resolveObject(value, blobs, seen);
+    }
+    return Effect.succeed(value);
+}
+function resolveObject(value, blobs, seen) {
+    return Effect.gen(function* () {
+        const next = {};
+        for (const key of Object.keys(value)) {
+            const child = value[key];
+            if (child === undefined) {
+                continue;
+            }
+            next[key] = yield* resolveNode(child, blobs, seen);
+        }
+        return next;
+    });
+}
+function decodeBody(body) {
+    return decodeJsonText(new TextDecoder().decode(body)).pipe(Effect.mapError((cause) => new BlobStoreError('blob body is not JSON', cause)));
+}

package/dist/core/definition-resolver.d.ts

@@ -0,0 +1,40 @@
+import { Effect } from 'effect';
+import type { VersionedDefinition } from './definition-store';
+import type { EventEnvelope } from './envelope';
+declare const DefinitionResolveError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "DefinitionResolveError";
+} & Readonly<A>;
+export declare class DefinitionResolveError extends DefinitionResolveError_base<{
+    readonly kind: string;
+    readonly name: string;
+    readonly version: string;
+    readonly message: string;
+}> {
+    constructor(kind: string, name: string, version: string, message: string);
+}
+/**
+ * Loads one definition by the identity pinned on a run.
+ * `peek` is synchronous and only returns a definition this resolver already holds.
+ * `resolve` may fetch, and it caches through {@link cachedDefinitionResolver}.
+ */
+export interface DefinitionResolver {
+    readonly peek: (kind: string, name: string, version: string) => VersionedDefinition | undefined;
+    readonly resolve: (kind: string, name: string, version: string) => Effect.Effect<VersionedDefinition | undefined, DefinitionResolveError>;
+}
+export declare const DEFAULT_DEFINITION_CACHE_CAPACITY = 256;
+/** Definitions registered on a module. A miss is `undefined`, not an error. */
+export declare function staticDefinitionResolver(definitions: ReadonlyArray<VersionedDefinition>): DefinitionResolver;
+/** First resolver that returns a definition wins. Later resolvers are not called. */
+export declare function chainDefinitionResolvers(resolvers: ReadonlyArray<DefinitionResolver>): DefinitionResolver;
+/**
+ * Remembers successful lookups and misses. A miss stays cached so recovery does not
+ * fetch a deleted definition on every wake. Failures are not cached.
+ */
+export declare function cachedDefinitionResolver(inner: DefinitionResolver, options?: {
+    readonly capacity?: number;
+}): DefinitionResolver;
+/** Identities pinned by `runtime.run.started` and `runtime.thread.started`. */
+export declare function pinnedDefinitionRefs(events: readonly EventEnvelope[]): readonly VersionedDefinition[];
+/** Resolves every pinned identity so a later synchronous lookup can see it. */
+export declare function primePinnedDefinitions(resolver: DefinitionResolver, events: readonly EventEnvelope[]): Effect.Effect<void, DefinitionResolveError>;
+export {};

package/dist/core/definition-resolver.js

@@ -0,0 +1,129 @@
+import { Data, Effect, Predicate } from 'effect';
+import { DEFAULT_DEFINITION_VERSION, definitionKey } from './module.js';
+export class DefinitionResolveError extends Data.TaggedError('DefinitionResolveError') {
+    constructor(kind, name, version, message) {
+        super({ kind, name, version, message });
+        this.name = 'DefinitionResolveError';
+    }
+}
+export const DEFAULT_DEFINITION_CACHE_CAPACITY = 256;
+/** Definitions registered on a module. A miss is `undefined`, not an error. */
+export function staticDefinitionResolver(definitions) {
+    const byKey = new Map(definitions.map((definition) => [definitionKey(definition.kind, definition.name, definition.version), definition]));
+    const lookup = (kind, name, version) => byKey.get(definitionKey(kind, name, version));
+    return {
+        peek: lookup,
+        resolve: (kind, name, version) => Effect.succeed(lookup(kind, name, version)),
+    };
+}
+/** First resolver that returns a definition wins. Later resolvers are not called. */
+export function chainDefinitionResolvers(resolvers) {
+    return {
+        peek(kind, name, version) {
+            for (const resolver of resolvers) {
+                const found = resolver.peek(kind, name, version);
+                if (found) {
+                    return found;
+                }
+            }
+            return undefined;
+        },
+        resolve(kind, name, version) {
+            return Effect.gen(function* () {
+                for (const resolver of resolvers) {
+                    const found = yield* resolver.resolve(kind, name, version);
+                    if (found) {
+                        return found;
+                    }
+                }
+                return undefined;
+            });
+        },
+    };
+}
+/**
+ * Remembers successful lookups and misses. A miss stays cached so recovery does not
+ * fetch a deleted definition on every wake. Failures are not cached.
+ */
+export function cachedDefinitionResolver(inner, options) {
+    const capacity = options?.capacity ?? DEFAULT_DEFINITION_CACHE_CAPACITY;
+    const entries = new Map();
+    const remember = (key, value) => {
+        entries.delete(key);
+        entries.set(key, value);
+        while (entries.size > capacity) {
+            const oldest = entries.keys().next().value;
+            if (oldest === undefined) {
+                return;
+            }
+            entries.delete(oldest);
+        }
+    };
+    return {
+        peek(kind, name, version) {
+            const key = definitionKey(kind, name, version);
+            if (!entries.has(key)) {
+                return inner.peek(kind, name, version);
+            }
+            const value = entries.get(key);
+            remember(key, value);
+            return value;
+        },
+        resolve(kind, name, version) {
+            const key = definitionKey(kind, name, version);
+            if (entries.has(key)) {
+                const value = entries.get(key);
+                remember(key, value);
+                return Effect.succeed(value);
+            }
+            return inner.resolve(kind, name, version).pipe(Effect.tap((value) => Effect.sync(() => {
+                remember(key, value);
+            })));
+        },
+    };
+}
+/** Identities pinned by `runtime.run.started` and `runtime.thread.started`. */
+export function pinnedDefinitionRefs(events) {
+    const refs = [];
+    const seen = new Set();
+    for (const event of events) {
+        if (event.type !== 'runtime.run.started' && event.type !== 'runtime.thread.started') {
+            continue;
+        }
+        const pinned = readPinned(event.payload);
+        if (!pinned) {
+            continue;
+        }
+        const key = definitionKey(pinned.kind, pinned.name, pinned.version);
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        refs.push(pinned);
+    }
+    return refs;
+}
+/** Resolves every pinned identity so a later synchronous lookup can see it. */
+export function primePinnedDefinitions(resolver, events) {
+    return Effect.gen(function* () {
+        for (const ref of pinnedDefinitionRefs(events)) {
+            yield* resolver.resolve(ref.kind, ref.name, ref.version);
+        }
+    });
+}
+function readPinned(payload) {
+    if (!Predicate.isObject(payload) || !('kind' in payload) || !('definitionName' in payload)) {
+        return undefined;
+    }
+    const kind = payload.kind;
+    const name = payload.definitionName;
+    const version = 'definitionVersion' in payload ? payload.definitionVersion : undefined;
+    if (!Predicate.isString(kind) || !Predicate.isString(name)) {
+        return undefined;
+    }
+    return {
+        kind,
+        name,
+        version: Predicate.isString(version) ? version : DEFAULT_DEFINITION_VERSION,
+    };
+}

package/dist/core/definition-store.d.ts

@@ -1,3 +1,5 @@
+import { Effect } from 'effect';
+import type { DefinitionResolveError, DefinitionResolver } from './definition-resolver';
 export interface VersionedDefinition {
     readonly kind: string;
     readonly name: string;
@@ -6,6 +8,10 @@ export interface VersionedDefinition {
 /** Lookup of one definition kind by name and version. */
 export interface DefinitionStore<TDefinition> {
     get(name: string, version: string): TDefinition | undefined;
+    /** Static definition, or a definition the bound resolver has already loaded. */
+    resolve(name: string, version: string): Effect.Effect<TDefinition | undefined, DefinitionResolveError>;
+    /** Called by the runtime before the module's services are built. */
+    useResolver(resolver: DefinitionResolver): void;
 }
 /** Indexes definitions of one kind by `definitionKey`. */
 export declare function makeDefinitionStore<TDefinition extends VersionedDefinition>(kind: TDefinition['kind'], definitions: ReadonlyArray<TDefinition>): DefinitionStore<TDefinition>;

package/dist/core/definition-store.js

@@ -1,6 +1,42 @@
+import { Effect } from 'effect';
 import { definitionKey } from './module.js';
 /** Indexes definitions of one kind by `definitionKey`. */
 export function makeDefinitionStore(kind, definitions) {
     const byKey = new Map(definitions.map((definition) => [definitionKey(definition.kind, definition.name, definition.version), definition]));
-    return { get: (name, version) => byKey.get(definitionKey(kind, name, version)) };
+    let resolver;
+    const get = (name, version) => {
+        const stored = byKey.get(definitionKey(kind, name, version));
+        if (stored) {
+            return stored;
+        }
+        const peeked = resolver?.peek(kind, name, version);
+        if (peeked && isStoredDefinition(kind, peeked)) {
+            return peeked;
+        }
+        return undefined;
+    };
+    return {
+        get,
+        useResolver(next) {
+            resolver = next;
+        },
+        resolve(name, version) {
+            const stored = get(name, version);
+            if (stored) {
+                return Effect.succeed(stored);
+            }
+            if (!resolver) {
+                // Effect.void is Effect<void>, which is not a missing definition.
+                // oxlint-disable-next-line effecttsgo/effect-succeed-with-void
+                return Effect.succeed(undefined);
+            }
+            return resolver
+                .resolve(kind, name, version)
+                .pipe(Effect.map((found) => found && isStoredDefinition(kind, found) ? found : undefined));
+        },
+    };
+}
+// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- the predicate returns the store's definition type
+function isStoredDefinition(kind, value) {
+    return value.kind === kind;
 }