@ewhauser/world-celld 0.5.0 → 0.7.0

package/dist/index.js

@@ -1,5 +1,5 @@
-import { A as MAX_STREAM_BATCH_BYTES, B as decodeStreamWriteResult, C as boundedIntegerOption, F as MAX_STREAM_READ_CHUNKS, H as encodeStreamWriteBatch, I as MAX_STREAM_WRITE_CHUNKS, L as STREAM_BATCH_CONTENT_TYPE, M as MAX_STREAM_ERROR_BYTES, N as MAX_STREAM_LONG_POLL_MS, O as queueDelayDeadline, P as MAX_STREAM_READ_BYTES, R as decodeStreamReadResult, W as normalizeStreamError, b as MAX_QUEUE_DELIVERY_TIMEOUT_MS, d as rpcParse, f as rpcStringify, h as createWorkflowIndex, j as MAX_STREAM_CHUNK_BYTES, k as strictIntegerSetting, m as stringify, p as parse, r as nativeQueueDelaySeconds, u as compact, w as checkedQueueTimestampAdd, x as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, y as MAX_FLEET_RPC_TIMEOUT_MS } from "./queue-protocol-CmJNMH_5.js";
-import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
+import { B as STREAM_BATCH_CONTENT_TYPE, C as MAX_QUEUE_DELIVERY_TIMEOUT_MS, D as checkedQueueTimestampAdd, E as boundedIntegerOption, F as MAX_STREAM_ERROR_BYTES, H as decodeStreamReadResult, I as MAX_STREAM_LONG_POLL_MS, J as isChunkLimitRejection, K as encodeStreamWriteBatch, L as MAX_STREAM_READ_BYTES, M as strictIntegerSetting, N as MAX_STREAM_BATCH_BYTES, P as MAX_STREAM_CHUNK_BYTES, R as MAX_STREAM_READ_CHUNKS, S as MAX_FLEET_RPC_TIMEOUT_MS, V as STREAM_CHUNK_LIMIT_HEADER, W as decodeStreamWriteResult, Y as normalizeStreamError, _ as createWorkflowIndex, a as nativeQueueDelaySeconds, f as compact, g as stringify, h as parse, j as queueDelayDeadline, m as rpcStringify, n as listRunsPage, p as rpcParse, v as runIndexMetadata, w as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, z as MAX_STREAM_WRITE_CHUNKS } from "./run-list-B__jKuE7.js";
+import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, parseQueueName } from "@workflow/world";
 import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
 import { monotonicFactory } from "ulid";
 //#region src/config.ts
@@ -132,6 +132,23 @@ const READ_ATTEMPTS = 3;
 const MAX_WRITE_RESPONSE_BYTES = 64;
 const MAX_READ_RESPONSE_BYTES = MAX_STREAM_READ_BYTES + 65536;
 const MAX_ERROR_RESPONSE_BYTES = 65536;
+/**
+* Per-request chunk limit the fleet has advertised, per transport. Absent
+* until a worker advertises one, so a client never sends an older worker more
+* than the baseline it accepts; a response without the header drops back.
+*/
+const advertisedChunkLimits = /* @__PURE__ */ new WeakMap();
+function streamChunkLimit(transport) {
+	return advertisedChunkLimits.get(transport) ?? 32;
+}
+function resetStreamChunkLimit(transport) {
+	advertisedChunkLimits.delete(transport);
+}
+function recordStreamChunkLimit(transport, response) {
+	const advertised = Number(response.headers.get(STREAM_CHUNK_LIMIT_HEADER));
+	if (Number.isSafeInteger(advertised) && advertised > 32) advertisedChunkLimits.set(transport, Math.min(advertised, 512));
+	else advertisedChunkLimits.delete(transport);
+}
 function delayMs(ms, signal) {
 	if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));
 	if (signal.aborted) return Promise.reject(aborted(signal));
@@ -202,7 +219,7 @@ function streamUrl(transport, name) {
 async function writeStreamChunks(transport, name, runId, chunks) {
 	const timeoutMs = resolveFleetTimeoutMs(transport);
 	const body = encodeStreamWriteBatch(chunks);
-	if (body.byteLength > 8389632) throw new Error("world-celld: encoded stream batch exceeds configured limit");
+	if (body.byteLength > 8391680) throw new Error("world-celld: encoded stream batch exceeds configured limit");
 	const url = new URL(streamUrl(transport, name));
 	url.searchParams.set("runId", runId);
 	const doFetch = transport.fetchImpl ?? fetch;
@@ -221,6 +238,7 @@ async function writeStreamChunks(transport, name, runId, chunks) {
 		throw new FleetTransportError(`world-celld: fleet unreachable at ${url.href}`, error);
 	}
 	if (!response.ok) throw await errorFromResponse(response);
+	recordStreamChunkLimit(transport, response);
 	try {
 		return decodeStreamWriteResult(await readBoundedResponse(response, MAX_WRITE_RESPONSE_BYTES));
 	} catch (error) {
@@ -257,13 +275,16 @@ async function readStreamChunks(transport, name, request, signal) {
 			await response.body?.cancel(aborted(signal)).catch(() => void 0);
 			throw aborted(signal);
 		}
-		if (response.ok) try {
-			return decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
-		} catch (error) {
-			if (signal?.aborted) throw aborted(signal);
-			lastError = error instanceof FleetTransportError ? error : new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
-			if (attempt < READ_ATTEMPTS) continue;
-			throw lastError;
+		if (response.ok) {
+			recordStreamChunkLimit(transport, response);
+			try {
+				return decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
+			} catch (error) {
+				if (signal?.aborted) throw aborted(signal);
+				lastError = error instanceof FleetTransportError ? error : new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
+				if (attempt < READ_ATTEMPTS) continue;
+				throw lastError;
+			}
 		}
 		let error;
 		try {
@@ -276,6 +297,12 @@ async function readStreamChunks(transport, name, request, signal) {
 			lastError = error;
 			continue;
 		}
+		if (request.maxChunks > 32 && isChunkLimitRejection(error) && attempt < READ_ATTEMPTS) {
+			resetStreamChunkLimit(transport);
+			url.searchParams.set("maxChunks", String(32));
+			lastError = error;
+			continue;
+		}
 		throw error;
 	}
 	throw lastError instanceof Error ? lastError : new FleetTransportError("world-celld: stream read failed", lastError);
@@ -335,6 +362,8 @@ function makeStreamNamespace(transport) {
 			const control = makeStub(transport, "streams", name, STREAMS);
 			control.writeChunks = (runId, chunks) => writeStreamChunks(transport, name, runId, chunks);
 			control.readChunks = (request, signal) => readStreamChunks(transport, name, request, signal);
+			control.streamChunkLimit = () => streamChunkLimit(transport);
+			control.resetStreamChunkLimit = () => resetStreamChunkLimit(transport);
 			return control;
 		}
 	};
@@ -382,6 +411,20 @@ const HOOK_IDS = {
 	],
 	mutating: /* @__PURE__ */ new Set()
 };
+/** Transports whose worker answered the fleet-side run listing with 404. */
+const legacyRunListing = /* @__PURE__ */ new WeakSet();
+async function listResolvedRuns(transport, request) {
+	if (legacyRunListing.has(transport)) return null;
+	try {
+		return await callFleetRoute(transport, "/v1/index/runs/list-resolved", [request], { idempotent: true });
+	} catch (error) {
+		if (error.status === 404) {
+			legacyRunListing.add(transport);
+			return null;
+		}
+		throw error;
+	}
+}
 function createRemoteEnv(transport) {
 	const runCatalog = makeNamespace(transport, "run-catalog", RUN_CATALOG);
 	const hookTokens = makeNamespace(transport, "hook-tokens", HOOK_TOKENS);
@@ -402,6 +445,7 @@ function createRemoteEnv(transport) {
 				publicationExpiresAt
 			], { idempotent: true }),
 			listRuns: (options) => callFleetRoute(transport, "/v1/index/runs/list", [options], { idempotent: true }),
+			listResolvedRuns: (request) => listResolvedRuns(transport, request),
 			expireRun: (request) => callFleetRoute(transport, "/v1/index/runs/expire", [request], { idempotent: true }),
 			reserveHook: (token, owner) => callFleetRoute(transport, "/v1/index/hooks/reserve", [token, owner], { idempotent: true }),
 			finalizeHookIndexes: (token, hookId, serializedHook, owner, reservation) => callFleetRoute(transport, "/v1/index/hooks/finalize", [
@@ -458,7 +502,7 @@ const debug = createDebugLogger("celld-world");
 * Vendored from vinnymac/worlds packages/world-cloudflare/src/queue.ts
 * (Apache-2.0, see NOTICE), modified for celld:
 *
-* - Production publishes a small pointer through celld v0.5.0 native Queues.
+* - Production publishes a small pointer through celld v0.6.0 native Queues.
 *   The deployed Queue consumer forwards deliveries to the same HTTP handler
 *   used by the in-process test pump.
 * - Run-bearing message bodies live in the fleet object store. This preserves
@@ -828,6 +872,7 @@ function parseApplyEventOutcome(value) {
 	for (const releasedHook of value.releasedHooks) if (!isRecord(releasedHook) || typeof releasedHook.hookId !== "string" || releasedHook.hookId.length === 0 || typeof releasedHook.token !== "string" || releasedHook.token.length === 0) malformedApplyEventOutcome("releasedHooks entries must contain hookId and token strings");
 	const event = parseSuccessEntity(value, "event", EventSchema);
 	const run = parseSuccessEntity(value, "run", WorkflowRunSchema);
+	const replayRun = parseSuccessEntity(value, "replayRun", WorkflowRunSchema);
 	const step = parseSuccessEntity(value, "step", StepSchema);
 	const hook = parseSuccessEntity(value, "hook", HookSchema);
 	const wait = parseSuccessEntity(value, "wait", WaitSchema);
@@ -851,6 +896,7 @@ function parseApplyEventOutcome(value) {
 	if (value.cursor !== void 0 && value.cursor !== null && typeof value.cursor !== "string") malformedApplyEventOutcome("cursor must be a string or null");
 	if (value.hasMore !== void 0 && typeof value.hasMore !== "boolean") malformedApplyEventOutcome("hasMore must be a boolean");
 	if (value.maxEvents !== void 0 && (typeof value.maxEvents !== "number" || !Number.isSafeInteger(value.maxEvents) || value.maxEvents < 0)) malformedApplyEventOutcome("maxEvents must be a non-negative safe integer");
+	for (const flag of ["runIndexPublished", "hookIndexesReleased"]) if (value[flag] !== void 0 && value[flag] !== true) malformedApplyEventOutcome(`${flag} must be true when present`);
 	if (run !== void 0 && value.indexPublicationExpiresAt === void 0) malformedApplyEventOutcome("indexPublicationExpiresAt is required when run is present");
 	if (value.indexPublicationExpiresAt !== void 0 && (typeof value.indexPublicationExpiresAt !== "number" || !Number.isFinite(value.indexPublicationExpiresAt))) malformedApplyEventOutcome("indexPublicationExpiresAt must be a finite number");
 	return {
@@ -859,6 +905,7 @@ function parseApplyEventOutcome(value) {
 			...value,
 			event,
 			run,
+			replayRun,
 			step,
 			hook,
 			wait,
@@ -951,19 +998,6 @@ function unwrapRead(outcome) {
 	if (!outcome.ok) throw new RunExpiredError(outcome.message);
 	return outcome.value;
 }
-/** Bound cross-run fanout so large list pages cannot create an RPC burst. */
-const RUN_LIST_CONCURRENCY = 8;
-/**
-* Run statuses only move pending -> running -> terminal, and terminal statuses
-* are immutable. An older index value may therefore safely exclude a filter
-* only when it is already terminal, or when the caller asks for pending and
-* the index has advanced beyond pending. Earlier non-terminal metadata cannot
-* exclude a later status because a post-commit index write may need replay.
-*/
-function indexStatusExcludes(indexed, requested) {
-	if (requested === void 0 || indexed === requested) return false;
-	return isTerminalWorkflowRunStatus(indexed) || requested === "pending";
-}
 function createStorage(config) {
 	const { env } = config;
 	const cleanup = { retentionMs: config.runRetentionMs ?? 0 };
@@ -981,60 +1015,31 @@ function createStorage(config) {
 		runs: {
 			get: runsGet,
 			async list(params) {
-				const limit = params?.pagination?.limit ?? 20;
-				const prefix = params?.workflowName ? `run:${params.workflowName}:` : "runall:";
-				const reverse = params?.pagination?.sortOrder === "desc";
-				const matches = [];
-				let scanCursor = params?.pagination?.cursor;
-				let exhausted = false;
-				while (matches.length <= limit && !exhausted) {
-					const kvList = await env.WORKFLOW_INDEX.listRuns({
-						prefix,
-						limit: Math.min(1e3, Math.max(50, limit)),
-						cursor: scanCursor,
-						reverse
-					});
-					if (kvList.keys.length === 0) {
-						exhausted = true;
-						break;
-					}
-					const candidates = [];
-					for (const key of kvList.keys) {
-						const metadata = JSON.parse(key.value);
-						if (indexStatusExcludes(metadata.status, params?.status)) continue;
-						candidates.push({
-							key: key.name,
-							metadata
-						});
-					}
-					for (let offset = 0; offset < candidates.length && matches.length <= limit;) {
-						const needed = limit + 1 - matches.length;
-						const batch = candidates.slice(offset, offset + Math.min(RUN_LIST_CONCURRENCY, needed));
-						const resolved = await Promise.all(batch.map(async ({ key, metadata }) => {
-							try {
-								const run = await runsGet(metadata.runId, { resolveData: params?.resolveData });
-								return !params?.status || run.status === params.status ? {
-									key,
-									run
-								} : null;
-							} catch (error) {
-								if (!WorkflowRunNotFoundError.is(error) && !RunExpiredError.is(error)) throw error;
-								return null;
-							}
-						}));
-						for (const match of resolved) if (match) matches.push(match);
-						offset += batch.length;
-					}
-					exhausted = kvList.list_complete;
-					scanCursor = kvList.cursor ?? kvList.keys.at(-1)?.name;
-				}
-				const hasMore = matches.length > limit;
-				const page = matches.slice(0, limit);
-				return {
-					data: page.map(({ run }) => run),
-					cursor: hasMore ? page.at(-1)?.key ?? null : null,
-					hasMore
+				const request = {
+					workflowName: params?.workflowName,
+					statuses: params?.status === void 0 ? void 0 : Array.isArray(params.status) ? params.status : [params.status],
+					limit: params?.pagination?.limit ?? 20,
+					cursor: params?.pagination?.cursor,
+					sortOrder: params?.pagination?.sortOrder,
+					resolveData: params?.resolveData
+				};
+				const fleetPage = Number.isSafeInteger(request.limit) && request.limit >= 1 && request.limit <= 1e3 ? await env.WORKFLOW_INDEX.listResolvedRuns?.(compact(request)) : void 0;
+				if (fleetPage) return {
+					data: fleetPage.data.map((run) => filterData(parseRun(run), params?.resolveData, ["input", "output"])),
+					cursor: fleetPage.cursor,
+					hasMore: fleetPage.hasMore
 				};
+				return await listRunsPage({
+					listRuns: (options) => env.WORKFLOW_INDEX.listRuns(options),
+					readRun: async (runId) => {
+						try {
+							return await runsGet(runId);
+						} catch (error) {
+							if (WorkflowRunNotFoundError.is(error) || RunExpiredError.is(error)) return null;
+							throw error;
+						}
+					}
+				}, request);
 			}
 		},
 		events: {
@@ -1094,20 +1099,18 @@ function createStorage(config) {
 					throwOutcomeError(parsedOutcome.outcome, effectiveRunId, data);
 				}
 				const outcome = parsedOutcome.outcome;
-				if (outcome.run) {
-					const meta = JSON.stringify({
-						runId: effectiveRunId,
-						createdAt: outcome.run.createdAt.toISOString(),
-						status: outcome.run.status
-					});
+				if (outcome.run && !outcome.runIndexPublished) {
 					if (outcome.indexPublicationExpiresAt === void 0) throw new Error("world-celld: authoritative run mutation omitted its index lease");
-					await env.WORKFLOW_INDEX.commitRun(outcome.run, meta, outcome.indexPublicationExpiresAt);
+					await env.WORKFLOW_INDEX.commitRun(outcome.run, runIndexMetadata({
+						...outcome.run,
+						runId: effectiveRunId
+					}), outcome.indexPublicationExpiresAt);
 				}
 				if (outcome.hookToIndex) {
 					const serialized = stringify(outcome.hookToIndex);
 					await env.WORKFLOW_INDEX.finalizeHookIndexes(outcome.hookToIndex.token, outcome.hookToIndex.hookId, serialized, hookOwner(outcome.hookToIndex), hookAdmission?.reservation);
 				} else if (hookAdmission) await env.WORKFLOW_INDEX.releaseHookReservation(hookAdmission.token, hookAdmission.owner, hookAdmission.reservation);
-				if (outcome.releasedHooks.length > 0) await env.WORKFLOW_INDEX.releaseHookIndexes({
+				if (outcome.releasedHooks.length > 0 && !outcome.hookIndexesReleased) await env.WORKFLOW_INDEX.releaseHookIndexes({
 					runId: effectiveRunId,
 					hooks: outcome.releasedHooks
 				});
@@ -1118,7 +1121,7 @@ function createStorage(config) {
 				};
 				return {
 					event: outcome.event,
-					run: outcome.run,
+					run: outcome.run ?? outcome.replayRun,
 					step: outcome.step,
 					hook: outcome.hook,
 					wait: outcome.wait,
@@ -1275,18 +1278,28 @@ function createStreamer(config) {
 		const chunks = input.map(toBytes);
 		for (const chunk of chunks) if (chunk.byteLength > 1048576) throw new Error(`Stream chunk exceeds ${MAX_STREAM_CHUNK_BYTES} bytes`);
 		await registerStreamForRun(name, resolvedRunId);
-		let batch = [];
-		let batchBytes = 0;
-		for (const chunk of chunks) {
-			if (batch.length > 0 && (batch.length === 32 || batchBytes + chunk.byteLength > 8388608)) {
-				await getStreamDO(name).writeChunks(resolvedRunId, batch);
-				batch = [];
-				batchBytes = 0;
+		const stub = getStreamDO(name);
+		let next = 0;
+		while (next < chunks.length) {
+			const chunkLimit = stub.streamChunkLimit?.() ?? 32;
+			let end = next;
+			let batchBytes = 0;
+			while (end < chunks.length && end - next < chunkLimit && (end === next || batchBytes + chunks[end].byteLength <= 8388608)) {
+				batchBytes += chunks[end].byteLength;
+				end++;
+			}
+			const batch = chunks.slice(next, end);
+			try {
+				await stub.writeChunks(resolvedRunId, batch);
+			} catch (error) {
+				if (batch.length > 32 && isChunkLimitRejection(error)) {
+					stub.resetStreamChunkLimit?.();
+					continue;
+				}
+				throw error;
 			}
-			batch.push(chunk);
-			batchBytes += chunk.byteLength;
+			next = end;
 		}
-		if (batch.length > 0) await getStreamDO(name).writeChunks(resolvedRunId, batch);
 	};
 	const writeToStream = (name, runId, chunk) => writeChunksToStream(name, runId, [chunk]);
 	const closeStream = async (name, runId) => {
@@ -1324,7 +1337,7 @@ function createStreamer(config) {
 						result = await stub.readChunks({
 							runId,
 							startIndex: nextIndex,
-							maxChunks: 32,
+							maxChunks: stub.streamChunkLimit?.() ?? 32,
 							maxBytes: MAX_STREAM_READ_BYTES,
 							waitMs: streamLongPollMs
 						}, abortController.signal);
@@ -1354,10 +1367,11 @@ function createStreamer(config) {
 	const getStreamChunks = async (name, runId, options) => {
 		const requestedLimit = options?.limit ?? DEFAULT_CHUNK_PAGE_SIZE;
 		if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1) throw new Error("Stream chunk limit must be a positive safe integer");
-		const limit = Math.min(requestedLimit, MAX_CHUNK_PAGE_SIZE);
+		const stub = getStreamDO(name);
+		const limit = Math.min(requestedLimit, stub.streamChunkLimit?.() ?? MAX_CHUNK_PAGE_SIZE);
 		const startIndex = options?.cursor ? Number.parseInt(options.cursor, 10) : 0;
 		if (Number.isNaN(startIndex) || startIndex < 0) throw new Error(`Invalid stream cursor: ${options?.cursor}`);
-		const result = await getStreamDO(name).readChunks({
+		const result = await stub.readChunks({
 			runId,
 			startIndex,
 			maxChunks: limit,