@shivaedev/work-fleet 0.2.1 → 0.4.0

package/src/host/configSettings.ts

@@ -0,0 +1,126 @@
+import { Equal, Schema } from "effect";
+import { type FleetHost, HostConfig } from "#host/config.ts";
+import type { AppliedConfig } from "#host/liveConfig.ts";
+import { FleetViews } from "#model.ts";
+
+// A running daemon applies these settings from an edited file; every other setting waits for a restart.
+export const RELOADABLE_SETTINGS = [
+	"areaMergeWaitMs",
+	"backlog",
+	"concurrency",
+	"intervalSeconds",
+	"launch",
+	"openPullRequests",
+	"quota",
+	"readRetryMs",
+	"stallMinutes",
+	"worker.reviewerGuidance",
+	"worker.workerGuidance",
+] as const;
+
+export const ConfigStatus = Schema.Struct({
+	file: Schema.optional(Schema.String),
+	loadedAt: Schema.Number,
+	rejected: Schema.optional(Schema.Struct({ field: Schema.optional(Schema.String), message: Schema.String })),
+	restartRequired: Schema.Array(Schema.String),
+	settings: HostConfig,
+});
+export type ConfigStatus = typeof ConfigStatus.Type;
+
+export const DaemonStatus = Schema.Struct({ ...FleetViews.fields, config: Schema.optional(ConfigStatus) });
+
+function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function setting(config: HostConfig, name: string): unknown {
+	return name.split(".").reduce<unknown>((value, key) => (isRecord(value) ? value[key] : undefined), config);
+}
+
+function appliedSettings(host: FleetHost): HostConfig {
+	return {
+		areaMergeWaitMs: host.areaMergeWaitMs,
+		backlog: host.backlog,
+		...(host.cloud === undefined ? {} : { cloud: host.cloud }),
+		concurrency: host.concurrency,
+		database: { maxConnections: host.maxConnections },
+		intervalSeconds: host.interval,
+		launch: host.launch,
+		...(host.local === undefined ? {} : { local: host.local }),
+		namespace: host.namespace,
+		...(host.openPullRequests === undefined ? {} : { openPullRequests: host.openPullRequests }),
+		provider: host.provider,
+		quota: host.quota,
+		readRetryMs: host.readRetryMs,
+		repository: host.repository,
+		...(host.result === undefined ? {} : { result: host.result }),
+		stallMinutes: host.stallMinutes,
+		...(host.worker === undefined ? {} : { worker: host.worker }),
+	};
+}
+
+function write(target: Record<string, unknown>, [key, ...rest]: readonly string[], value: unknown) {
+	if (key === undefined) {
+		return;
+	}
+	if (rest.length > 0) {
+		const child = target[key];
+		if (isRecord(child)) {
+			const copy = { ...child };
+			write(copy, rest, value);
+			target[key] = copy;
+		}
+		return;
+	}
+	if (value === undefined) {
+		delete target[key];
+	} else {
+		target[key] = value;
+	}
+}
+
+// A reloadable setting nested under a section that the running config lacks stays unapplied until a restart adds the section.
+export function withReloadable(applied: HostConfig, next: HostConfig): Readonly<Record<string, unknown>> {
+	const merged: Record<string, unknown> = { ...applied };
+	for (const name of RELOADABLE_SETTINGS) {
+		write(merged, name.split("."), setting(next, name));
+	}
+	return merged;
+}
+
+export function changedSettings(left: unknown, right: unknown, prefix = ""): readonly string[] {
+	if (isRecord(left) && isRecord(right)) {
+		return [...new Set([...Object.keys(left), ...Object.keys(right)])]
+			.sort((first, second) => first.localeCompare(second))
+			.flatMap((key) => changedSettings(left[key], right[key], prefix === "" ? key : `${prefix}.${key}`));
+	}
+	return Equal.equals(left, right) ? [] : [prefix];
+}
+
+export function configStatus({ host, loadedAt, rejected, restartRequired }: AppliedConfig): ConfigStatus {
+	return {
+		...(host.configFile === undefined ? {} : { file: host.configFile }),
+		loadedAt,
+		...(rejected === undefined
+			? {}
+			: { rejected: { ...(rejected.field === undefined ? {} : { field: rejected.field }), message: rejected.message } }),
+		restartRequired,
+		settings: appliedSettings(host),
+	};
+}
+
+export function renderConfig(config: ConfigStatus): string {
+	const applying = RELOADABLE_SETTINGS.flatMap((name) => {
+		const value = setting(config.settings, name);
+		return value === undefined ? [] : [`${name} ${typeof value === "object" ? JSON.stringify(value) : String(value)}`];
+	});
+	const rejected = config.rejected;
+	return [
+		`Config loaded ${new Date(config.loadedAt).toISOString()}${config.file === undefined ? "" : ` from ${config.file}`}`,
+		`  provider ${config.settings.provider}, ${applying.join(", ")}`,
+		...(config.restartRequired.length === 0 ? [] : [`  Restart to apply: ${config.restartRequired.join(", ")}`]),
+		...(rejected === undefined
+			? []
+			: [`  Rejected edit${rejected.field === undefined ? "" : ` of ${rejected.field}`}, still applying the config above: ${rejected.message}`]),
+	].join("\n");
+}

package/src/host/daemon.ts

@@ -5,6 +5,7 @@ import { type HostParts, hostLayer, hostScheduler, productionParts } from "#host
 import type { FleetHost } from "#host/config.ts";
 import { DaemonContract, RPC_PATH } from "#host/daemonContract.ts";
 import { daemonHandlers } from "#host/daemonHandlers.ts";
+import { watchedConfig } from "#host/liveConfig.ts";
 import { fleetHandlers } from "#operations.ts";
 
 const { DAEMON_PORT: DEFAULT_DAEMON_PORT } = diagnosticsApi;
@@ -21,7 +22,7 @@ export function daemonHostOptions(host: FleetHost, parts: HostParts = production
 		handlers: Layer.merge(fleetHandlers, Layer.unwrap(daemonHandlers(host))),
 		policy: "loopback" as const,
 		rpcPath: RPC_PATH,
-		services: hostLayer(host, parts),
+		services: hostLayer(host, parts, watchedConfig(host)),
 	};
 }
 

package/src/host/daemonClient.ts

@@ -19,18 +19,53 @@ export function daemonClient(port: number = DAEMON_PORT) {
 	);
 }
 
-export function daemonRequest<A, E, R>(port: number, use: (client: Effect.Success<ReturnType<typeof daemonClient>>) => Effect.Effect<A, E, R>) {
-	return Effect.scoped(daemonClient(port).pipe(Effect.flatMap(use))).pipe(Effect.mapError((error) => daemonError(error, port)));
+export function daemonRequest<A, E, R>(
+	port: number,
+	use: (client: Effect.Success<ReturnType<typeof daemonClient>>) => Effect.Effect<A, E, R>,
+	request = "request",
+	timeoutMs = 5000,
+) {
+	return Effect.scoped(daemonClient(port).pipe(Effect.flatMap(use))).pipe(
+		Effect.timeout(timeoutMs),
+		Effect.mapError((error) => daemonError(error, port, request)),
+	);
 }
 
-export function daemonError(error: unknown, port: number) {
-	if (isFleetError(error)) {
+export function daemonError(error: unknown, port: number, request = "request") {
+	if (isFleetError(error) && (error._tag !== "ServiceUnavailable" || request === "request")) {
 		return error;
 	}
-	const transport = error instanceof RpcClientError && error.reason._tag === "HttpError" && error.reason.kind === "TransportError";
+	const refused = error instanceof RpcClientError && connectionRefused(error.reason);
+	const timeout = requestTimedOut(error);
+	const detail = error instanceof Error ? error.message : String(error);
+	const recording = request.startsWith("answer ") ? "; answer recording is unconfirmed; check status or watch before retrying" : "";
 	return new ServiceUnavailable({
-		message: transport
+		message: refused
 			? `No Fleet daemon is running at 127.0.0.1:${port}; start work-fleet serve <board-dir> --port ${port}`
-			: `Fleet daemon communication failed at 127.0.0.1:${port}: ${error instanceof Error ? error.message : String(error)}`,
+			: `Fleet daemon ${request} ${timeout ? "timed out" : "failed"} at 127.0.0.1:${port}: ${detail}${recording}`,
 	});
 }
+
+function requestTimedOut(value: unknown, depth = 0): boolean {
+	if (typeof value !== "object" || value === null || depth > 8) {
+		return false;
+	}
+	return (
+		("_tag" in value && value._tag === "TimeoutError")
+		|| ("name" in value && value.name === "TimeoutError")
+		|| ("code" in value && ["ETIMEDOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT", "UND_ERR_CONNECT_TIMEOUT"].includes(String(value.code)))
+		|| ("cause" in value && requestTimedOut(value.cause, depth + 1))
+		|| ("reason" in value && requestTimedOut(value.reason, depth + 1))
+	);
+}
+
+function connectionRefused(value: unknown, depth = 0): boolean {
+	if (typeof value !== "object" || value === null || depth > 8) {
+		return false;
+	}
+	return (
+		("code" in value && value.code === "ECONNREFUSED")
+		|| ("cause" in value && connectionRefused(value.cause, depth + 1))
+		|| ("reason" in value && connectionRefused(value.reason, depth + 1))
+	);
+}

package/src/host/daemonContract.ts

@@ -3,8 +3,10 @@ import { contract } from "@shivaedev/effect-contract/contract.ts";
 import { command, query, stream } from "@shivaedev/effect-contract/operation.ts";
 import { RequestTracing } from "@shivaedev/platform/rpc/middleware.ts";
 import { diagnostics as diagnosticsApi } from "@shivaedev/work-board/integration/diagnostics.ts";
+import { DispatchRequest } from "#DispatchRequest.ts";
 import { fleetRejections } from "#errors.ts";
-import { FleetRecord, FleetViews } from "#model.ts";
+import { DaemonStatus } from "#host/configSettings.ts";
+import { FleetRecord } from "#model.ts";
 import { FleetContract, fleetRecords } from "#operations.ts";
 
 const { DAEMON_RPC_PATH, DiagnosticsContract } = diagnosticsApi;
@@ -31,17 +33,23 @@ export const ClientContract = contract("daemon", {
 		}),
 		command("dispatch", {
 			invalidates: ({ workId }) => [fleetRecords.list, fleetRecords.item(workId)],
-			payload: { workId: Schema.String },
+			payload: { ignoreAreas: Schema.optional(Schema.Boolean), now: Schema.optional(Schema.Boolean), workId: Schema.String },
 			rejections: fleetRejections,
-			success: FleetRecord,
+			success: DispatchRequest,
 		}),
 	],
 	queries: [
+		query("dispatch-status", {
+			payload: { workId: Schema.String },
+			reads: ({ workId }) => [fleetRecords.item(workId)],
+			rejections: fleetRejections,
+			success: Schema.NullOr(DispatchRequest),
+		}),
 		query("status", {
 			payload: { workId: Schema.optional(Schema.String) },
 			reads: ({ workId }) => [workId === undefined ? fleetRecords.list : fleetRecords.item(workId)],
 			rejections: fleetRejections,
-			success: FleetViews,
+			success: DaemonStatus,
 		}),
 	],
 	streams: [

package/src/host/daemonDiagnostics.ts

@@ -1,5 +1,7 @@
 import { Effect } from "effect";
+import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import { diagnostics as diagnosticsApi } from "@shivaedev/work-board/integration/diagnostics.ts";
+import { failureMessage } from "#engine/failureMessage.ts";
 import { HealthSnapshot } from "#engine/health.ts";
 import { fleetItems } from "#host/fleetItems.ts";
 
@@ -7,14 +9,20 @@ const { codexLimits, DiagnosticsContract, DiagnosticsSources, diagnostics, githu
 
 export const daemonDiagnostics = Effect.gen(function* () {
 	const provided = yield* Effect.serviceOption(DiagnosticsSources);
+	const fleet = yield* HealthSnapshot;
 	const sources = {
 		codex: provided._tag === "Some" ? provided.value.codex : codexLimits(),
-		fleet: yield* HealthSnapshot,
-		github: provided._tag === "Some" ? provided.value.github : githubLimits(),
+		fleet,
+		github: githubLimits(fleet),
 	};
 	const read = yield* diagnostics(sources);
 	const items = yield* fleetItems;
 	return DiagnosticsContract.toLayer(
-		Effect.succeed(DiagnosticsContract.of({ "daemon.diagnostics": () => read, "daemon.fleetItems": () => items.pipe(Effect.orDie) })),
+		Effect.succeed(
+			DiagnosticsContract.of({
+				"daemon.diagnostics": () => read,
+				"daemon.fleetItems": () => items.pipe(Effect.mapError((error) => new ServiceUnavailable({ message: failureMessage(error) }))),
+			}),
+		),
 	);
 });

package/src/host/daemonHandlers.ts

@@ -1,31 +1,48 @@
-import { Effect, Layer } from "effect";
+import { Effect, Fiber, Layer } from "effect";
 import { exposeFailure } from "#exposeFailure.ts";
-import { dispatchApproved } from "#host/admission.ts";
 import { answerWork } from "#host/answerWork.ts";
 import type { FleetHost } from "#host/config.ts";
+import { configStatus } from "#host/configSettings.ts";
 import { ClientContract } from "#host/daemonContract.ts";
 import { daemonDiagnostics } from "#host/daemonDiagnostics.ts";
 import { daemonSnapshots } from "#host/daemonSnapshots.ts";
+import { dispatchRequests } from "#host/dispatchRequests.ts";
+import { LiveConfig } from "#host/liveConfig.ts";
 import { nameStoredThreads } from "#host/nameThreads.ts";
-import { releaseItems } from "#host/releaseWork.ts";
+import { decisionResponses, releaseItems } from "#host/releaseWork.ts";
 import { fleetStatus } from "#host/status.ts";
 
 export const daemonHandlers = Effect.fn("FleetDaemon.daemonHandlers")(function* (host: FleetHost) {
+	const scope = yield* Effect.scope;
 	const follow = yield* daemonSnapshots(host);
 	const diagnostics = yield* daemonDiagnostics;
+	const dispatch = yield* dispatchRequests(host);
+	const live = yield* LiveConfig;
+	const status = Effect.zipWith(fleetStatus(host), live.current, (views, applied) => ({ ...views, config: configStatus(applied) }));
+	// The server already accepts connections while handlers build, so the Board watcher starts beside them instead of delaying them.
+	const responses = yield* Layer.buildWithScope(decisionResponses(host), scope).pipe(Effect.forkIn(scope));
+	// A client that stops waiting closes its request; the Board response it asked for must still be recorded and applied.
+	function settle<A, E, R>(effect: Effect.Effect<A, E, R>) {
+		return Fiber.join(responses).pipe(
+			Effect.flatMap((context) => effect.pipe(Effect.provide(context))),
+			Effect.forkIn(scope),
+			Effect.flatMap(Fiber.join),
+		);
+	}
 	return Layer.merge(
 		diagnostics,
 		ClientContract.toLayer(
 			Effect.succeed(
 				ClientContract.of({
 					"daemon.answer": ({ workId, text, release }) =>
-						exposeFailure(answerWork(host, workId, text, release).pipe(Effect.map(({ value }) => value))),
-					"daemon.dispatch": ({ workId }) => exposeFailure(dispatchApproved(host, workId).pipe(Effect.map(({ value }) => value))),
+						exposeFailure(settle(answerWork(host, workId, text, release, true)).pipe(Effect.map(({ value }) => value))),
+					"daemon.dispatch": ({ workId, now, ignoreAreas }) => exposeFailure(dispatch.accept(workId, { ignoreAreas, now })),
+					"daemon.dispatch-status": ({ workId }) => exposeFailure(dispatch.get(workId)),
 					"daemon.follow": ({ workIds, follow: continuous }) => follow(workIds, continuous),
 					"daemon.name-threads": () => exposeFailure(nameStoredThreads(host)),
 					"daemon.release": ({ workIds }) =>
-						exposeFailure(releaseItems(host, workIds).pipe(Effect.map((records) => records.map(({ value }) => value)))),
-					"daemon.status": ({ workId }) => exposeFailure(fleetStatus(host, workId)),
+						exposeFailure(settle(releaseItems(host, workIds)).pipe(Effect.map((records) => records.map(({ value }) => value)))),
+					"daemon.status": ({ workId }) => exposeFailure(workId === undefined ? status : fleetStatus(host, workId)),
 				}),
 			),
 		),

package/src/host/daemonSnapshots.ts

@@ -2,15 +2,21 @@ import { Effect, Schedule, Stream } from "effect";
 import { SqlClient } from "effect/sql";
 import { AppChanges } from "@shivaedev/platform/appHost.ts";
 import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
+import type { DispatchRequest } from "#DispatchRequest.ts";
 import { fleetRecords } from "#fleetRecords.ts";
 import type { FleetHost } from "#host/config.ts";
 import { readWaitStates } from "#host/readWaitStates.ts";
 import { snapshotStream } from "#host/snapshotStream.ts";
+import type { State } from "#host/waitOutcome.ts";
+import { dispatchStore } from "#storage/dispatchStore.ts";
 
 export const daemonSnapshots = Effect.fn("FleetDaemon.daemonSnapshots")(function* (host: FleetHost) {
 	const sql = yield* SqlClient.SqlClient;
 	const changes = yield* AppChanges;
-	const read = readWaitStates(host).pipe(
+	const dispatches = yield* dispatchStore(host.namespace);
+	const read = Effect.gen(function* () {
+		return dispatchStates(yield* readWaitStates(host), yield* dispatches.list());
+	}).pipe(
 		Effect.provideService(SqlClient.SqlClient, sql),
 		Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 19 }),
 		Effect.mapError(() => new ServiceUnavailable({ message: "Fleet daemon could not read work states after 20 consecutive attempts" })),
@@ -22,3 +28,20 @@ export const daemonSnapshots = Effect.fn("FleetDaemon.daemonSnapshots")(function
 	);
 	return (ids: readonly string[], continuous = false) => snapshotStream(snapshots, ids, continuous);
 });
+
+function dispatchStates(states: readonly (typeof State.Type)[], requests: readonly DispatchRequest[]) {
+	const result = new Map<string, typeof State.Type>(states.map((state) => [state.workId, state]));
+	for (const request of requests) {
+		const state = result.get(request.workId);
+		if (request.status === "completed" || (request.status === "failed" && state !== undefined && state.stage !== "prepared")) {
+			continue;
+		}
+		result.set(request.workId, {
+			attentionOwner: request.status === "pending" ? "automation" : "orchestrator",
+			blocker: request.status === "pending" ? "Dispatch recorded; preparing in the background" : `Dispatch failed: ${request.error}`,
+			stage: request.status === "pending" ? "dispatching" : "needs-human",
+			workId: request.workId,
+		});
+	}
+	return [...result.values()].sort((left, right) => left.workId.localeCompare(right.workId));
+}

package/src/host/dispatchRequests.ts

@@ -0,0 +1,53 @@
+import { Cause, Effect } from "effect";
+import { BadRequest, Forbidden } from "@shivaedev/platform/errors/taxonomy.ts";
+import type { DispatchRequest } from "#DispatchRequest.ts";
+import { failureMessage } from "#engine/failureMessage.ts";
+import { dispatchApproved } from "#host/admission.ts";
+import type { FleetHost } from "#host/config.ts";
+import { BoardGateway } from "#policy.ts";
+import { dispatchStore } from "#storage/dispatchStore.ts";
+
+export const dispatchRequests = Effect.fn("FleetDaemon.dispatchRequests")(function* (host: FleetHost) {
+	const scope = yield* Effect.scope;
+	const board = yield* BoardGateway;
+	const store = yield* dispatchStore(host.namespace);
+	const running = new Set<string>();
+	const start = Effect.fn("FleetDaemon.start")(function* (request: DispatchRequest) {
+		if (running.has(request.id)) {
+			return;
+		}
+		running.add(request.id);
+		yield* dispatchApproved(host, request.workId, request.now ? { ignoreAreas: request.ignoreAreas === true } : undefined).pipe(
+			Effect.andThen(store.finish(request)),
+			Effect.catchCause((cause) =>
+				Cause.hasInterrupts(cause) ? Effect.failCause(cause) : store.finish(request, failureMessage(Cause.squash(cause))),
+			),
+			Effect.catchCause((cause) => (Cause.hasInterrupts(cause) ? Effect.void : Effect.logError(cause))),
+			Effect.ensuring(Effect.sync(() => running.delete(request.id))),
+			Effect.forkIn(scope),
+		);
+	});
+	for (const request of yield* store.list()) {
+		if (request.status === "pending") {
+			yield* start(request);
+		}
+	}
+	return {
+		accept: Effect.fn("FleetDaemon.accept")(function* (
+			workId: string,
+			options: { readonly ignoreAreas?: boolean | undefined; readonly now?: boolean | undefined } = {},
+		) {
+			if (options.ignoreAreas === true && options.now !== true) {
+				return yield* new BadRequest({ message: "Ignoring area claims needs a dispatch now request" });
+			}
+			const work = yield* board.get(workId);
+			if (!work.approved) {
+				return yield* new Forbidden({ message: `${workId} needs "fleet: approved" in its Board frontmatter` });
+			}
+			const request = yield* store.accept(workId, options.now === true ? { ignoreAreas: options.ignoreAreas === true } : undefined);
+			yield* start(request);
+			return request;
+		}),
+		get: store.get,
+	};
+});

package/src/host/follow.ts

@@ -34,7 +34,7 @@ export function follow(options: FollowOptions, workIds: readonly string[]) {
 			Stream.catch((error) => {
 				failures += 1;
 				if (!connected || isFleetError(error) || failures >= 20) {
-					return Stream.fail(daemonError(error, port));
+					return Stream.fail(daemonError(error, port, "follow"));
 				}
 				return Stream.unwrap(Effect.sleep(options.intervalMs).pipe(Effect.as(connect())));
 			}),

package/src/host/followState.ts

@@ -20,9 +20,7 @@ export function decodeFollowState(value: unknown): FollowState {
 
 export function unrecognizedState(state: FollowState): boolean {
 	return (
-		state.undecoded !== undefined
-		|| !Schema.is(FleetRecord.fields.stage)(state.stage)
-		|| (state.decision?.code !== undefined && !Schema.is(DecisionCode)(state.decision.code))
+		state.undecoded !== undefined || !knownStage(state.stage) || (state.decision?.code !== undefined && !Schema.is(DecisionCode)(state.decision.code))
 	);
 }
 
@@ -35,7 +33,9 @@ export function followOutcome(state: FollowState): WaitOutcome | undefined {
 	if (code !== undefined && !Schema.is(DecisionCode)(code)) {
 		return { code: outcome?.code ?? WAIT_EXIT.error, summary: `${outcome?.summary ?? `${state.workId} ${state.stage}`}: ${code}` };
 	}
-	return (
-		outcome ?? (Schema.is(FleetRecord.fields.stage)(state.stage) ? undefined : { code: WAIT_EXIT.error, summary: `${state.workId} ${state.stage}` })
-	);
+	return outcome ?? (knownStage(state.stage) ? undefined : { code: WAIT_EXIT.error, summary: `${state.workId} ${state.stage}` });
+}
+
+function knownStage(stage: string) {
+	return stage === "dispatching" || Schema.is(FleetRecord.fields.stage)(stage);
 }

package/src/host/github.ts

@@ -1,6 +1,6 @@
 import { Effect, Schema } from "effect";
-import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import type { FleetError } from "#errors.ts";
+import { githubFailure } from "#host/githubRateLimit.ts";
 
 const Sha = Schema.String.check(Schema.isPattern(/^[a-f\d]{40}$/u));
 const Positive = Schema.Int.check(Schema.isGreaterThan(0));
@@ -20,13 +20,15 @@ export const FilePages = Schema.Array(
 		),
 	),
 );
-export const Repository = Schema.Struct({ nameWithOwner: Schema.String });
 
-export type GithubRead = (args: readonly string[]) => Effect.Effect<string, FleetError>;
+export interface GithubRead {
+	readonly get: (endpoint: string) => Effect.Effect<string, FleetError>;
+	readonly pages: (endpoint: string) => Effect.Effect<string, FleetError>;
+}
 
-export function githubJson<TSchema extends Schema.Top>(schema: TSchema, read: GithubRead, args: readonly string[]) {
-	return read(args).pipe(
+export function githubJson<TSchema extends Schema.Top>(schema: TSchema, read: Effect.Effect<string, FleetError>) {
+	return read.pipe(
 		Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(schema))),
-		Effect.mapError(() => new ServiceUnavailable({ message: "GitHub observation failed or returned invalid data" })),
+		Effect.mapError(githubFailure("GitHub observation failed or returned invalid data")),
 	);
 }

package/src/host/githubApi.ts

@@ -0,0 +1,142 @@
+import { Clock, Effect, Layer } from "effect";
+import { defineService } from "@shivaedev/effect-service/define-service.ts";
+import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
+import {
+	type GithubObservation,
+	type GithubPause,
+	githubPaused,
+	pace,
+	type RecordedObservation,
+	rateLimitWait,
+	readObservation,
+	reservedUntil,
+} from "#host/githubRateLimit.ts";
+import { nextLink, type ParsedResponse, parseResponse, requestResource } from "#host/githubResponse.ts";
+import { GithubTransport, ghTransport } from "#host/githubTransport.ts";
+
+interface Page {
+	readonly body: string;
+	readonly etag: string | undefined;
+	readonly next: string | undefined;
+}
+interface Block {
+	readonly failures: number;
+	readonly until: number;
+}
+
+const CACHE_LIMIT = 1000;
+const PAGE_LIMIT = 100;
+
+export const GithubApi = defineService({
+	id: "Fleet/GithubApi",
+	initialize: Effect.gen(function* () {
+		return {
+			blocks: new Map<string, Block>(),
+			cache: new Map<string, Page>(),
+			observations: new Map<string, RecordedObservation>(),
+			transport: yield* GithubTransport,
+		};
+	}),
+	methods: (state) => {
+		function remember(url: string, page: Page) {
+			state.cache.delete(url);
+			state.cache.set(url, page);
+			for (const key of state.cache.keys()) {
+				if (state.cache.size <= CACHE_LIMIT) {
+					break;
+				}
+				state.cache.delete(key);
+			}
+			return page;
+		}
+		function paused(resource: string, now: number): { readonly by: GithubPause; readonly until: number } | undefined {
+			const blocked = state.blocks.get(resource)?.until;
+			if (blocked !== undefined && blocked > now) {
+				return { by: "rate-limit", until: blocked };
+			}
+			const reserved = reservedUntil(state.observations.get(resource), now);
+			return reserved === undefined ? undefined : { by: "reserve", until: reserved };
+		}
+		function limited(observation: RecordedObservation, now: number): GithubObservation {
+			const pause = paused(observation.resource, now);
+			return { ...observation, limitedBy: pause?.by ?? null, limitedUntil: pause?.until ?? null };
+		}
+		const settle = Effect.fn("Fleet.settle")(function* (url: string, exitCode: number, response: ParsedResponse, now: number) {
+			const observation = readObservation(response.headers, now);
+			const resource = observation?.resource ?? requestResource(url);
+			if (observation !== undefined) {
+				state.observations.set(resource, observation);
+			}
+			const previous = state.blocks.get(resource);
+			const wait = rateLimitWait(response, observation, now, previous?.failures ?? 0);
+			if (wait !== undefined) {
+				const until = Math.max(wait, previous?.until ?? 0);
+				state.blocks.set(resource, { failures: (previous?.failures ?? 0) + 1, until });
+				return yield* githubPaused("rate-limit", until);
+			}
+			state.blocks.delete(resource);
+			const cached = state.cache.get(url);
+			if (response.status === 304 && cached !== undefined) {
+				return remember(url, cached);
+			}
+			if (exitCode !== 0 || response.status < 200 || response.status > 299) {
+				return yield* new ServiceUnavailable({ message: `GitHub request failed with HTTP ${response.status}` });
+			}
+			return remember(url, { body: response.body, etag: response.headers.get("etag"), next: nextLink(response.headers) });
+		});
+		const request = Effect.fn("Fleet.request")(function* (url: string) {
+			const pause = paused(requestResource(url), yield* Clock.currentTimeMillis);
+			if (pause !== undefined) {
+				return yield* githubPaused(pause.by, pause.until);
+			}
+			const etag = state.cache.get(url)?.etag;
+			const { exitCode, output } = yield* state.transport([
+				"api",
+				"--include",
+				"--method",
+				"GET",
+				...(etag === undefined ? [] : ["--header", `If-None-Match: ${etag}`]),
+				url,
+			]);
+			const response = parseResponse(output);
+			if (response === undefined) {
+				return yield* new ServiceUnavailable({ message: "GitHub returned a response without an HTTP status" });
+			}
+			return yield* settle(url, exitCode, response, yield* Clock.currentTimeMillis);
+		});
+		return {
+			get: (endpoint: string) => request(endpoint).pipe(Effect.map(({ body }) => body)),
+			observations: () =>
+				Effect.gen(function* () {
+					const now = yield* Clock.currentTimeMillis;
+					return [...state.observations.values()]
+						.map((observation) => limited(observation, now))
+						.sort((left, right) => left.resource.localeCompare(right.resource));
+				}),
+			pace: (intervalMs: number) =>
+				Effect.gen(function* () {
+					const now = yield* Clock.currentTimeMillis;
+					const core = state.observations.get("core");
+					return pace(core === undefined ? undefined : limited(core, now), intervalMs, now);
+				}),
+			pages: (endpoint: string) =>
+				Effect.gen(function* () {
+					const bodies: string[] = [];
+					let next: string | undefined = endpoint;
+					while (next !== undefined) {
+						if (bodies.length >= PAGE_LIMIT) {
+							return yield* new ServiceUnavailable({ message: `GitHub returned more than ${PAGE_LIMIT} pages` });
+						}
+						const page: Page = yield* request(next);
+						bodies.push(page.body);
+						next = page.next;
+					}
+					return `[${bodies.join(",")}]`;
+				}),
+		};
+	},
+	requires: [GithubTransport],
+});
+export type GithubApi = typeof GithubApi.Identifier;
+
+export const githubApiLayer = GithubApi.layer.pipe(Layer.provide(ghTransport));