@shivaedev/work-fleet 0.2.1 → 0.3.0

package/src/host/githubChecks.ts

@@ -2,8 +2,9 @@ import { Effect, Schema } from "effect";
 import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import { withFailureContext } from "#engine/failureContext.ts";
 import type { FleetError } from "#errors.ts";
+import type { GithubRead } from "#host/github.ts";
 import { CheckPages, githubCheckState, StatusPages } from "#host/githubCheckState.ts";
-import { githubRead } from "#host/githubRead.ts";
+import { githubFailure } from "#host/githubRateLimit.ts";
 import { githubRequiredChecks } from "#host/githubRequiredChecks.ts";
 import type { WorkResult } from "#policy.ts";
 
@@ -30,11 +31,11 @@ export function makeGithubChecks<R>(read: (endpoint: string) => Effect.Effect<st
 		const commit = `repos/${owner}/${repository}/commits/${result.head}`;
 		const checks = yield* read(`${commit}/check-runs?per_page=100&filter=latest`).pipe(
 			Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(CheckPages))),
-			Effect.mapError(() => failure("GitHub check runs are unavailable or invalid")),
+			Effect.mapError(githubFailure("GitHub check runs are unavailable or invalid")),
 		);
 		const statuses = yield* read(`${commit}/status?per_page=100`).pipe(
 			Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(StatusPages))),
-			Effect.mapError(() => failure("GitHub commit statuses are unavailable or invalid")),
+			Effect.mapError(githubFailure("GitHub commit statuses are unavailable or invalid")),
 		);
 		if (
 			statuses.length === 0
@@ -48,6 +49,6 @@ export function makeGithubChecks<R>(read: (endpoint: string) => Effect.Effect<st
 	});
 }
 
-export function githubChecks(result: WorkResult, branch = "main") {
-	return makeGithubChecks((endpoint) => githubRead(["api", "--method", "GET", "--paginate", "--slurp", endpoint]), branch)(result);
+export function githubChecks(read: GithubRead, branch = "main") {
+	return makeGithubChecks(read.pages, branch);
 }

package/src/host/githubDelivery.ts

@@ -1,10 +1,10 @@
-import * as NodeServices from "@effect/platform-node/NodeServices";
 import { Effect, Schema } from "effect";
 import { BadRequest, ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import { conflictingPr, movedPrHead } from "#engine/deliveryFailure.ts";
 import { withFailureContext } from "#engine/failureContext.ts";
 import type { FleetError } from "#errors.ts";
-import { githubRead } from "#host/githubRead.ts";
+import type { GithubRead } from "#host/github.ts";
+import { githubFailure } from "#host/githubRateLimit.ts";
 import type { ChangeResult } from "#policy.ts";
 
 const PullRequest = Schema.Struct({
@@ -21,20 +21,57 @@ function failure(message: string, reason: "integration" | "invalid" = "invalid")
 	return reason === "integration" ? new ServiceUnavailable({ message }) : new BadRequest({ message });
 }
 
-function readPullRequest(cwd: string, pr: string) {
-	return githubRead(["pr", "view", pr, "--json", "url,state,headRefOid,mergedAt,mergeCommit,mergeable,baseRefOid"], cwd).pipe(
-		Effect.provide(NodeServices.layer),
-		Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))),
-		Effect.mapError(() => failure("PR delivery observation failed; check GitHub CLI access and response", "integration")),
-	);
+const Sha = Schema.String.check(Schema.isPattern(/^[a-f\d]{40}$/u));
+const RestPullRequest = Schema.Struct({
+	base: Schema.Struct({ sha: Sha }),
+	head: Schema.Struct({ sha: Sha }),
+	htmlUrl: Schema.NonEmptyString,
+	mergeable: Schema.NullOr(Schema.Boolean),
+	mergeCommitSha: Schema.NullOr(Schema.String),
+	merged: Schema.Boolean,
+	mergedAt: Schema.NullOr(Schema.NonEmptyString),
+	state: Schema.Literals(["open", "closed"]),
+}).pipe(Schema.encodeKeys({ htmlUrl: "html_url", mergeCommitSha: "merge_commit_sha", mergedAt: "merged_at" }));
+
+function mergeability(mergeable: boolean | null) {
+	if (mergeable === null) {
+		return "UNKNOWN";
+	}
+	return mergeable ? "MERGEABLE" : "CONFLICTING";
+}
+
+function deliveryState(observed: typeof RestPullRequest.Type) {
+	if (observed.merged) {
+		return "MERGED";
+	}
+	return observed.state === "open" ? "OPEN" : "CLOSED";
+}
+
+export function readPullRequest(read: GithubRead["get"]) {
+	return (pr: string) => {
+		const match = /^https:\/\/github\.com\/(?<repository>[^/]+\/[^/]+)\/pull\/(?<number>[1-9]\d*)$/u.exec(pr)?.groups;
+		return read(`repos/${match?.repository}/pulls/${match?.number}`).pipe(
+			Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(RestPullRequest))),
+			Effect.map((observed): typeof PullRequest.Encoded => ({
+				baseRefOid: observed.base.sha,
+				headRefOid: observed.head.sha,
+				mergeable: mergeability(observed.mergeable),
+				mergeCommit: observed.merged && observed.mergeCommitSha !== null ? { oid: observed.mergeCommitSha } : null,
+				mergedAt: observed.mergedAt,
+				state: deliveryState(observed),
+				url: observed.htmlUrl,
+			})),
+			Effect.mapError(githubFailure("PR delivery observation failed; check GitHub CLI access and response")),
+		);
+	};
 }
 
-export function githubDelivery(cwd: string, read: (cwd: string, pr: string) => Effect.Effect<unknown, FleetError> = readPullRequest) {
+export function githubDelivery(read: (pr: string) => Effect.Effect<unknown, FleetError>) {
 	return Effect.fn("FleetHost.githubDelivery")(function* (result: typeof ChangeResult.Type) {
 		if (result.pr.match(/^https:\/\/github\.com\/[^/]+\/[^/]+\/pull\/[1-9]\d*$/u) === null) {
 			return yield* withFailureContext(new BadRequest({ message: "Delivery requires the adopted GitHub PR URL" }), { code: "delivery-pr-required" });
 		}
-		const pr = yield* Schema.decodeUnknownEffect(PullRequest)(yield* read(cwd, result.pr)).pipe(
+		const pr = yield* Schema.decodeUnknownEffect(PullRequest)(yield* read(result.pr)).pipe(
 			Effect.mapError(() => failure("PR delivery observation returned an unknown shape", "integration")),
 		);
 		yield* validateDelivery(pr, result);

package/src/host/githubRateLimit.ts

@@ -0,0 +1,91 @@
+import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
+import { failureContext, withFailureContext } from "#engine/failureContext.ts";
+import { headerCount, type ParsedResponse } from "#host/githubResponse.ts";
+
+export type GithubPause = "rate-limit" | "reserve";
+export interface GithubObservation {
+	readonly limit: number;
+	readonly limitedBy: GithubPause | null;
+	readonly limitedUntil: number | null;
+	readonly observedAt: number;
+	readonly remaining: number;
+	readonly reset: number;
+	readonly resource: string;
+	readonly used: number;
+}
+export type RecordedObservation = Omit<GithubObservation, "limitedBy" | "limitedUntil">;
+
+// Every local tool shares one GitHub token, so Fleet leaves the last fifth of each budget to agents and people.
+export const RESERVE_SHARE = 0.2;
+const FULL_SPEED_SHARE = 0.5;
+const SLOWEST_PACE_MS = 5 * 60_000;
+const BACKOFF_MS = 60_000;
+const BACKOFF_LIMIT_MS = 15 * 60_000;
+
+export function githubRateLimited(error: unknown): error is ServiceUnavailable {
+	return error instanceof ServiceUnavailable && failureContext(error).retryAt !== undefined;
+}
+
+export function githubFailure(message: string) {
+	return (error: unknown): ServiceUnavailable => (githubRateLimited(error) ? error : new ServiceUnavailable({ message }));
+}
+
+const PAUSE_MESSAGES: Readonly<Record<GithubPause, string>> = {
+	"rate-limit": "GitHub is rate-limited until",
+	reserve: `GitHub paused below the ${RESERVE_SHARE * 100}% reserve until`,
+};
+
+export function githubPaused(by: GithubPause, until: number) {
+	return withFailureContext(new ServiceUnavailable({ message: `${PAUSE_MESSAGES[by]} ${new Date(until).toISOString()}` }), {
+		attentionOwner: "automation",
+		retryAt: until,
+	});
+}
+
+export function reservedUntil(observation: RecordedObservation | undefined, now: number) {
+	if (observation === undefined || observation.reset * 1000 <= now) {
+		return undefined;
+	}
+	return observation.remaining < observation.limit * RESERVE_SHARE ? observation.reset * 1000 : undefined;
+}
+
+export function readObservation(headers: ReadonlyMap<string, string>, observedAt: number): RecordedObservation | undefined {
+	const resource = headers.get("x-ratelimit-resource");
+	const limit = headerCount(headers, "x-ratelimit-limit");
+	const remaining = headerCount(headers, "x-ratelimit-remaining");
+	const reset = headerCount(headers, "x-ratelimit-reset");
+	if (resource === undefined || limit === undefined || remaining === undefined || reset === undefined) {
+		return undefined;
+	}
+	return { limit, observedAt, remaining, reset, resource, used: headerCount(headers, "x-ratelimit-used") ?? limit - remaining };
+}
+
+export function rateLimitWait(response: ParsedResponse, observation: RecordedObservation | undefined, now: number, failures: number) {
+	const retryAfter = headerCount(response.headers, "retry-after");
+	const exhausted = observation?.remaining === 0 && observation.reset * 1000 > now ? observation.reset * 1000 : undefined;
+	const limited = observation?.remaining === 0 || retryAfter !== undefined || /rate limit/iu.test(response.body);
+	if (!([403, 429].includes(response.status) && limited)) {
+		return undefined;
+	}
+	if (retryAfter !== undefined) {
+		return now + retryAfter * 1000;
+	}
+	return exhausted ?? now + Math.min(BACKOFF_LIMIT_MS, BACKOFF_MS * 2 ** failures);
+}
+
+export function pace(observation: GithubObservation | undefined, intervalMs: number, now: number) {
+	if (observation === undefined) {
+		return intervalMs;
+	}
+	if (observation.limitedUntil !== null && observation.limitedUntil > now) {
+		return Math.max(intervalMs, observation.limitedUntil - now);
+	}
+	const reset = observation.reset * 1000;
+	const share = observation.remaining / Math.max(1, observation.limit);
+	if (reset <= now || share >= FULL_SPEED_SHARE) {
+		return intervalMs;
+	}
+	const stretch = Math.min(1, (FULL_SPEED_SHARE - share) / (FULL_SPEED_SHARE - RESERVE_SHARE)) ** 2;
+	const paced = intervalMs + (Math.max(intervalMs, SLOWEST_PACE_MS) - intervalMs) * stretch;
+	return Math.round(Math.max(intervalMs, Math.min(paced, reset - now)));
+}

package/src/host/githubRequiredChecks.ts

@@ -1,6 +1,6 @@
 import { Effect, Schema } from "effect";
-import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import type { FleetError } from "#errors.ts";
+import { githubFailure } from "#host/githubRateLimit.ts";
 
 const RulePages = Schema.Array(Schema.Array(Schema.Struct({ parameters: Schema.optional(Schema.Unknown), type: Schema.String })));
 const RequiredParameters = Schema.Struct({ requiredStatusChecks: Schema.Array(Schema.Struct({ context: Schema.String })) }).pipe(
@@ -15,5 +15,5 @@ export function githubRequiredChecks<R>(read: (endpoint: string) => Effect.Effec
 		const rules = pages.flat().filter((rule) => rule.type === "required_status_checks");
 		const parameters = yield* Effect.forEach(rules, (rule) => Schema.decodeUnknownEffect(RequiredParameters)(rule.parameters));
 		return [...new Set(parameters.flatMap((rule) => rule.requiredStatusChecks.map((check) => check.context)))];
-	}).pipe(Effect.mapError(() => new ServiceUnavailable({ message: "GitHub required status checks are unavailable or invalid" })));
+	}).pipe(Effect.mapError(githubFailure("GitHub required status checks are unavailable or invalid")));
 }

package/src/host/githubResponse.ts

@@ -0,0 +1,37 @@
+export interface ParsedResponse {
+	readonly body: string;
+	readonly headers: ReadonlyMap<string, string>;
+	readonly status: number;
+}
+
+export function parseResponse(output: string): ParsedResponse | undefined {
+	const split = /\r?\n\r?\n/u.exec(output);
+	const head = split === null ? output : output.slice(0, split.index);
+	const [statusLine = "", ...lines] = head.split(/\r?\n/u);
+	const status = Number(/^HTTP\/\S+ (?<status>\d{3})/u.exec(statusLine)?.groups?.status);
+	if (!Number.isInteger(status)) {
+		return undefined;
+	}
+	const headers = new Map<string, string>();
+	for (const line of lines) {
+		const colon = line.indexOf(":");
+		if (colon > 0) {
+			headers.set(line.slice(0, colon).trim().toLowerCase(), line.slice(colon + 1).trim());
+		}
+	}
+	return { body: split === null ? "" : output.slice(split.index + split[0].length), headers, status };
+}
+
+export function headerCount(headers: ReadonlyMap<string, string>, name: string) {
+	const value = headers.get(name);
+	return value !== undefined && /^\d+$/u.test(value) ? Number(value) : undefined;
+}
+
+export function nextLink(headers: ReadonlyMap<string, string>) {
+	return /<(?<url>[^>]+)>;\s*rel="next"/u.exec(headers.get("link") ?? "")?.groups?.url;
+}
+
+export function requestResource(endpoint: string) {
+	const path = endpoint.replace(/^https:\/\/api\.github\.com\//u, "");
+	return path.startsWith("search/") ? "search" : "core";
+}

package/src/host/githubResult.ts

@@ -1,21 +1,14 @@
-import * as NodeServices from "@effect/platform-node/NodeServices";
 import { Effect } from "effect";
 import { BadRequest, Conflict, ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
 import { withFailureContext } from "#engine/failureContext.ts";
 import type { FleetError } from "#errors.ts";
 import type { FleetHost } from "#host/config.ts";
-import { FilePages, type GithubRead, githubJson, PullRequest, Repository } from "#host/github.ts";
-import { githubRead } from "#host/githubRead.ts";
-import type { MainSource } from "#host/mainSource.ts";
+import { FilePages, type GithubRead, githubJson, PullRequest } from "#host/github.ts";
 import { resultReport } from "#host/resultReport.ts";
 import type { BoardWork, WorkResult } from "#policy.ts";
 import type { SessionObservation } from "#session/schema.ts";
 
-export function githubResult(
-	host: FleetHost,
-	main: MainSource["Service"],
-	read: GithubRead = (args) => githubRead(args, main.repositoryRoot).pipe(Effect.provide(NodeServices.layer)),
-) {
+export function githubResult(host: FleetHost, read: GithubRead, repository: Effect.Effect<string, FleetError>) {
 	return Effect.fn("FleetHost.githubResult")(function* (_work: BoardWork, observation: SessionObservation): Effect.fn.Return<WorkResult, FleetError> {
 		if (observation.execution !== "completed") {
 			return yield* new BadRequest({ message: "Result adoption requires completed worker execution" });
@@ -24,25 +17,30 @@ export function githubResult(
 		if (report.pr.kind === "none") {
 			return { evidence: [observation.output], head: report.head, kind: "no-change" };
 		}
-		return yield* adoptPullRequest(host, read, report.head, report.pr.url);
+		return yield* adoptPullRequest(host, read, yield* repository, report.head, report.pr.url);
 	});
 }
 
-const adoptPullRequest = Effect.fn("FleetHost.adoptPullRequest")(function* (host: FleetHost, read: GithubRead, head: string, prUrl: string) {
-	const repository = yield* githubJson(Repository, read, ["repo", "view", "--json", "nameWithOwner"]);
+const adoptPullRequest = Effect.fn("FleetHost.adoptPullRequest")(function* (
+	host: FleetHost,
+	read: GithubRead,
+	repository: string,
+	head: string,
+	prUrl: string,
+) {
 	const url = new URL(prUrl);
 	const parts = url.pathname.split("/");
 	const owner = parts[1];
 	const name = parts[2];
 	const number = Number(parts[4]);
-	if (!Number.isSafeInteger(number) || `${owner}/${name}`.toLowerCase() !== repository.nameWithOwner.toLowerCase()) {
+	if (!Number.isSafeInteger(number) || `${owner}/${name}`.toLowerCase() !== repository.toLowerCase()) {
 		return yield* withFailureContext(new BadRequest({ message: "Reported PR must belong to the host repository" }), { code: "foreign-pr" });
 	}
-	const endpoint = `repos/${repository.nameWithOwner}/pulls/${number}`;
-	const before = yield* githubJson(PullRequest, read, ["api", endpoint]);
+	const endpoint = `repos/${repository}/pulls/${number}`;
+	const before = yield* githubJson(PullRequest, read.get(endpoint));
 	yield* validatePullRequest(before, head, number, prUrl, host.repository.branch);
-	const files = (yield* githubJson(FilePages, read, ["api", `${endpoint}/files?per_page=100`, "--paginate", "--slurp"])).flat();
-	const after = yield* githubJson(PullRequest, read, ["api", endpoint]);
+	const files = (yield* githubJson(FilePages, read.pages(`${endpoint}/files?per_page=100`))).flat();
+	const after = yield* githubJson(PullRequest, read.get(endpoint));
 	yield* validatePullRequest(after, head, number, prUrl, host.repository.branch);
 	if (before.changedFiles !== after.changedFiles) {
 		return yield* new Conflict({ message: "PR changed paths moved during observation" });

package/src/host/githubTransport.ts

@@ -0,0 +1,31 @@
+import { Context, Effect, Layer, Stream } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
+import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
+import type { FleetError } from "#errors.ts";
+
+export interface GithubResponse {
+	readonly exitCode: number;
+	readonly output: string;
+}
+export class GithubTransport extends Context.Service<GithubTransport, (args: readonly string[]) => Effect.Effect<GithubResponse, FleetError>>()(
+	"Fleet/GithubTransport",
+) {}
+
+export const ghTransport = Layer.effect(GithubTransport)(
+	Effect.gen(function* () {
+		const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+		return (args: readonly string[]) =>
+			Effect.scoped(
+				Effect.gen(function* () {
+					const process = yield* spawner.spawn(ChildProcess.make("gh", args));
+					const output = yield* process.stdout.pipe(Stream.decodeText, Stream.mkString);
+					return { exitCode: Number(yield* process.exitCode), output };
+				}),
+			).pipe(
+				Effect.timeout("60 seconds"),
+				Effect.mapError(
+					() => new ServiceUnavailable({ message: "GitHub evidence could not be read; check gh authentication and repository access" }),
+				),
+			);
+	}),
+);

package/src/host/integrations.ts

@@ -0,0 +1,40 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { Effect, Layer } from "effect";
+import { defineService } from "@shivaedev/effect-service/define-service.ts";
+import { ServiceUnavailable } from "@shivaedev/platform/errors/taxonomy.ts";
+import type { DecisionCode } from "#DecisionCode.ts";
+import { withFailureContext } from "#engine/failureContext.ts";
+import type { FleetHost } from "#host/config.ts";
+import { GithubApi } from "#host/githubApi.ts";
+import { githubChecks } from "#host/githubChecks.ts";
+import { githubDelivery, readPullRequest } from "#host/githubDelivery.ts";
+import { githubResult } from "#host/githubResult.ts";
+import { MainSource } from "#host/mainSource.ts";
+import { reportRepository } from "#host/reportPublication.ts";
+import { FleetIntegrations } from "#policy.ts";
+
+function unconfigured(message: string, code: DecisionCode = "fleet-integration") {
+	return Effect.fail(withFailureContext(new ServiceUnavailable({ message }), { attentionOwner: "automation", code, retryable: false }));
+}
+
+export function hostIntegrations(host: FleetHost) {
+	const HostIntegrations = defineService({
+		id: "Fleet/HostIntegrations",
+		initialize: Effect.all({ github: GithubApi, main: MainSource }),
+		methods: ({ github, main }) =>
+			FleetIntegrations.of({
+				checks: (_work, result) =>
+					result.kind === "no-change"
+						? unconfigured("No-change validation is not configured in this Fleet host; inspect the worker session", "no-change-checks-unconfigured")
+						: githubChecks(github, host.repository.branch)(result),
+				delivery: githubDelivery(readPullRequest(github.get)),
+				main: main.observe,
+				result:
+					host.result === "github"
+						? githubResult(host, github, reportRepository(main.repositoryRoot, host.repository.remote).pipe(Effect.provide(NodeServices.layer)))
+						: () => unconfigured("Worker result adoption is not configured in this Fleet host; inspect the worker session"),
+			}),
+		requires: [MainSource, GithubApi],
+	});
+	return Layer.effect(FleetIntegrations)(HostIntegrations).pipe(Layer.provide(HostIntegrations.layer));
+}

package/src/host/liveConfig.ts

@@ -0,0 +1,134 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import {
+	Clock,
+	Context,
+	Deferred,
+	Effect,
+	Equal,
+	Fiber,
+	FileSystem,
+	Layer,
+	Result,
+	Schedule,
+	Schema,
+	Semaphore,
+	Stream,
+	SubscriptionRef,
+} from "effect";
+import { type FleetHost, HostConfig, HostConfigRejected, type HostFailure, readHostConfig, resolveHost } from "#host/config.ts";
+import { changedSettings, withReloadable } from "#host/configSettings.ts";
+import { hostGuidance } from "#host/guidance.ts";
+
+type Role = "reviewer" | "worker";
+type Guidance = Effect.Effect<string | undefined, HostFailure>;
+
+export interface AppliedConfig {
+	readonly guidance: Readonly<Record<Role, Guidance>>;
+	readonly host: FleetHost;
+	readonly loadedAt: number;
+	readonly rejected?: HostConfigRejected;
+	readonly restartRequired: readonly string[];
+}
+
+export class LiveConfig extends Context.Service<
+	LiveConfig,
+	{ readonly changes: Stream.Stream<AppliedConfig>; readonly current: Effect.Effect<AppliedConfig> }
+>()("@shivaedev/work-fleet/LiveConfig") {}
+
+const SETTLE = "100 millis";
+const RETRY = Schedule.min([Schedule.exponential("100 millis"), Schedule.spaced("5 seconds")]);
+
+const applyHost = Effect.fn("FleetHost.applyHost")(function* <E>(host: FleetHost, rejected: (role: Role, failure: HostFailure) => E) {
+	const guidance = yield* Effect.all({
+		reviewer: hostGuidance(host, "reviewer").pipe(Effect.mapError((failure) => rejected("reviewer", failure))),
+		worker: hostGuidance(host, "worker").pipe(Effect.mapError((failure) => rejected("worker", failure))),
+	});
+	const applied: AppliedConfig = { guidance, host, loadedAt: yield* Clock.currentTimeMillis, restartRequired: [] };
+	return applied;
+});
+
+function served(ref: SubscriptionRef.SubscriptionRef<AppliedConfig>) {
+	return LiveConfig.of({ changes: SubscriptionRef.changes(ref), current: SubscriptionRef.get(ref) });
+}
+
+export function staticConfig(host: FleetHost) {
+	return Layer.effect(LiveConfig)(applyHost(host, (_role, failure) => failure).pipe(Effect.flatMap(SubscriptionRef.make), Effect.map(served)));
+}
+
+export function watchedConfig(host: FleetHost) {
+	const file = host.configFile;
+	return file === undefined ? staticConfig(host) : Layer.effect(LiveConfig)(watchConfig(host, file)).pipe(Layer.provide(NodeServices.layer));
+}
+
+const watchConfig = Effect.fn("FleetHost.watchConfig")(function* (host: FleetHost, file: string) {
+	const fs = yield* FileSystem.FileSystem;
+	const ref = yield* SubscriptionRef.make(yield* applyHost(host, (_role, failure) => failure));
+	const lock = yield* Semaphore.make(1);
+	let seen = host.config;
+	const loadEdit = Effect.fn("FleetHost.loadEdit")(function* (previous: AppliedConfig, next: HostConfig) {
+		const config = yield* Schema.decodeUnknownEffect(HostConfig)(withReloadable(previous.host.config, next)).pipe(
+			Effect.mapError((error) => new HostConfigRejected({ file, message: `Fleet host configuration ${file} cannot be applied: ${error.message}` })),
+		);
+		const applied = yield* applyHost(
+			resolveHost(previous.host.boardDir, config, file),
+			(role, failure) => new HostConfigRejected({ field: `worker.${role}Guidance`, file, message: failure.message }),
+		);
+		return { ...applied, restartRequired: changedSettings(config, next) };
+	});
+	const reload = lock.withPermits(1)(
+		Effect.gen(function* () {
+			const previous = yield* SubscriptionRef.get(ref);
+			const outcome = yield* readHostConfig(file).pipe(
+				Effect.flatMap((edited) =>
+					Equal.equals(edited, seen) && previous.rejected === undefined
+						? Effect.succeed(undefined)
+						: Effect.map(loadEdit(previous, edited), (loaded) => ({ applied: loaded, next: edited })),
+				),
+				Effect.result,
+			);
+			if (Result.isFailure(outcome)) {
+				if (previous.rejected?.message !== outcome.failure.message) {
+					yield* Effect.logWarning("Fleet kept its last good config after a rejected edit", outcome.failure);
+					yield* SubscriptionRef.set(ref, { ...previous, rejected: outcome.failure });
+				}
+				return;
+			}
+			if (outcome.success === undefined) {
+				return;
+			}
+			const { applied, next } = outcome.success;
+			seen = next;
+			if (applied.restartRequired.length > 0 && !Equal.equals(applied.restartRequired, previous.restartRequired)) {
+				yield* Effect.logWarning(`Fleet keeps its running values until a daemon restart for: ${applied.restartRequired.join(", ")}`);
+			}
+			yield* SubscriptionRef.set(ref, applied);
+		}),
+	);
+	const ready = yield* Deferred.make<void>();
+	// An editor that saves by renaming replaces the watched file, so each replacement starts a new watch on the new file.
+	const watching = Effect.scoped(
+		Effect.gen(function* () {
+			const events = yield* fs.watch(file).pipe(
+				Stream.takeUntil((event) => event._tag !== "Update"),
+				Stream.debounce(SETTLE),
+				Stream.runForEach(() => reload),
+				Effect.forkScoped,
+			);
+			// The watch registers once its fiber runs; reading after that applies an edit made while no watch was active.
+			yield* Effect.sleep(SETTLE);
+			yield* reload;
+			yield* Deferred.succeed(ready, undefined);
+			yield* Fiber.join(events);
+		}),
+	);
+	yield* watching.pipe(
+		Effect.tapError((error) =>
+			Deferred.succeed(ready, undefined).pipe(Effect.andThen(Effect.logWarning("Watching the Fleet config failed; retrying", error))),
+		),
+		Effect.retry(RETRY),
+		Effect.forever,
+		Effect.forkScoped,
+	);
+	yield* Deferred.await(ready);
+	return served(ref);
+});

package/src/host/providers.ts

@@ -23,8 +23,10 @@ export function hostProvider(host: FleetHost) {
 					Effect.fail(new HostFailure({ message: 'provider "cloud" needs cloud settings in the host configuration' })),
 				);
 			}
-			const { requestTimeoutMs = 60_000, ...cloud } = host.cloud;
-			return codexCloudLayer.pipe(Layer.provide([Layer.succeed(CodexCloudConfig)({ ...cloud, requestTimeoutMs }), NodeServices.layer]));
+			const { recoveryGraceMs = 5000, requestTimeoutMs = 60_000, ...cloud } = host.cloud;
+			return codexCloudLayer.pipe(
+				Layer.provide([Layer.succeed(CodexCloudConfig)({ ...cloud, recoveryGraceMs, requestTimeoutMs }), NodeServices.layer]),
+			);
 		}
 	}
 }

package/src/host/readWaitStates.ts

@@ -16,25 +16,13 @@ export const readWaitStates = Effect.fn("FleetReader.readWaitStates")(function*
 		...Object.fromEntries(Object.entries(shapeColumns(shape)).map(([name, kind]) => [name, columnSchema(kind)])),
 		"work_id": Schema.String,
 	}) {}
-	const read = Effect.gen(function* () {
-		const repository = yield* makeRepository(SnapshotRow, {
-			idColumn: "work_id",
-			spanPrefix: "FleetReader",
-			tableName: `${options.namespace}_records`,
-		});
-		const rows = yield* repository
-			.findMany({
-				orderBy: { direction: "asc", field: "work_id" },
-			})
-			.pipe(
-				Effect.catchTag("SqlError", (error) => {
-					const cause = error.reason.cause;
-					return typeof cause === "object" && cause !== null && "code" in cause && cause.code === "42P01" ? Effect.succeed([]) : Effect.fail(error);
-				}),
-			);
-		const selected = rows.filter((row) => ids === undefined || ids.includes(row.work_id));
-		const states = yield* Schema.decodeUnknownEffect(Schema.Array(State))(yield* recordValues(selected, shape, options.namespace));
-		return states.map((state) => ({ ...state, outcome: state.outcome ?? {} }));
+	const repository = yield* makeRepository(SnapshotRow, {
+		idColumn: "work_id",
+		spanPrefix: "FleetReader",
+		tableName: `${options.namespace}_records`,
 	});
-	return yield* read;
+	const rows = yield* repository.findMany({ orderBy: { direction: "asc", field: "work_id" } });
+	const selected = rows.filter((row) => ids === undefined || ids.includes(row.work_id));
+	const states = yield* Schema.decodeUnknownEffect(Schema.Array(State))(yield* recordValues(selected, shape, options.namespace));
+	return states.map((state) => ({ ...state, outcome: state.outcome ?? {} }));
 });

package/src/host/releaseWork.ts

@@ -13,26 +13,29 @@ const { searchSnapshot, watchChanges } = board;
 const { ResponseConfig, Responses } = responses;
 
 export function releaseWork(host: FleetHost, workIds: readonly string[], parts: HostParts = productionParts(host)) {
-	return releaseItems(host, workIds).pipe(Effect.provide(hostLayer(host, parts)));
+	return withDecisionResponses(host, releaseItems(host, workIds)).pipe(Effect.provide(hostLayer(host, parts)));
 }
 
 export function releaseItems(host: FleetHost, workIds: readonly string[]) {
-	return withDecisionResponses(host, Effect.forEach(workIds, releaseItem(host.boardDir)));
+	return Effect.forEach(workIds, releaseItem(host.boardDir));
 }
 
-export function withDecisionResponses<A, E, R>(host: FleetHost, effect: Effect.Effect<A, E, R>) {
-	return Effect.scoped(
+// Indexing a large Board takes seconds, so the watched index starts warming at once and later responses read only changed files.
+export function decisionResponses(host: FleetHost) {
+	return Layer.unwrap(
 		Effect.gen(function* () {
 			const changes = yield* watchChanges(host.boardDir);
-			const read = yield* searchSnapshot(host.boardDir, undefined, changes);
-			const index = read.pipe(Effect.provide(NodeServices.layer));
-			return yield* effect.pipe(
-				Effect.provide(Responses.layer.pipe(Layer.provide(Layer.succeed(ResponseConfig, { changes, enabled: true, index, root: host.boardDir })))),
-			);
+			const index = (yield* searchSnapshot(host.boardDir, undefined, changes)).pipe(Effect.provide(NodeServices.layer));
+			yield* index.pipe(Effect.ignore, Effect.forkScoped);
+			return Responses.layer.pipe(Layer.provide(Layer.succeed(ResponseConfig, { changes, enabled: true, index, root: host.boardDir })));
 		}),
 	);
 }
 
+export function withDecisionResponses<A, E, R>(host: FleetHost, effect: Effect.Effect<A, E, R>) {
+	return effect.pipe(Effect.provide(decisionResponses(host)));
+}
+
 function releaseItem(root: string) {
 	return Effect.fn("FleetHost.releaseItem")(function* (workId: string) {
 		const fleet = yield* Fleet;