@sshkeda/task 0.0.0-stage → 1.0.1

package/src/lib/main-thread/lib/run-tasks-on-workers/internal/handle-worker-events.ts

@@ -0,0 +1,348 @@
+import { Clock, Deferred, Effect, Exit, Fiber, FiberSet, Match, Option, Schema } from "effect";
+
+import {
+  dieOnInvariantFailure,
+  InternalInvariantBroken,
+} from "../../../../../shared/invariant-errors.ts";
+import { TASK_TRACE_LABEL_PREFIX } from "../../../../../shared/names-from-package-json.ts";
+import { printLine, printTaskLines } from "../../../../../shared/print-to-standard-error.ts";
+import { WorkerExited } from "../../../../../shared/process-errors.ts";
+import { TaskFiberResult } from "../../../../../shared/rpc.ts";
+import {
+  TaskCleanupFailed,
+  ServedTaskFailed,
+  TaskBodyFailed,
+} from "../../../../../shared/task-errors.ts";
+import type { AwaitedTasksFailure, TaskRequestFailure } from "../../../../../shared/task-errors.ts";
+import { TraceFile } from "../../../../../shared/write-trace-file.ts";
+import { failRun, failTask } from "../../../shared/fail-run.ts";
+import { ProcessGroups } from "../../../shared/provide-process-groups.ts";
+import { StopState } from "../../../shared/provide-stop-state.ts";
+import { FromWorker, WorkerEvent, WorkerPool } from "../../../shared/provide-worker-pool.ts";
+import { reportFailure } from "../../../shared/report-task-failure.ts";
+import { findAssignedWorkerId, TaskRunStatus } from "../shared/define-task-run-state.ts";
+import type { makeTaskRunsState } from "../shared/define-task-run-state.ts";
+
+const sendDependencyResultToWorker = Effect.fn("sendDependencyResultToWorker")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  requestDependencyTaskResult: (
+    dependencyTaskName: string,
+    requestingTaskName: string,
+  ) => Effect.Effect<Schema.Json | undefined, typeof TaskRequestFailure.Type>,
+  messageFromWorker: typeof FromWorker.cases.TaskFiberRequestedTask.Type,
+) {
+  yield* Effect.annotateCurrentSpan({
+    [`${TASK_TRACE_LABEL_PREFIX}.task`]: messageFromWorker.taskName,
+    [`${TASK_TRACE_LABEL_PREFIX}.wanted_task`]: messageFromWorker.dependencyTaskName,
+  });
+  const requestingTaskRun = yield* Effect.fromNullishOr(
+    taskRunsState.runsByTaskName.get(messageFromWorker.taskName),
+  ).pipe(dieOnInvariantFailure("task requests", "a task that requests another task has a run"));
+  const answer = yield* requestDependencyTaskResult(
+    messageFromWorker.dependencyTaskName,
+    messageFromWorker.taskName,
+  ).pipe(
+    Effect.exit,
+    Effect.raceFirst(
+      Deferred.await(requestingTaskRun.bodyFinishedDeferred).pipe(Effect.andThen(Effect.interrupt)),
+    ),
+  );
+  const workerId = findAssignedWorkerId(requestingTaskRun.status);
+  if (workerId === null) {
+    return;
+  }
+
+  yield* (yield* WorkerPool).answerDependencyRequest(messageFromWorker.dependencyRequestId, answer);
+});
+
+const isTaskBodyFailed = Schema.is(TaskBodyFailed);
+
+const isWorkerExited = Schema.is(WorkerExited);
+
+const isTaskCleanupFailed = Schema.is(TaskCleanupFailed);
+
+const isTaskEnded = TaskRunStatus.isAnyOf(["Failed", "Cancelled"]);
+
+const bodyEnding = (taskName: string, result: typeof TaskFiberResult.Type) =>
+  TaskFiberResult.match(result, {
+    Returned: (
+      returned,
+    ): Exit.Exit<
+      typeof returned,
+      TaskBodyFailed | TaskCleanupFailed | WorkerExited | typeof AwaitedTasksFailure.Type
+    > => Exit.succeed(returned),
+    Failed: (failed) =>
+      Exit.fail(
+        new TaskBodyFailed({
+          taskName,
+          failures: failed.failures,
+          details: failed.details,
+          failedCommandExitCode: failed.failedCommandExitCode,
+        }),
+      ),
+    AwaitedTasksFailed: (awaitedTasksFailed) => Exit.fail(awaitedTasksFailed.failure),
+    CleanupFailed: (cleanupFailure) =>
+      Exit.fail(
+        new TaskCleanupFailed({
+          taskName,
+          failures: cleanupFailure.failures,
+          details: cleanupFailure.details,
+        }),
+      ),
+    Interrupted: () => Exit.interrupt(),
+  });
+
+const settleBodyOutcome = Effect.fn("settleBodyOutcome")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  taskName: string,
+  ending: Exit.Exit<
+    Effect.Success<ReturnType<typeof bodyEnding>>,
+    Effect.Error<ReturnType<typeof bodyEnding>>
+  >,
+) {
+  const taskRun = taskRunsState.runsByTaskName.get(taskName);
+  const isOutcomeAlreadySettled =
+    taskRun === undefined || Deferred.isDoneUnsafe(taskRun.bodyOutcomeDeferred);
+  if (isOutcomeAlreadySettled) {
+    return;
+  }
+  const failedAfterTaskEnded = isTaskEnded(taskRun.status)
+    ? Option.filter(Exit.findErrorOption(ending), isTaskBodyFailed)
+    : Option.none();
+  Deferred.doneUnsafe(taskRun.bodyOutcomeDeferred, ending);
+  if (Option.isSome(failedAfterTaskEnded)) {
+    yield* failTask(failedAfterTaskEnded.value, taskRunsState.markFailed);
+  }
+});
+
+const endTask = Effect.fn("endTask")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  taskName: string,
+  ending: Exit.Exit<
+    Effect.Success<ReturnType<typeof bodyEnding>>,
+    Effect.Error<ReturnType<typeof bodyEnding>>
+  >,
+) {
+  const taskRun = taskRunsState.runsByTaskName.get(taskName);
+  const isBodyAlreadyFinished =
+    taskRun === undefined || Deferred.isDoneUnsafe(taskRun.bodyFinishedDeferred);
+  if (isBodyAlreadyFinished) {
+    return;
+  }
+
+  Deferred.doneUnsafe(taskRun.bodyFinishedDeferred, Exit.void);
+  taskRunsState.refreshBodyClocks(taskName, yield* Clock.currentTimeMillis);
+  const servedBeforeEnding = taskRunsState.markClosed(taskName);
+  if (servedBeforeEnding) {
+    yield* printTaskLines("Info", taskName, "closed");
+  }
+  if (!Deferred.isDoneUnsafe(taskRun.bodyOutcomeDeferred)) {
+    return yield* settleBodyOutcome(taskRunsState, taskName, ending);
+  }
+  const failure = Exit.findErrorOption(ending);
+  if (Option.isNone(failure)) {
+    return;
+  }
+  const failureEndsRun = isTaskCleanupFailed(failure.value) || isWorkerExited(failure.value);
+  if (failureEndsRun) {
+    yield* (yield* StopState).recordFirstFailure(failure.value);
+    return yield* failTask(failure.value, taskRunsState.markFailed);
+  }
+  if (!servedBeforeEnding) {
+    return;
+  }
+
+  yield* failRun(
+    new ServedTaskFailed({
+      taskName,
+      failures: isTaskBodyFailed(failure.value)
+        ? failure.value.failures
+        : reportFailure(failure.value),
+      details: isTaskBodyFailed(failure.value) ? failure.value.details : undefined,
+    }),
+    taskRunsState.markFailed,
+  );
+});
+
+const endTasksOfExitedWorker = Effect.fn("endTasksOfExitedWorker")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  workerEvent: typeof WorkerEvent.cases.ExitedWithCode.Type,
+) {
+  const processGroups = yield* ProcessGroups;
+  const unfinished = [...taskRunsState.runsByTaskName].filter(
+    ([, taskRun]) =>
+      findAssignedWorkerId(taskRun.status) === workerEvent.workerId &&
+      !Deferred.isDoneUnsafe(taskRun.bodyFinishedDeferred),
+  );
+  if (!workerEvent.terminationRequested) {
+    for (const [taskName] of unfinished) {
+      yield* processGroups.killForTask(taskName);
+    }
+  }
+  const workerExit = {
+    workerId: workerEvent.workerId,
+    exitCode: workerEvent.exitCode,
+    uncaughtError: workerEvent.uncaughtError,
+  };
+  const exitedUnexpectedlyWithoutTasks =
+    !workerEvent.terminationRequested && unfinished.length === 0;
+  if (exitedUnexpectedlyWithoutTasks) {
+    return yield* failRun(
+      new WorkerExited({ taskName: null, ...workerExit }),
+      taskRunsState.markFailed,
+    );
+  }
+  for (const [taskName] of unfinished) {
+    yield* endTask(
+      taskRunsState,
+      taskName,
+      workerEvent.terminationRequested
+        ? Exit.interrupt()
+        : Exit.fail(new WorkerExited({ taskName, ...workerExit })),
+    );
+  }
+});
+
+const onMessageAboutTask = Effect.fn("onMessageAboutTask")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  requestDependencyTaskResult: Parameters<typeof sendDependencyResultToWorker>[1],
+  messageFromWorker: Exclude<
+    typeof FromWorker.Type,
+    | typeof FromWorker.cases.StandardErrorLineRequested.Type
+    | typeof FromWorker.cases.TraceWriteFailed.Type
+  >,
+) {
+  const processGroups = yield* ProcessGroups;
+  const clock = yield* Clock.Clock;
+  const taskRun = yield* Effect.fromNullishOr(
+    taskRunsState.runsByTaskName.get(messageFromWorker.taskName),
+  ).pipe(dieOnInvariantFailure("worker protocol", "messages refer to a started task"));
+  yield* Match.value(messageFromWorker).pipe(
+    Match.tagsExhaustive({
+      TaskFiberStartedProcess: (started) =>
+        processGroups
+          .record(started.processId, started.taskName, started.startTime)
+          .pipe(
+            Effect.andThen(
+              WorkerPool.use((workerPool) =>
+                workerPool.acknowledgeProcessRecorded(started.recordId),
+              ),
+            ),
+          ),
+      TaskFiberProcessLeaderExited: (exited) =>
+        processGroups.recordLeaderExit(exited.processId, exited.latestMemberStartTime),
+      TaskFiberClaimingProcessReports: (claiming) =>
+        processGroups
+          .claimProcessesToReport(claiming.processes)
+          .pipe(
+            Effect.flatMap((claimed) =>
+              WorkerPool.use((workerPool) =>
+                workerPool.answerReportClaim(claiming.claimId, claimed),
+              ),
+            ),
+          ),
+      TaskFiberLoading: () => Deferred.done(taskRun.bodyLoadingDeferred, Exit.void),
+      TaskFiberStarted: () => {
+        taskRun.stepBeforeBody = null;
+        return Deferred.complete(
+          taskRun.bodyStartedAtMillisecondsDeferred,
+          Clock.currentTimeMillis,
+        );
+      },
+      TaskFiberStartedCleanUp: () => Deferred.done(taskRun.bodyCleanupStartedDeferred, Exit.void),
+      TaskFiberRequestedTask: (requested) => {
+        taskRun.bodyRequestedTaskNames.add(requested.dependencyTaskName);
+        return FiberSet.run(
+          taskRunsState.taskFibers,
+          sendDependencyResultToWorker(taskRunsState, requestDependencyTaskResult, requested),
+          { startImmediately: false },
+        ).pipe(
+          Effect.flatMap((requestFiber) =>
+            Effect.sync(() => {
+              taskRun.requestFibersByDependencyRequestId.set(
+                requested.dependencyRequestId,
+                requestFiber,
+              );
+              requestFiber.addObserver(() => {
+                taskRun.requestFibersByDependencyRequestId.delete(requested.dependencyRequestId);
+                taskRunsState.refreshBodyClocks(
+                  requested.taskName,
+                  clock.currentTimeMillisUnsafe(),
+                );
+              });
+            }),
+          ),
+        );
+      },
+      TaskFiberRequestedStandardInput: (requested) =>
+        taskRunsState.rootTaskStandardInput.answerNextRead((read) =>
+          WorkerPool.use((workerPool) =>
+            workerPool.answerStandardInputRead(requested.readId, read),
+          ),
+        ),
+      TaskFiberReceivedAwaitedFailure: (received) =>
+        Effect.sync(() => {
+          taskRunsState.recordFailureReachedAwaitingTask(received.awaitedTaskName);
+        }),
+      TaskFiberCancelledRequest: (cancelled) => {
+        const requestFiber = taskRun.requestFibersByDependencyRequestId.get(
+          cancelled.dependencyRequestId,
+        );
+        return requestFiber === undefined ? Effect.void : Fiber.interrupt(requestFiber);
+      },
+      TaskFiberFailedBeforeCleanup: (failed) =>
+        Deferred.done(taskRun.bodyFailedBeforeCleanupDeferred, Exit.void).pipe(
+          Effect.andThen(Effect.exit(bodyEnding(failed.taskName, failed.failure))),
+          Effect.flatMap((ending) => settleBodyOutcome(taskRunsState, failed.taskName, ending)),
+        ),
+      TaskFiberServed: (served) =>
+        Effect.sync(() => {
+          taskRunsState.markServing(served.taskName);
+          Deferred.doneUnsafe(taskRun.bodyOutcomeDeferred, Exit.succeed(served));
+        }),
+      TaskFiberEnded: (ended) => {
+        taskRun.logMessages = [...ended.logs];
+        return Effect.exit(bodyEnding(ended.taskName, ended.result)).pipe(
+          Effect.flatMap((ending) => endTask(taskRunsState, messageFromWorker.taskName, ending)),
+        );
+      },
+    }),
+  );
+  taskRunsState.refreshBodyClocks(messageFromWorker.taskName, yield* Clock.currentTimeMillis);
+});
+
+export const handleWorkerEvent = Effect.fn("handleWorkerEvent")(function* (
+  taskRunsState: Effect.Success<typeof makeTaskRunsState>,
+  requestDependencyTaskResult: Parameters<typeof sendDependencyResultToWorker>[1],
+  workerEvent: typeof WorkerEvent.Type,
+) {
+  yield* Effect.annotateCurrentSpan(`${TASK_TRACE_LABEL_PREFIX}.worker`, workerEvent.workerId);
+  if (WorkerEvent.guards.ExitedWithCode(workerEvent)) {
+    return yield* endTasksOfExitedWorker(taskRunsState, workerEvent);
+  }
+  if (WorkerEvent.guards.MessageUnreadable(workerEvent)) {
+    return yield* Effect.die(
+      new InternalInvariantBroken({
+        area: "worker protocol",
+        invariant: "messages match the worker protocol",
+        cause: workerEvent.reason,
+      }),
+    );
+  }
+  const messageFromWorker = workerEvent.message;
+  if (FromWorker.guards.StandardErrorLineRequested(messageFromWorker)) {
+    return yield* printLine(messageFromWorker.logLevel, messageFromWorker.text);
+  }
+  if (FromWorker.guards.TraceWriteFailed(messageFromWorker)) {
+    return yield* TraceFile.use((traceFile) =>
+      traceFile.markIncomplete(
+        `worker ${workerEvent.workerId} reported that ${messageFromWorker.reason}`,
+      ),
+    );
+  }
+  yield* Effect.annotateCurrentSpan({
+    [`${TASK_TRACE_LABEL_PREFIX}.message`]: messageFromWorker._tag,
+    [`${TASK_TRACE_LABEL_PREFIX}.task`]: messageFromWorker.taskName,
+  });
+  yield* onMessageAboutTask(taskRunsState, requestDependencyTaskResult, messageFromWorker);
+});

package/src/lib/main-thread/lib/run-tasks-on-workers/internal/make-task-runs.ts

@@ -0,0 +1,207 @@
+import { Context, Deferred, Effect, Exit, FiberSet } from "effect";
+import type { FileSystem, Schema } from "effect";
+
+import { printLine, printTaskLines } from "../../../../../shared/print-to-standard-error.ts";
+import type { TaskRequestFailure } from "../../../../../shared/task-errors.ts";
+import { TASK_PACKAGE } from "../../../../../shared/task-package.ts";
+import type { TraceFile } from "../../../../../shared/write-trace-file.ts";
+import { failRunOnFailure, failTask } from "../../../shared/fail-run.ts";
+import { ServingTaskRequestsItself } from "../../../shared/main-thread-errors.ts";
+import type { ProcessGroups } from "../../../shared/provide-process-groups.ts";
+import type { RunRootTaskConfig } from "../../../shared/provide-run-root-task-config.ts";
+import { StopState } from "../../../shared/provide-stop-state.ts";
+import type { WorkerEvent, WorkerPool } from "../../../shared/provide-worker-pool.ts";
+import {
+  runRequestedTask,
+  runRequestedDependencyTask,
+} from "../lib/run-requested-task-once/index.ts";
+import { makeTaskRunsState, TaskRunStatus } from "../shared/define-task-run-state.ts";
+import { stopTask } from "../shared/stop-task.ts";
+import { handleWorkerEvent } from "./handle-worker-events.ts";
+
+export const makeTaskRuns = Effect.gen(function* () {
+  const taskExecutionServices = yield* Effect.context<
+    FileSystem.FileSystem | ProcessGroups | RunRootTaskConfig | StopState | TraceFile | WorkerPool
+  >();
+  const taskRunsState = yield* makeTaskRunsState;
+  const resultDeferreds: Parameters<typeof runRequestedTask>[1] = new Map();
+  const {
+    discoveredTasksByName,
+    taskNamesWhoseFailureReachedAnAwaitingTask,
+    runsByTaskName: taskRunsByName,
+    markFailed,
+  } = taskRunsState;
+  const requestDependencyTaskResult = (
+    dependencyTaskName: string,
+    requestingTaskName: string,
+  ): Effect.Effect<Schema.Json | undefined, typeof TaskRequestFailure.Type> =>
+    runRequestedDependencyTask(
+      taskRunsState,
+      resultDeferreds,
+      dependencyTaskName,
+      requestingTaskName,
+    ).pipe(Effect.provide(taskExecutionServices));
+  return {
+    registerDiscoveredTasks: (
+      entries: ReadonlyMap<string, Parameters<typeof discoveredTasksByName.set>[1]>,
+    ) =>
+      Effect.sync(() => {
+        for (const [taskName, discoveredTask] of entries) {
+          discoveredTasksByName.set(taskName, discoveredTask);
+        }
+      }),
+    requestRootTaskResult: (taskName: string) =>
+      runRequestedTask(taskRunsState, resultDeferreds, taskName).pipe(
+        Effect.tapError(() =>
+          Effect.sync(() => taskNamesWhoseFailureReachedAnAwaitingTask.add(taskName)),
+        ),
+        Effect.provide(taskExecutionServices),
+      ),
+    requestDependencyTaskResult,
+    handleWorkerEvent: (event: typeof WorkerEvent.Type) =>
+      handleWorkerEvent(taskRunsState, requestDependencyTaskResult, event).pipe(
+        failRunOnFailure(markFailed),
+        Effect.provide(taskExecutionServices),
+      ),
+    stopAll: Effect.gen(function* () {
+      const stopState = yield* StopState;
+      const failedBeforeStopping = yield* stopState.begin;
+      const unfinishedTaskNames = new Set(
+        [...taskRunsByName]
+          .filter(
+            ([, taskRun]) =>
+              TaskRunStatus.guards.Preparing(taskRun.status) ||
+              TaskRunStatus.guards.Running(taskRun.status),
+          )
+          .map(([taskName]) => taskName),
+      );
+      yield* FiberSet.clear(taskRunsState.taskFibers);
+      const workerWatches = yield* FiberSet.make<void, never>();
+      const unorderedStops = Array.from(taskRunsByName, ([taskName, taskRun]) => ({
+        taskName,
+        isServing: TaskRunStatus.guards.Serving(taskRun.status),
+        isCompleted: TaskRunStatus.guards.Passed(taskRun.status),
+        requestedTaskNames: taskRun.bodyRequestedTaskNames,
+        stopped: Deferred.makeUnsafe<void>(),
+      }));
+      const findTasksRequestedBy = (requester: (typeof unorderedStops)[number]) =>
+        unorderedStops.filter((stop) => requester.requestedTaskNames.has(stop.taskName));
+      const findServingTasksReachedBy = (requester: (typeof unorderedStops)[number]) => {
+        const reached = new Set<(typeof unorderedStops)[number]>();
+        const reach = (stop: (typeof unorderedStops)[number]) => {
+          for (const requested of findTasksRequestedBy(stop)) {
+            if (!reached.has(requested)) {
+              reached.add(requested);
+              if (requested.isCompleted) {
+                reach(requested);
+              }
+            }
+          }
+        };
+        reach(requester);
+        return [...reached].filter((stop) => stop.isServing);
+      };
+      const requestedBeforeRequesters = new Array<(typeof unorderedStops)[number]>();
+      const visitedStops = new Set<(typeof unorderedStops)[number]>();
+      const requestPath = new Array<(typeof unorderedStops)[number]>();
+      const servingCycles = new Array<ServingTaskRequestsItself>();
+      const visit = (stop: (typeof unorderedStops)[number]) => {
+        if (visitedStops.has(stop)) {
+          return;
+        }
+        visitedStops.add(stop);
+        requestPath.push(stop);
+        for (const requested of findServingTasksReachedBy(stop)) {
+          if (requestPath.includes(requested)) {
+            servingCycles.push(
+              new ServingTaskRequestsItself({
+                taskName: stop.taskName,
+                cycle: [stop, ...requestPath.slice(requestPath.indexOf(requested))].map(
+                  (cycleStop) => cycleStop.taskName,
+                ),
+              }),
+            );
+          }
+          visit(requested);
+        }
+        requestPath.pop();
+        requestedBeforeRequesters.push(stop);
+      };
+      for (const stop of unorderedStops) {
+        visit(stop);
+      }
+      yield* Effect.forEach(
+        servingCycles,
+        (servingCycle) =>
+          stopState
+            .recordFirstFailure(servingCycle)
+            .pipe(Effect.andThen(failTask(servingCycle, markFailed))),
+        { discard: true },
+      );
+      const stops = requestedBeforeRequesters.toReversed();
+      yield* Effect.forEach(
+        stops,
+        (stop, index) =>
+          Effect.forEach(
+            stops
+              .slice(0, index)
+              .filter((requester) => findServingTasksReachedBy(requester).includes(stop)),
+            (requester) => Deferred.await(requester.stopped),
+            { discard: true },
+          ).pipe(
+            Effect.andThen(
+              stopTask(taskRunsState, stop.taskName, (watch) =>
+                FiberSet.run(workerWatches, watch).pipe(Effect.asVoid),
+              ),
+            ),
+            failRunOnFailure(markFailed),
+            Effect.ensuring(
+              unfinishedTaskNames.has(stop.taskName)
+                ? printTaskLines("Info", stop.taskName, "stopped: the run ended before it finished")
+                : Effect.void,
+            ),
+            Effect.ensuring(Deferred.done(stop.stopped, Exit.void)),
+          ),
+        { concurrency: failedBeforeStopping ? "unbounded" : 1, discard: true },
+      );
+    }).pipe(Effect.scoped, Effect.provide(taskExecutionServices)),
+    awaitServingTaskClosure: (taskName: string) =>
+      Effect.gen(function* () {
+        const taskRun = taskRunsByName.get(taskName);
+        const isNotServing = taskRun === undefined || !TaskRunStatus.guards.Serving(taskRun.status);
+        if (isNotServing) {
+          return;
+        }
+        yield* printLine("Info", `${taskName} is serving`);
+        yield* Deferred.await(taskRun.bodyFinishedDeferred);
+      }),
+    markFailed,
+    counts: Effect.sync(() => {
+      const taskRuns = [...taskRunsByName];
+      return {
+        total: taskRunsByName.size,
+        failedTaskNames: [...discoveredTasksByName.keys()].filter((taskName) => {
+          const taskRun = taskRunsByName.get(taskName);
+          return (
+            taskRun !== undefined &&
+            TaskRunStatus.guards.Failed(taskRun.status) &&
+            !taskRun.status.causedByAwaitedTasks
+          );
+        }),
+        stopped: taskRuns.filter(([, taskRun]) => TaskRunStatus.guards.Cancelled(taskRun.status))
+          .length,
+        unawaitedFailedTaskNames: taskRuns
+          .filter(
+            ([taskName, taskRun]) =>
+              TaskRunStatus.guards.Failed(taskRun.status) &&
+              !taskNamesWhoseFailureReachedAnAwaitingTask.has(taskName),
+          )
+          .map(([taskName]) => taskName),
+      };
+    }),
+  };
+});
+
+export class TaskRuns extends Context.Service<TaskRuns, Effect.Success<typeof makeTaskRuns>>()(
+  `${TASK_PACKAGE.name}/TaskRuns`,
+) {}

package/src/lib/main-thread/lib/run-tasks-on-workers/lib/run-requested-task-once/index.ts

@@ -0,0 +1 @@
+export { runRequestedDependencyTask, runRequestedTask } from "./internal/run-requested-tasks.ts";

package/src/lib/main-thread/lib/run-tasks-on-workers/lib/run-requested-task-once/internal/await-within-counted-limit.ts

@@ -0,0 +1,29 @@
+import { Clock, Duration, Effect, Option } from "effect";
+
+const SHORTEST_RECHECK_SHARE_OF_LIMIT = 0.05;
+
+export const awaitWithinCountedLimit = <Success, Failure, Services>(
+  awaiting: Effect.Effect<Success, Failure, Services>,
+  limitMilliseconds: number,
+  countMilliseconds: (nowMilliseconds: number) => Effect.Effect<number>,
+): Effect.Effect<Option.Option<Success>, Failure, Services> =>
+  Effect.gen(function* () {
+    const remainingMilliseconds =
+      limitMilliseconds - (yield* countMilliseconds(yield* Clock.currentTimeMillis));
+    if (remainingMilliseconds <= 0) {
+      return Option.none();
+    }
+    const exit = yield* Effect.exit(awaiting).pipe(
+      Effect.timeoutOption(
+        Duration.millis(
+          Math.max(remainingMilliseconds, limitMilliseconds * SHORTEST_RECHECK_SHARE_OF_LIMIT),
+        ),
+      ),
+    );
+    if (Option.isNone(exit)) {
+      return yield* awaitWithinCountedLimit(awaiting, limitMilliseconds, countMilliseconds);
+    }
+    const arrivedPastLimit =
+      (yield* countMilliseconds(yield* Clock.currentTimeMillis)) > limitMilliseconds;
+    return arrivedPastLimit ? Option.none() : Option.some(yield* exit.value);
+  });