@sshkeda/task 0.0.0-stage → 1.0.1

package/src/lib/kill-leftover-processes/internal/kill-process-groups.ts

@@ -0,0 +1,270 @@
+import { Effect } from "effect";
+
+import { processIdentityKey } from "../../../shared/claim-processes-to-report.ts";
+import type { ProcessIdentity } from "../../../shared/claim-processes-to-report.ts";
+import { formatFailureLines } from "../../../shared/format-error-lines.ts";
+import { TASK_TRACE_LABEL_PREFIX } from "../../../shared/names-from-package-json.ts";
+import { printLine } from "../../../shared/print-to-standard-error.ts";
+import type { RunMarker } from "../../../shared/run-marker.ts";
+import { findDescendantProcessIds, findSessionProcessIds } from "./find-process-ids.ts";
+import { isStillListedAs, listProcesses } from "./list-processes.ts";
+import { findMarkedProcessIds } from "./mark-run-processes.ts";
+import { ProcessNotKillable } from "./process-cleanup-errors.ts";
+import { sendSignal, signalUnlessReplaced } from "./send-signal.ts";
+import { findStartedGroupWitness } from "./started-process-groups.ts";
+import type { startedProcessGroup } from "./started-process-groups.ts";
+
+const reportKilledProcesses = (
+  reportedResults: readonly {
+    readonly processId: number;
+    readonly command: string;
+    readonly ownerTaskName: string | null;
+    readonly result: "EPERM" | "ESRCH" | "sent";
+  }[],
+) =>
+  Effect.gen(function* () {
+    for (const killResult of reportedResults) {
+      const startedBy = `started by ${killResult.ownerTaskName === null ? "no known task" : `"${killResult.ownerTaskName}"`}`;
+      yield* killResult.result === "sent"
+        ? printLine("Info", `killed ${killResult.processId} (${killResult.command}), ${startedBy}`)
+        : printLine(
+            "Error",
+            formatFailureLines(
+              new ProcessNotKillable({
+                processId: killResult.processId,
+                command: killResult.command,
+                ownerTaskName: killResult.ownerTaskName,
+              }),
+              null,
+            ),
+          );
+      yield* Effect.logInfo(
+        killResult.result === "sent" ? "process killed" : "process could not be killed",
+      ).pipe(
+        Effect.annotateLogs({
+          "process.pid": killResult.processId,
+          "process.executable.name": killResult.command,
+          ...(killResult.ownerTaskName === null
+            ? {}
+            : { [`${TASK_TRACE_LABEL_PREFIX}.task`]: killResult.ownerTaskName }),
+        }),
+      );
+    }
+  });
+
+const findLiveProcessGroups = Effect.fn("findLiveProcessGroups")(function* (
+  taskName: string | null,
+  selectedProcessGroups: ReadonlyMap<number, ReturnType<typeof startedProcessGroup>>,
+) {
+  const liveProcessGroups = new Map<number, ReturnType<typeof startedProcessGroup>>();
+  const endedProcessGroupIds = new Set<number>();
+  for (const [processGroupId, startedGroup] of selectedProcessGroups) {
+    if ((yield* sendSignal(taskName, -processGroupId, 0)) === "sent") {
+      liveProcessGroups.set(processGroupId, startedGroup);
+    } else {
+      endedProcessGroupIds.add(processGroupId);
+    }
+  }
+  return { liveProcessGroups, endedProcessGroupIds };
+});
+
+const terminateSelectedProcesses = Effect.fn("terminateSelectedProcesses")(function* (input: {
+  readonly startedProcessGroups: ReadonlyMap<number, ReturnType<typeof startedProcessGroup>>;
+  readonly liveProcessGroups: ReadonlyMap<number, ReturnType<typeof startedProcessGroup>>;
+  readonly ownsProcess: (ownerTaskName: string | null) => boolean;
+  readonly taskName: string | null;
+  readonly sessionId: number | null;
+  readonly runMarker: typeof RunMarker.Type | null;
+  readonly adoptingProcessId: number | null;
+  readonly listProcessesNow: typeof listProcesses;
+  readonly claimProcessesToReport: (
+    processes: readonly (typeof ProcessIdentity.Type)[],
+  ) => Effect.Effect<readonly (typeof ProcessIdentity.Type)[]>;
+}) {
+  const listedProcesses = yield* input.listProcessesNow(input.taskName);
+  const findChildProcessIds = (parentProcessIds: readonly number[]) =>
+    [...listedProcesses]
+      .filter(([, listedProcess]) => parentProcessIds.includes(listedProcess.parentProcessId))
+      .map(([processId]) => processId);
+  const childProcessIds = findChildProcessIds([process.pid]);
+  const adoptedProcessIds =
+    input.adoptingProcessId === null
+      ? []
+      : findChildProcessIds([input.adoptingProcessId]).filter(
+          (processId) => processId !== process.pid,
+        );
+  const killableGroups = new Map<number, string>();
+  const groupWitnesses = new Map<number, Parameters<typeof isStillListedAs>[1]>();
+  const endedProcessGroupIds = new Set<number>();
+  for (const [group, startedGroup] of input.liveProcessGroups) {
+    const witness = findStartedGroupWitness(listedProcesses, group, startedGroup);
+    if (witness === null) {
+      endedProcessGroupIds.add(group);
+    } else {
+      killableGroups.set(group, startedGroup.taskName);
+      groupWitnesses.set(group, witness);
+    }
+  }
+
+  const processIds = [
+    ...findDescendantProcessIds(listedProcesses, [...childProcessIds, ...adoptedProcessIds]),
+    ...findSessionProcessIds(listedProcesses, input.sessionId),
+    ...(yield* findMarkedProcessIds(input.runMarker, listedProcesses.keys())),
+    ...[...listedProcesses]
+      .filter(([, listedProcess]) => killableGroups.has(listedProcess.processGroupId))
+      .map(([processId]) => processId),
+  ];
+  const processRows = new Map(
+    processIds.flatMap((processId) => {
+      const listedProcess = listedProcesses.get(processId);
+      const isGone = listedProcess === undefined || listedProcess.hasEnded;
+      return isGone ? [] : [[processId, listedProcess] as const];
+    }),
+  );
+  yield* Effect.annotateCurrentSpan(
+    `${TASK_TRACE_LABEL_PREFIX}.processes.listed`,
+    processRows.size,
+  );
+
+  const findOwner = (processId: number): string | null => {
+    const row = processRows.get(processId);
+    if (row === undefined) {
+      return null;
+    }
+
+    return (
+      input.startedProcessGroups.get(processId)?.taskName ??
+      findOwner(row.parentProcessId) ??
+      killableGroups.get(row.processGroupId) ??
+      null
+    );
+  };
+  const leftovers = (yield* Effect.forEach(
+    [...new Set(processIds)].flatMap((processId) => {
+      const row = processRows.get(processId);
+      const ownerTaskName = findOwner(processId);
+      const isGoneOrNotOwned = row === undefined || !input.ownsProcess(ownerTaskName);
+      return isGoneOrNotOwned ? [] : [{ processId, row, ownerTaskName }];
+    }),
+    (leftover) =>
+      Effect.gen(function* () {
+        const command = yield* leftover.row.readCommandLineUnlessGone;
+        return command === null
+          ? []
+          : [
+              {
+                processId: leftover.processId,
+                startTime: leftover.row.startTime,
+                processGroupId: leftover.row.processGroupId,
+                command,
+                ownerTaskName: leftover.ownerTaskName,
+              },
+            ];
+      }),
+  )).flat();
+  const groupsStillStarted = yield* Effect.filter([...groupWitnesses], ([, witness]) =>
+    isStillListedAs(input.taskName, witness),
+  );
+  const killResults = yield* Effect.forEach(leftovers, (leftover) =>
+    Effect.map(signalUnlessReplaced(input.taskName, leftover, "SIGKILL"), (result) => ({
+      ...leftover,
+      result,
+    })),
+  );
+  for (const [group] of groupsStillStarted) {
+    yield* sendSignal(input.taskName, -group, "SIGKILL");
+  }
+  for (const group of killableGroups.keys()) {
+    endedProcessGroupIds.add(group);
+  }
+  const signaledResults = killResults.filter((killResult) => killResult.result !== "ESRCH");
+  const firstReportedProcessKeys = new Set(
+    (yield* input.claimProcessesToReport(
+      signaledResults.map((killResult) => ({
+        processId: killResult.processId,
+        startTime: killResult.startTime,
+      })),
+    )).map((identity) => processIdentityKey(identity)),
+  );
+  const reportedResults = signaledResults.filter((killResult) =>
+    firstReportedProcessKeys.has(processIdentityKey(killResult)),
+  );
+  const killed = reportedResults.filter((killResult) => killResult.result === "sent");
+  yield* Effect.annotateCurrentSpan({
+    [`${TASK_TRACE_LABEL_PREFIX}.processes.killed`]: killed.length,
+    [`${TASK_TRACE_LABEL_PREFIX}.processes.killed.executable_names`]: killed.map(
+      (killResult) => killResult.command,
+    ),
+  });
+  yield* reportKilledProcesses(reportedResults);
+  return endedProcessGroupIds;
+});
+
+export const forgetEndedProcessGroups =
+  (startedProcessGroups: Map<number, ReturnType<typeof startedProcessGroup>>) =>
+  (endedProcessGroupIds: ReadonlySet<number>) =>
+    Effect.sync(() => {
+      for (const endedProcessGroupId of endedProcessGroupIds) {
+        startedProcessGroups.delete(endedProcessGroupId);
+      }
+    });
+
+export const killTaskProcessGroups = Effect.fn("killTaskProcessGroups")(function* (
+  startedProcessGroups: ReadonlyMap<number, ReturnType<typeof startedProcessGroup>>,
+  taskName: string,
+  claimProcessesToReport: Parameters<
+    typeof terminateSelectedProcesses
+  >[0]["claimProcessesToReport"],
+  listProcessesNow: typeof listProcesses = listProcesses,
+) {
+  yield* Effect.annotateCurrentSpan(`${TASK_TRACE_LABEL_PREFIX}.task`, taskName);
+  const selectedProcessGroups = new Map(
+    [...startedProcessGroups].filter(([, startedGroup]) => startedGroup.taskName === taskName),
+  );
+  const { liveProcessGroups, endedProcessGroupIds } = yield* findLiveProcessGroups(
+    taskName,
+    selectedProcessGroups,
+  );
+  if (liveProcessGroups.size === 0) {
+    return endedProcessGroupIds;
+  }
+  const terminatedProcessGroupIds = yield* terminateSelectedProcesses({
+    startedProcessGroups,
+    liveProcessGroups,
+    ownsProcess: (ownerTaskName) => ownerTaskName === taskName,
+    taskName,
+    sessionId: null,
+    runMarker: null,
+    adoptingProcessId: null,
+    listProcessesNow,
+    claimProcessesToReport,
+  });
+  return new Set([...endedProcessGroupIds, ...terminatedProcessGroupIds]);
+});
+
+export const killAllProcessGroups = Effect.fn("killAllProcessGroups")(function* (
+  startedProcessGroups: ReadonlyMap<number, ReturnType<typeof startedProcessGroup>>,
+  claimProcessesToReport: Parameters<
+    typeof terminateSelectedProcesses
+  >[0]["claimProcessesToReport"],
+  sessionId: number | null,
+  runMarker: typeof RunMarker.Type | null,
+  adoptingProcessId: number | null,
+) {
+  const { liveProcessGroups, endedProcessGroupIds } = yield* findLiveProcessGroups(
+    null,
+    startedProcessGroups,
+  );
+  const terminatedProcessGroupIds = yield* terminateSelectedProcesses({
+    startedProcessGroups,
+    liveProcessGroups,
+    ownsProcess: () => true,
+    taskName: null,
+    sessionId,
+    runMarker,
+    adoptingProcessId,
+    listProcessesNow: listProcesses,
+    claimProcessesToReport,
+  });
+  return new Set([...endedProcessGroupIds, ...terminatedProcessGroupIds]);
+});

package/src/lib/kill-leftover-processes/internal/list-linux-processes-from-proc.ts

@@ -0,0 +1,191 @@
+import { Effect, FileSystem, Predicate, Schema } from "effect";
+import type { PlatformError } from "effect";
+
+import { TASK_TRACE_LABEL_PREFIX } from "../../../shared/names-from-package-json.ts";
+import {
+  ProcCommandLineUnreadable,
+  ProcDirectoryUnlistable,
+  ProcStatMalformed,
+  ProcStatUnreadable,
+} from "../../../shared/process-errors.ts";
+import { availableParallelism } from "../../../shared/read-environment-config.ts";
+import { commandWordsOf, programNameOf } from "./command-words.ts";
+
+const PROC_DIRECTORY_PATH = "/proc";
+const PROC_STAT_STATE_FIELD_NUMBER = 3;
+const PROC_STAT_START_TIME_FIELD_NUMBER = 22;
+
+const ProcessIdFromText = Schema.FiniteFromString.check(Schema.isInt());
+
+const decodeProcStatFields = Schema.decodeUnknownEffect(
+  Schema.TupleWithRest(
+    Schema.Tuple([Schema.NonEmptyString, ProcessIdFromText, ProcessIdFromText, ProcessIdFromText]),
+    [Schema.String],
+  ),
+);
+
+const decodeProcStatStartTime = Schema.decodeUnknownEffect(
+  Schema.String.check(Schema.isPattern(/^\d+$/u)),
+);
+
+export const decodeProcStatText = (statText: string) =>
+  Effect.gen(function* () {
+    const fieldsAfterCommand = statText
+      .slice(statText.lastIndexOf(")") + ") ".length)
+      .trim()
+      .split(" ");
+    const [state, parentProcessId, processGroupId, sessionId] =
+      yield* decodeProcStatFields(fieldsAfterCommand);
+    return {
+      processName: statText.slice(statText.indexOf("(") + 1, statText.lastIndexOf(")")),
+      state,
+      parentProcessId,
+      processGroupId,
+      sessionId,
+      startTime: yield* decodeProcStatStartTime(
+        fieldsAfterCommand[PROC_STAT_START_TIME_FIELD_NUMBER - PROC_STAT_STATE_FIELD_NUMBER],
+      ),
+    };
+  });
+
+const isEndedState = (state: string) => state.startsWith("Z") || state.startsWith("X");
+
+const HIDDEN_PROCESS_ERROR_CODES = new Set<unknown>(["EACCES", "EPERM"]);
+
+const errorCodeOf = (error: PlatformError.PlatformError) =>
+  Predicate.hasProperty(error.reason.cause, "code") ? error.reason.cause.code : undefined;
+
+export const nullWhenProcessGoneOrHidden = <Success, Services>(
+  readProcessFile: Effect.Effect<Success, PlatformError.PlatformError, Services>,
+  onHidden: Effect.Effect<void>,
+) =>
+  readProcessFile.pipe(
+    Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(null)),
+    Effect.catchTag("PlatformError", (error) => {
+      const code = errorCodeOf(error);
+      if (code === "ESRCH") {
+        return Effect.succeed(null);
+      }
+      return HIDDEN_PROCESS_ERROR_CODES.has(code) ? Effect.as(onHidden, null) : Effect.fail(error);
+    }),
+  );
+
+const readStatFile = (
+  fileSystem: FileSystem.FileSystem,
+  taskName: string | null,
+  statFilePath: string,
+  onHidden: Effect.Effect<void>,
+) =>
+  nullWhenProcessGoneOrHidden(fileSystem.readFileString(statFilePath), onHidden).pipe(
+    Effect.mapError((cause) => new ProcStatUnreadable({ taskName, filePath: statFilePath, cause })),
+    Effect.flatMap((statText) =>
+      statText === null
+        ? Effect.succeed(null)
+        : decodeProcStatText(statText).pipe(
+            Effect.mapError(
+              (cause) => new ProcStatMalformed({ taskName, filePath: statFilePath, cause }),
+            ),
+          ),
+    ),
+  );
+
+export const readProcessFromProcDirectory = Effect.fn("readProcessFromProcDirectory")(function* (
+  taskName: string | null,
+  processId: number,
+) {
+  const fileSystem = yield* FileSystem.FileSystem;
+  const stat = yield* readStatFile(
+    fileSystem,
+    taskName,
+    `${PROC_DIRECTORY_PATH}/${processId}/stat`,
+    Effect.void,
+  );
+  return stat === null
+    ? null
+    : {
+        processGroupId: stat.processGroupId,
+        startTime: stat.startTime,
+        hasEnded: isEndedState(stat.state),
+      };
+});
+
+export const listProcessesFromProcDirectory = Effect.fn("listProcessesFromProcDirectory")(
+  function* (taskName: string | null) {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const unreadableProcessIds: number[] = [];
+    const processIdTexts = (yield* fileSystem
+      .readDirectory(PROC_DIRECTORY_PATH)
+      .pipe(
+        Effect.mapError(
+          (cause) =>
+            new ProcDirectoryUnlistable({ taskName, directoryPath: PROC_DIRECTORY_PATH, cause }),
+        ),
+      )).filter((name) => /^\d+$/u.test(name));
+    const listedProcesses = yield* Effect.forEach(
+      processIdTexts,
+      (processIdText) =>
+        Effect.gen(function* () {
+          const statFilePath = `${PROC_DIRECTORY_PATH}/${processIdText}/stat`;
+          const commandLineFilePath = `${PROC_DIRECTORY_PATH}/${processIdText}/cmdline`;
+          const readStat = (onHidden: Effect.Effect<void>) =>
+            readStatFile(fileSystem, taskName, statFilePath, onHidden);
+          const stat = yield* readStat(
+            Effect.sync(() => unreadableProcessIds.push(Number(processIdText))),
+          );
+          if (stat === null) {
+            return [];
+          }
+          const { processName, state, parentProcessId, processGroupId, sessionId, startTime } =
+            stat;
+          const readCommandLineUnlessGone = Effect.gen(function* () {
+            const commandLine = yield* nullWhenProcessGoneOrHidden(
+              fileSystem
+                .readFile(commandLineFilePath)
+                .pipe(
+                  Effect.map((commandLineBytes) =>
+                    new TextDecoder("utf-8", { ignoreBOM: true }).decode(commandLineBytes),
+                  ),
+                ),
+              Effect.void,
+            ).pipe(
+              Effect.mapError(
+                (cause) =>
+                  new ProcCommandLineUnreadable({ taskName, filePath: commandLineFilePath, cause }),
+              ),
+            );
+            if (commandLine === null) {
+              return null;
+            }
+            const argumentWords = commandLine.split("\0");
+            if (commandWordsOf(argumentWords).length > 0) {
+              return programNameOf(argumentWords, processName);
+            }
+            const statNow = yield* readStat(Effect.void);
+            const isGone = statNow === null || isEndedState(statNow.state);
+            return isGone ? null : processName;
+          });
+          return [
+            [
+              Number(processIdText),
+              {
+                parentProcessId,
+                processGroupId,
+                sessionId,
+                startTime,
+                hasEnded: isEndedState(state),
+                readCommandLineUnlessGone,
+              },
+            ] as const,
+          ];
+        }),
+      { concurrency: yield* availableParallelism },
+    );
+    if (unreadableProcessIds.length > 0) {
+      yield* Effect.annotateCurrentSpan(
+        `${TASK_TRACE_LABEL_PREFIX}.unreadableProcessIds`,
+        unreadableProcessIds,
+      );
+    }
+    return new Map(listedProcesses.flat());
+  },
+);