@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.7

package/src/cli/check.ts

@@ -0,0 +1,977 @@
+import { existsSync, watch } from "node:fs";
+import { spawn } from "node:child_process";
+import { createRequire } from "node:module";
+import { once } from "node:events";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { assert } from "es-toolkit";
+import {
+  createBundledTaskUiTypechecker,
+  findPostprocessKeyCollisions,
+  type TaskUiDiagnosticsReport,
+  type TaskUiSource,
+} from "../checks";
+import {
+  TEST_EVIDENCE_DIR_ENV,
+  TEST_NODE_FILTER_ENV,
+  TEST_TYPED_ONLY_ENV,
+  TEST_PIPELINE_FILTER_ENV,
+  pipelineFingerprint,
+  readTestEvidence,
+  typedTestNamePattern,
+  type TestEvidence,
+  type TestEvidenceKind,
+} from "../testing/test-evidence";
+import { loadProjectSource, type ProjectSource } from "./source";
+
+export type CheckStatus = "passed" | "failed" | "missing" | "partial" | "infrastructure_error";
+export type CheckKind =
+  | "source"
+  | "project-types"
+  | "ui-types"
+  | "code-test"
+  | "ui-test"
+  | "pipeline-flow"
+  | "test-runner";
+
+export type CheckDiagnostic = {
+  readonly message: string;
+  readonly severity?: "error" | "warning";
+  readonly setting?: string;
+  readonly line?: number;
+  readonly column?: number;
+  readonly code?: string;
+};
+
+export type CheckResult = {
+  readonly scope: "project" | "pipeline" | "node";
+  readonly kind: CheckKind;
+  readonly status: CheckStatus;
+  readonly sourceDirectory?: string;
+  readonly pipelineFingerprint?: string;
+  readonly nodeId?: string;
+  readonly diagnostics: readonly CheckDiagnostic[];
+  readonly truncatedDiagnosticCount?: number;
+};
+
+export type CheckInventoryNode = {
+  readonly id: string;
+  readonly name: string;
+  readonly kind: "code" | "ui";
+};
+
+export type CheckInventoryPipeline = {
+  readonly sourceDirectory: string;
+  readonly name: string;
+  readonly pipelineFingerprint: string;
+  readonly nodes: readonly CheckInventoryNode[];
+};
+
+export type ToolRunResult = {
+  readonly exitCode: number;
+  readonly stdout: string;
+  readonly stderr: string;
+  readonly infrastructureError?: boolean;
+};
+
+export type RunVitestOptions = {
+  readonly sourceRoot: string;
+  readonly evidenceDirectory: string;
+  readonly environment: NodeJS.ProcessEnv;
+  readonly inventory: readonly CheckInventoryPipeline[];
+  readonly signal?: AbortSignal;
+};
+
+export type CheckProjectDependencies = {
+  readonly loadSource: (
+    sourceRoot: string,
+    environment: NodeJS.ProcessEnv,
+  ) => Promise<ProjectSource>;
+  readonly runTypeScript: (
+    sourceRoot: string,
+    environment: NodeJS.ProcessEnv,
+    signal?: AbortSignal,
+  ) => Promise<ToolRunResult>;
+  readonly runVitest: (options: RunVitestOptions) => Promise<ToolRunResult>;
+  readonly typecheckUi: (source: TaskUiSource) => Promise<TaskUiDiagnosticsReport>;
+};
+
+export type CheckProjectOptions = {
+  readonly sourceRoot: string;
+  readonly pipeline?: string;
+  readonly node?: string;
+  readonly json?: boolean;
+  readonly environment?: NodeJS.ProcessEnv;
+  readonly signal?: AbortSignal;
+};
+
+export type CheckProjectResult = {
+  readonly exitCode: 0 | 1 | 2;
+  readonly results: readonly CheckResult[];
+  readonly source?: ProjectSource;
+  readonly output: string;
+};
+const MAX_DIAGNOSTIC_CHARS = 5_000;
+const MAX_TOOL_OUTPUT_BYTES = 64 * 1024;
+export const CHECK_RESULT_FILE_ENV = "TOLOKA_INTERNAL_CHECK_RESULT_FILE";
+
+const truncateDiagnostic = (diagnostic: CheckDiagnostic): CheckDiagnostic =>
+  diagnostic.message.length <= MAX_DIAGNOSTIC_CHARS
+    ? diagnostic
+    : {
+        ...diagnostic,
+        message: `${diagnostic.message.slice(0, MAX_DIAGNOSTIC_CHARS)}\n…output truncated`,
+      };
+
+const MAX_DIAGNOSTICS = 20;
+
+export const collectOutput = async (stream: NodeJS.ReadableStream): Promise<string> => {
+  const chunks: Buffer[] = [];
+  let retainedBytes = 0;
+  let truncated = false;
+  for await (const value of stream) {
+    const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
+    const remaining = MAX_TOOL_OUTPUT_BYTES - retainedBytes;
+    if (remaining > 0) {
+      const retained = chunk.subarray(0, remaining);
+      chunks.push(retained);
+      retainedBytes += retained.length;
+    }
+    if (chunk.length > remaining) truncated = true;
+  }
+  const output = Buffer.concat(chunks).toString("utf8");
+  return truncated ? `${output}\n…output truncated` : output;
+};
+
+const runModule = async (
+  sourceRoot: string,
+  modulePath: string,
+  args: readonly string[],
+  environment: NodeJS.ProcessEnv = process.env,
+  signal?: AbortSignal,
+  executable: string = process.execPath,
+): Promise<ToolRunResult> => {
+  signal?.throwIfAborted();
+  const child = spawn(executable, [modulePath, ...args], {
+    cwd: sourceRoot,
+    env: environment,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  const abort = () => child.kill("SIGTERM");
+  signal?.addEventListener("abort", abort, { once: true });
+  if (signal?.aborted === true) abort();
+  try {
+    const [close, stdout, stderr] = await Promise.all([
+      once(child, "close"),
+      collectOutput(child.stdout),
+      collectOutput(child.stderr),
+    ]);
+    const exitCode = close[0];
+    return typeof exitCode === "number"
+      ? { exitCode, stdout, stderr }
+      : {
+          exitCode: 2,
+          stdout,
+          stderr: [stderr, `Process terminated by ${String(close[1])}`].filter(Boolean).join("\n"),
+          infrastructureError: true,
+        };
+  } finally {
+    signal?.removeEventListener("abort", abort);
+  }
+};
+
+const resolveProjectModule = (sourceRoot: string, packageName: string, entry: string): string => {
+  const require = createRequire(join(sourceRoot, "package.json"));
+  return join(dirname(require.resolve(`${packageName}/package.json`)), entry);
+};
+
+const runTypeScript = async (
+  sourceRoot: string,
+  environment: NodeJS.ProcessEnv,
+  signal?: AbortSignal,
+): Promise<ToolRunResult> => {
+  try {
+    return await runModule(
+      sourceRoot,
+      resolveProjectModule(sourceRoot, "typescript", "bin/tsc"),
+      ["--noEmit"],
+      environment,
+      signal,
+    );
+  } catch (error) {
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: `Unable to run project TypeScript: ${error instanceof Error ? error.message : String(error)}. Install it with \`bun add --dev typescript\`.`,
+      infrastructureError: true,
+    };
+  }
+};
+
+export const classifyVitestRun = (run: ToolRunResult): ToolRunResult => {
+  if (run.exitCode === 0) return run;
+  const output = `${run.stderr}\n${run.stdout}`;
+  return /Local Task UI testing requires Playwright and Chromium|Unable to load the Task UI launcher|Cannot find (?:package|module).*playwright/iu.test(
+    output,
+  )
+    ? { ...run, infrastructureError: true }
+    : run;
+};
+
+const runVitest = async ({
+  sourceRoot,
+  environment,
+  signal,
+}: RunVitestOptions): Promise<ToolRunResult> => {
+  try {
+    const fingerprint = environment[TEST_PIPELINE_FILTER_ENV];
+    const focusedPattern =
+      environment[TEST_TYPED_ONLY_ENV] === "1" && fingerprint !== undefined
+        ? typedTestNamePattern(fingerprint, environment[TEST_NODE_FILTER_ENV])
+        : undefined;
+    return classifyVitestRun(
+      await runModule(
+        sourceRoot,
+        resolveProjectModule(sourceRoot, "vitest", "vitest.mjs"),
+        ["run", ...(focusedPattern === undefined ? [] : ["--testNamePattern", focusedPattern])],
+        environment,
+        signal,
+      ),
+    );
+  } catch (error) {
+    return {
+      exitCode: 2,
+      stdout: "",
+      stderr: `Unable to run project Vitest: ${error instanceof Error ? error.message : String(error)}. Install it with \`bun add --dev vitest playwright\`.`,
+      infrastructureError: true,
+    };
+  }
+};
+
+const bundledTypechecker = createBundledTaskUiTypechecker();
+
+const loadSourceWithoutPlatformCredentials = async (
+  sourceRoot: string,
+  environment: NodeJS.ProcessEnv,
+): Promise<ProjectSource> => {
+  const environments = new Map<NodeJS.ProcessEnv, string | undefined>([
+    [process.env, process.env.POTATO_API_KEY],
+    [environment, environment.POTATO_API_KEY],
+  ]);
+  for (const current of environments.keys()) delete current.POTATO_API_KEY;
+  try {
+    return await loadProjectSource(sourceRoot);
+  } finally {
+    for (const [current, apiKey] of environments) {
+      if (apiKey === undefined) delete current.POTATO_API_KEY;
+      else current.POTATO_API_KEY = apiKey;
+    }
+  }
+};
+
+const defaultDependencies: CheckProjectDependencies = {
+  loadSource: loadSourceWithoutPlatformCredentials,
+  runTypeScript,
+  runVitest,
+  typecheckUi: (source) => bundledTypechecker.performTypeCheck(source),
+};
+const taskUiSource = (value: unknown): TaskUiSource | undefined => {
+  if (typeof value !== "object" || value === null) return undefined;
+  const code = Reflect.get(value, "code");
+  const data = Reflect.get(value, "data");
+  const constants = Reflect.get(value, "constants");
+  const preprocess = Reflect.get(value, "preprocess");
+  const postprocess = Reflect.get(value, "postprocess");
+  if (
+    typeof code !== "string" ||
+    typeof data !== "string" ||
+    typeof constants !== "string" ||
+    typeof preprocess !== "string" ||
+    (postprocess !== undefined && typeof postprocess !== "string")
+  ) {
+    return undefined;
+  }
+  return { code, data, constants, preprocess, postprocess: postprocess ?? "return {};" };
+};
+
+const nodeName = (
+  node: ProjectSource["pipelines"][number]["definition"]["graph"]["nodes"][number],
+) => {
+  const about = Reflect.get(node.settings, "about");
+  const name = typeof about === "object" && about !== null ? Reflect.get(about, "name") : undefined;
+  return typeof name === "string" && name.length > 0 ? name : node.id;
+};
+
+const inventoryFor = (source: ProjectSource): CheckInventoryPipeline[] =>
+  source.pipelines.map((pipeline) => ({
+    sourceDirectory: pipeline.sourceDirectory,
+    name: pipeline.name,
+    pipelineFingerprint: pipelineFingerprint(pipeline.definition),
+    nodes: pipeline.definition.graph.nodes
+      .filter(
+        (node) => node.type === "code" || node.type === "generation" || node.type === "quorum",
+      )
+      .map(
+        (node): CheckInventoryNode => ({
+          id: node.id,
+          name: nodeName(node),
+          kind: node.type === "code" ? "code" : "ui",
+        }),
+      )
+      .toSorted((left, right) => left.id.localeCompare(right.id)),
+  }));
+
+type SelectedInventory = {
+  readonly pipelines: readonly CheckInventoryPipeline[];
+  readonly nodeId?: string;
+};
+
+const filterInventory = (
+  inventory: readonly CheckInventoryPipeline[],
+  pipelineSelector: string | undefined,
+  nodeSelector: string | undefined,
+): SelectedInventory => {
+  const pipelines =
+    pipelineSelector === undefined
+      ? [...inventory]
+      : inventory.filter(
+          (pipeline) =>
+            pipeline.sourceDirectory === pipelineSelector ||
+            pipeline.name === pipelineSelector ||
+            pipeline.sourceDirectory.split("/").at(-1) === pipelineSelector,
+        );
+  if (pipelineSelector !== undefined) {
+    assert(
+      pipelines.length > 0,
+      `Unknown pipeline ${JSON.stringify(pipelineSelector)}. Available: ${inventory.map((item) => item.sourceDirectory).join(", ")}`,
+    );
+    assert(
+      pipelines.length === 1,
+      `Pipeline selector ${JSON.stringify(pipelineSelector)} is ambiguous: ${pipelines.map((item) => item.sourceDirectory).join(", ")}`,
+    );
+  }
+  if (nodeSelector === undefined) return { pipelines };
+  const matches = pipelines.flatMap((pipeline) =>
+    pipeline.nodes
+      .filter((node) => node.id === nodeSelector || node.name === nodeSelector)
+      .map((node) => ({ pipeline, node })),
+  );
+  assert(
+    matches.length === 1,
+    matches.length === 0
+      ? `Unknown checkable node ${JSON.stringify(nodeSelector)}.`
+      : `Node selector ${JSON.stringify(nodeSelector)} is ambiguous across pipelines.`,
+  );
+  return { pipelines: [matches[0]!.pipeline], nodeId: matches[0]!.node.id };
+};
+
+const diagnosticFromText = (
+  message: string,
+  options: Pick<CheckDiagnostic, "severity" | "setting"> = {},
+): CheckDiagnostic => {
+  const location = /^(\d+):(\d+) - \[[^\]]+\](?: (TS\d+))?/u.exec(message);
+  return {
+    message,
+    ...options,
+    ...(location === null
+      ? {}
+      : {
+          line: Number(location[1]),
+          column: Number(location[2]),
+          ...(location[3] === undefined ? {} : { code: location[3] }),
+        }),
+  };
+};
+
+const boundedDiagnostics = (diagnostics: readonly CheckDiagnostic[]) => ({
+  diagnostics: diagnostics.slice(0, MAX_DIAGNOSTICS).map(truncateDiagnostic),
+  ...(diagnostics.length <= MAX_DIAGNOSTICS
+    ? {}
+    : { truncatedDiagnosticCount: diagnostics.length - MAX_DIAGNOSTICS }),
+});
+
+const projectToolResult = (
+  kind: "project-types" | "test-runner",
+  run: ToolRunResult,
+): CheckResult => {
+  const status =
+    run.infrastructureError === true
+      ? "infrastructure_error"
+      : run.exitCode === 0
+        ? "passed"
+        : "failed";
+  const details = [run.stderr.trim(), run.stdout.trim()].filter(Boolean).join("\n");
+  return {
+    scope: "project",
+    kind,
+    status,
+    ...boundedDiagnostics(
+      status === "passed" || details.length === 0 ? [] : [{ message: details }],
+    ),
+  };
+};
+
+const uiTypeResults = async (
+  source: ProjectSource,
+  selected: readonly CheckInventoryPipeline[],
+  nodeSelector: string | undefined,
+  typecheckUi: CheckProjectDependencies["typecheckUi"],
+): Promise<CheckResult[]> => {
+  const selectedDirectories = new Set(selected.map((pipeline) => pipeline.sourceDirectory));
+  const results: CheckResult[] = [];
+  for (const pipeline of source.pipelines) {
+    const fingerprint = pipelineFingerprint(pipeline.definition);
+    if (!selectedDirectories.has(pipeline.sourceDirectory)) continue;
+    for (const node of pipeline.definition.graph.nodes) {
+      if (node.type !== "generation" && node.type !== "quorum") continue;
+      if (nodeSelector !== undefined && node.id !== nodeSelector) continue;
+      const ui = taskUiSource(Reflect.get(node.settings, "ui"));
+      if (ui === undefined) {
+        results.push({
+          scope: "node",
+          kind: "ui-types",
+          status: "failed",
+          sourceDirectory: pipeline.sourceDirectory,
+          pipelineFingerprint: fingerprint,
+          nodeId: node.id,
+          diagnostics: [{ message: "No labeling interface is configured on this node" }],
+        });
+        continue;
+      }
+      let report: TaskUiDiagnosticsReport;
+      try {
+        report = await typecheckUi(ui);
+      } catch (error) {
+        results.push({
+          scope: "node",
+          kind: "ui-types",
+          status: "infrastructure_error",
+          sourceDirectory: pipeline.sourceDirectory,
+          pipelineFingerprint: fingerprint,
+          nodeId: node.id,
+          diagnostics: [{ message: error instanceof Error ? error.message : String(error) }],
+        });
+        continue;
+      }
+      const diagnostics = Object.entries(report).flatMap(([setting, entries]) =>
+        entries.map((entry) =>
+          diagnosticFromText(entry.text, { severity: entry.severity, setting }),
+        ),
+      );
+      diagnostics.push(
+        ...findPostprocessKeyCollisions(ui).map((entry) =>
+          diagnosticFromText(entry.text, { severity: entry.severity, setting: "postprocess" }),
+        ),
+      );
+      results.push({
+        scope: "node",
+        kind: "ui-types",
+        status: diagnostics.some((diagnostic) => diagnostic.severity === "error")
+          ? "failed"
+          : "passed",
+        sourceDirectory: pipeline.sourceDirectory,
+        pipelineFingerprint: fingerprint,
+        nodeId: node.id,
+        ...boundedDiagnostics(diagnostics),
+      });
+    }
+  }
+  return results;
+};
+
+const evidenceStatus = (
+  evidence: readonly TestEvidence[],
+): Pick<CheckResult, "status" | "diagnostics" | "truncatedDiagnosticCount"> => {
+  if (evidence.some((entry) => entry.status === "infrastructure_error")) {
+    return {
+      status: "infrastructure_error",
+      ...boundedDiagnostics(
+        evidence
+          .filter((entry) => entry.status === "infrastructure_error")
+          .map((entry) => ({
+            message: entry.diagnostic ?? `Infrastructure error in ${entry.testName}`,
+          })),
+      ),
+    };
+  }
+  if (evidence.some((entry) => entry.status === "failed")) {
+    return {
+      status: "failed",
+      ...boundedDiagnostics(
+        evidence
+          .filter((entry) => entry.status === "failed")
+          .map((entry) => ({ message: entry.diagnostic ?? `Failed test ${entry.testName}` })),
+      ),
+    };
+  }
+  const passing = evidence.filter((entry) => entry.status === "passed");
+  if (passing.length === 0) return { status: "missing", diagnostics: [] };
+  if (passing.every((entry) => entry.partial === true)) {
+    return {
+      status: "partial",
+      diagnostics: [
+        {
+          message: `Hardware-backed fields not fully covered: ${[
+            ...new Set(passing.flatMap((entry) => entry.untestedHardwareFields ?? [])),
+          ].join(", ")}`,
+        },
+      ],
+    };
+  }
+  return { status: "passed", diagnostics: [] };
+};
+
+const evidenceFor = (
+  evidence: readonly TestEvidence[],
+  pipeline: CheckInventoryPipeline,
+  kind: TestEvidenceKind,
+  nodeId?: string,
+) =>
+  evidence.filter(
+    (entry) =>
+      entry.pipelineFingerprint === pipeline.pipelineFingerprint &&
+      entry.kind === kind &&
+      (nodeId === undefined || entry.nodeId === nodeId),
+  );
+
+const coverageResults = (
+  inventory: readonly CheckInventoryPipeline[],
+  evidence: readonly TestEvidence[],
+  nodeSelector: string | undefined,
+): CheckResult[] =>
+  inventory.flatMap((pipeline) => {
+    const result: CheckResult[] = [];
+    for (const node of pipeline.nodes) {
+      if (nodeSelector !== undefined && node.id !== nodeSelector) continue;
+      const kind = node.kind === "code" ? "code-test" : "ui-test";
+      const evidenceKind = node.kind === "code" ? "code" : "ui";
+      result.push({
+        scope: "node",
+        kind,
+        sourceDirectory: pipeline.sourceDirectory,
+        pipelineFingerprint: pipeline.pipelineFingerprint,
+        nodeId: node.id,
+        ...evidenceStatus(evidenceFor(evidence, pipeline, evidenceKind, node.id)),
+      });
+    }
+    if (nodeSelector === undefined) {
+      result.push({
+        scope: "pipeline",
+        kind: "pipeline-flow",
+        sourceDirectory: pipeline.sourceDirectory,
+        pipelineFingerprint: pipeline.pipelineFingerprint,
+        ...evidenceStatus(evidenceFor(evidence, pipeline, "pipeline")),
+      });
+    }
+    return result;
+  });
+
+const statusLabel: Record<CheckStatus, string> = {
+  passed: "PASS",
+  failed: "FAIL",
+  missing: "MISSING",
+  partial: "PARTIAL",
+  infrastructure_error: "ERROR",
+};
+
+const resultTarget = (result: CheckResult) =>
+  [result.sourceDirectory, result.nodeId, result.kind].filter(Boolean).join("/");
+
+const orderCheckResults = (results: readonly CheckResult[]): CheckResult[] => {
+  const scopeOrder: Record<CheckResult["scope"], number> = { project: 0, pipeline: 1, node: 2 };
+  return [...results].toSorted(
+    (left, right) =>
+      scopeOrder[left.scope] - scopeOrder[right.scope] ||
+      (left.sourceDirectory ?? "").localeCompare(right.sourceDirectory ?? "") ||
+      (left.nodeId ?? "").localeCompare(right.nodeId ?? "") ||
+      left.kind.localeCompare(right.kind),
+  );
+};
+
+const formatDiagnostic = (diagnostic: CheckDiagnostic): string[] =>
+  diagnostic.message.split("\n").map((line) => `  ${line}`);
+
+export const formatCheckResults = (results: readonly CheckResult[]): string =>
+  results
+    .flatMap((result) => [
+      `${statusLabel[result.status]} ${resultTarget(result)}`,
+      ...result.diagnostics.flatMap(formatDiagnostic),
+      ...(result.truncatedDiagnosticCount === undefined
+        ? []
+        : [`  ${result.truncatedDiagnosticCount} more diagnostics not shown`]),
+    ])
+    .join("\n");
+
+const exitCodeForResults = (results: readonly CheckResult[]): 0 | 1 | 2 => {
+  if (results.some((result) => result.status === "infrastructure_error")) return 2;
+  if (
+    results.some(
+      (result) =>
+        result.status === "failed" || result.status === "missing" || result.status === "partial",
+    )
+  )
+    return 1;
+  return 0;
+};
+
+export async function checkProject(
+  options: CheckProjectOptions,
+  dependencies: CheckProjectDependencies = defaultDependencies,
+): Promise<CheckProjectResult> {
+  let source: ProjectSource;
+  try {
+    options.signal?.throwIfAborted();
+    source = await dependencies.loadSource(options.sourceRoot, options.environment ?? process.env);
+    options.signal?.throwIfAborted();
+  } catch (error) {
+    const results: CheckResult[] = [
+      {
+        scope: "project",
+        kind: "source",
+        status: "failed",
+        diagnostics: [{ message: error instanceof Error ? error.message : String(error) }],
+      },
+    ];
+    return {
+      exitCode: 1,
+      results,
+      output:
+        options.json === true
+          ? JSON.stringify({ exitCode: 1, results })
+          : formatCheckResults(results),
+    };
+  }
+
+  if (source.pipelines.length === 0) {
+    const results: CheckResult[] = [
+      {
+        scope: "project",
+        kind: "source",
+        status: "failed",
+        diagnostics: [
+          { message: `No pipeline source found under ${options.sourceRoot}/pipelines` },
+        ],
+      },
+    ];
+    return {
+      exitCode: 1,
+      results,
+      output:
+        options.json === true
+          ? JSON.stringify({ exitCode: 1, results })
+          : formatCheckResults(results),
+    };
+  }
+
+  const selected = filterInventory(inventoryFor(source), options.pipeline, options.node);
+  const evidenceDirectory = await mkdtemp(join(tmpdir(), "toloka-check-"));
+  try {
+    const selectedPipeline = selected.pipelines.length === 1 ? selected.pipelines[0] : undefined;
+    const checkEnvironment = { ...(options.environment ?? process.env) };
+    delete checkEnvironment.POTATO_API_KEY;
+    const environment = {
+      ...checkEnvironment,
+      [TEST_EVIDENCE_DIR_ENV]: evidenceDirectory,
+      ...(selectedPipeline === undefined
+        ? {}
+        : { [TEST_PIPELINE_FILTER_ENV]: selectedPipeline.pipelineFingerprint }),
+      ...(selected.nodeId === undefined ? {} : { [TEST_NODE_FILTER_ENV]: selected.nodeId }),
+      ...(options.pipeline === undefined && options.node === undefined
+        ? {}
+        : { [TEST_TYPED_ONLY_ENV]: "1" }),
+    };
+    const [typeScriptRun, uiResults, vitestRun] = await Promise.all([
+      dependencies.runTypeScript(options.sourceRoot, checkEnvironment, options.signal),
+      uiTypeResults(source, selected.pipelines, selected.nodeId, dependencies.typecheckUi),
+      dependencies.runVitest({
+        sourceRoot: options.sourceRoot,
+        evidenceDirectory,
+        environment,
+        inventory: selected.pipelines,
+        signal: options.signal,
+      }),
+    ]);
+    const evidence = await readTestEvidence(evidenceDirectory);
+    const results = orderCheckResults([
+      { scope: "project", kind: "source", status: "passed", diagnostics: [] },
+      projectToolResult("project-types", typeScriptRun),
+      ...uiResults,
+      projectToolResult("test-runner", vitestRun),
+      ...coverageResults(selected.pipelines, evidence, selected.nodeId),
+    ]);
+    const exitCode = exitCodeForResults(results);
+    return {
+      exitCode,
+      results,
+      source,
+      output:
+        options.json === true ? JSON.stringify({ exitCode, results }) : formatCheckResults(results),
+    };
+  } catch (error) {
+    const results: CheckResult[] = [
+      {
+        scope: "project",
+        kind: "test-runner",
+        status: "infrastructure_error",
+        diagnostics: [{ message: error instanceof Error ? error.message : String(error) }],
+      },
+    ];
+    return {
+      exitCode: 2,
+      results,
+      source,
+      output:
+        options.json === true
+          ? JSON.stringify({ exitCode: 2, results })
+          : formatCheckResults(results),
+    };
+  } finally {
+    await rm(evidenceDirectory, { recursive: true, force: true });
+  }
+}
+
+export const relevantCheckPath = (filename: string | Buffer | null): boolean => {
+  if (filename === null) return true;
+  const path = filename.toString();
+  if (
+    path.includes("node_modules/") ||
+    path.includes(".git/") ||
+    path.includes("test-results/") ||
+    path.includes("coverage/") ||
+    path.includes("dist/") ||
+    path.includes(".cache/") ||
+    path.includes(".toloka-check-project-") ||
+    path.includes(".toloka-project-") ||
+    path.includes(".pipeline-source-")
+  ) {
+    return false;
+  }
+  return (
+    /(?:^|\/)(?:package\.json|tsconfig[^/]*\.json)$/u.test(path) ||
+    /\.(?:ts|tsx|mts|cts|js|jsx|mjs|cjs|json|jsonl|csv)$/u.test(path)
+  );
+};
+
+const isCheckScope = (value: unknown): value is CheckResult["scope"] =>
+  value === "project" || value === "pipeline" || value === "node";
+
+const isCheckKind = (value: unknown): value is CheckKind =>
+  value === "source" ||
+  value === "project-types" ||
+  value === "ui-types" ||
+  value === "code-test" ||
+  value === "ui-test" ||
+  value === "pipeline-flow" ||
+  value === "test-runner";
+
+const isCheckStatus = (value: unknown): value is CheckStatus =>
+  value === "passed" ||
+  value === "failed" ||
+  value === "missing" ||
+  value === "partial" ||
+  value === "infrastructure_error";
+
+const parseFreshCheckDiagnostic = (value: unknown): CheckDiagnostic => {
+  assert(typeof value === "object" && value !== null, "Fresh check returned invalid diagnostics");
+  const message = Reflect.get(value, "message");
+  const severity = Reflect.get(value, "severity");
+  const setting = Reflect.get(value, "setting");
+  const line = Reflect.get(value, "line");
+  const column = Reflect.get(value, "column");
+  const code = Reflect.get(value, "code");
+  assert(typeof message === "string", "Fresh check diagnostic has no message");
+  assert(
+    severity === undefined || severity === "error" || severity === "warning",
+    "Fresh check diagnostic has invalid severity",
+  );
+  assert(
+    setting === undefined || typeof setting === "string",
+    "Fresh check diagnostic has invalid setting",
+  );
+  assert(line === undefined || Number.isInteger(line), "Fresh check diagnostic has invalid line");
+  assert(
+    column === undefined || Number.isInteger(column),
+    "Fresh check diagnostic has invalid column",
+  );
+  assert(code === undefined || typeof code === "string", "Fresh check diagnostic has invalid code");
+  return {
+    message,
+    ...(severity === undefined ? {} : { severity }),
+    ...(setting === undefined ? {} : { setting }),
+    ...(line === undefined ? {} : { line: Number(line) }),
+    ...(column === undefined ? {} : { column: Number(column) }),
+    ...(code === undefined ? {} : { code }),
+  };
+};
+
+const parseFreshCheckResultItem = (value: unknown): CheckResult => {
+  assert(typeof value === "object" && value !== null, "Fresh check returned an invalid result");
+  const scope = Reflect.get(value, "scope");
+  const kind = Reflect.get(value, "kind");
+  const status = Reflect.get(value, "status");
+  const diagnostics = Reflect.get(value, "diagnostics");
+  const sourceDirectory = Reflect.get(value, "sourceDirectory");
+  const pipeline = Reflect.get(value, "pipelineFingerprint");
+  const nodeId = Reflect.get(value, "nodeId");
+  const truncated = Reflect.get(value, "truncatedDiagnosticCount");
+  assert(isCheckScope(scope), "Fresh check returned an invalid scope");
+  assert(isCheckKind(kind), "Fresh check returned an invalid kind");
+  assert(isCheckStatus(status), "Fresh check returned an invalid status");
+  assert(Array.isArray(diagnostics), "Fresh check returned invalid diagnostics");
+  assert(
+    sourceDirectory === undefined || typeof sourceDirectory === "string",
+    "Fresh check returned an invalid source directory",
+  );
+  assert(
+    pipeline === undefined || typeof pipeline === "string",
+    "Fresh check returned an invalid pipeline fingerprint",
+  );
+  assert(
+    nodeId === undefined || typeof nodeId === "string",
+    "Fresh check returned an invalid node id",
+  );
+  assert(
+    truncated === undefined || (Number.isInteger(truncated) && Number(truncated) >= 0),
+    "Fresh check returned an invalid truncated count",
+  );
+  return {
+    scope,
+    kind,
+    status,
+    diagnostics: diagnostics.map(parseFreshCheckDiagnostic),
+    ...(sourceDirectory === undefined ? {} : { sourceDirectory }),
+    ...(pipeline === undefined ? {} : { pipelineFingerprint: pipeline }),
+    ...(nodeId === undefined ? {} : { nodeId }),
+    ...(truncated === undefined ? {} : { truncatedDiagnosticCount: Number(truncated) }),
+  };
+};
+
+const parseFreshCheckResult = (value: unknown, json: boolean): CheckProjectResult => {
+  assert(typeof value === "object" && value !== null, "Fresh check returned invalid JSON");
+  const exitCode = Reflect.get(value, "exitCode");
+  const rawResults = Reflect.get(value, "results");
+  assert(
+    exitCode === 0 || exitCode === 1 || exitCode === 2,
+    "Fresh check returned an invalid exit code",
+  );
+  assert(Array.isArray(rawResults), "Fresh check returned no results");
+  const results = rawResults.map(parseFreshCheckResultItem);
+  return {
+    exitCode,
+    results,
+    output: json ? JSON.stringify({ exitCode, results }) : formatCheckResults(results),
+  };
+};
+export const runFreshCheck = async (options: CheckProjectOptions): Promise<CheckProjectResult> => {
+  const sourceEntry = fileURLToPath(new URL("./bin.ts", import.meta.url));
+  const entry = existsSync(sourceEntry) ? sourceEntry : fileURLToPath(import.meta.url);
+  const resultDirectory = await mkdtemp(join(tmpdir(), "toloka-check-watch-result-"));
+  const resultPath = join(resultDirectory, "result.json");
+  let run: ToolRunResult | undefined;
+  try {
+    run = await runModule(
+      options.sourceRoot,
+      entry,
+      [
+        "--source-root",
+        options.sourceRoot,
+        "check",
+        ...(options.pipeline === undefined ? [] : [options.pipeline]),
+        ...(options.node === undefined ? [] : ["--node", options.node]),
+        "--json",
+      ],
+      {
+        ...(process.env.PATH === undefined ? {} : { PATH: process.env.PATH }),
+        ...(options.environment ?? process.env),
+        [CHECK_RESULT_FILE_ENV]: resultPath,
+      },
+      options.signal,
+      "bun",
+    );
+    return parseFreshCheckResult(
+      JSON.parse(await readFile(resultPath, "utf8")),
+      options.json === true,
+    );
+  } catch (error) {
+    const detail = [
+      run?.stderr,
+      run?.stdout,
+      error instanceof Error ? error.message : String(error),
+    ]
+      .filter(Boolean)
+      .join("\n");
+    throw new Error(`Fresh check process returned unreadable output: ${detail}`, { cause: error });
+  } finally {
+    await rm(resultDirectory, { recursive: true, force: true });
+  }
+};
+
+export async function watchProjectChecks(
+  options: CheckProjectOptions,
+  onResult: (result: CheckProjectResult) => void | Promise<void>,
+  dependencies: CheckProjectDependencies = defaultDependencies,
+  signal?: AbortSignal,
+): Promise<void> {
+  const { promise, resolve, reject } = Promise.withResolvers<void>();
+  let running = false;
+  let dirty = false;
+  let stopping = false;
+  let timer: NodeJS.Timeout | undefined;
+
+  const stop = () => {
+    stopping = true;
+    dirty = false;
+    clearTimeout(timer);
+    watcher.close();
+    if (!running) resolve();
+  };
+  const fail = (error: unknown) => {
+    clearTimeout(timer);
+    watcher.close();
+    reject(error);
+  };
+  const run = async (): Promise<void> => {
+    if (running) {
+      dirty = true;
+      return;
+    }
+    running = true;
+    try {
+      do {
+        dirty = false;
+        const result =
+          dependencies === defaultDependencies
+            ? await runFreshCheck({ ...options, signal })
+            : await checkProject({ ...options, signal }, dependencies);
+        if (!stopping) await onResult(result);
+      } while (dirty && signal?.aborted !== true);
+    } catch (error) {
+      if (!stopping) fail(error);
+    } finally {
+      running = false;
+      if (stopping) resolve();
+    }
+  };
+
+  const watcher = watch(options.sourceRoot, { recursive: true }, (_event, filename) => {
+    if (!relevantCheckPath(filename)) return;
+    dirty = true;
+    if (running) return;
+    clearTimeout(timer);
+    timer = setTimeout(() => void run(), 100);
+  });
+  watcher.once("error", fail);
+  if (signal?.aborted === true) stop();
+  else {
+    signal?.addEventListener("abort", stop, { once: true });
+    void run();
+  }
+  try {
+    await promise;
+  } finally {
+    signal?.removeEventListener("abort", stop);
+  }
+}