@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.7

package/src/testing/task-ui-validation.ts

@@ -0,0 +1,135 @@
+import Ajv2020 from "ajv/dist/2020.js";
+import addFormats from "ajv-formats";
+import { assert } from "es-toolkit";
+import { query, type JsonValue } from "jsonpath-rfc9535";
+import { Script } from "node:vm";
+import type { PipelineDefinition } from "../authoring";
+import type { TaskUiEntityError, TaskUiNode } from "./types";
+
+export const resolveTaskNode = (pipeline: PipelineDefinition, node: TaskUiNode) => {
+  const graphNode = pipeline.graph.nodes.find((candidate) => candidate.id === node.id);
+  assert(graphNode, `Node "${node.id}" is not part of this pipeline`);
+  assert(
+    graphNode.type === "generation" || graphNode.type === "quorum",
+    `Node "${node.id}" does not have a Task UI`,
+  );
+  return graphNode;
+};
+
+export type ResolvedTaskNode = ReturnType<typeof resolveTaskNode>;
+type TaskUiOutputEntity = ResolvedTaskNode["settings"]["entities"]["output"][number];
+type PipelineField = PipelineDefinition["graph"]["fields"][number];
+
+export const runtimeJavaScriptSyntaxError = (
+  source: string,
+  block: "preprocess" | "postprocess",
+): SyntaxError | null => {
+  try {
+    new Script(`(function (data) {\n${source}\n})`, { filename: `${block}.js` });
+    return null;
+  } catch (error) {
+    return new SyntaxError(
+      `${block} is not valid runtime JavaScript: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+};
+
+export const normalizePath = (path: string): string =>
+  path.replace(/^\$\.?/u, "").replace(/\[\*\]$/u, "");
+
+const resolveEntitySchema = (node: ResolvedTaskNode, field: PipelineField): unknown => {
+  const schema = field.jsonSchema;
+  if (
+    node.type !== "quorum" ||
+    schema === null ||
+    typeof schema !== "object" ||
+    Array.isArray(schema) ||
+    Reflect.get(schema, "type") !== "array"
+  ) {
+    return schema;
+  }
+  return Reflect.get(schema, "items");
+};
+
+const extractEntityValues = (
+  data: Record<string, unknown>,
+  entity: TaskUiOutputEntity,
+  field: PipelineField,
+): JsonValue[] => {
+  if (entity.jsonPath !== "$") return query(data as JsonValue, entity.jsonPath);
+  return Object.hasOwn(data, field.name) ? [data[field.name] as JsonValue] : [];
+};
+
+const validateEntity = ({
+  ajv,
+  node,
+  data,
+  entity,
+  field,
+}: {
+  ajv: Ajv2020;
+  node: ResolvedTaskNode;
+  data: Record<string, unknown>;
+  entity: TaskUiOutputEntity;
+  field: PipelineField;
+}): TaskUiEntityError | undefined => {
+  const expectedSchema = resolveEntitySchema(node, field);
+  const extracted = extractEntityValues(data, entity, field);
+  if (extracted.length === 0) {
+    return entity.optional
+      ? undefined
+      : {
+          entityName: field.name,
+          path: entity.jsonPath,
+          reason: `No data found at path "${entity.jsonPath}"`,
+          submittedValue: null,
+          expectedSchema,
+        };
+  }
+
+  assert(
+    typeof expectedSchema === "boolean" ||
+      (expectedSchema !== null &&
+        typeof expectedSchema === "object" &&
+        !Array.isArray(expectedSchema)),
+    `Invalid schema for "${field.name}"`,
+  );
+  const submittedValue = entity.dynamic ? extracted : extracted[0];
+  const validate = ajv.compile(expectedSchema as boolean | object);
+  if (validate(submittedValue)) return undefined;
+
+  return {
+    entityName: field.name,
+    path: entity.jsonPath,
+    reason:
+      validate.errors
+        ?.map((error) => error.message)
+        .filter(Boolean)
+        .join(", ") || "Schema mismatch",
+    submittedValue,
+    expectedSchema,
+  };
+};
+
+export const validateEntities = (
+  pipeline: PipelineDefinition,
+  node: ResolvedTaskNode,
+  data: Record<string, unknown>,
+  ignoredPaths: Set<string>,
+): TaskUiEntityError[] => {
+  const fieldsById = new Map(pipeline.graph.fields.map((field) => [field.id, field]));
+  const ajv = new Ajv2020({ strict: false });
+  addFormats(ajv);
+  const errors: TaskUiEntityError[] = [];
+
+  for (const entity of node.settings.entities.output) {
+    const field = fieldsById.get(entity.fieldProxyId);
+    assert(field, `Output entity "${entity.fieldProxyId}" has no declared field`);
+    const path = entity.jsonPath === "$" ? field.name : normalizePath(entity.jsonPath);
+    if (ignoredPaths.has(path)) continue;
+    const error = validateEntity({ ajv, node, data, entity, field });
+    if (error !== undefined) errors.push(error);
+  }
+
+  return errors;
+};

package/src/testing/task-ui.ts

@@ -0,0 +1,557 @@
+import { assert } from "es-toolkit";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { Browser } from "playwright";
+import { expect, test as baseTest, type TestAPI } from "vitest";
+import type { AuthoredCodeNode, PipelineDefinition } from "../authoring";
+import {
+  DEFAULT_TBX_LAUNCHER_URL,
+  componentIds,
+  evaluateWithApi,
+  launchDefaultBrowser,
+  readDataPath,
+  selectComponentDocumentation,
+  waitForTaskUiApi,
+} from "./task-ui-browser";
+import {
+  normalizePath,
+  resolveTaskNode,
+  runtimeJavaScriptSyntaxError,
+  validateEntities,
+  type ResolvedTaskNode,
+} from "./task-ui-validation";
+import {
+  TaskUiInfrastructureError,
+  TaskUiValidationError,
+  type RenderTaskUiOptions,
+  type TaskUiCoverage,
+  type TaskUiFormError,
+  type TaskUiNode,
+  type TaskUiSession,
+  type TbxTemplate,
+  type TypedTestOptions,
+  type UiTestFixtures,
+  type UiWorkerFixtures,
+} from "./types";
+import { CodeNodeInfrastructureError, runCodeNode, type RunCodeNodeOptions } from "./code-node";
+import {
+  assertTestNodeInPipeline,
+  assertTypedTestCompletion,
+  matchesTestFilter,
+  pipelineFingerprint,
+  writeTestEvidence,
+  typedTestNamePrefix,
+  TEST_TYPED_ONLY_ENV,
+  type TestEvidenceKind,
+} from "./test-evidence";
+
+const FILE_COMPONENT_PATTERN = /<Field\.(?:Video(?:Record|Upload)|FileUpload)\b[^>]*>/gu;
+const FIELD_PATH_PATTERN = /\bpath=\{?['"]([^'"]+)['"]\}?/u;
+const FILE_ENTITY_TYPES = new Set(["VIDEO", "IMAGE", "AUDIO", "DOCUMENT"]);
+
+type CreateTaskUiTestOptions = {
+  launcherUrl?: string;
+  launchBrowser?: () => Promise<Browser>;
+};
+
+const createTaskUiTemplate = (
+  graphNode: ResolvedTaskNode,
+  options: RenderTaskUiOptions,
+): TbxTemplate => {
+  const template = {
+    ...graphNode.settings.ui,
+    data: options.data === undefined ? graphNode.settings.ui.data : JSON.stringify(options.data),
+    postprocess: graphNode.settings.ui.postprocess ?? "return {};",
+  } satisfies TbxTemplate;
+  const syntaxError =
+    runtimeJavaScriptSyntaxError(template.preprocess, "preprocess") ??
+    runtimeJavaScriptSyntaxError(template.postprocess, "postprocess");
+  if (syntaxError) throw syntaxError;
+  return template;
+};
+
+const analyzeHardwareCoverage = (
+  pipeline: PipelineDefinition,
+  graphNode: ResolvedTaskNode,
+  template: TbxTemplate,
+) => {
+  const hardwareComponents = [...template.code.matchAll(FILE_COMPONENT_PATTERN)];
+  const hasHardwareComponents = hardwareComponents.length > 0;
+  const fieldsById = new Map(pipeline.graph.fields.map((field) => [field.id, field]));
+  const hardwareEntityPaths = new Set(
+    hasHardwareComponents
+      ? graphNode.settings.entities.output
+          .filter((entity) => FILE_ENTITY_TYPES.has(entity.entityType))
+          .map((entity) => {
+            const field = fieldsById.get(entity.fieldProxyId);
+            assert(field, `Output entity "${entity.fieldProxyId}" has no declared field`);
+            return entity.jsonPath === "$" ? field.name : normalizePath(entity.jsonPath);
+          })
+      : [],
+  );
+  const hardwareFieldPaths = new Set(
+    hardwareComponents
+      .map((component) => component[0].match(FIELD_PATH_PATTERN)?.[1])
+      .filter((path): path is string => path !== undefined)
+      .map(normalizePath),
+  );
+  return {
+    coverage: {
+      complete: !hasHardwareComponents,
+      untestedHardwareFields: hasHardwareComponents ? [...hardwareEntityPaths] : [],
+    } satisfies TaskUiCoverage,
+    hardwareEntityPaths,
+    hardwareFieldPaths,
+  };
+};
+
+const openTaskUiSession = async ({
+  browser,
+  launcherUrl,
+  pipeline,
+  node,
+  options,
+}: {
+  browser: Browser;
+  launcherUrl: string;
+  pipeline: PipelineDefinition;
+  node: TaskUiNode;
+  options: RenderTaskUiOptions;
+}): Promise<TaskUiSession> => {
+  const graphNode = resolveTaskNode(pipeline, node);
+  const template = createTaskUiTemplate(graphNode, options);
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  let closed = false;
+
+  try {
+    try {
+      await page.goto(launcherUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
+      await waitForTaskUiApi(page);
+    } catch (error) {
+      throw new TaskUiInfrastructureError(`Unable to load the Task UI launcher at ${launcherUrl}`, {
+        cause: error,
+      });
+    }
+    let tree = await evaluateWithApi(page, { operation: "setConfig", config: template });
+    const { coverage, hardwareEntityPaths, hardwareFieldPaths } = analyzeHardwareCoverage(
+      pipeline,
+      graphNode,
+      template,
+    );
+
+    const componentForPath = async (componentTree: string, path: string) => {
+      const states = await Promise.all(
+        componentIds(componentTree).map(async (componentId) => ({
+          componentId,
+          state: await evaluateWithApi(page, { operation: "getComponentState", componentId }),
+        })),
+      );
+      return states.find(({ state }) => readDataPath(state) === path);
+    };
+    const resolveComponentId = async (path: string): Promise<string> => {
+      const deadline = Date.now() + 5_000;
+      await new Promise<void>((resolve) => {
+        setTimeout(resolve, 500);
+      });
+      do {
+        try {
+          tree = await evaluateWithApi(page, { operation: "getComponentTree" });
+          const component = await componentForPath(tree, path);
+          if (component !== undefined) return component.componentId;
+        } catch (error) {
+          if (Date.now() >= deadline) throw error;
+        }
+        await new Promise<void>((resolve) => {
+          setTimeout(resolve, 250);
+        });
+      } while (Date.now() < deadline);
+      assert(false, `No Task UI component is bound to path "${path}"`);
+    };
+
+    const validationErrors = async (): Promise<TaskUiFormError[]> => {
+      const errors = await evaluateWithApi(page, { operation: "getValidationErrors" });
+      return Promise.all(
+        errors.map(async (error) => {
+          const state = await evaluateWithApi(page, {
+            operation: "getComponentState",
+            componentId: error.componentId,
+          });
+          const path = readDataPath(state);
+          return { ...error, ...(path === undefined ? {} : { path }) };
+        }),
+      );
+    };
+
+    return {
+      componentTree: async () => {
+        tree = await evaluateWithApi(page, { operation: "getComponentTree" });
+        return tree;
+      },
+      componentDocumentation: async (names) =>
+        selectComponentDocumentation(
+          await evaluateWithApi(page, { operation: "getComponentsDescriptions" }),
+          names,
+        ),
+      getValue: async (path) => {
+        const state = await evaluateWithApi(page, {
+          operation: "getComponentState",
+          componentId: await resolveComponentId(path),
+        });
+        return Reflect.get(state, "value");
+      },
+      setValue: async (path, value) => {
+        await evaluateWithApi(page, {
+          operation: "setComponentValue",
+          componentId: await resolveComponentId(path),
+          value,
+        });
+      },
+      validationErrors,
+      submit: async () => {
+        if (!coverage.complete && options.allowPartialCoverage !== true) {
+          throw new TaskUiValidationError({
+            message: `Task UI contains hardware-backed fields that were not tested: ${coverage.untestedHardwareFields.join(", ") || "unknown"}`,
+            coverage,
+          });
+        }
+
+        const formErrors = (await validationErrors()).filter(
+          (error) => error.path === undefined || !hardwareFieldPaths.has(normalizePath(error.path)),
+        );
+        if (formErrors.length > 0) {
+          throw new TaskUiValidationError({
+            message: "Task UI browser validation failed",
+            formErrors,
+            coverage,
+          });
+        }
+
+        const data = await evaluateWithApi(page, { operation: "getData" });
+        assert(
+          data && typeof data === "object" && !Array.isArray(data),
+          "Task UI produced no submission data",
+        );
+        const { _context, ...output } = data;
+        void _context;
+        const entityErrors = validateEntities(pipeline, graphNode, output, hardwareEntityPaths);
+        if (entityErrors.length > 0) {
+          throw new TaskUiValidationError({
+            message: `Task UI output entity validation failed: ${entityErrors
+              .map((error) => `${error.entityName} (${error.path}): ${error.reason}`)
+              .join("; ")}`,
+            entityErrors,
+            coverage,
+          });
+        }
+        return output;
+      },
+      screenshot: async (path) => {
+        await mkdir(dirname(path), { recursive: true });
+        await writeFile(path, await page.screenshot({ type: "png", fullPage: true }));
+        return path;
+      },
+      coverage,
+      close: async () => {
+        if (closed) return;
+        closed = true;
+        await page.close();
+        await context.close();
+      },
+    };
+  } catch (error) {
+    await page.close().catch(() => undefined);
+    await context.close().catch(() => undefined);
+    throw error;
+  }
+};
+
+const createTaskUiTest = ({
+  launcherUrl = DEFAULT_TBX_LAUNCHER_URL,
+  launchBrowser = launchDefaultBrowser,
+}: CreateTaskUiTestOptions = {}): TestAPI<UiTestFixtures & UiWorkerFixtures> =>
+  baseTest.extend<UiTestFixtures & UiWorkerFixtures>({
+    _uiConfig: [{ launcherUrl, launchBrowser }, { scope: "worker" }],
+    _uiBrowser: [
+      async ({ _uiConfig }, use) => {
+        const browser = await _uiConfig.launchBrowser();
+        await use(browser);
+        await browser.close();
+      },
+      { scope: "worker" },
+    ],
+    ui: async ({ _uiBrowser, _uiConfig }, use) => {
+      const sessions = new Set<TaskUiSession>();
+      await use({
+        componentDocumentation: async (names) => {
+          const context = await _uiBrowser.newContext();
+          const page = await context.newPage();
+          try {
+            await page.goto(_uiConfig.launcherUrl, {
+              waitUntil: "domcontentloaded",
+              timeout: 30_000,
+            });
+            await waitForTaskUiApi(page);
+            return selectComponentDocumentation(
+              await evaluateWithApi(page, { operation: "getComponentsDescriptions" }),
+              names,
+            );
+          } finally {
+            await page.close();
+            await context.close();
+          }
+        },
+        render: async (pipeline, node, options = {}) => {
+          const session = await openTaskUiSession({
+            browser: _uiBrowser,
+            launcherUrl: _uiConfig.launcherUrl,
+            pipeline,
+            node,
+            options,
+          });
+          sessions.add(session);
+          return session;
+        },
+      });
+      await Promise.all([...sessions].map((session) => session.close()));
+    },
+  });
+
+const vitestTest = createTaskUiTest();
+
+type CodeTestContext<Node extends AuthoredCodeNode> = {
+  readonly node: Node;
+  readonly run: (
+    inputs: Record<string, unknown>,
+    options?: RunCodeNodeOptions,
+  ) => Promise<Record<string, unknown>>;
+};
+
+type UiTestContext<Node extends TaskUiNode> = {
+  readonly node: Node;
+  readonly pipeline: PipelineDefinition;
+  readonly render: (options?: RenderTaskUiOptions) => Promise<TaskUiSession>;
+};
+
+type PipelineTestContext = {
+  readonly pipeline: PipelineDefinition;
+  readonly runCode: (
+    node: AuthoredCodeNode,
+    inputs: Record<string, unknown>,
+    options?: RunCodeNodeOptions,
+  ) => Promise<Record<string, unknown>>;
+  readonly renderUi: (node: TaskUiNode, options?: RenderTaskUiOptions) => Promise<TaskUiSession>;
+};
+
+const errorMessage = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const runTypedTest = async ({
+  pipeline,
+  nodeId,
+  kind,
+  name,
+  execute,
+  wasExecuted,
+  coverage,
+}: {
+  pipeline: PipelineDefinition;
+  nodeId?: string;
+  kind: TestEvidenceKind;
+  name: string;
+  execute: () => Promise<void>;
+  wasExecuted: () => boolean;
+  coverage?: () => TaskUiCoverage[];
+}) => {
+  const assertionsBefore = expect.getState().assertionCalls;
+  try {
+    await execute();
+    assertTypedTestCompletion({
+      kind,
+      name,
+      executed: wasExecuted(),
+      assertionsBefore,
+      assertionsAfter: expect.getState().assertionCalls,
+    });
+    const sessions = coverage?.() ?? [];
+    const untestedHardwareFields = [
+      ...new Set(sessions.flatMap((entry) => entry.untestedHardwareFields)),
+    ];
+    await writeTestEvidence({
+      version: 1,
+      pipelineFingerprint: pipelineFingerprint(pipeline),
+      kind,
+      status: "passed",
+      testName: name,
+      ...(nodeId === undefined ? {} : { nodeId }),
+      ...(sessions.length === 0
+        ? {}
+        : {
+            partial: sessions.some((entry) => !entry.complete),
+            untestedHardwareFields,
+          }),
+    });
+  } catch (error) {
+    await writeTestEvidence({
+      version: 1,
+      pipelineFingerprint: pipelineFingerprint(pipeline),
+      kind,
+      status:
+        error instanceof TaskUiInfrastructureError || error instanceof CodeNodeInfrastructureError
+          ? "infrastructure_error"
+          : "failed",
+
+      testName: name,
+      diagnostic: errorMessage(error),
+      ...(nodeId === undefined ? {} : { nodeId }),
+    });
+    throw error;
+  }
+};
+const DEFAULT_CODE_TEST_TIMEOUT_MS = 30_000;
+
+const codeTest = <Node extends AuthoredCodeNode>(
+  pipeline: PipelineDefinition,
+  node: Node,
+  name: string,
+  callback: (context: CodeTestContext<Node>) => Promise<void> | void,
+  options: TypedTestOptions = {},
+) => {
+  const register = matchesTestFilter(pipeline, node.id) ? vitestTest : vitestTest.skip;
+  register(
+    `${typedTestNamePrefix(pipelineFingerprint(pipeline), node.id)} ${name}`,
+    { timeout: options.timeout ?? DEFAULT_CODE_TEST_TIMEOUT_MS },
+    async () => {
+      assertTestNodeInPipeline(pipeline, node);
+      let executed = false;
+      await runTypedTest({
+        pipeline,
+        nodeId: node.id,
+        kind: "code",
+        name,
+        wasExecuted: () => executed,
+        execute: async () =>
+          callback({
+            node,
+            run: async (inputs, runOptions) => {
+              executed = true;
+              return runCodeNode(node, inputs, runOptions);
+            },
+          }),
+      });
+    },
+  );
+};
+
+const wrapUiSession = (session: TaskUiSession, onSubmit: () => void): TaskUiSession => ({
+  ...session,
+  submit: async () => {
+    const output = await session.submit();
+    onSubmit();
+    return output;
+  },
+});
+const DEFAULT_UI_TEST_TIMEOUT_MS = 60_000;
+const DEFAULT_PIPELINE_TEST_TIMEOUT_MS = 120_000;
+
+const uiTest = <Node extends TaskUiNode>(
+  pipeline: PipelineDefinition,
+  node: Node,
+  name: string,
+  callback: (context: UiTestContext<Node>) => Promise<void> | void,
+  options: TypedTestOptions = {},
+) => {
+  const register = matchesTestFilter(pipeline, node.id) ? vitestTest : vitestTest.skip;
+  register(
+    `${typedTestNamePrefix(pipelineFingerprint(pipeline), node.id)} ${name}`,
+    { timeout: options.timeout ?? DEFAULT_UI_TEST_TIMEOUT_MS },
+    async ({ ui }) => {
+      assertTestNodeInPipeline(pipeline, node);
+      let submitted = false;
+      const sessions: TaskUiCoverage[] = [];
+      await runTypedTest({
+        pipeline,
+        nodeId: node.id,
+        kind: "ui",
+        name,
+        wasExecuted: () => submitted,
+        coverage: () => sessions,
+        execute: async () =>
+          callback({
+            node,
+            pipeline,
+            render: async (options) => {
+              const session = await ui.render(pipeline, node, options);
+              sessions.push(session.coverage);
+              return wrapUiSession(session, () => {
+                submitted = true;
+              });
+            },
+          }),
+      });
+    },
+  );
+};
+
+const pipelineTest = (
+  pipeline: PipelineDefinition,
+  name: string,
+  callback: (context: PipelineTestContext) => Promise<void> | void,
+  options: TypedTestOptions = {},
+) => {
+  const register = matchesTestFilter(pipeline, undefined) ? vitestTest : vitestTest.skip;
+  const execute = async (ui?: UiTestFixtures["ui"]) => {
+    let executed = pipeline.graph.nodes.every(
+      (node) => node.type !== "code" && node.type !== "generation" && node.type !== "quorum",
+    );
+    const sessions: TaskUiCoverage[] = [];
+    await runTypedTest({
+      pipeline,
+      kind: "pipeline",
+      name,
+      wasExecuted: () => executed,
+      coverage: () => sessions,
+      execute: async () =>
+        callback({
+          pipeline,
+          runCode: async (node, inputs, options) => {
+            assertTestNodeInPipeline(pipeline, node);
+            executed = true;
+            return runCodeNode(node, inputs, options);
+          },
+          renderUi: async (node, options) => {
+            assertTestNodeInPipeline(pipeline, node);
+            assert(ui, `Pipeline test "${name}" cannot render UI without a human-work node`);
+            const session = await ui.render(pipeline, node, options);
+            sessions.push(session.coverage);
+            return wrapUiSession(session, () => {
+              executed = true;
+            });
+          },
+        }),
+    });
+  };
+  const hasUiNode = pipeline.graph.nodes.some(
+    (node) => node.type === "generation" || node.type === "quorum",
+  );
+  const registeredName = `${typedTestNamePrefix(pipelineFingerprint(pipeline))} ${name}`;
+  if (hasUiNode)
+    register(
+      registeredName,
+      { timeout: options.timeout ?? DEFAULT_PIPELINE_TEST_TIMEOUT_MS },
+      async ({ ui }) => execute(ui),
+    );
+  else
+    register(registeredName, { timeout: options.timeout ?? DEFAULT_PIPELINE_TEST_TIMEOUT_MS }, () =>
+      execute(),
+    );
+};
+
+const exportedTest = process.env[TEST_TYPED_ONLY_ENV] === "1" ? vitestTest.skip : vitestTest;
+
+export const test = Object.assign(exportedTest, {
+  code: codeTest,
+  ui: uiTest,
+  pipeline: pipelineTest,
+});
+export { expect };