@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8

package/src/testing/code-node.ts

@@ -0,0 +1,160 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AuthoredCodeNode } from "../authoring";
+
+export type RunCodeNodeOptions = {
+  env?: Record<string, string>;
+  pythonExecutable?: string;
+  timeoutMs?: number;
+};
+
+export class CodeNodeExecutionError extends Error {
+  readonly exitCode: number | null;
+  readonly stderr: string;
+
+  constructor(message: string, exitCode: number | null, stderr: string) {
+    super(message);
+    this.name = "CodeNodeExecutionError";
+    this.exitCode = exitCode;
+    this.stderr = stderr;
+  }
+}
+
+export class CodeNodeInfrastructureError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "CodeNodeInfrastructureError";
+  }
+}
+
+const collectOutput = async (stream: NodeJS.ReadableStream): Promise<string> => {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks).toString("utf8");
+};
+
+const staticEnvironment = (node: AuthoredCodeNode): Record<string, string> =>
+  Object.fromEntries(
+    (node.settings.envVars ?? [])
+      .filter((entry) => entry.type === "static")
+      .map((entry) => [entry.name, entry.value]),
+  );
+
+const decodeCodeNodeOutput = (
+  nodeId: string,
+  serialized: string,
+  exitCode: number | null,
+  stderr: string,
+): Record<string, unknown> => {
+  let output: unknown;
+  try {
+    output = JSON.parse(serialized);
+  } catch (error) {
+    throw new CodeNodeExecutionError(
+      `Code node ${nodeId} returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
+      exitCode,
+      stderr,
+    );
+  }
+  if (output === null || typeof output !== "object" || Array.isArray(output)) {
+    throw new CodeNodeExecutionError(
+      `Code node ${nodeId} returned a non-object value`,
+      exitCode,
+      stderr,
+    );
+  }
+  return output as Record<string, unknown>;
+};
+
+export const runCodeNode = async (
+  node: AuthoredCodeNode,
+  inputs: Record<string, unknown>,
+  { env = {}, pythonExecutable = "python3", timeoutMs = 10_000 }: RunCodeNodeOptions = {},
+): Promise<Record<string, unknown>> => {
+  const directory = await mkdtemp(join(tmpdir(), "toloka-code-node-"));
+  const sourcePath = join(directory, "node.py");
+  const mainTypesPath = join(directory, "main_types.py");
+  const runnerPath = join(directory, "runner.py");
+  const outputPath = join(directory, "output.json");
+
+  try {
+    await Promise.all([
+      writeFile(sourcePath, node.settings.code.code, "utf8"),
+      writeFile(
+        mainTypesPath,
+        "from typing import Any\n\nInputs = dict[str, Any]\nOutputs = dict[str, Any]\n",
+        "utf8",
+      ),
+      writeFile(
+        runnerPath,
+        `import asyncio\nimport json\nimport sys\n\nfrom node import main\n\nasync def run():\n    result = await main(json.load(sys.stdin))\n    with open(sys.argv[1], "w", encoding="utf-8") as output:\n        json.dump(result, output)\n\nasyncio.run(run())\n`,
+        "utf8",
+      ),
+    ]);
+
+    const child = spawn(pythonExecutable, [runnerPath, outputPath], {
+      cwd: directory,
+      env: { ...process.env, ...staticEnvironment(node), ...env },
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    const {
+      promise: closed,
+      resolve: resolveClose,
+      reject: rejectClose,
+    } = Promise.withResolvers<number | null>();
+    child.once("close", resolveClose);
+    child.once("error", (error) =>
+      rejectClose(
+        new CodeNodeInfrastructureError(
+          `Unable to start Python executable ${JSON.stringify(pythonExecutable)}`,
+          { cause: error },
+        ),
+      ),
+    );
+    let timedOut = false;
+    const timeout = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGKILL");
+    }, timeoutMs);
+    child.stdin.end(JSON.stringify(inputs));
+
+    const [exitCode, , stderr] = await Promise.all([
+      closed,
+      collectOutput(child.stdout),
+      collectOutput(child.stderr),
+    ]).finally(() => clearTimeout(timeout));
+
+    if (timedOut) {
+      throw new CodeNodeExecutionError(
+        `Code node ${node.id} exceeded the ${timeoutMs}ms timeout`,
+        exitCode,
+        stderr,
+      );
+    }
+    if (exitCode !== 0) {
+      throw new CodeNodeExecutionError(
+        `Code node ${node.id} exited with code ${exitCode ?? "unknown"}`,
+        exitCode,
+        stderr,
+      );
+    }
+
+    let serialized: string;
+    try {
+      serialized = await readFile(outputPath, "utf8");
+    } catch (error) {
+      throw new CodeNodeExecutionError(
+        `Code node ${node.id} did not produce an output file: ${error instanceof Error ? error.message : String(error)}`,
+        exitCode,
+        stderr,
+      );
+    }
+    return decodeCodeNodeOutput(node.id, serialized, exitCode, stderr);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+};

package/src/testing/index.ts

@@ -0,0 +1,15 @@
+export { CodeNodeExecutionError, CodeNodeInfrastructureError, runCodeNode } from "./code-node";
+export type { RunCodeNodeOptions } from "./code-node";
+export { expect, test } from "./task-ui";
+export { TaskUiInfrastructureError, TaskUiValidationError } from "./types";
+export type {
+  RenderTaskUiOptions,
+  TaskUiComponentDescription,
+  TaskUiCoverage,
+  TaskUiEntityError,
+  TaskUiFixture,
+  TaskUiFormError,
+  TaskUiNode,
+  TaskUiSession,
+  TypedTestOptions,
+} from "./types";

package/src/testing/task-ui-browser.ts

@@ -0,0 +1,120 @@
+import { assert } from "es-toolkit";
+import { chromium, type Browser, type Page } from "playwright";
+import {
+  TaskUiInfrastructureError,
+  type TaskUiComponentDescription,
+  type TbxBrowserRequest,
+  type TbxTemplate,
+  type TbxWindowApi,
+} from "./types";
+
+export const DEFAULT_TBX_LAUNCHER_URL =
+  "https://tlkfrontprodpublic.blob.core.windows.net/template-builder-production/tb-jsx.agents/index.html?postprocess=true";
+
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "setConfig" }>,
+): Promise<string>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getComponentTree" }>,
+): Promise<string>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getComponentState" }>,
+): Promise<Record<string, unknown>>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getComponentValue" }>,
+): Promise<unknown>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "setComponentValue" }>,
+): Promise<void>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getValidationErrors" }>,
+): Promise<Array<{ componentId: string; error: string }>>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getComponentsDescriptions" }>,
+): Promise<TaskUiComponentDescription[]>;
+export function evaluateWithApi(
+  page: Page,
+  request: Extract<TbxBrowserRequest, { operation: "getData" }>,
+): Promise<Record<string, unknown> | undefined>;
+export async function evaluateWithApi(page: Page, request: TbxBrowserRequest): Promise<unknown> {
+  assert(!page.isClosed(), "Task UI browser page is closed");
+
+  return page.evaluate(async (input) => {
+    const api: TbxWindowApi | undefined = Reflect.get(globalThis, "LAUNCHER_EMBED_API");
+    if (api === undefined) throw new Error("Task UI launcher API is unavailable");
+
+    switch (input.operation) {
+      case "setConfig":
+        return api.agents.setConfig(input.config);
+      case "getComponentTree":
+        return api.agents.getComponentTree("xml");
+      case "getComponentState":
+        return api.agents.getComponentState(input.componentId);
+      case "getComponentValue":
+        return api.agents.getComponentValue(input.componentId);
+      case "setComponentValue":
+        return api.agents.setComponentValue(input.componentId, input.value);
+      case "getValidationErrors":
+        return api.agents.getValidationErrors();
+      case "getComponentsDescriptions":
+        return api.agents.getComponentsDescriptions();
+      case "getData":
+        return api.getData();
+    }
+  }, request);
+}
+
+export const selectComponentDocumentation = (
+  descriptions: TaskUiComponentDescription[],
+  names?: string[],
+): Record<string, TaskUiComponentDescription> => {
+  const selected =
+    names === undefined
+      ? descriptions
+      : descriptions.filter((description) => names.includes(description.name));
+  const byName = Object.fromEntries(selected.map((description) => [description.name, description]));
+  if (names !== undefined) {
+    const missing = names.filter((name) => !(name in byName));
+    assert(missing.length === 0, `Unknown Task UI components: ${missing.join(", ")}`);
+  }
+  return byName;
+};
+
+export const readDataPath = (state: Record<string, unknown>): string | undefined => {
+  const path = state.dataPath;
+  return typeof path === "string" ? path : undefined;
+};
+
+export const componentIds = (tree: string): string[] =>
+  [...tree.matchAll(/\bid="([^"]+)"/gu)].map((match) => match[1]!);
+
+export const launchDefaultBrowser = async (): Promise<Browser> => {
+  try {
+    return await chromium.launch({ headless: true });
+  } catch (error) {
+    throw new TaskUiInfrastructureError(
+      "Local Task UI testing requires Playwright and Chromium. Run `bun add --dev playwright vitest` and `bunx playwright install chromium`.",
+      { cause: error },
+    );
+  }
+};
+
+export const waitForTaskUiApi = async (page: Page): Promise<void> => {
+  await page.waitForFunction(
+    () => {
+      const api: TbxWindowApi | undefined = Reflect.get(globalThis, "LAUNCHER_EMBED_API");
+      return api?.agents !== undefined;
+    },
+    undefined,
+    { timeout: 15_000 },
+  );
+};
+
+export type { Browser, Page, TbxTemplate };

package/src/testing/task-ui-validation.ts

@@ -0,0 +1,135 @@
+import Ajv2020 from "ajv/dist/2020.js";
+import addFormats from "ajv-formats";
+import { assert } from "es-toolkit";
+import { query, type JsonValue } from "jsonpath-rfc9535";
+import { Script } from "node:vm";
+import type { PipelineDefinition } from "../authoring";
+import type { TaskUiEntityError, TaskUiNode } from "./types";
+
+export const resolveTaskNode = (pipeline: PipelineDefinition, node: TaskUiNode) => {
+  const graphNode = pipeline.graph.nodes.find((candidate) => candidate.id === node.id);
+  assert(graphNode, `Node "${node.id}" is not part of this pipeline`);
+  assert(
+    graphNode.type === "generation" || graphNode.type === "quorum",
+    `Node "${node.id}" does not have a Task UI`,
+  );
+  return graphNode;
+};
+
+export type ResolvedTaskNode = ReturnType<typeof resolveTaskNode>;
+type TaskUiOutputEntity = ResolvedTaskNode["settings"]["entities"]["output"][number];
+type PipelineField = PipelineDefinition["graph"]["fields"][number];
+
+export const runtimeJavaScriptSyntaxError = (
+  source: string,
+  block: "preprocess" | "postprocess",
+): SyntaxError | null => {
+  try {
+    new Script(`(function (data) {\n${source}\n})`, { filename: `${block}.js` });
+    return null;
+  } catch (error) {
+    return new SyntaxError(
+      `${block} is not valid runtime JavaScript: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+};
+
+export const normalizePath = (path: string): string =>
+  path.replace(/^\$\.?/u, "").replace(/\[\*\]$/u, "");
+
+const resolveEntitySchema = (node: ResolvedTaskNode, field: PipelineField): unknown => {
+  const schema = field.jsonSchema;
+  if (
+    node.type !== "quorum" ||
+    schema === null ||
+    typeof schema !== "object" ||
+    Array.isArray(schema) ||
+    Reflect.get(schema, "type") !== "array"
+  ) {
+    return schema;
+  }
+  return Reflect.get(schema, "items");
+};
+
+const extractEntityValues = (
+  data: Record<string, unknown>,
+  entity: TaskUiOutputEntity,
+  field: PipelineField,
+): JsonValue[] => {
+  if (entity.jsonPath !== "$") return query(data as JsonValue, entity.jsonPath);
+  return Object.hasOwn(data, field.name) ? [data[field.name] as JsonValue] : [];
+};
+
+const validateEntity = ({
+  ajv,
+  node,
+  data,
+  entity,
+  field,
+}: {
+  ajv: Ajv2020;
+  node: ResolvedTaskNode;
+  data: Record<string, unknown>;
+  entity: TaskUiOutputEntity;
+  field: PipelineField;
+}): TaskUiEntityError | undefined => {
+  const expectedSchema = resolveEntitySchema(node, field);
+  const extracted = extractEntityValues(data, entity, field);
+  if (extracted.length === 0) {
+    return entity.optional
+      ? undefined
+      : {
+          entityName: field.name,
+          path: entity.jsonPath,
+          reason: `No data found at path "${entity.jsonPath}"`,
+          submittedValue: null,
+          expectedSchema,
+        };
+  }
+
+  assert(
+    typeof expectedSchema === "boolean" ||
+      (expectedSchema !== null &&
+        typeof expectedSchema === "object" &&
+        !Array.isArray(expectedSchema)),
+    `Invalid schema for "${field.name}"`,
+  );
+  const submittedValue = entity.dynamic ? extracted : extracted[0];
+  const validate = ajv.compile(expectedSchema as boolean | object);
+  if (validate(submittedValue)) return undefined;
+
+  return {
+    entityName: field.name,
+    path: entity.jsonPath,
+    reason:
+      validate.errors
+        ?.map((error) => error.message)
+        .filter(Boolean)
+        .join(", ") || "Schema mismatch",
+    submittedValue,
+    expectedSchema,
+  };
+};
+
+export const validateEntities = (
+  pipeline: PipelineDefinition,
+  node: ResolvedTaskNode,
+  data: Record<string, unknown>,
+  ignoredPaths: Set<string>,
+): TaskUiEntityError[] => {
+  const fieldsById = new Map(pipeline.graph.fields.map((field) => [field.id, field]));
+  const ajv = new Ajv2020({ strict: false });
+  addFormats(ajv);
+  const errors: TaskUiEntityError[] = [];
+
+  for (const entity of node.settings.entities.output) {
+    const field = fieldsById.get(entity.fieldProxyId);
+    assert(field, `Output entity "${entity.fieldProxyId}" has no declared field`);
+    const path = entity.jsonPath === "$" ? field.name : normalizePath(entity.jsonPath);
+    if (ignoredPaths.has(path)) continue;
+    const error = validateEntity({ ajv, node, data, entity, field });
+    if (error !== undefined) errors.push(error);
+  }
+
+  return errors;
+};