@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.7

package/src/testing/test-evidence.ts

@@ -0,0 +1,170 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { assert, isEqual } from "es-toolkit";
+import type { PipelineDefinition } from "../authoring";
+
+export const TEST_EVIDENCE_DIR_ENV = "TOLOKA_CHECK_EVIDENCE_DIR";
+export const TEST_PIPELINE_FILTER_ENV = "TOLOKA_CHECK_PIPELINE_FINGERPRINT";
+export const TEST_NODE_FILTER_ENV = "TOLOKA_CHECK_NODE_ID";
+export const TEST_TYPED_ONLY_ENV = "TOLOKA_CHECK_TYPED_ONLY";
+
+export type TestEvidenceKind = "code" | "ui" | "pipeline";
+export type TestEvidenceStatus = "passed" | "failed" | "infrastructure_error";
+
+export type TestEvidence = {
+  readonly version: 1;
+  readonly pipelineFingerprint: string;
+  readonly kind: TestEvidenceKind;
+  readonly status: TestEvidenceStatus;
+  readonly testName: string;
+  readonly nodeId?: string;
+  readonly diagnostic?: string;
+  readonly partial?: boolean;
+  readonly untestedHardwareFields?: readonly string[];
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isStringArray = (value: unknown): value is string[] =>
+  Array.isArray(value) && value.every((entry) => typeof entry === "string");
+
+const canonicalize = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (typeof value !== "object" || value === null) return value;
+  return Object.fromEntries(
+    Object.entries(value)
+      .toSorted(([left], [right]) => left.localeCompare(right))
+      .map(([key, entryValue]) => [key, canonicalize(entryValue)]),
+  );
+};
+
+export const pipelineFingerprint = (pipeline: PipelineDefinition): string =>
+  createHash("sha256")
+    .update(JSON.stringify(canonicalize(pipeline)))
+    .digest("hex");
+export const typedTestNamePrefix = (fingerprint: string, nodeId?: string): string =>
+  `[toloka:${fingerprint}:${nodeId ?? "pipeline"}]`;
+
+const escapeRegExp = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+
+export const typedTestNamePattern = (fingerprint: string, nodeId?: string): string =>
+  nodeId === undefined
+    ? escapeRegExp(`[toloka:${fingerprint}:`)
+    : escapeRegExp(typedTestNamePrefix(fingerprint, nodeId));
+
+export const assertTypedTestCompletion = ({
+  kind,
+  name,
+  executed,
+  assertionsBefore,
+  assertionsAfter,
+}: {
+  readonly kind: TestEvidenceKind;
+  readonly name: string;
+  readonly executed: boolean;
+  readonly assertionsBefore: number;
+  readonly assertionsAfter: number;
+}): void => {
+  assert(executed, `${kind} test "${name}" did not run its bound executor`);
+  assert(assertionsAfter > assertionsBefore, `${kind} test "${name}" made no assertions`);
+};
+
+export const assertTestNodeInPipeline = (
+  pipeline: PipelineDefinition,
+  node: { readonly id: string; readonly type: string; readonly settings: unknown },
+): void => {
+  const graphNode = pipeline.graph.nodes.find((candidate) => candidate.id === node.id);
+  assert(
+    graphNode?.type === node.type && isEqual(graphNode.settings, node.settings),
+    `Node ${JSON.stringify(node.id)} does not belong to the bound pipeline`,
+  );
+};
+
+const isEvidenceKind = (value: unknown): value is TestEvidenceKind =>
+  value === "code" || value === "ui" || value === "pipeline";
+
+const isEvidenceStatus = (value: unknown): value is TestEvidenceStatus =>
+  value === "passed" || value === "failed" || value === "infrastructure_error";
+
+const parseEvidence = (value: unknown, path: string): TestEvidence => {
+  assert(isRecord(value), `Invalid test evidence ${path}`);
+  const record = value;
+  assert(record.version === 1, `Unsupported test evidence version in ${path}`);
+  assert(typeof record.pipelineFingerprint === "string", `Missing pipeline fingerprint in ${path}`);
+  assert(isEvidenceKind(record.kind), `Invalid test evidence kind in ${path}`);
+  assert(isEvidenceStatus(record.status), `Invalid test evidence status in ${path}`);
+  assert(typeof record.testName === "string", `Missing test name in ${path}`);
+  assert(
+    record.nodeId === undefined || typeof record.nodeId === "string",
+    `Invalid node id in ${path}`,
+  );
+  assert(
+    record.diagnostic === undefined || typeof record.diagnostic === "string",
+    `Invalid diagnostic in ${path}`,
+  );
+  assert(
+    record.partial === undefined || typeof record.partial === "boolean",
+    `Invalid partial coverage in ${path}`,
+  );
+  assert(
+    record.untestedHardwareFields === undefined || isStringArray(record.untestedHardwareFields),
+    `Invalid hardware fields in ${path}`,
+  );
+  const untestedHardwareFields = record.untestedHardwareFields;
+  return {
+    version: 1,
+    pipelineFingerprint: record.pipelineFingerprint,
+    kind: record.kind,
+    status: record.status,
+    testName: record.testName,
+    ...(record.nodeId === undefined ? {} : { nodeId: record.nodeId }),
+    ...(record.diagnostic === undefined ? {} : { diagnostic: record.diagnostic }),
+    ...(record.partial === undefined ? {} : { partial: record.partial }),
+    ...(untestedHardwareFields === undefined ? {} : { untestedHardwareFields }),
+  };
+};
+
+export const writeTestEvidence = async (
+  evidence: TestEvidence,
+  environment: NodeJS.ProcessEnv = process.env,
+): Promise<void> => {
+  const directory = environment[TEST_EVIDENCE_DIR_ENV];
+  if (directory === undefined) return;
+  await mkdir(directory, { recursive: true });
+  await writeFile(join(directory, `${process.pid}-${randomUUID()}.json`), JSON.stringify(evidence));
+};
+
+export const readTestEvidence = async (directory: string): Promise<TestEvidence[]> => {
+  const entries = await readdir(directory, { withFileTypes: true });
+  const files = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+    .map((entry) => entry.name)
+    .toSorted();
+  return Promise.all(
+    files.map(async (file) => {
+      const path = join(directory, file);
+      let value: unknown;
+      try {
+        value = JSON.parse(await readFile(path, "utf8"));
+      } catch (error) {
+        throw new Error(`Invalid test evidence ${path}`, { cause: error });
+      }
+      return parseEvidence(value, path);
+    }),
+  );
+};
+
+export const matchesTestFilter = (
+  pipeline: PipelineDefinition,
+  nodeId: string | undefined,
+  environment: NodeJS.ProcessEnv = process.env,
+): boolean => {
+  const expectedPipeline = environment[TEST_PIPELINE_FILTER_ENV];
+  if (expectedPipeline !== undefined && expectedPipeline !== pipelineFingerprint(pipeline))
+    return false;
+  const expectedNode = environment[TEST_NODE_FILTER_ENV];
+  return expectedNode === undefined || expectedNode === nodeId;
+};

package/src/testing/types.ts

@@ -0,0 +1,141 @@
+import type { Browser } from "playwright";
+import type { AuthoredGenerationNode, AuthoredQuorumNode, PipelineDefinition } from "../authoring";
+
+export type TaskUiNode = AuthoredGenerationNode | AuthoredQuorumNode;
+
+export type TaskUiFormError = {
+  componentId: string;
+  error: string;
+  path?: string;
+};
+
+export type TaskUiEntityError = {
+  entityName: string;
+  path: string;
+  reason: string;
+  submittedValue?: unknown;
+  expectedSchema?: unknown;
+};
+
+export type TaskUiCoverage = {
+  complete: boolean;
+  untestedHardwareFields: string[];
+};
+
+export type RenderTaskUiOptions = {
+  data?: Record<string, unknown>;
+  allowPartialCoverage?: boolean;
+};
+export type TypedTestOptions = {
+  readonly timeout?: number;
+};
+
+export type TaskUiComponentDescription = {
+  name: string;
+  description: string;
+  whenToUse: string;
+  capabilities: string[];
+  limitations: string[];
+  bestPractices: string[];
+  propsType: string;
+  valueType?: string;
+  example?: {
+    jsxTemplate: string;
+    output: string;
+  };
+};
+
+export type TaskUiSession = {
+  componentTree: () => Promise<string>;
+  componentDocumentation: (names?: string[]) => Promise<Record<string, TaskUiComponentDescription>>;
+  getValue: (path: string) => Promise<unknown>;
+  setValue: (path: string, value: unknown) => Promise<void>;
+  validationErrors: () => Promise<TaskUiFormError[]>;
+  submit: () => Promise<Record<string, unknown>>;
+  screenshot: (path: string) => Promise<string>;
+  coverage: TaskUiCoverage;
+  close: () => Promise<void>;
+};
+
+export type TaskUiFixture = {
+  componentDocumentation: (names?: string[]) => Promise<Record<string, TaskUiComponentDescription>>;
+  render: (
+    pipeline: PipelineDefinition,
+    node: TaskUiNode,
+    options?: RenderTaskUiOptions,
+  ) => Promise<TaskUiSession>;
+};
+
+export class TaskUiInfrastructureError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "TaskUiInfrastructureError";
+  }
+}
+
+export class TaskUiValidationError extends Error {
+  readonly formErrors: TaskUiFormError[];
+  readonly entityErrors: TaskUiEntityError[];
+  readonly coverage: TaskUiCoverage;
+
+  constructor({
+    message,
+    formErrors = [],
+    entityErrors = [],
+    coverage,
+  }: {
+    message: string;
+    formErrors?: TaskUiFormError[];
+    entityErrors?: TaskUiEntityError[];
+    coverage: TaskUiCoverage;
+  }) {
+    super(message);
+    this.name = "TaskUiValidationError";
+    this.formErrors = formErrors;
+    this.entityErrors = entityErrors;
+    this.coverage = coverage;
+  }
+}
+
+export type TbxTemplate = {
+  code: string;
+  data: string;
+  preprocess: string;
+  postprocess: string;
+  constants: string;
+};
+
+export type TbxWindowApi = {
+  getData: () => Promise<Record<string, unknown> | undefined>;
+  agents: {
+    setConfig: (config: TbxTemplate, mode?: "xml" | "json") => Promise<string>;
+    getComponentTree: (mode?: "xml" | "json") => Promise<string>;
+    getComponentState: (componentId: string) => Promise<Record<string, unknown>>;
+    getComponentValue: (componentId: string) => Promise<unknown>;
+    setComponentValue: (componentId: string, value: unknown) => Promise<void>;
+    getValidationErrors: () => Promise<Array<{ componentId: string; error: string }>>;
+    getComponentsDescriptions: () => Promise<TaskUiComponentDescription[]>;
+  };
+};
+
+export type UiWorkerFixtures = {
+  _uiConfig: {
+    launcherUrl: string;
+    launchBrowser: () => Promise<Browser>;
+  };
+  _uiBrowser: Browser;
+};
+
+export type UiTestFixtures = {
+  ui: TaskUiFixture;
+};
+
+export type TbxBrowserRequest =
+  | { operation: "setConfig"; config: TbxTemplate }
+  | { operation: "getComponentTree" }
+  | { operation: "getComponentState"; componentId: string }
+  | { operation: "getComponentValue"; componentId: string }
+  | { operation: "setComponentValue"; componentId: string; value: unknown }
+  | { operation: "getValidationErrors" }
+  | { operation: "getComponentsDescriptions" }
+  | { operation: "getData" };

package/src/transport.ts

@@ -1,7 +1,7 @@
-import { assert } from 'es-toolkit';
-import { PotatoApiError } from './errors';
+import { assert } from "es-toolkit";
+import { PotatoApiError } from "./errors";
 
-const BASE_URL_HEADER = 'x-potato-sdk-base-url';
+const BASE_URL_HEADER = "x-potato-sdk-base-url";
 
 type TransportRequestInit = RequestInit & {
   readonly fetch?: typeof globalThis.fetch;
@@ -10,18 +10,18 @@ type TransportRequestInit = RequestInit & {
 export const createRequestOptions = (
   baseUrl: string,
   apiKey: string,
-  fetch?: typeof globalThis.fetch
+  fetch?: typeof globalThis.fetch,
 ): TransportRequestInit => {
-  assert(baseUrl.length > 0, 'The SDK base URL cannot be empty');
-  assert(apiKey.length > 0, 'The SDK API key cannot be empty');
+  assert(baseUrl.length > 0, "The SDK base URL cannot be empty");
+  assert(apiKey.length > 0, "The SDK API key cannot be empty");
 
   return {
     headers: {
-      Accept: 'application/json',
+      Accept: "application/json",
       Authorization: `ApiKey ${apiKey}`,
-      [BASE_URL_HEADER]: new URL(baseUrl).toString()
+      [BASE_URL_HEADER]: new URL(baseUrl).toString(),
     },
-    fetch
+    fetch,
   };
 };
 
@@ -30,7 +30,7 @@ export async function potatoFetch(url: string, options: TransportRequestInit): P
   const { fetch: fetchImplementation = globalThis.fetch, ...requestOptions } = options;
   const headers = new Headers(requestOptions.headers);
   const baseUrl = headers.get(BASE_URL_HEADER);
-  assert(baseUrl, 'The SDK base URL is missing');
+  assert(baseUrl, "The SDK base URL is missing");
   headers.delete(BASE_URL_HEADER);
 
   const response = await fetchImplementation(new URL(url, baseUrl), { ...requestOptions, headers });
@@ -41,13 +41,16 @@ export async function potatoFetch(url: string, options: TransportRequestInit): P
 
     if (text.length === 0) {
       body = undefined;
-    } else if (response.headers.get('content-type')?.includes('application/json')) {
+    } else if (response.headers.get("content-type")?.includes("application/json")) {
       body = JSON.parse(text);
     }
 
-    throw PotatoApiError.fromResponse(response.status, body);
+    throw PotatoApiError.fromResponse(
+      response.status,
+      body,
+      response.headers.get("x-request-id") ?? undefined,
+    );
   }
 
   return text.length === 0 ? undefined : JSON.parse(text);
 }
-