@toloka-platform/sdk 0.1.0-preview.3

package/src/importer.ts

@@ -0,0 +1,606 @@
+import { assert } from "es-toolkit";
+import type { JsonSchema, PipelineGraphV1 } from "../openapi/gen/models";
+import type { AuthoringIdOptions } from "./authoring";
+
+export type PipelineSourceTree = Readonly<Record<string, string>>;
+export type GeneratePipelineSourceTreeOptions = {
+  readonly dataset: string;
+};
+
+export type FieldSourceDefinition = {
+  readonly name: string;
+  readonly jsonSchema: JsonSchema;
+  readonly options?: AuthoringIdOptions;
+};
+
+type GraphNode = PipelineGraphV1["nodes"][number];
+type ConnectionEndpoint = PipelineGraphV1["connections"][number]["from"];
+type NameInput = {
+  readonly key: string;
+  readonly name: string;
+  readonly fallback: string;
+};
+type NodeEmission = {
+  readonly binding: string;
+  readonly path: string;
+  readonly source: string;
+};
+type FieldReference = (id: string) => SourceReference;
+
+const SOURCE_REFERENCE = Symbol("sourceReference");
+const BLOCKED_BINDINGS: Record<string, true> = {
+  code: true,
+  connect: true,
+  definePipeline: true,
+  field: true,
+  generation: true,
+  match: true,
+  pipeline: true,
+  quorum: true,
+  root: true,
+};
+const RESERVED_IDENTIFIERS: Record<string, true> = {
+  arguments: true,
+  await: true,
+  break: true,
+  case: true,
+  catch: true,
+  class: true,
+  const: true,
+  continue: true,
+  debugger: true,
+  default: true,
+  delete: true,
+  do: true,
+  else: true,
+  enum: true,
+  eval: true,
+  export: true,
+  extends: true,
+  false: true,
+  finally: true,
+  for: true,
+  function: true,
+  if: true,
+  implements: true,
+  import: true,
+  in: true,
+  instanceof: true,
+  interface: true,
+  let: true,
+  new: true,
+  null: true,
+  package: true,
+  private: true,
+  protected: true,
+  public: true,
+  return: true,
+  static: true,
+  super: true,
+  switch: true,
+  this: true,
+  throw: true,
+  true: true,
+  try: true,
+  typeof: true,
+  var: true,
+  void: true,
+  while: true,
+  with: true,
+  yield: true,
+};
+
+interface SourceReference {
+  readonly name: string;
+  readonly [SOURCE_REFERENCE]: true;
+}
+
+function compareText(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function hash(value: string): string {
+  let result = 0x811c9dc5;
+  for (let index = 0; index < value.length; index += 1) {
+    result ^= value.charCodeAt(index);
+    result = Math.imul(result, 0x01000193);
+  }
+  return (result >>> 0).toString(16).padStart(8, "0");
+}
+
+function words(value: string): string[] {
+  return value
+    .normalize("NFKD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .split(/[^A-Za-z0-9]+/)
+    .filter(Boolean);
+}
+
+function identifierBase(value: string, fallback: string): string {
+  const [first, ...rest] = words(value);
+  const firstWord = first ?? fallback;
+  const identifier = `${firstWord.toLowerCase()}${rest
+    .map((word) => `${word.slice(0, 1).toUpperCase()}${word.slice(1).toLowerCase()}`)
+    .join("")}`;
+  const prefixed = /^\d/.test(identifier) ? `_${identifier}` : identifier;
+  return Object.hasOwn(RESERVED_IDENTIFIERS, prefixed) ? `${prefixed}Value` : prefixed;
+}
+
+function directoryBase(value: string, fallback: string): string {
+  return (
+    words(value)
+      .map((word) => word.toLowerCase())
+      .join("-") || fallback
+  );
+}
+
+function allocateNames(
+  inputs: ReadonlyArray<NameInput>,
+  baseFor: (input: NameInput) => string,
+  blocked?: Readonly<Record<string, true>>,
+): ReadonlyMap<string, string> {
+  const groups = new Map<string, NameInput[]>();
+  const keys = new Set<string>();
+  for (const input of inputs) {
+    assert(!keys.has(input.key), `Naming key "${input.key}" is duplicated`);
+    keys.add(input.key);
+    const base = baseFor(input);
+    const group = groups.get(base) ?? [];
+    group.push(input);
+    groups.set(base, group);
+  }
+
+  const reservedBases = new Set(groups.keys());
+  const used = new Set<string>();
+  const allocated = new Map<string, string>();
+  for (const [base, group] of [...groups.entries()].toSorted(([left], [right]) =>
+    compareText(left, right),
+  )) {
+    const orderedGroup = group.toSorted((left, right) => compareText(left.key, right.key));
+    for (const input of orderedGroup) {
+      let candidate = base;
+      if (orderedGroup.length > 1 || (blocked !== undefined && Object.hasOwn(blocked, candidate))) {
+        candidate = `${base}_${hash(input.key)}`;
+      }
+      let collisionIndex = 2;
+      while (
+        used.has(candidate) ||
+        (blocked !== undefined && Object.hasOwn(blocked, candidate)) ||
+        (candidate !== base && reservedBases.has(candidate))
+      ) {
+        candidate = `${base}_${hash(input.key)}_${collisionIndex}`;
+        collisionIndex += 1;
+      }
+      used.add(candidate);
+      allocated.set(input.key, candidate);
+    }
+  }
+  return allocated;
+}
+
+function isSourceReference(value: unknown): value is SourceReference {
+  return typeof value === "object" && value !== null && SOURCE_REFERENCE in value;
+}
+
+function indentation(depth: number): string {
+  return "  ".repeat(depth);
+}
+
+function formatSource(value: unknown, depth = 0): string {
+  if (isSourceReference(value)) return value.name;
+  assert(value !== undefined, "Cannot emit an undefined source value");
+  if (typeof value === "number") assert(Number.isFinite(value), "Cannot emit a non-finite number");
+  if (value === null || typeof value !== "object") {
+    const serialized = JSON.stringify(value);
+    assert(serialized !== undefined, `Cannot emit a ${typeof value} source value`);
+    return serialized;
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) return "[]";
+    const items = value
+      .map((item) => `${indentation(depth + 1)}${formatSource(item, depth + 1)}`)
+      .join(",\n");
+    return `[\n${items}\n${indentation(depth)}]`;
+  }
+  const entries = Object.entries(value)
+    .filter(([, item]) => item !== undefined)
+    .toSorted(([left], [right]) => compareText(left, right));
+  if (entries.length === 0) return "{}";
+  const properties = entries
+    .map(([key, item]) => {
+      const formattedKey =
+        key === "__proto__"
+          ? `[${JSON.stringify(key)}]`
+          : /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
+            ? key
+            : JSON.stringify(key);
+      return `${indentation(depth + 1)}${formattedKey}: ${formatSource(item, depth + 1)}`;
+    })
+    .join(",\n");
+  return `{\n${properties}\n${indentation(depth)}}`;
+}
+
+function transformFieldProxy(
+  reference: { id: string; optional: boolean },
+  fieldReference: FieldReference,
+): unknown {
+  return { field: fieldReference(reference.id), optional: reference.optional };
+}
+
+function transformEntity<
+  Entity extends { fieldProxyId: string; relations?: Array<{ id: string }> },
+>(entity: Entity, fieldReference: FieldReference): unknown {
+  const { fieldProxyId, relations, ...settings } = entity;
+  return {
+    ...settings,
+    field: fieldReference(fieldProxyId),
+    ...(relations === undefined
+      ? {}
+      : {
+          relations: relations.map(({ id, ...relation }) => ({
+            ...relation,
+            field: fieldReference(id),
+          })),
+        }),
+  };
+}
+
+function transformTarget<Target extends { targetEntity: string }>(
+  target: Target,
+  fieldReference: FieldReference,
+): unknown {
+  const { targetEntity, ...settings } = target;
+  return { ...settings, targetEntity: fieldReference(targetEntity) };
+}
+
+function transformExam<Exam extends { gradedEntities: Array<{ targetEntity: string }> }>(
+  exam: Exam,
+  fieldReference: FieldReference,
+): unknown {
+  return {
+    ...exam,
+    gradedEntities: exam.gradedEntities.map((entity) => transformTarget(entity, fieldReference)),
+  };
+}
+
+function authoringSettings(node: GraphNode, fieldReference: FieldReference): unknown {
+  switch (node.type) {
+    case "root": {
+      const {
+        datasetId: _datasetId,
+        itemFilter: _itemFilter,
+        startedFieldProxies,
+        completedFieldProxies,
+        ...settings
+      } = node.settings;
+      return {
+        ...settings,
+        started: startedFieldProxies.output.map((reference) =>
+          transformFieldProxy(reference, fieldReference),
+        ),
+        completed: completedFieldProxies.input.map((reference) =>
+          transformFieldProxy(reference, fieldReference),
+        ),
+      };
+    }
+    case "code": {
+      const { fieldProxies, ...settings } = node.settings;
+      return {
+        ...settings,
+        fieldProxies: {
+          input: fieldProxies.input.map((reference) =>
+            transformFieldProxy(reference, fieldReference),
+          ),
+          output: fieldProxies.output.map((reference) =>
+            transformFieldProxy(reference, fieldReference),
+          ),
+        },
+      };
+    }
+    case "generation": {
+      const { entities, qualityCriteria, exam, ...settings } = node.settings;
+      return {
+        ...settings,
+        entities: {
+          input: entities.input.map((entity) => transformEntity(entity, fieldReference)),
+          output: entities.output.map((entity) => transformEntity(entity, fieldReference)),
+        },
+        qualityCriteria: qualityCriteria.map((criterion) =>
+          transformTarget(criterion, fieldReference),
+        ),
+        ...(exam === undefined ? {} : { exam: transformExam(exam, fieldReference) }),
+      };
+    }
+    case "match": {
+      const { conditions, fieldProxies, ...settings } = node.settings;
+      return {
+        ...settings,
+        conditions: conditions.map(({ predicates, ...condition }) => ({
+          ...condition,
+          predicates: predicates.map(({ fieldProxyId, ...predicate }) => ({
+            ...predicate,
+            field: fieldReference(fieldProxyId),
+          })),
+        })),
+        fieldProxies: {
+          input: fieldProxies.input.map((reference) =>
+            transformFieldProxy(reference, fieldReference),
+          ),
+        },
+      };
+    }
+    case "quorum": {
+      const { entities, qualityCriteria, exam, ...settings } = node.settings;
+      return {
+        ...settings,
+        entities: {
+          input: entities.input.map((entity) => transformEntity(entity, fieldReference)),
+          output: entities.output.map((entity) => transformEntity(entity, fieldReference)),
+        },
+        qualityCriteria: qualityCriteria.map((criterion) =>
+          transformTarget(criterion, fieldReference),
+        ),
+        ...(exam === undefined ? {} : { exam: transformExam(exam, fieldReference) }),
+      };
+    }
+  }
+}
+
+function nodeSource(
+  node: GraphNode,
+  binding: string,
+  path: string,
+  fieldBindings: ReadonlyMap<string, string>,
+): NodeEmission {
+  const referencedFields = new Set<string>();
+  const fieldReference = (id: string): SourceReference => {
+    const referencedBinding = fieldBindings.get(id);
+    assert(referencedBinding, `Node "${node.id}" references unknown field "${id}"`);
+    referencedFields.add(referencedBinding);
+    return { name: referencedBinding, [SOURCE_REFERENCE]: true };
+  };
+  const settings = authoringSettings(node, fieldReference);
+  const fieldImport = [...referencedFields].toSorted(compareText);
+  const imports = [
+    `import { ${node.type} } from '@toloka-platform/sdk';`,
+    ...(fieldImport.length === 0
+      ? []
+      : ["", `import { ${fieldImport.join(", ")} } from '../../fields';`]),
+  ];
+  const argumentsSource =
+    node.type === "root"
+      ? [
+          `${indentation(1)}${formatSource(settings, 1)}`,
+          `${indentation(1)}${formatSource({ id: node.id }, 1)}`,
+        ]
+      : [
+          `${indentation(1)}${formatSource(node.settings.about.name, 1)}`,
+          `${indentation(1)}${formatSource(settings, 1)}`,
+          `${indentation(1)}${formatSource({ id: node.id }, 1)}`,
+        ];
+  return {
+    binding,
+    path,
+    source: `${imports.join("\n")}\n\nexport const ${binding} = ${node.type}(\n${argumentsSource.join(",\n")}\n);\n`,
+  };
+}
+
+function fieldDeclarationsSource(
+  fields: ReadonlyArray<FieldSourceDefinition>,
+  bindingFor: (fieldDefinition: FieldSourceDefinition) => string,
+): string {
+  if (fields.length === 0) return "export {};\n";
+  const declarations = fields
+    .map((fieldDefinition) => {
+      const binding = bindingFor(fieldDefinition);
+      const argumentsSource = [
+        `${indentation(1)}${formatSource(fieldDefinition.name, 1)}`,
+        `${indentation(1)}${formatSource(fieldDefinition.jsonSchema, 1)}`,
+        ...(fieldDefinition.options === undefined
+          ? []
+          : [`${indentation(1)}${formatSource(fieldDefinition.options, 1)}`]),
+      ];
+      return {
+        binding,
+        source: `export const ${binding} = field(\n${argumentsSource.join(",\n")}\n);`,
+      };
+    })
+    .toSorted((left, right) => compareText(left.binding, right.binding));
+  return `import { field } from '@toloka-platform/sdk';\n\n${declarations.map(({ source }) => source).join("\n\n")}\n`;
+}
+
+export function generateFieldDeclarationsSource(
+  fields: ReadonlyArray<FieldSourceDefinition>,
+): string {
+  const sourceKeys = new Map(
+    fields.map((fieldDefinition) => [
+      fieldDefinition,
+      `field:${fieldDefinition.options?.id ?? fieldDefinition.name}`,
+    ]),
+  );
+  const bindings = allocateNames(
+    fields.map((fieldDefinition) => ({
+      key: sourceKeys.get(fieldDefinition) ?? "",
+      name: fieldDefinition.name,
+      fallback: "field",
+    })),
+    ({ name, fallback }) => identifierBase(name, fallback),
+    BLOCKED_BINDINGS,
+  );
+  return fieldDeclarationsSource(fields, (fieldDefinition) => {
+    const sourceKey = sourceKeys.get(fieldDefinition);
+    assert(sourceKey, `Field "${fieldDefinition.name}" has no source key`);
+    const binding = bindings.get(sourceKey);
+    assert(binding, `Field "${fieldDefinition.name}" has no source binding`);
+    return binding;
+  });
+}
+
+function fieldSource(graph: PipelineGraphV1, fieldBindings: ReadonlyMap<string, string>): string {
+  return fieldDeclarationsSource(
+    graph.fields.map((fieldDefinition) => ({
+      name: fieldDefinition.name,
+      jsonSchema: fieldDefinition.jsonSchema,
+      options: { id: fieldDefinition.id },
+    })),
+    (fieldDefinition) => {
+      const fieldId = fieldDefinition.options?.id;
+      assert(fieldId, `Field "${fieldDefinition.name}" has no source ID`);
+      const binding = fieldBindings.get(fieldId);
+      assert(binding, `Field "${fieldId}" has no source binding`);
+      return binding;
+    },
+  );
+}
+
+function endpointSource(
+  endpoint: ConnectionEndpoint,
+  direction: "input" | "output",
+  nodesById: ReadonlyMap<string, GraphNode>,
+  nodeBindings: ReadonlyMap<string, string>,
+): string {
+  const node = nodesById.get(endpoint.node);
+  assert(node, `Connection references unknown node "${endpoint.node}"`);
+  const binding = nodeBindings.get(node.id);
+  assert(binding, `Node "${node.id}" has no source binding`);
+  switch (node.type) {
+    case "root":
+      assert(
+        endpoint.handle === (direction === "output" ? "started" : "completed"),
+        `Root ${direction} handle "${endpoint.handle}" cannot be authored`,
+      );
+      return `${binding}.${direction === "output" ? "started" : "completed"}`;
+    case "code":
+    case "generation":
+    case "quorum":
+      assert(
+        endpoint.handle === direction,
+        `${node.type} ${direction} handle "${endpoint.handle}" cannot be authored`,
+      );
+      return `${binding}.${direction}`;
+    case "match":
+      if (direction === "input") {
+        assert(
+          endpoint.handle === "input",
+          `Match input handle "${endpoint.handle}" cannot be authored`,
+        );
+        return `${binding}.input`;
+      }
+      if (endpoint.handle === "default") return `${binding}.default`;
+      const matchedHandle = /^matched_(\d+)$/.exec(endpoint.handle);
+      assert(matchedHandle, `Match output handle "${endpoint.handle}" cannot be authored`);
+      const matchedIndex = matchedHandle[1];
+      assert(matchedIndex !== undefined, `Match output handle "${endpoint.handle}" has no index`);
+      assert(
+        String(Number(matchedIndex)) === matchedIndex,
+        `Match output handle "${endpoint.handle}" is not canonical`,
+      );
+      return `${binding}.matched(${matchedIndex})`;
+  }
+}
+
+function pipelineSource(
+  graph: PipelineGraphV1,
+  dataset: string,
+  nodeEmissions: ReadonlyArray<NodeEmission>,
+  fieldBindings: ReadonlyMap<string, string>,
+  nodeBindings: ReadonlyMap<string, string>,
+): string {
+  const nodesById = new Map(graph.nodes.map((node) => [node.id, node]));
+  const imports = [
+    `import { ${graph.connections.length === 0 ? "definePipeline" : "connect, definePipeline"} } from '@toloka-platform/sdk';`,
+    ...(graph.fields.length === 0
+      ? []
+      : [
+          "",
+          `import { ${[...fieldBindings.values()].toSorted(compareText).join(", ")} } from './fields';`,
+        ]),
+    ...nodeEmissions
+      .toSorted((left, right) => compareText(left.path, right.path))
+      .flatMap(({ binding, path }) => [`import { ${binding} } from './${path.slice(0, -3)}';`]),
+  ];
+  const fieldReferences = graph.fields.map(({ id }) => {
+    const binding = fieldBindings.get(id);
+    assert(binding, `Field "${id}" has no source binding`);
+    return binding;
+  });
+  const nodeReferences = graph.nodes.map(({ id }) => {
+    const binding = nodeBindings.get(id);
+    assert(binding, `Node "${id}" has no source binding`);
+    return binding;
+  });
+  const connectionReferences = graph.connections.map((connection) => {
+    const from = endpointSource(connection.from, "output", nodesById, nodeBindings);
+    const to = endpointSource(connection.to, "input", nodesById, nodeBindings);
+    if (Object.keys(connection.settings).length === 0) return `connect(${from}, ${to})`;
+    return `connect(\n${indentation(3)}${from},\n${indentation(3)}${to},\n${indentation(3)}${formatSource(
+      connection.settings,
+      3,
+    )}\n${indentation(2)})`;
+  });
+  const formatList = (values: ReadonlyArray<string>, depth: number): string => {
+    if (values.length === 0) return "[]";
+    return `[\n${values.map((value) => `${indentation(depth + 1)}${value}`).join(",\n")}\n${indentation(depth)}]`;
+  };
+  return `${imports.join("\n")}\n\nexport default definePipeline({\n  dataset: ${formatSource(dataset)},\n  fields: ${formatList(fieldReferences, 1)},\n  nodes: ${formatList(nodeReferences, 1)},\n  connections: ${formatList(connectionReferences, 1)}\n});\n`;
+}
+
+export function generatePipelineSourceTree(
+  graph: PipelineGraphV1,
+  options: GeneratePipelineSourceTreeOptions = { dataset: "imported-dataset" },
+): PipelineSourceTree {
+  const bindingInputs: NameInput[] = [
+    ...graph.fields.map((fieldDefinition) => ({
+      key: `field:${fieldDefinition.id}`,
+      name: fieldDefinition.name,
+      fallback: "field",
+    })),
+    ...graph.nodes.map((node) => ({
+      key: `node:${node.id}`,
+      name: node.type === "root" ? "root" : node.settings.about.name || node.type,
+      fallback: node.type,
+    })),
+  ];
+  const bindings = allocateNames(
+    bindingInputs,
+    ({ name, fallback }) => identifierBase(name, fallback),
+    BLOCKED_BINDINGS,
+  );
+  const directories = allocateNames(
+    graph.nodes.map((node) => ({
+      key: `node:${node.id}`,
+      name: node.type === "root" ? "root" : node.settings.about.name || node.type,
+      fallback: node.type,
+    })),
+    ({ name, fallback }) => directoryBase(name, fallback),
+  );
+  const fieldBindings = new Map(
+    graph.fields.map((fieldDefinition) => {
+      const binding = bindings.get(`field:${fieldDefinition.id}`);
+      assert(binding, `Field "${fieldDefinition.id}" has no allocated binding`);
+      return [fieldDefinition.id, binding];
+    }),
+  );
+  const nodeBindings = new Map(
+    graph.nodes.map((node) => {
+      const binding = bindings.get(`node:${node.id}`);
+      assert(binding, `Node "${node.id}" has no allocated binding`);
+      return [node.id, binding];
+    }),
+  );
+  const nodeEmissions = graph.nodes.map((node) => {
+    const binding = nodeBindings.get(node.id);
+    const directory = directories.get(`node:${node.id}`);
+    assert(binding && directory, `Node "${node.id}" has no allocated source name`);
+    return nodeSource(node, binding, `nodes/${directory}/node.ts`, fieldBindings);
+  });
+  const entries = [
+    { path: "fields.ts", source: fieldSource(graph, fieldBindings) },
+    ...nodeEmissions.map(({ path, source }) => ({ path, source })),
+    {
+      path: "pipeline.ts",
+      source: pipelineSource(graph, options.dataset, nodeEmissions, fieldBindings, nodeBindings),
+    },
+  ].toSorted((left, right) => compareText(left.path, right.path));
+  return Object.fromEntries(entries.map(({ path, source }) => [path, source]));
+}