@toloka-platform/sdk 0.1.0-preview.3 → 0.1.0-preview.5

package/src/cli/skill.ts

@@ -0,0 +1,363 @@
+import { assert } from "es-toolkit";
+import { createHash } from "node:crypto";
+import { copyFile, mkdir, mkdtemp, readFile, realpath, rename, rm, stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { assertSafeDestination, entryStats, hasErrorCode } from "./file-system";
+import { formatCliError } from "./format-error";
+
+export const BUNDLED_SKILL_CATALOG_ROOT = fileURLToPath(new URL("../../skills", import.meta.url));
+export const SKILL_CATALOG_MANIFEST = "catalog.json";
+export const SKILL_INSTALL_ROOT = ".agents/skills";
+export const SKILL_ENTRY_POINT = "SKILL.md";
+
+const SKILL_CATALOG_DIRECTORY = "skills";
+const MAX_SKILL_COUNT = 500;
+const MAX_MANIFEST_BYTES = 1024 * 1024;
+const MAX_SKILL_ID_LENGTH = 64;
+const SKILL_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
+const DIGEST_PATTERN = /^[a-f0-9]{64}$/u;
+
+export type SkillDescriptor = {
+  readonly id: string;
+  readonly name: string;
+  readonly description: string;
+  readonly tags: readonly string[];
+  readonly version: string;
+  readonly digest: string;
+  readonly files: readonly string[];
+};
+
+export type SkillCatalog = {
+  readonly root: string;
+  readonly manifestPath: string;
+  readonly skills: readonly SkillDescriptor[];
+};
+
+export type SkillStatus = "not-installed" | "current" | "stale";
+
+export type SkillListEntry = SkillDescriptor & { readonly status: SkillStatus };
+
+export type InstalledSkill = {
+  readonly skill: SkillDescriptor;
+  readonly directory: string;
+  readonly replaced: boolean;
+};
+
+export type InstallSkillOptions = {
+  readonly catalog: SkillCatalog;
+  readonly sourceRoot: string;
+  readonly skillId: string;
+  readonly force?: boolean;
+  readonly rename?: typeof rename;
+};
+
+export type SkillIo = {
+  readonly stdout: (text: string) => void;
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isStringArray = (value: unknown): value is string[] =>
+  Array.isArray(value) && value.every((item) => typeof item === "string");
+
+const isNonEmptyString = (value: unknown): value is string =>
+  typeof value === "string" && value.trim().length > 0;
+
+const isSafeSkillId = (value: unknown): value is string =>
+  typeof value === "string" && value.length <= MAX_SKILL_ID_LENGTH && SKILL_ID_PATTERN.test(value);
+
+const isSafeRelativePath = (path: string) =>
+  !path.includes("\\") &&
+  !path.includes("\0") &&
+  path.split("/").every((segment) => segment.length > 0 && segment !== "." && segment !== "..");
+
+const parseDescriptor = (value: unknown, invalid: (detail: string) => string): SkillDescriptor => {
+  assert(isRecord(value), invalid("every skill must be an object."));
+  const { id, name, description, tags, version, digest, files } = value;
+  assert(
+    isSafeSkillId(id),
+    invalid(
+      `skill id ${JSON.stringify(id)} must be at most ${MAX_SKILL_ID_LENGTH} lowercase letters, digits, and single hyphens.`,
+    ),
+  );
+  assert(isNonEmptyString(name), invalid(`skill ${id} needs a name.`));
+  assert(isNonEmptyString(description), invalid(`skill ${id} needs a description.`));
+  assert(isStringArray(tags), invalid(`skill ${id} tags must be strings.`));
+  assert(isNonEmptyString(version), invalid(`skill ${id} needs a version.`));
+  assert(
+    typeof digest === "string" && DIGEST_PATTERN.test(digest),
+    invalid(`skill ${id} digest must be the sha256 hex digest of its files.`),
+  );
+  assert(
+    isStringArray(files) && files.includes(SKILL_ENTRY_POINT),
+    invalid(`skill ${id} files must include ${SKILL_ENTRY_POINT}.`),
+  );
+  assert(
+    files.every(isSafeRelativePath),
+    invalid(`skill ${id} files must be relative paths inside its directory.`),
+  );
+  assert(new Set(files).size === files.length, invalid(`skill ${id} lists a file more than once.`));
+  return { id, name, description, tags, version, digest, files };
+};
+
+const parseManifest = (source: string, manifestPath: string): unknown => {
+  try {
+    return JSON.parse(source);
+  } catch (error) {
+    throw new Error(`Invalid skill catalog ${manifestPath}: ${formatCliError(error)}`);
+  }
+};
+
+const isFileAt = async (path: string): Promise<boolean> => {
+  try {
+    return (await stat(path)).isFile();
+  } catch (error) {
+    if (hasErrorCode(error, "ENOENT", "ENOTDIR")) return false;
+    throw error;
+  }
+};
+
+const findSkillCatalogRootAbove = async (directory: string): Promise<string | undefined> => {
+  const candidate = join(directory, SKILL_CATALOG_DIRECTORY);
+  if (await isFileAt(join(candidate, SKILL_CATALOG_MANIFEST))) return candidate;
+  const parent = dirname(directory);
+  return parent === directory ? undefined : findSkillCatalogRootAbove(parent);
+};
+
+export async function resolveBundledSkillCatalogRoot(
+  runtimeEntry: string | undefined = process.argv[1],
+): Promise<string> {
+  const entry = runtimeEntry === undefined ? undefined : await realpath(runtimeEntry);
+  const runtimeRoot =
+    entry === undefined ? undefined : await findSkillCatalogRootAbove(dirname(entry));
+  if (runtimeRoot !== undefined) return runtimeRoot;
+  const searched =
+    entry === undefined
+      ? "process.argv[1] names no runtime entry"
+      : `none exists between ${entry} and the filesystem root`;
+  assert(
+    await isFileAt(join(BUNDLED_SKILL_CATALOG_ROOT, SKILL_CATALOG_MANIFEST)),
+    `Cannot locate the bundled skill catalog. An installed @toloka-platform/sdk must ship ${SKILL_CATALOG_DIRECTORY}/${SKILL_CATALOG_MANIFEST} in a directory above its runtime entry, but ${searched}, and the source checkout catalog ${BUNDLED_SKILL_CATALOG_ROOT} is absent. Reinstall @toloka-platform/sdk.`,
+  );
+  return BUNDLED_SKILL_CATALOG_ROOT;
+}
+
+const loadBundledSkillCatalog = async (): Promise<SkillCatalog> =>
+  loadSkillCatalog(await resolveBundledSkillCatalogRoot());
+
+export async function loadSkillCatalog(catalogRoot: string): Promise<SkillCatalog> {
+  const manifestPath = join(catalogRoot, SKILL_CATALOG_MANIFEST);
+  const invalid = (detail: string) => `Invalid skill catalog ${manifestPath}: ${detail}`;
+  const { size } = await stat(manifestPath);
+  assert(
+    size <= MAX_MANIFEST_BYTES,
+    invalid(`the manifest is ${size} bytes; the limit is ${MAX_MANIFEST_BYTES} bytes.`),
+  );
+  const manifest = parseManifest(await readFile(manifestPath, "utf8"), manifestPath);
+  assert(
+    isRecord(manifest) && Array.isArray(manifest.skills),
+    invalid("expected an object with a skills array."),
+  );
+  assert(
+    manifest.skills.length <= MAX_SKILL_COUNT,
+    invalid(`${manifest.skills.length} skills exceed the limit of ${MAX_SKILL_COUNT}.`),
+  );
+  const skills = manifest.skills.map((skill) => parseDescriptor(skill, invalid));
+  const ids = new Set<string>();
+  for (const skill of skills) {
+    assert(!ids.has(skill.id), invalid(`skill id ${skill.id} appears more than once.`));
+    ids.add(skill.id);
+  }
+  return { root: catalogRoot, manifestPath, skills };
+}
+
+export async function computeSkillDigest(
+  directory: string,
+  files: readonly string[],
+): Promise<string> {
+  const hash = createHash("sha256");
+  for (const path of files.toSorted()) {
+    hash.update(path);
+    hash.update("\0");
+    hash.update(await readFile(join(directory, path)));
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+export const searchSkills = (
+  skills: readonly SkillDescriptor[],
+  query: string,
+): SkillDescriptor[] => {
+  const needle = query.trim().toLowerCase();
+  return skills.filter((skill) =>
+    [skill.id, skill.name, skill.description, ...skill.tags].some((field) =>
+      field.toLowerCase().includes(needle),
+    ),
+  );
+};
+
+export async function skillStatus(
+  sourceRoot: string,
+  skill: SkillDescriptor,
+): Promise<SkillStatus> {
+  const directory = join(sourceRoot, SKILL_INSTALL_ROOT, skill.id);
+  if ((await entryStats(directory)) === undefined) return "not-installed";
+  try {
+    return (await computeSkillDigest(directory, skill.files)) === skill.digest
+      ? "current"
+      : "stale";
+  } catch (error) {
+    if (hasErrorCode(error, "ENOENT", "ENOTDIR")) return "stale";
+    throw error;
+  }
+}
+
+export const listSkills = ({
+  sourceRoot,
+  skills,
+}: {
+  readonly sourceRoot: string;
+  readonly skills: readonly SkillDescriptor[];
+}): Promise<SkillListEntry[]> =>
+  Promise.all(
+    skills.map(async (skill) => ({ ...skill, status: await skillStatus(sourceRoot, skill) })),
+  );
+
+const swapDirectories = async (
+  renameEntry: typeof rename,
+  replacement: string,
+  destination: string,
+  backup: string,
+) => {
+  await renameEntry(destination, backup);
+  try {
+    await renameEntry(replacement, destination);
+  } catch (error) {
+    try {
+      await renameEntry(backup, destination);
+    } catch (restoreError) {
+      throw new Error(
+        `Failed to replace ${destination}; the previous version is preserved at ${backup}. ${formatCliError(restoreError)}`,
+        { cause: error },
+      );
+    }
+    throw error;
+  }
+  await rm(backup, { recursive: true, force: true });
+};
+
+export async function installSkill({
+  catalog,
+  sourceRoot,
+  skillId,
+  force = false,
+  rename: renameEntry = rename,
+}: InstallSkillOptions): Promise<InstalledSkill> {
+  const skill = catalog.skills.find((candidate) => candidate.id === skillId);
+  assert(
+    skill,
+    `Skill ${JSON.stringify(skillId)} is not in the catalog. Run \`toloka skill list\` to see the available skills.`,
+  );
+  const relativeDestination = `${SKILL_INSTALL_ROOT}/${skill.id}`;
+  await assertSafeDestination(sourceRoot, relativeDestination);
+  const destination = join(sourceRoot, relativeDestination);
+  const existing = await entryStats(destination);
+  assert(
+    existing === undefined || force,
+    `${relativeDestination} already exists. Re-run with --force to replace it with the catalog version of ${skill.id}.`,
+  );
+
+  const skillsRoot = dirname(destination);
+  await mkdir(skillsRoot, { recursive: true });
+  const temporary = await mkdtemp(join(skillsRoot, `.${skill.id}.tmp-`));
+  try {
+    for (const path of skill.files) {
+      const target = join(temporary, path);
+      await mkdir(dirname(target), { recursive: true });
+      await copyFile(join(catalog.root, skill.id, path), target);
+    }
+    assert(
+      (await computeSkillDigest(temporary, skill.files)) === skill.digest,
+      `Skill ${skill.id} failed its digest check: the packaged files do not match the catalog digest ${skill.digest}. Nothing was installed.`,
+    );
+    if (existing === undefined) {
+      await renameEntry(temporary, destination);
+    } else {
+      await swapDirectories(renameEntry, temporary, destination, `${temporary}.backup`);
+    }
+  } catch (error) {
+    await rm(temporary, { recursive: true, force: true });
+    throw error;
+  }
+  return { skill, directory: destination, replaced: existing !== undefined };
+}
+
+const formatEntries = (entries: readonly SkillListEntry[], json: boolean): string => {
+  if (json) return JSON.stringify(entries, undefined, 2);
+  const idWidth = Math.max(...entries.map((entry) => entry.id.length));
+  const statusWidth = Math.max(...entries.map((entry) => entry.status.length));
+  return entries
+    .flatMap((entry) => [
+      `${entry.id.padEnd(idWidth)}  ${entry.status.padEnd(statusWidth)}  ${entry.name}`,
+      `  ${entry.description}`,
+    ])
+    .join("\n");
+};
+
+export async function skillListCommand({
+  sourceRoot,
+  json,
+  io,
+}: {
+  readonly sourceRoot: string;
+  readonly json: boolean;
+  readonly io: SkillIo;
+}): Promise<void> {
+  const catalog = await loadBundledSkillCatalog();
+  io.stdout(formatEntries(await listSkills({ sourceRoot, skills: catalog.skills }), json));
+}
+
+export async function skillSearchCommand({
+  sourceRoot,
+  query,
+  json,
+  io,
+}: {
+  readonly sourceRoot: string;
+  readonly query: string;
+  readonly json: boolean;
+  readonly io: SkillIo;
+}): Promise<void> {
+  const catalog = await loadBundledSkillCatalog();
+  const matches = await listSkills({ sourceRoot, skills: searchSkills(catalog.skills, query) });
+  if (json || matches.length > 0) {
+    io.stdout(formatEntries(matches, json));
+    return;
+  }
+  io.stdout(
+    `No skills match ${JSON.stringify(query)}. Run \`toloka skill list\` to see the catalog.`,
+  );
+}
+
+export async function skillInstallCommand({
+  sourceRoot,
+  skillId,
+  force,
+  io,
+}: {
+  readonly sourceRoot: string;
+  readonly skillId: string;
+  readonly force: boolean;
+  readonly io: SkillIo;
+}): Promise<void> {
+  const catalog = await loadBundledSkillCatalog();
+  const installed = await installSkill({ catalog, sourceRoot, skillId, force });
+  const relativeDestination = `${SKILL_INSTALL_ROOT}/${installed.skill.id}`;
+  const fileCount = installed.skill.files.length;
+  io.stdout(
+    `${installed.replaced ? "Replaced" : "Installed"} ${installed.skill.id} in ${relativeDestination} (${fileCount} ${fileCount === 1 ? "file" : "files"}). Start with ${relativeDestination}/${SKILL_ENTRY_POINT}.`,
+  );
+}

package/src/pipeline-graph-v1.contract.json

@@ -1633,10 +1633,14 @@
                           "llmInstruction": {
                             "default": "",
                             "type": "string"
+                          },
+                          "maxCostPerAttempt": {
+                            "type": "number"
                           }
                         },
                         "required": [
-                          "audienceType"
+                          "audienceType",
+                          "maxCostPerAttempt"
                         ],
                         "additionalProperties": false
                       }