@toloka-platform/sdk 0.1.0-preview.3

package/src/cli/init.ts

@@ -0,0 +1,361 @@
+import { assert } from "es-toolkit";
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, sep } from "node:path";
+import packageMetadata from "../../package.json" with { type: "json" };
+import { readTolokaState, writeTolokaState } from "../state";
+
+export const SDK_VERSION = packageMetadata.version;
+
+const SDK_PACKAGE_NAME = "@toloka-platform/sdk";
+const SKILL_NAME = "toloka-build";
+const CLAUDE_SKILL_PATH = `.claude/skills/${SKILL_NAME}/SKILL.md`;
+const CURSOR_RULE_PATH = `.cursor/rules/${SKILL_NAME}.mdc`;
+const AGENTS_PATH = "AGENTS.md";
+const BLOCK_BEGIN = "<!-- toloka:begin -->";
+const BLOCK_END = "<!-- toloka:end -->";
+const VERSION_PATTERN = /<!-- toloka:sdk-version ([^\s>]+) -->/u;
+const DIGEST_PATTERN = /<!-- toloka:skill-digest sha256:([a-f0-9]{64}) -->/u;
+const AGENT_ENTRY_PATHS = [CLAUDE_SKILL_PATH, CURSOR_RULE_PATH, AGENTS_PATH];
+const LEGACY_SKILL_NAME = "toloka-pipeline-as-code";
+const LEGACY_SKILL_DIRECTORY = join(".claude", "skills", LEGACY_SKILL_NAME);
+const LEGACY_CURSOR_RULE = join(".cursor", "rules", `${LEGACY_SKILL_NAME}.mdc`);
+const LEGACY_PATHS = [LEGACY_SKILL_DIRECTORY, LEGACY_CURSOR_RULE];
+const LEGACY_ENTRY_PATHS = [join(LEGACY_SKILL_DIRECTORY, "SKILL.md"), LEGACY_CURSOR_RULE];
+const BUNDLED_SKILL_FILES = [
+  ["SKILL.md", new URL("../../skills/SKILL.md", import.meta.url)],
+  [
+    "references/node-semantics.md",
+    new URL("../../skills/references/node-semantics.md", import.meta.url),
+  ],
+  [
+    "references/workflow-and-cli.md",
+    new URL("../../skills/references/workflow-and-cli.md", import.meta.url),
+  ],
+] satisfies ReadonlyArray<readonly [string, URL]>;
+
+export type InitProjectResult = {
+  readonly targetDirectory: string;
+  readonly files: readonly string[];
+  readonly writtenFiles: readonly string[];
+  readonly sdkVersion: string;
+  readonly skillDigest: string;
+  readonly dependencyReady: boolean;
+  readonly output: string;
+  readonly warning?: string;
+};
+
+type AgentEntry = {
+  readonly relativePath: string;
+  readonly source: string;
+};
+
+const isMissingFileError = (error: unknown) =>
+  error instanceof Error && "code" in error && error.code === "ENOENT";
+
+const readOptionalFile = async (path: string) => {
+  try {
+    return await readFile(path, "utf8");
+  } catch (error) {
+    if (isMissingFileError(error)) return undefined;
+    throw error;
+  }
+};
+
+const isDirectory = async (path: string) => {
+  try {
+    return (await lstat(path)).isDirectory();
+  } catch (error) {
+    if (isMissingFileError(error)) return false;
+    throw error;
+  }
+};
+
+const isFile = async (path: string) => {
+  try {
+    return (await lstat(path)).isFile();
+  } catch (error) {
+    if (isMissingFileError(error)) return false;
+    throw error;
+  }
+};
+
+const assertSafeDestination = async (targetDirectory: string, relativePath: string) => {
+  let currentPath = targetDirectory;
+  for (const segment of relativePath.split("/")) {
+    currentPath = join(currentPath, segment);
+    try {
+      const entry = await lstat(currentPath);
+      assert(!entry.isSymbolicLink(), `Refusing to write through symbolic link ${currentPath}.`);
+    } catch (error) {
+      if (isMissingFileError(error)) return;
+      throw error;
+    }
+  }
+};
+
+export async function computeBundledSkillDigest(): Promise<string> {
+  const digest = createHash("sha256");
+  for (const [relativePath, url] of BUNDLED_SKILL_FILES) {
+    digest.update(relativePath);
+    digest.update("\0");
+    digest.update(await readFile(url));
+    digest.update("\0");
+  }
+  return digest.digest("hex");
+}
+
+const BUNDLED_SKILL_BODY = new URL("../../skills/SKILL.md", import.meta.url);
+const INSTALLED_REFERENCE_PREFIX = `node_modules/${SDK_PACKAGE_NAME}/skills/references/`;
+
+const bundledSkillBody = async (sectionHeading: "##" | "###"): Promise<string> => {
+  const source = await readFile(BUNDLED_SKILL_BODY, "utf8");
+  const withoutFrontmatter = source.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/u, "");
+  const withoutTitle = withoutFrontmatter.replace(/^#\s.*\r?\n+/u, "");
+  const withInstalledReferences = withoutTitle
+    .replaceAll("](references/", `](${INSTALLED_REFERENCE_PREFIX}`)
+    .replaceAll("](../examples/", `](node_modules/${SDK_PACKAGE_NAME}/examples/`);
+  return sectionHeading === "##"
+    ? withInstalledReferences
+    : withInstalledReferences.replaceAll(/^## /gmu, "### ");
+};
+
+const guidance = async (
+  sectionHeading: "##" | "###",
+  skillDigest: string,
+) => `Generated by \`${SDK_PACKAGE_NAME}\` version \`${SDK_VERSION}\`.
+<!-- toloka:sdk-version ${SDK_VERSION} -->
+<!-- toloka:skill-digest sha256:${skillDigest} -->
+
+${await bundledSkillBody(sectionHeading)}
+Use the globally installed \`toloka\` binary. Install it with \`bun add --global ${SDK_PACKAGE_NAME}@preview\` if it is not on \`PATH\`.
+
+If the reference paths above are missing, keep following the workflow and constraints in this file, do not invent SDK helpers or CLI commands, and install the project's declared dependencies before validating, planning, or applying. When no project is selected, use \`toloka project add\` to adopt one or \`toloka project create\` to create one. Adopt live source with \`toloka import\`.
+
+`;
+
+const claudeSkillSource = async (skillDigest: string) => `---
+name: ${SKILL_NAME}
+description: Build, inspect, and change Toloka platform pipeline projects as TypeScript source with @toloka-platform/sdk and toloka. Use when the user wants to create, adopt, inspect, plan, or apply a pipeline project as code.
+---
+
+# Build Toloka pipelines as code
+
+${await guidance("##", skillDigest)}`;
+
+const cursorRuleSource = async (skillDigest: string) => `---
+description: Build, inspect, and change Toloka platform pipeline projects as TypeScript source with @toloka-platform/sdk and toloka
+alwaysApply: false
+---
+
+# Build Toloka pipelines as code
+
+${await guidance("##", skillDigest)}`;
+
+const agentsBlockSource = async (skillDigest: string) => `${BLOCK_BEGIN}
+## Build Toloka pipelines as code
+
+${(await guidance("###", skillDigest)).trimEnd()}
+${BLOCK_END}`;
+
+const mergeAgentsBlock = (source: string, block: string) => {
+  const beginIndex = source.indexOf(BLOCK_BEGIN);
+  const endIndex = source.indexOf(BLOCK_END);
+  const hasBegin = beginIndex !== -1;
+  const hasEnd = endIndex !== -1;
+  const lineEnding = source.includes("\r\n") ? "\r\n" : "\n";
+  const normalizedBlock = block.replaceAll("\n", lineEnding);
+  if (!hasBegin && !hasEnd) {
+    if (source.length === 0) return `${normalizedBlock}${lineEnding}`;
+    const gap = source.endsWith(`${lineEnding}${lineEnding}`)
+      ? ""
+      : source.endsWith(lineEnding)
+        ? lineEnding
+        : `${lineEnding}${lineEnding}`;
+    return `${source}${gap}${normalizedBlock}${lineEnding}`;
+  }
+  assert(hasBegin && hasEnd && beginIndex < endIndex, `Malformed toloka block in ${AGENTS_PATH}.`);
+  assert(
+    !source.includes(BLOCK_BEGIN, beginIndex + BLOCK_BEGIN.length) &&
+      !source.includes(BLOCK_END, endIndex + BLOCK_END.length),
+    `Multiple toloka blocks found in ${AGENTS_PATH}.`,
+  );
+  return `${source.slice(0, beginIndex)}${normalizedBlock}${source.slice(endIndex + BLOCK_END.length)}`;
+};
+
+const hasSdkDependency = async (targetDirectory: string) => {
+  if (await isDirectory(join(targetDirectory, "node_modules", SDK_PACKAGE_NAME))) return true;
+
+  const source = await readOptionalFile(join(targetDirectory, "package.json"));
+  if (source === undefined) return false;
+  const manifest: unknown = JSON.parse(source);
+  assert(
+    typeof manifest === "object" && manifest !== null,
+    "package.json must contain a JSON object.",
+  );
+  return ["dependencies", "devDependencies"].some((key) => {
+    const dependencies = Reflect.get(manifest, key);
+    return (
+      typeof dependencies === "object" &&
+      dependencies !== null &&
+      typeof Reflect.get(dependencies, SDK_PACKAGE_NAME) === "string"
+    );
+  });
+};
+
+const writeIfChanged = async (path: string, source: string) => {
+  if ((await readOptionalFile(path)) === source) return false;
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, source, "utf8");
+  return true;
+};
+
+const entryStats = async (path: string) => {
+  try {
+    return await lstat(path);
+  } catch (error) {
+    if (isMissingFileError(error)) return undefined;
+    throw error;
+  }
+};
+
+const removeLegacyEntryPoints = async (targetRoot: string): Promise<string[]> => {
+  const removed = await Promise.all(
+    LEGACY_PATHS.map(async (relativePath) => {
+      const path = join(targetRoot, relativePath);
+      const entry = await entryStats(path);
+      if (entry === undefined) return undefined;
+      assert(
+        !entry.isSymbolicLink(),
+        `Refusing to remove legacy agent entry point through symbolic link ${path}.`,
+      );
+      await assertSafeDestination(targetRoot, relativePath.split(sep).join("/"));
+      await rm(path, { recursive: true, force: true });
+      return relativePath;
+    }),
+  );
+  return removed.filter((relativePath): relativePath is string => relativePath !== undefined);
+};
+
+const formatInitOutput = (
+  targetRoot: string,
+  files: readonly string[],
+  writtenFiles: readonly string[],
+  removedLegacyFiles: readonly string[],
+): string =>
+  [
+    `${writtenFiles.length === 0 ? "Agent entry points are up to date" : "Initialized agent entry points"} for ${SDK_PACKAGE_NAME} ${SDK_VERSION} in ${targetRoot}:`,
+    ...files.map((file) => `  ${file}`),
+    ...removedLegacyFiles.map((file) => `  removed ${file} (renamed to ${SKILL_NAME})`),
+  ].join("\n");
+
+const bindCheckout = async (
+  targetRoot: string,
+  projectId: string | undefined,
+): Promise<string | undefined> => {
+  if (projectId === undefined) return undefined;
+  const existing = await readTolokaState(targetRoot);
+  assert(
+    existing === undefined || existing.projectId === projectId,
+    `This directory is already bound to ${existing?.projectId}.`,
+  );
+  if (existing !== undefined) return undefined;
+  await writeTolokaState(targetRoot, { projectId, pipelines: {}, datasets: {} });
+  return "toloka.json";
+};
+
+export async function initProject(
+  targetDirectory: string,
+  options: { readonly projectId?: string } = {},
+): Promise<InitProjectResult> {
+  await mkdir(targetDirectory, { recursive: true });
+  const targetRoot = await realpath(targetDirectory);
+  const boundCheckout = await bindCheckout(targetRoot, options.projectId);
+  const [hasClaude, hasCursor, hasAgents, skillDigest] = await Promise.all([
+    isDirectory(join(targetRoot, ".claude")),
+    isDirectory(join(targetRoot, ".cursor", "rules")),
+    isFile(join(targetRoot, AGENTS_PATH)),
+    computeBundledSkillDigest(),
+  ]);
+  const useDefaults = !hasClaude && !hasCursor && !hasAgents;
+  const candidates = [
+    {
+      relativePath: CLAUDE_SKILL_PATH,
+      wanted: hasClaude || useDefaults,
+      source: () => claudeSkillSource(skillDigest),
+    },
+    {
+      relativePath: CURSOR_RULE_PATH,
+      wanted: hasCursor,
+      source: () => cursorRuleSource(skillDigest),
+    },
+    {
+      relativePath: AGENTS_PATH,
+      wanted: hasAgents || useDefaults,
+      source: async () => {
+        const existingSource = await readOptionalFile(join(targetRoot, AGENTS_PATH));
+        return mergeAgentsBlock(existingSource ?? "", await agentsBlockSource(skillDigest));
+      },
+    },
+  ];
+  const selected = candidates.filter((candidate) => candidate.wanted);
+  await Promise.all(
+    selected.map((candidate) => assertSafeDestination(targetRoot, candidate.relativePath)),
+  );
+  const entries: AgentEntry[] = await Promise.all(
+    selected.map(async (candidate) => ({
+      relativePath: candidate.relativePath,
+      source: await candidate.source(),
+    })),
+  );
+
+  const writtenFiles = boundCheckout === undefined ? [] : [boundCheckout];
+  for (const entry of entries) {
+    if (await writeIfChanged(join(targetRoot, entry.relativePath), entry.source)) {
+      writtenFiles.push(entry.relativePath);
+    }
+  }
+  const removedLegacyFiles = await removeLegacyEntryPoints(targetRoot);
+  const dependencyReady = await hasSdkDependency(targetRoot);
+  const files = [
+    ...entries.map((entry) => entry.relativePath),
+    ...(boundCheckout === undefined ? [] : [boundCheckout]),
+  ];
+  const warning = dependencyReady
+    ? undefined
+    : `Warning: ${SDK_PACKAGE_NAME} is not listed in dependencies or devDependencies in ${join(targetRoot, "package.json")}. Run \`bun add --dev ${SDK_PACKAGE_NAME}\` before using \`toloka\`, then re-run \`toloka init\`.`;
+  return {
+    targetDirectory: targetRoot,
+    files,
+    writtenFiles,
+    sdkVersion: SDK_VERSION,
+    skillDigest,
+    dependencyReady,
+    output: formatInitOutput(targetRoot, files, writtenFiles, removedLegacyFiles),
+    ...(warning === undefined ? {} : { warning }),
+  };
+}
+
+export async function readAgentSkillStalenessWarning(
+  targetDirectory: string,
+): Promise<string | undefined> {
+  const currentDigest = await computeBundledSkillDigest();
+  const staleDigests = new Set<string>();
+  const generatedVersions = new Set<string>();
+  for (const relativePath of [...AGENT_ENTRY_PATHS, ...LEGACY_ENTRY_PATHS]) {
+    const source = await readOptionalFile(join(targetDirectory, relativePath));
+    if (source === undefined) continue;
+    const recordedDigest = DIGEST_PATTERN.exec(source)?.[1];
+    if (recordedDigest !== undefined && recordedDigest !== currentDigest) {
+      staleDigests.add(recordedDigest.slice(0, 12));
+      const recordedVersion = VERSION_PATTERN.exec(source)?.[1];
+      if (recordedVersion !== undefined) generatedVersions.add(recordedVersion);
+    }
+  }
+  if (staleDigests.size === 0) return undefined;
+  const versions =
+    generatedVersions.size === 0
+      ? "an earlier build"
+      : [...generatedVersions].toSorted().join(", ");
+  return `Warning: Agent instructions generated by ${SDK_PACKAGE_NAME} ${versions} do not match the installed ${SDK_VERSION} skill content (${[...staleDigests].toSorted().join(", ")} -> ${currentDigest.slice(0, 12)}). Re-run \`toloka init\`.`;
+}