@toloka-platform/sdk 0.1.0-preview.5 → 0.1.0-preview.7

package/src/cli/program.ts

@@ -1,11 +1,20 @@
 import { Command, CommanderError, InvalidArgumentError } from "commander";
 import { assert } from "es-toolkit";
+import { writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { createInterface } from "node:readline/promises";
 import { createClient } from "../client";
 import type { PotatoClient } from "../client";
 import { readTolokaState } from "../state";
 import type { TolokaState } from "../state";
+import {
+  CHECK_RESULT_FILE_ENV,
+  checkProject,
+  watchProjectChecks,
+  runFreshCheck,
+  type CheckProjectOptions,
+  type CheckProjectResult,
+} from "./check";
 import { authStatus, login, logout } from "./auth";
 import type { AuthPrompt } from "./auth";
 import { commandNameFromArgv, createCliAnalytics } from "./analytics";
@@ -33,6 +42,16 @@ import {
   projectStatus,
   validateProject,
 } from "./project";
+import {
+  archiveModel,
+  createModel,
+  getBaseModel,
+  getModel,
+  listBaseModels,
+  listModels,
+  listModelVersions,
+  parseBaseModelId,
+} from "./models";
 import {
   MISSING_PROJECT_ID_MESSAGE,
   addProject,
@@ -73,6 +92,7 @@ export type ProgramContext = {
   readonly environment: TolokaEnvironment;
   readonly io: CliIo;
   readonly injectedClient?: PotatoClient;
+  readonly runChecks?: typeof checkProject;
   readonly analytics?: CliAnalytics;
 };
 
@@ -192,6 +212,29 @@ export type CreatedProgram = {
 export function createProgram(context: ProgramContext): CreatedProgram {
   const program = new Command("toloka");
   const exitCode = { value: 0 };
+  const runChecks = context.runChecks ?? checkProject;
+  const runChecksInterruptibly = async (
+    options: CheckProjectOptions,
+    runner: (options: CheckProjectOptions) => Promise<CheckProjectResult> = runChecks,
+  ): Promise<{ result: CheckProjectResult; interruptedExitCode?: number }> => {
+    const abortController = new AbortController();
+    let interruptedExitCode: number | undefined;
+    const interrupt = (exitCode: number) => {
+      interruptedExitCode = exitCode;
+      abortController.abort();
+    };
+    const interruptSigint = () => interrupt(130);
+    const interruptSigterm = () => interrupt(143);
+    process.on("SIGINT", interruptSigint);
+    process.on("SIGTERM", interruptSigterm);
+    try {
+      const result = await runner({ ...options, signal: abortController.signal });
+      return { result, ...(interruptedExitCode === undefined ? {} : { interruptedExitCode }) };
+    } finally {
+      process.off("SIGINT", interruptSigint);
+      process.off("SIGTERM", interruptSigterm);
+    }
+  };
   let warnedAboutStaleSkills = false;
 
   program
@@ -314,8 +357,9 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .description("select and manage the current platform project");
   project
     .command("list")
+    .option("--limit <n>", "maximum projects to list", positiveInteger)
     .description("list remote projects this key can see")
-    .action(async () => {
+    .action(async (options: { limit?: number }) => {
       const flags = globals();
       const { client, stored, state } = await resolveContext(flags, context);
       const selected = resolveSelectedProject(flags, context, stored, state);
@@ -323,6 +367,7 @@ export function createProgram(context: ProgramContext): CreatedProgram {
         client,
         io: context.io,
         ...(selected === undefined ? {} : { context: selected }),
+        ...(options.limit === undefined ? {} : { limit: options.limit }),
       });
     });
   project
@@ -430,9 +475,12 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .action(async () => {
       const flags = globals();
       const sourceRoot = resolveInitRoot(flags);
-      const stored = await readCommandCredentials(flags, context);
       const state = await readTolokaState(sourceRoot);
-      const selected = resolveSelectedProject(flags, context, stored, state);
+      const selected = resolveProjectContext({
+        flags,
+        environment: context.environment,
+        ...(state === undefined ? {} : { state }),
+      });
       const result = await initProject(
         sourceRoot,
         selected === undefined ? {} : { projectId: selected.id },
@@ -490,13 +538,89 @@ export function createProgram(context: ProgramContext): CreatedProgram {
       if (!result.valid) exitCode.value = 1;
     });
 
+  program
+    .command("check")
+    .argument("[pipeline]", "pipeline source directory or name")
+    .option("--node <node>", "check one code or human-work node by name or id")
+    .option("--json", "print the structured check report", false)
+    .option("--watch", "rerun checks when project files change", false)
+    .description("run the complete local source, type, and test gate")
+    .action(
+      async (
+        pipeline: string | undefined,
+        options: { node?: string; json: boolean; watch: boolean },
+      ) => {
+        const sourceRoot = await resolveSourceRoot(globals(), context.io);
+        const checkOptions = {
+          environment: context.environment,
+          sourceRoot,
+          ...(pipeline === undefined ? {} : { pipeline }),
+          ...(options.node === undefined ? {} : { node: options.node }),
+          ...(options.json ? { json: true } : {}),
+        };
+        const writeCheckOutput = async (output: string): Promise<void> => {
+          const resultPath = context.environment[CHECK_RESULT_FILE_ENV];
+          if (resultPath === undefined) context.io.stdout(output);
+          else await writeFile(resultPath, output, "utf8");
+        };
+        const checkRunner =
+          context.runChecks === undefined &&
+          context.environment[CHECK_RESULT_FILE_ENV] === undefined
+            ? runFreshCheck
+            : runChecks;
+        if (options.watch) {
+          const abortController = new AbortController();
+          const stop = () => abortController.abort();
+          process.on("SIGINT", stop);
+          process.on("SIGTERM", stop);
+          try {
+            await watchProjectChecks(
+              checkOptions,
+              async (result) => {
+                await writeCheckOutput(result.output);
+                exitCode.value = result.exitCode;
+              },
+              undefined,
+              abortController.signal,
+            );
+          } finally {
+            process.off("SIGINT", stop);
+            process.off("SIGTERM", stop);
+          }
+          return;
+        }
+        const { result, interruptedExitCode } = await runChecksInterruptibly(
+          checkOptions,
+          checkRunner,
+        );
+        await writeCheckOutput(result.output);
+        exitCode.value = interruptedExitCode ?? result.exitCode;
+      },
+    );
+
   program
     .command("apply")
-    .description("make the platform match source")
+    .description("check source, then make the platform match it")
     .action(async () => {
-      const { sourceRoot, client, projectId } = await resolveContext(globals(), context);
+      const sourceRoot = await resolveSourceRoot(globals(), context.io);
+      const { result: checked, interruptedExitCode } = await runChecksInterruptibly({
+        sourceRoot,
+        environment: context.environment,
+      });
+      context.io.stdout(checked.output);
+      if (interruptedExitCode !== undefined) {
+        exitCode.value = interruptedExitCode;
+        return;
+      }
+      if (checked.exitCode !== 0) {
+        exitCode.value = checked.exitCode;
+        return;
+      }
+      assert(checked.source, "Passing checks must return project source");
+      const { client, projectId } = await resolveContext(globals(), context);
       const result = await applyProject(client, {
         sourceRoot,
+        source: checked.source,
         ...(projectId === undefined ? {} : { projectId }),
       });
       context.io.stdout(result.output);
@@ -533,6 +657,94 @@ export function createProgram(context: ProgramContext): CreatedProgram {
       }
     });
 
+  const baseModel = program
+    .command("base-model")
+    .description("inspect the global base model catalog");
+  baseModel
+    .command("list")
+    .option("--json", "print JSON", false)
+    .description("list base models")
+    .action(async (options: { json: boolean }) => {
+      const { client } = await resolveContext(globals(), context);
+      await listBaseModels(client, { json: options.json }, context.io);
+    });
+  baseModel
+    .command("get")
+    .argument("<id>", "base model ID")
+    .option("--json", "print JSON", false)
+    .description("show one base model")
+    .action(async (id: string, options: { json: boolean }) => {
+      const { client } = await resolveContext(globals(), context);
+      await getBaseModel(client, { id: parseBaseModelId(id), json: options.json }, context.io);
+    });
+
+  const model = program
+    .command("model")
+    .description("manage project Models and immutable versions");
+  model
+    .command("list")
+    .option("--json", "print JSON", false)
+    .description("list Models owned by the selected project")
+    .action(async (options: { json: boolean }) => {
+      const { client, projectId } = await resolveContext(globals(), context);
+      assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+      await listModels(client, { projectId, json: options.json }, context.io);
+    });
+  model
+    .command("get")
+    .argument("<name-or-id>", "project Model name or ID")
+    .option("--json", "print JSON", false)
+    .description("show one project Model")
+    .action(async (nameOrId: string, options: { json: boolean }) => {
+      const { client, projectId } = await resolveContext(globals(), context);
+      assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+      await getModel(client, { projectId, nameOrId, json: options.json }, context.io);
+    });
+  model
+    .command("versions")
+    .argument("<name-or-id>", "project Model name or ID")
+    .option("--json", "print JSON", false)
+    .description("list immutable versions of a model")
+    .action(async (nameOrId: string, options: { json: boolean }) => {
+      const { client, projectId } = await resolveContext(globals(), context);
+      assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+      await listModelVersions(client, { projectId, nameOrId, json: options.json }, context.io);
+    });
+  model
+    .command("create")
+    .argument("<name>", "new Model name")
+    .requiredOption("--base <id>", "base model ID")
+    .option("--description <text>", "Model description")
+    .option("--json", "print JSON", false)
+    .description("create project Model metadata without a Model Version")
+    .action(
+      async (name: string, options: { base: string; description?: string; json: boolean }) => {
+        const { client, projectId } = await resolveContext(globals(), context);
+        assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+        await createModel(
+          client,
+          {
+            projectId,
+            name,
+            description: options.description,
+            baseModelId: parseBaseModelId(options.base),
+            json: options.json,
+          },
+          context.io,
+        );
+      },
+    );
+  model
+    .command("archive")
+    .option("--json", "print JSON", false)
+    .argument("<name-or-id>", "project Model name or ID")
+    .description("archive a project Model")
+    .action(async (nameOrId: string, options: { json: boolean }) => {
+      const { client, projectId } = await resolveContext(globals(), context);
+      assert(projectId, MISSING_PROJECT_ID_MESSAGE);
+      await archiveModel(client, { projectId, nameOrId, json: options.json }, context.io);
+    });
+
   const dataset = program
     .command("dataset")
     .description("inspect and move dataset catalog and rows");
@@ -580,7 +792,7 @@ export function createProgram(context: ProgramContext): CreatedProgram {
   dataset
     .command("upload")
     .argument("<name>", "dataset name or id")
-    .argument("<file>", "JSONL file, one JSON object per line")
+    .argument("<file>", "JSON array, JSONL, or CSV file")
     .description("append rows to a dataset")
     .action(async (name: string, file: string) => {
       const { sourceRoot, client, projectId } = await resolveContext(globals(), context);
@@ -601,11 +813,17 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .description("operate on a live pipeline in the selected project");
   pipeline
     .command("list")
+    .option("--limit <n>", "maximum pipelines to list", positiveInteger)
     .description("list live pipelines in the selected project")
-    .action(async () => {
+    .action(async (options: { limit?: number }) => {
       const { client, projectId, state } = await resolveContext(globals(), context);
       assert(projectId, MISSING_PROJECT_ID_MESSAGE);
-      const result = await listLivePipelines({ client, projectId, state });
+      const result = await listLivePipelines({
+        client,
+        projectId,
+        state,
+        ...(options.limit === undefined ? {} : { limit: options.limit }),
+      });
       context.io.stdout(result.output);
     });
   pipeline
@@ -907,27 +1125,35 @@ export async function runTolokaCli(
   environment: TolokaEnvironment = process.env,
   io: CliIo = terminalIo(),
   injectedClient?: PotatoClient,
-  analytics: CliAnalytics = createCliAnalytics({ environment }),
+  analytics?: CliAnalytics,
+  runChecks?: typeof checkProject,
 ): Promise<number> {
+  const activeAnalytics =
+    environment[CHECK_RESULT_FILE_ENV] === undefined
+      ? (analytics ?? createCliAnalytics({ environment }))
+      : undefined;
   const { program, exitCode } = createProgram({
     environment,
     io,
-    analytics,
+    ...(activeAnalytics === undefined ? {} : { analytics: activeAnalytics }),
     ...(injectedClient === undefined ? {} : { injectedClient }),
+    ...(runChecks === undefined ? {} : { runChecks }),
   });
 
   const startedAt = Date.now();
   const result = await runProgram(program, exitCode, argv, io);
-  try {
-    analytics.captureCommand({
-      command: commandNameFromArgv(argv),
-      durationMs: Date.now() - startedAt,
-      exitCode: result.code,
-      outcome: result.outcome,
-      ...(result.error === undefined ? {} : { error: result.error }),
-    });
-    await analytics.shutdown();
-  } catch {}
+  if (activeAnalytics !== undefined) {
+    try {
+      activeAnalytics.captureCommand({
+        command: commandNameFromArgv(argv),
+        durationMs: Date.now() - startedAt,
+        exitCode: result.code,
+        outcome: result.outcome,
+        ...(result.error === undefined ? {} : { error: result.error }),
+      });
+      await activeAnalytics.shutdown();
+    } catch {}
+  }
 
   return result.code;
 }

package/src/cli/project-context.ts

@@ -11,7 +11,10 @@ type ProjectRecord = { readonly id: string; readonly name: string; readonly team
 
 export type ProjectListApi = {
   readonly projects: {
-    readonly listAll: (params?: object) => AsyncIterable<ProjectRecord>;
+    readonly listPage: (params: { readonly limit: number }) => Promise<{
+      readonly projects: readonly ProjectRecord[];
+      readonly pagination: { readonly total: number; readonly hasMore: boolean };
+    }>;
   };
 };
 
@@ -225,15 +228,22 @@ export async function listRemoteProjects(options: {
   readonly client: ProjectListApi;
   readonly context?: ProjectContext;
   readonly io: ProjectIo;
+  readonly limit?: number;
 }): Promise<{ readonly output: string }> {
-  const lines: string[] = [];
-  for await (const project of options.client.projects.listAll()) {
+  const limit = options.limit ?? 20;
+  const page = await options.client.projects.listPage({ limit });
+  const lines = page.projects.map((project) => {
     const mark = options.context?.id === project.id ? "*" : " ";
     const alias =
       options.context?.id === project.id && options.context.alias !== undefined
         ? `\t${options.context.alias}`
         : "";
-    lines.push(`${mark}\t${project.id}\t${project.name}${alias}`);
+    return `${mark}\t${project.id}\t${project.name}${alias}`;
+  });
+  if (page.pagination.hasMore) {
+    lines.push(
+      `Showing ${page.projects.length} of ${page.pagination.total} projects. Raise --limit to see more.`,
+    );
   }
   const output = lines.length === 0 ? "No projects visible." : lines.join("\n");
   options.io.stdout(output);

package/src/cli/runs.ts

@@ -84,21 +84,31 @@ export async function listLivePipelines(options: {
   readonly client: PotatoClient;
   readonly projectId: string;
   readonly state?: TolokaState;
+  readonly limit?: number;
 }): Promise<RunCommandResult> {
-  const live = await collectLivePipelineRefs(options.client, options.projectId, options.state);
+  const limit = options.limit ?? 20;
+  const page = await options.client.pipelines.listPage({ projectId: options.projectId, limit });
+  const sourceDirectoryById = bindingByLiveId(options.projectId, options.state);
+  const live = page.pipelines
+    .map((pipeline) => toRef(pipeline, sourceDirectoryById))
+    .toSorted((left, right) => {
+      const byName = left.name.localeCompare(right.name);
+      return byName === 0 ? left.pipelineId.localeCompare(right.pipelineId) : byName;
+    });
   if (live.length === 0) {
     return { outcome: "reported", output: "This project has no pipelines." };
   }
-  return {
-    outcome: "reported",
-    output: live
-      .map((ref) =>
-        ref.sourceDirectory === undefined
-          ? `${ref.name}\t${ref.pipelineId}`
-          : `${ref.name}\t${ref.pipelineId}\t${ref.sourceDirectory}`,
-      )
-      .join("\n"),
-  };
+  const lines = live.map((ref) =>
+    ref.sourceDirectory === undefined
+      ? `${ref.name}\t${ref.pipelineId}`
+      : `${ref.name}\t${ref.pipelineId}\t${ref.sourceDirectory}`,
+  );
+  if (page.pagination.hasMore) {
+    lines.push(
+      `Showing ${live.length} of ${page.pagination.total} pipelines. Raise --limit to see more.`,
+    );
+  }
+  return { outcome: "reported", output: lines.join("\n") };
 }
 
 const collectLivePipelineRefs = async (

package/src/cli/skill.ts

@@ -191,12 +191,23 @@ export const searchSkills = (
   skills: readonly SkillDescriptor[],
   query: string,
 ): SkillDescriptor[] => {
-  const needle = query.trim().toLowerCase();
-  return skills.filter((skill) =>
-    [skill.id, skill.name, skill.description, ...skill.tags].some((field) =>
-      field.toLowerCase().includes(needle),
-    ),
-  );
+  const needles = query.trim().toLowerCase().split(/\s+/u).filter(Boolean);
+  if (needles.length === 0) return [];
+
+  return skills
+    .map((skill, index) => {
+      const searchable = [skill.id, skill.name, skill.description, ...skill.tags]
+        .join("\n")
+        .toLowerCase();
+      return {
+        skill,
+        index,
+        score: needles.filter((needle) => searchable.includes(needle)).length,
+      };
+    })
+    .filter(({ score }) => score > 0)
+    .toSorted((left, right) => right.score - left.score || left.index - right.index)
+    .map(({ skill }) => skill);
 };
 
 export async function skillStatus(