@toloka-platform/sdk 0.1.0-preview.6 → 0.1.0-preview.8

package/src/cli/credentials.ts

@@ -52,6 +52,9 @@ export const credentialsPath = (environment: CliEnvironment = process.env): stri
 export const legacyCredentialsPath = (environment: CliEnvironment = process.env): string =>
   join(home(environment), ".toloka", "credentials.json");
 
+export const configDirectory = (environment: CliEnvironment = process.env): string =>
+  dirname(credentialsPath(environment));
+
 const removeTrailingSlashes = (value: string): string => {
   let end = value.length;
   while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;

package/src/cli/program.ts

@@ -1,15 +1,26 @@
 import { Command, CommanderError, InvalidArgumentError } from "commander";
 import { assert } from "es-toolkit";
+import { writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { createInterface } from "node:readline/promises";
 import { createClient } from "../client";
 import type { PotatoClient } from "../client";
 import { readTolokaState } from "../state";
 import type { TolokaState } from "../state";
+import {
+  CHECK_RESULT_FILE_ENV,
+  checkProject,
+  watchProjectChecks,
+  runFreshCheck,
+  type CheckProjectOptions,
+  type CheckProjectResult,
+} from "./check";
 import { authStatus, login, logout } from "./auth";
 import type { AuthPrompt } from "./auth";
 import { commandNameFromArgv, createCliAnalytics } from "./analytics";
 import type { CliAnalytics, CliAnalyticsCommandOutcome } from "./analytics";
+import type { AutoUpdateHooks } from "./auto-update";
+import { maybeAutoUpdate } from "./auto-update";
 import { PRODUCTION_BASE_URL, resolveInternalBaseUrl, resolveTolokaConfig } from "./config";
 import type { TolokaEnvironment } from "./config";
 import {
@@ -18,6 +29,7 @@ import {
   requireStoredCredentials,
 } from "./credentials";
 import type { StoredCredentials } from "./credentials";
+import { getCliSetting, readCliSettings, setCliSetting } from "./settings";
 import {
   generateDatasetFieldDeclarations,
   listProjectDatasets,
@@ -83,6 +95,7 @@ export type ProgramContext = {
   readonly environment: TolokaEnvironment;
   readonly io: CliIo;
   readonly injectedClient?: PotatoClient;
+  readonly runChecks?: typeof checkProject;
   readonly analytics?: CliAnalytics;
 };
 
@@ -202,6 +215,29 @@ export type CreatedProgram = {
 export function createProgram(context: ProgramContext): CreatedProgram {
   const program = new Command("toloka");
   const exitCode = { value: 0 };
+  const runChecks = context.runChecks ?? checkProject;
+  const runChecksInterruptibly = async (
+    options: CheckProjectOptions,
+    runner: (options: CheckProjectOptions) => Promise<CheckProjectResult> = runChecks,
+  ): Promise<{ result: CheckProjectResult; interruptedExitCode?: number }> => {
+    const abortController = new AbortController();
+    let interruptedExitCode: number | undefined;
+    const interrupt = (exitCode: number) => {
+      interruptedExitCode = exitCode;
+      abortController.abort();
+    };
+    const interruptSigint = () => interrupt(130);
+    const interruptSigterm = () => interrupt(143);
+    process.on("SIGINT", interruptSigint);
+    process.on("SIGTERM", interruptSigterm);
+    try {
+      const result = await runner({ ...options, signal: abortController.signal });
+      return { result, ...(interruptedExitCode === undefined ? {} : { interruptedExitCode }) };
+    } finally {
+      process.off("SIGINT", interruptSigint);
+      process.off("SIGTERM", interruptSigterm);
+    }
+  };
   let warnedAboutStaleSkills = false;
 
   program
@@ -236,7 +272,8 @@ export function createProgram(context: ProgramContext): CreatedProgram {
       commandNames.includes("help") ||
       commandNames.includes("report") ||
       commandNames.includes("url") ||
-      commandNames.includes("version")
+      commandNames.includes("version") ||
+      commandNames.includes("config")
     )
       return;
     const sourceRoot = await resolveSourceRoot(globals(), context.io);
@@ -250,6 +287,29 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .description("print the current CLI version")
     .action(() => context.io.stdout(SDK_VERSION));
 
+  const config = program.command("config").description("read and write machine CLI settings");
+  config
+    .command("get")
+    .argument("[name]", "setting name; omit to print every setting")
+    .description("print a CLI setting, or every setting")
+    .action(async (name?: string) => {
+      const settings = await readCliSettings(context.environment);
+      context.io.stdout(getCliSetting(settings, name).output);
+    });
+  config
+    .command("set")
+    .argument("<name>", "setting name")
+    .argument("<value>", "setting value")
+    .description("write a CLI setting for this machine")
+    .action(async (name: string, value: string) => {
+      const result = await setCliSetting(context.environment, name, value);
+      context.io.stdout(result.output);
+    });
+  config.action(async () => {
+    const settings = await readCliSettings(context.environment);
+    context.io.stdout(getCliSetting(settings, undefined).output);
+  });
+
   program
     .command("login")
     .description("store an API key for this machine after verifying it")
@@ -324,8 +384,9 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .description("select and manage the current platform project");
   project
     .command("list")
+    .option("--limit <n>", "maximum projects to list", positiveInteger)
     .description("list remote projects this key can see")
-    .action(async () => {
+    .action(async (options: { limit?: number }) => {
       const flags = globals();
       const { client, stored, state } = await resolveContext(flags, context);
       const selected = resolveSelectedProject(flags, context, stored, state);
@@ -333,6 +394,7 @@ export function createProgram(context: ProgramContext): CreatedProgram {
         client,
         io: context.io,
         ...(selected === undefined ? {} : { context: selected }),
+        ...(options.limit === undefined ? {} : { limit: options.limit }),
       });
     });
   project
@@ -440,9 +502,12 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .action(async () => {
       const flags = globals();
       const sourceRoot = resolveInitRoot(flags);
-      const stored = await readCommandCredentials(flags, context);
       const state = await readTolokaState(sourceRoot);
-      const selected = resolveSelectedProject(flags, context, stored, state);
+      const selected = resolveProjectContext({
+        flags,
+        environment: context.environment,
+        ...(state === undefined ? {} : { state }),
+      });
       const result = await initProject(
         sourceRoot,
         selected === undefined ? {} : { projectId: selected.id },
@@ -500,13 +565,89 @@ export function createProgram(context: ProgramContext): CreatedProgram {
       if (!result.valid) exitCode.value = 1;
     });
 
+  program
+    .command("check")
+    .argument("[pipeline]", "pipeline source directory or name")
+    .option("--node <node>", "check one code or human-work node by name or id")
+    .option("--json", "print the structured check report", false)
+    .option("--watch", "rerun checks when project files change", false)
+    .description("run the complete local source, type, and test gate")
+    .action(
+      async (
+        pipeline: string | undefined,
+        options: { node?: string; json: boolean; watch: boolean },
+      ) => {
+        const sourceRoot = await resolveSourceRoot(globals(), context.io);
+        const checkOptions = {
+          environment: context.environment,
+          sourceRoot,
+          ...(pipeline === undefined ? {} : { pipeline }),
+          ...(options.node === undefined ? {} : { node: options.node }),
+          ...(options.json ? { json: true } : {}),
+        };
+        const writeCheckOutput = async (output: string): Promise<void> => {
+          const resultPath = context.environment[CHECK_RESULT_FILE_ENV];
+          if (resultPath === undefined) context.io.stdout(output);
+          else await writeFile(resultPath, output, "utf8");
+        };
+        const checkRunner =
+          context.runChecks === undefined &&
+          context.environment[CHECK_RESULT_FILE_ENV] === undefined
+            ? runFreshCheck
+            : runChecks;
+        if (options.watch) {
+          const abortController = new AbortController();
+          const stop = () => abortController.abort();
+          process.on("SIGINT", stop);
+          process.on("SIGTERM", stop);
+          try {
+            await watchProjectChecks(
+              checkOptions,
+              async (result) => {
+                await writeCheckOutput(result.output);
+                exitCode.value = result.exitCode;
+              },
+              undefined,
+              abortController.signal,
+            );
+          } finally {
+            process.off("SIGINT", stop);
+            process.off("SIGTERM", stop);
+          }
+          return;
+        }
+        const { result, interruptedExitCode } = await runChecksInterruptibly(
+          checkOptions,
+          checkRunner,
+        );
+        await writeCheckOutput(result.output);
+        exitCode.value = interruptedExitCode ?? result.exitCode;
+      },
+    );
+
   program
     .command("apply")
-    .description("make the platform match source")
+    .description("check source, then make the platform match it")
     .action(async () => {
-      const { sourceRoot, client, projectId } = await resolveContext(globals(), context);
+      const sourceRoot = await resolveSourceRoot(globals(), context.io);
+      const { result: checked, interruptedExitCode } = await runChecksInterruptibly({
+        sourceRoot,
+        environment: context.environment,
+      });
+      context.io.stdout(checked.output);
+      if (interruptedExitCode !== undefined) {
+        exitCode.value = interruptedExitCode;
+        return;
+      }
+      if (checked.exitCode !== 0) {
+        exitCode.value = checked.exitCode;
+        return;
+      }
+      assert(checked.source, "Passing checks must return project source");
+      const { client, projectId } = await resolveContext(globals(), context);
       const result = await applyProject(client, {
         sourceRoot,
+        source: checked.source,
         ...(projectId === undefined ? {} : { projectId }),
       });
       context.io.stdout(result.output);
@@ -699,11 +840,17 @@ export function createProgram(context: ProgramContext): CreatedProgram {
     .description("operate on a live pipeline in the selected project");
   pipeline
     .command("list")
+    .option("--limit <n>", "maximum pipelines to list", positiveInteger)
     .description("list live pipelines in the selected project")
-    .action(async () => {
+    .action(async (options: { limit?: number }) => {
       const { client, projectId, state } = await resolveContext(globals(), context);
       assert(projectId, MISSING_PROJECT_ID_MESSAGE);
-      const result = await listLivePipelines({ client, projectId, state });
+      const result = await listLivePipelines({
+        client,
+        projectId,
+        state,
+        ...(options.limit === undefined ? {} : { limit: options.limit }),
+      });
       context.io.stdout(result.output);
     });
   pipeline
@@ -1005,27 +1152,44 @@ export async function runTolokaCli(
   environment: TolokaEnvironment = process.env,
   io: CliIo = terminalIo(),
   injectedClient?: PotatoClient,
-  analytics: CliAnalytics = createCliAnalytics({ environment }),
+  analytics?: CliAnalytics,
+  runChecks?: typeof checkProject,
+  autoUpdate?: AutoUpdateHooks,
 ): Promise<number> {
+  const autoUpdateResult = await maybeAutoUpdate({
+    argv,
+    environment,
+    io,
+    ...(autoUpdate === undefined ? {} : { hooks: autoUpdate }),
+  });
+  if (autoUpdateResult.status === "reexec") return autoUpdateResult.exitCode;
+
+  const activeAnalytics =
+    environment[CHECK_RESULT_FILE_ENV] === undefined
+      ? (analytics ?? createCliAnalytics({ environment }))
+      : undefined;
   const { program, exitCode } = createProgram({
     environment,
     io,
-    analytics,
+    ...(activeAnalytics === undefined ? {} : { analytics: activeAnalytics }),
     ...(injectedClient === undefined ? {} : { injectedClient }),
+    ...(runChecks === undefined ? {} : { runChecks }),
   });
 
   const startedAt = Date.now();
   const result = await runProgram(program, exitCode, argv, io);
-  try {
-    analytics.captureCommand({
-      command: commandNameFromArgv(argv),
-      durationMs: Date.now() - startedAt,
-      exitCode: result.code,
-      outcome: result.outcome,
-      ...(result.error === undefined ? {} : { error: result.error }),
-    });
-    await analytics.shutdown();
-  } catch {}
+  if (activeAnalytics !== undefined) {
+    try {
+      activeAnalytics.captureCommand({
+        command: commandNameFromArgv(argv),
+        durationMs: Date.now() - startedAt,
+        exitCode: result.code,
+        outcome: result.outcome,
+        ...(result.error === undefined ? {} : { error: result.error }),
+      });
+      await activeAnalytics.shutdown();
+    } catch {}
+  }
 
   return result.code;
 }

package/src/cli/project-context.ts

@@ -11,7 +11,10 @@ type ProjectRecord = { readonly id: string; readonly name: string; readonly team
 
 export type ProjectListApi = {
   readonly projects: {
-    readonly listAll: (params?: object) => AsyncIterable<ProjectRecord>;
+    readonly listPage: (params: { readonly limit: number }) => Promise<{
+      readonly projects: readonly ProjectRecord[];
+      readonly pagination: { readonly total: number; readonly hasMore: boolean };
+    }>;
   };
 };
 
@@ -225,15 +228,22 @@ export async function listRemoteProjects(options: {
   readonly client: ProjectListApi;
   readonly context?: ProjectContext;
   readonly io: ProjectIo;
+  readonly limit?: number;
 }): Promise<{ readonly output: string }> {
-  const lines: string[] = [];
-  for await (const project of options.client.projects.listAll()) {
+  const limit = options.limit ?? 20;
+  const page = await options.client.projects.listPage({ limit });
+  const lines = page.projects.map((project) => {
     const mark = options.context?.id === project.id ? "*" : " ";
     const alias =
       options.context?.id === project.id && options.context.alias !== undefined
         ? `\t${options.context.alias}`
         : "";
-    lines.push(`${mark}\t${project.id}\t${project.name}${alias}`);
+    return `${mark}\t${project.id}\t${project.name}${alias}`;
+  });
+  if (page.pagination.hasMore) {
+    lines.push(
+      `Showing ${page.projects.length} of ${page.pagination.total} projects. Raise --limit to see more.`,
+    );
   }
   const output = lines.length === 0 ? "No projects visible." : lines.join("\n");
   options.io.stdout(output);

package/src/cli/runs.ts

@@ -84,21 +84,31 @@ export async function listLivePipelines(options: {
   readonly client: PotatoClient;
   readonly projectId: string;
   readonly state?: TolokaState;
+  readonly limit?: number;
 }): Promise<RunCommandResult> {
-  const live = await collectLivePipelineRefs(options.client, options.projectId, options.state);
+  const limit = options.limit ?? 20;
+  const page = await options.client.pipelines.listPage({ projectId: options.projectId, limit });
+  const sourceDirectoryById = bindingByLiveId(options.projectId, options.state);
+  const live = page.pipelines
+    .map((pipeline) => toRef(pipeline, sourceDirectoryById))
+    .toSorted((left, right) => {
+      const byName = left.name.localeCompare(right.name);
+      return byName === 0 ? left.pipelineId.localeCompare(right.pipelineId) : byName;
+    });
   if (live.length === 0) {
     return { outcome: "reported", output: "This project has no pipelines." };
   }
-  return {
-    outcome: "reported",
-    output: live
-      .map((ref) =>
-        ref.sourceDirectory === undefined
-          ? `${ref.name}\t${ref.pipelineId}`
-          : `${ref.name}\t${ref.pipelineId}\t${ref.sourceDirectory}`,
-      )
-      .join("\n"),
-  };
+  const lines = live.map((ref) =>
+    ref.sourceDirectory === undefined
+      ? `${ref.name}\t${ref.pipelineId}`
+      : `${ref.name}\t${ref.pipelineId}\t${ref.sourceDirectory}`,
+  );
+  if (page.pagination.hasMore) {
+    lines.push(
+      `Showing ${live.length} of ${page.pagination.total} pipelines. Raise --limit to see more.`,
+    );
+  }
+  return { outcome: "reported", output: lines.join("\n") };
 }
 
 const collectLivePipelineRefs = async (

package/src/cli/settings.ts

@@ -0,0 +1,93 @@
+import { chmod, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { isFalseyFlag, isTruthyFlag } from "./boolean-flag";
+import type { CliEnvironment } from "./credentials";
+import { configDirectory, preparePrivateFilePath } from "./credentials";
+
+export const AUTO_UPDATE_SETTING = "auto-update";
+
+export type CliSettings = {
+  readonly autoUpdate: boolean;
+};
+
+export const DEFAULT_CLI_SETTINGS: CliSettings = {
+  autoUpdate: true,
+};
+
+export class SettingsError extends Error {
+  constructor(message: string, options?: { readonly cause?: unknown }) {
+    super(message, options);
+    this.name = "SettingsError";
+  }
+}
+
+export const settingsPath = (environment: CliEnvironment = process.env): string =>
+  join(configDirectory(environment), "settings.json");
+
+const asSettings = (payload: unknown): CliSettings => {
+  const autoUpdate =
+    typeof payload === "object" && payload !== null
+      ? Reflect.get(payload, "autoUpdate")
+      : undefined;
+  if (typeof autoUpdate !== "boolean") return DEFAULT_CLI_SETTINGS;
+  return { autoUpdate };
+};
+
+export async function readCliSettings(
+  environment: CliEnvironment = process.env,
+): Promise<CliSettings> {
+  try {
+    return asSettings(JSON.parse(await readFile(settingsPath(environment), "utf8")));
+  } catch {
+    return DEFAULT_CLI_SETTINGS;
+  }
+}
+
+export async function writeCliSettings(
+  settings: CliSettings,
+  environment: CliEnvironment = process.env,
+): Promise<string> {
+  const path = settingsPath(environment);
+  await preparePrivateFilePath(path);
+  await writeFile(path, `${JSON.stringify({ autoUpdate: settings.autoUpdate }, null, 2)}\n`, {
+    mode: 0o600,
+  });
+  await chmod(path, 0o600);
+  return path;
+}
+
+export const parseBooleanSetting = (value: string): boolean => {
+  if (isTruthyFlag(value)) return true;
+  if (isFalseyFlag(value)) return false;
+  throw new SettingsError(`Expected a boolean (true or false), got ${JSON.stringify(value)}.`);
+};
+
+export const formatCliSettings = (settings: CliSettings): string =>
+  `${AUTO_UPDATE_SETTING}=${settings.autoUpdate}`;
+
+export const getCliSetting = (
+  settings: CliSettings,
+  name: string | undefined,
+): { readonly output: string } => {
+  if (name === undefined || name.length === 0) return { output: formatCliSettings(settings) };
+  if (name !== AUTO_UPDATE_SETTING) {
+    throw new SettingsError(`Unknown setting "${name}". Known settings: ${AUTO_UPDATE_SETTING}.`);
+  }
+  return { output: String(settings.autoUpdate) };
+};
+
+export async function setCliSetting(
+  environment: CliEnvironment,
+  name: string,
+  value: string,
+): Promise<{ readonly output: string; readonly settings: CliSettings }> {
+  if (name !== AUTO_UPDATE_SETTING) {
+    throw new SettingsError(`Unknown setting "${name}". Known settings: ${AUTO_UPDATE_SETTING}.`);
+  }
+  const settings = { autoUpdate: parseBooleanSetting(value) };
+  await writeCliSettings(settings, environment);
+  return {
+    settings,
+    output: `Set ${AUTO_UPDATE_SETTING} to ${settings.autoUpdate}.`,
+  };
+}

package/src/cli/skill.ts

@@ -191,12 +191,23 @@ export const searchSkills = (
   skills: readonly SkillDescriptor[],
   query: string,
 ): SkillDescriptor[] => {
-  const needle = query.trim().toLowerCase();
-  return skills.filter((skill) =>
-    [skill.id, skill.name, skill.description, ...skill.tags].some((field) =>
-      field.toLowerCase().includes(needle),
-    ),
-  );
+  const needles = query.trim().toLowerCase().split(/\s+/u).filter(Boolean);
+  if (needles.length === 0) return [];
+
+  return skills
+    .map((skill, index) => {
+      const searchable = [skill.id, skill.name, skill.description, ...skill.tags]
+        .join("\n")
+        .toLowerCase();
+      return {
+        skill,
+        index,
+        score: needles.filter((needle) => searchable.includes(needle)).length,
+      };
+    })
+    .filter(({ score }) => score > 0)
+    .toSorted((left, right) => right.score - left.score || left.index - right.index)
+    .map(({ skill }) => skill);
 };
 
 export async function skillStatus(

package/src/importer.ts

@@ -220,7 +220,7 @@ function formatSource(value: unknown, depth = 0): string {
   return `{\n${properties}\n${indentation(depth)}}`;
 }
 
-function transformFieldProxy(
+function transformFieldReference(
   reference: { id: string; optional: boolean },
   fieldReference: FieldReference,
 ): unknown {
@@ -276,10 +276,10 @@ function authoringSettings(node: GraphNode, fieldReference: FieldReference): unk
       return {
         ...settings,
         started: startedFieldProxies.output.map((reference) =>
-          transformFieldProxy(reference, fieldReference),
+          transformFieldReference(reference, fieldReference),
         ),
         completed: completedFieldProxies.input.map((reference) =>
-          transformFieldProxy(reference, fieldReference),
+          transformFieldReference(reference, fieldReference),
         ),
       };
     }
@@ -287,14 +287,12 @@ function authoringSettings(node: GraphNode, fieldReference: FieldReference): unk
       const { fieldProxies, ...settings } = node.settings;
       return {
         ...settings,
-        fieldProxies: {
-          input: fieldProxies.input.map((reference) =>
-            transformFieldProxy(reference, fieldReference),
-          ),
-          output: fieldProxies.output.map((reference) =>
-            transformFieldProxy(reference, fieldReference),
-          ),
-        },
+        input: fieldProxies.input.map((reference) =>
+          transformFieldReference(reference, fieldReference),
+        ),
+        output: fieldProxies.output.map((reference) =>
+          transformFieldReference(reference, fieldReference),
+        ),
       };
     }
     case "generation": {
@@ -322,11 +320,9 @@ function authoringSettings(node: GraphNode, fieldReference: FieldReference): unk
             field: fieldReference(fieldProxyId),
           })),
         })),
-        fieldProxies: {
-          input: fieldProxies.input.map((reference) =>
-            transformFieldProxy(reference, fieldReference),
-          ),
-        },
+        input: fieldProxies.input.map((reference) =>
+          transformFieldReference(reference, fieldReference),
+        ),
       };
     }
     case "quorum": {