@toloka-platform/sdk 0.2.0 → 0.2.1

package/src/cli/auto-update.ts

@@ -1,9 +1,23 @@
+import { createHash, timingSafeEqual } from "node:crypto";
 import { spawn } from "node:child_process";
 import { once } from "node:events";
-import { realpathSync } from "node:fs";
-import { chmod, readFile, writeFile } from "node:fs/promises";
-import { join, sep } from "node:path";
-import { findCliEntry } from "./cli-entry";
+import { constants, createReadStream, realpathSync } from "node:fs";
+import {
+  chmod,
+  lstat,
+  mkdir,
+  mkdtemp,
+  open,
+  readFile,
+  realpath,
+  rename,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { dirname, join, sep } from "node:path";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { cliRelaunchCommand, findCliEntry } from "./cli-entry";
 import type { CliEnvironment } from "./credentials";
 import { configDirectory, preparePrivateFilePath } from "./credentials";
 import {
@@ -19,6 +33,7 @@ import { CHECK_RESULT_FILE_ENV } from "./check";
 import { isFalseyFlag, isTruthyFlag } from "./boolean-flag";
 import { TolokaCliError } from "./cli-error";
 import { formatCliError } from "./format-error";
+import { bunSubprocessEnvironment, isNativeCli } from "./native-runtime";
 
 export const AUTO_UPDATE_INTERVAL_MS = 24 * 60 * 60 * 1000;
 export const AUTO_UPDATE_QUARANTINE_MS = 24 * 60 * 60 * 1000;
@@ -34,9 +49,15 @@ export type AutoUpdateState = {
   readonly lastCheckedAt: string;
 };
 
+export type NpmPackageDistribution = {
+  readonly tarball: string;
+  readonly integrity: string;
+};
+
 export type NpmPackageView = {
   readonly "dist-tags"?: Readonly<Record<string, string>>;
   readonly time?: Readonly<Record<string, string>>;
+  readonly versions?: Readonly<Record<string, { readonly dist?: NpmPackageDistribution }>>;
 };
 
 export type AutoUpdateInstall = (options: {
@@ -44,6 +65,7 @@ export type AutoUpdateInstall = (options: {
   readonly version: string;
   readonly registry: string;
   readonly environment: CliEnvironment;
+  readonly distribution?: NpmPackageDistribution;
 }) => Promise<void>;
 
 export type AutoUpdateReexec = (options: {
@@ -54,9 +76,13 @@ export type AutoUpdateReexec = (options: {
 export type AutoUpdateHooks = {
   readonly now?: () => Date;
   readonly fetchRegistry?: (url: string) => Promise<NpmPackageView>;
+  readonly fetchTarball?: (url: string, init: RequestInit) => Promise<Response>;
   readonly install?: AutoUpdateInstall;
   readonly reexec?: AutoUpdateReexec;
   readonly isGlobalInstall?: () => boolean | Promise<boolean>;
+  readonly isNative?: () => boolean;
+  readonly executablePath?: () => string;
+  readonly runtimeExecutable?: string;
 };
 
 export type AutoUpdateResult =
@@ -71,7 +97,12 @@ type UpdateCheck = { readonly status: "skip" } | { readonly status: "check"; rea
 
 type UpdateCandidate =
   | { readonly status: "skip" }
-  | { readonly status: "update"; readonly currentVersion: string; readonly version: string };
+  | {
+      readonly status: "update";
+      readonly currentVersion: string;
+      readonly version: string;
+      readonly distribution?: NpmPackageDistribution;
+    };
 
 export const autoUpdateStatePath = (environment: CliEnvironment = process.env): string =>
   join(configDirectory(environment), "auto-update.json");
@@ -80,6 +111,31 @@ export const npmRegistryUrl = (): string => DEFAULT_NPM_REGISTRY;
 
 export const npmPackageMetadataUrl = (packageName: string): string =>
   new URL(encodeURIComponent(packageName), DEFAULT_NPM_REGISTRY).toString();
+const NATIVE_CLI_TARGETS: Readonly<Record<string, true>> = {
+  "darwin-arm64": true,
+  "darwin-x64": true,
+  "linux-arm64": true,
+  "linux-x64": true,
+};
+
+export const nativeCliPackageName = (
+  platform = process.platform,
+  architecture = process.arch,
+  libc?: "glibc" | "musl",
+): string => {
+  if (platform === "linux" && libc === "musl") {
+    throw new TolokaCliError("The standalone Toloka CLI supports glibc Linux, not musl Linux.", {
+      hint: { disabled: true },
+    });
+  }
+  if (!Object.hasOwn(NATIVE_CLI_TARGETS, `${platform}-${architecture}`)) {
+    throw new TolokaCliError(
+      `The standalone Toloka CLI does not support ${platform}-${architecture}.`,
+      { hint: { disabled: true } },
+    );
+  }
+  return `@toloka-platform/cli-${platform}-${architecture}`;
+};
 
 const asStringRecord = (value: unknown): Readonly<Record<string, string>> | undefined => {
   if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
@@ -90,15 +146,41 @@ const asStringRecord = (value: unknown): Readonly<Record<string, string>> | unde
   }
   return record;
 };
+const asNpmPackageDistribution = (value: unknown): NpmPackageDistribution | undefined => {
+  if (typeof value !== "object" || value === null) return undefined;
+  const tarball = Reflect.get(value, "tarball");
+  const integrity = Reflect.get(value, "integrity");
+  return typeof tarball === "string" &&
+    tarball.length > 0 &&
+    typeof integrity === "string" &&
+    integrity.length > 0
+    ? { tarball, integrity }
+    : undefined;
+};
+
+const asNpmPackageVersions = (
+  value: unknown,
+): Readonly<Record<string, { readonly dist?: NpmPackageDistribution }>> | undefined => {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+  const versions: Record<string, { readonly dist?: NpmPackageDistribution }> = {};
+  for (const [version, manifest] of Object.entries(value)) {
+    if (typeof manifest !== "object" || manifest === null) continue;
+    const distribution = asNpmPackageDistribution(Reflect.get(manifest, "dist"));
+    versions[version] = distribution === undefined ? {} : { dist: distribution };
+  }
+  return versions;
+};
 
 export const asNpmPackageView = (payload: unknown): NpmPackageView | undefined => {
   if (typeof payload !== "object" || payload === null) return undefined;
   const tags = asStringRecord(Reflect.get(payload, "dist-tags"));
   const time = asStringRecord(Reflect.get(payload, "time"));
-  if (tags === undefined && time === undefined) return {};
+  const versions = asNpmPackageVersions(Reflect.get(payload, "versions"));
+  if (tags === undefined && time === undefined && versions === undefined) return {};
   return {
     ...(tags === undefined ? {} : { "dist-tags": tags }),
     ...(time === undefined ? {} : { time }),
+    ...(versions === undefined ? {} : { versions }),
   };
 };
 
@@ -219,7 +301,228 @@ const defaultFetchRegistry = async (url: string): Promise<NpmPackageView> => {
   return view;
 };
 
-const defaultInstall: AutoUpdateInstall = async ({
+type IntegrityAlgorithm = "sha512" | "sha384" | "sha256";
+
+const INTEGRITY_ALGORITHMS: readonly IntegrityAlgorithm[] = ["sha512", "sha384", "sha256"];
+
+const parseIntegrity = (
+  integrity: string,
+): { readonly algorithm: IntegrityAlgorithm; readonly digest: Buffer } => {
+  const tokens = integrity.split(/\s+/u);
+  for (const algorithm of INTEGRITY_ALGORITHMS) {
+    const prefix = `${algorithm}-`;
+    const token = tokens.find((candidate) => candidate.startsWith(prefix));
+    if (token === undefined) continue;
+    const encoded = token.slice(prefix.length);
+    if (!/^[A-Za-z0-9+/]+={0,2}$/u.test(encoded)) continue;
+    const digest = Buffer.from(encoded, "base64");
+    if (
+      digest.length > 0 &&
+      digest.toString("base64").replace(/=+$/u, "") === encoded.replace(/=+$/u, "")
+    ) {
+      return { algorithm, digest };
+    }
+  }
+  throw new RegistryError("npm registry returned unsupported package integrity metadata");
+};
+
+const downloadVerifiedTarball = async (
+  distribution: NpmPackageDistribution,
+  target: string,
+  fetchTarball: (url: string, init: RequestInit) => Promise<Response>,
+): Promise<void> => {
+  const tarballUrl = new URL(distribution.tarball, DEFAULT_NPM_REGISTRY);
+  const registryUrl = new URL(DEFAULT_NPM_REGISTRY);
+  if (tarballUrl.protocol !== "https:" || tarballUrl.origin !== registryUrl.origin) {
+    throw new RegistryError("npm registry returned a tarball outside registry.npmjs.org");
+  }
+  const response = await fetchTarball(tarballUrl.toString(), {
+    headers: { accept: "application/octet-stream" },
+    redirect: "error",
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!response.ok || response.body === null) {
+    throw new RegistryError(`npm registry tarball download responded ${response.status}`);
+  }
+  const expected = parseIntegrity(distribution.integrity);
+  const hash = createHash(expected.algorithm);
+  const verify = new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      hash.update(chunk);
+      callback(null, chunk);
+    },
+  });
+  const handle = await open(
+    target,
+    constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,
+    0o600,
+  );
+  await pipeline(Readable.from(response.body), verify, handle.createWriteStream());
+  const actual = hash.digest();
+  if (actual.length !== expected.digest.length || !timingSafeEqual(actual, expected.digest)) {
+    throw new RegistryError("npm package tarball failed its registry integrity check");
+  }
+};
+
+const runEmbeddedBunInstall = async (options: {
+  readonly stagingDirectory: string;
+  readonly tarballPath: string;
+  readonly registry: string;
+  readonly environment: CliEnvironment;
+  readonly runtimeExecutable: string;
+}): Promise<void> => {
+  const { stagingDirectory, tarballPath, registry, environment } = options;
+  const home = join(stagingDirectory, "home");
+  const config = join(stagingDirectory, "bunfig.toml");
+  const npmConfig = join(stagingDirectory, "npmrc");
+  await mkdir(home);
+  await Promise.all([
+    writeFile(join(stagingDirectory, "package.json"), `${JSON.stringify({ private: true })}\n`),
+    writeFile(
+      config,
+      `env = false\ntelemetry = false\n\n[install]\nregistry = "${registry}"\nignoreScripts = true\noffline = true\noptional = false\npeer = false\ndev = false\n\n[install.cache]\ndir = "./cache"\ndisable = true\ndisableManifest = true\n\n[install.lockfile]\nsave = false\n`,
+    ),
+    writeFile(npmConfig, ""),
+  ]);
+  const child = spawn(
+    options.runtimeExecutable,
+    [
+      "add",
+      tarballPath,
+      `--cwd=${stagingDirectory}`,
+      `--config=${config}`,
+      `--registry=${registry}`,
+      "--ignore-scripts",
+      "--offline",
+      "--no-save",
+      "--linker=hoisted",
+    ],
+    {
+      cwd: stagingDirectory,
+      env: bunSubprocessEnvironment(
+        {
+          ...process.env,
+          ...environment,
+          HOME: home,
+          XDG_CONFIG_HOME: home,
+          NPM_CONFIG_USERCONFIG: npmConfig,
+          BUN_INSTALL_CACHE_DIR: join(stagingDirectory, "cache"),
+        },
+        true,
+      ),
+      stdio: ["ignore", "pipe", "pipe"],
+    },
+  );
+  const [close, stdout, stderr] = await Promise.all([
+    once(child, "close"),
+    collectOutput(child.stdout),
+    collectOutput(child.stderr),
+  ]);
+  const exitCode = close[0];
+  if (exitCode === 0) return;
+  const detail = [stderr.trim(), stdout.trim()].filter((part) => part.length > 0).join("\n");
+  throw new TolokaCliError(
+    detail.length > 0
+      ? `Embedded Bun package staging failed: ${detail}`
+      : `Embedded Bun package staging exited ${String(exitCode)}`,
+    {
+      hint: {
+        description: "Reinstall the Toloka CLI, or disable automatic updates in toloka config.",
+      },
+    },
+  );
+};
+
+const defaultNativeInstall = async (options: {
+  readonly packageName: string;
+  readonly version: string;
+  readonly registry: string;
+  readonly environment: CliEnvironment;
+  readonly distribution: NpmPackageDistribution;
+  readonly executablePath: string;
+  readonly fetchTarball: (url: string, init: RequestInit) => Promise<Response>;
+  readonly runtimeExecutable: string;
+}): Promise<void> => {
+  const executablePath = await realpath(options.executablePath);
+  const currentExecutable = await lstat(executablePath);
+  if (!currentExecutable.isFile()) {
+    throw new TolokaCliError("Refusing to replace a non-file Toloka executable.", {
+      hint: { disabled: true },
+    });
+  }
+  const stagingDirectory = await mkdtemp(join(dirname(executablePath), ".toloka-update-"));
+  try {
+    const tarballPath = join(stagingDirectory, "package.tgz");
+    await downloadVerifiedTarball(options.distribution, tarballPath, options.fetchTarball);
+    await runEmbeddedBunInstall({
+      stagingDirectory,
+      tarballPath,
+      registry: options.registry,
+      environment: options.environment,
+      runtimeExecutable: options.runtimeExecutable,
+    });
+    const packageDirectory = join(
+      stagingDirectory,
+      "node_modules",
+      ...options.packageName.split("/"),
+    );
+    const manifest: unknown = JSON.parse(
+      await readFile(join(packageDirectory, "package.json"), "utf8"),
+    );
+    if (
+      typeof manifest !== "object" ||
+      manifest === null ||
+      Reflect.get(manifest, "name") !== options.packageName ||
+      Reflect.get(manifest, "version") !== options.version
+    ) {
+      throw new TolokaCliError("Staged native CLI package has unexpected metadata.", {
+        hint: { disabled: true },
+      });
+    }
+    const candidate = join(packageDirectory, "bin", "toloka");
+    const checksumPath = join(packageDirectory, "bin", "toloka.sha256");
+    const [candidateStats, checksumStats, checksumSource] = await Promise.all([
+      lstat(candidate),
+      lstat(checksumPath),
+      readFile(checksumPath, "utf8"),
+    ]);
+    if (!candidateStats.isFile() || !checksumStats.isFile()) {
+      throw new TolokaCliError("Staged native CLI package has an invalid artifact layout.", {
+        hint: { disabled: true },
+      });
+    }
+    const expectedChecksum = /^([a-f0-9]{64})  toloka\n$/u.exec(checksumSource)?.[1];
+    if (expectedChecksum === undefined) {
+      throw new TolokaCliError("Staged native CLI package has an invalid checksum file.", {
+        hint: { disabled: true },
+      });
+    }
+    const hash = createHash("sha256");
+    for await (const chunk of createReadStream(candidate)) hash.update(chunk);
+    const actualChecksum = hash.digest();
+    const expectedChecksumBytes = Buffer.from(expectedChecksum, "hex");
+    if (
+      actualChecksum.length !== expectedChecksumBytes.length ||
+      !timingSafeEqual(actualChecksum, expectedChecksumBytes)
+    ) {
+      throw new TolokaCliError("Staged native CLI executable failed its checksum.", {
+        hint: { disabled: true },
+      });
+    }
+    await chmod(candidate, currentExecutable.mode & 0o777);
+    const candidateHandle = await open(candidate, "r");
+    try {
+      await candidateHandle.sync();
+    } finally {
+      await candidateHandle.close();
+    }
+    await rename(candidate, executablePath);
+  } finally {
+    await rm(stagingDirectory, { recursive: true, force: true });
+  }
+};
+
+const defaultJsInstall: AutoUpdateInstall = async ({
   packageName,
   version,
   registry,
@@ -255,9 +558,13 @@ const defaultInstall: AutoUpdateInstall = async ({
 };
 
 const defaultReexec: AutoUpdateReexec = async ({ argv, environment }) => {
-  const entry = process.argv[1];
-  const child = spawn(process.execPath, entry === undefined ? [...argv] : [entry, ...argv], {
-    env: { ...process.env, ...environment, TOLOKA_SKIP_AUTO_UPDATE: "1" },
+  const command = cliRelaunchCommand(argv, {
+    ...process.env,
+    ...environment,
+    TOLOKA_SKIP_AUTO_UPDATE: "1",
+  });
+  const child = spawn(command.executable, command.args, {
+    env: command.environment,
     stdio: "inherit",
   });
   const [code] = await once(child, "close");
@@ -270,11 +577,14 @@ const shouldCheckForUpdate = async (options: {
   readonly argv: readonly string[];
   readonly environment: CliEnvironment;
   readonly hooks: AutoUpdateHooks;
+  readonly native: boolean;
 }): Promise<UpdateCheck> => {
-  const { argv, environment, hooks } = options;
+  const { argv, environment, hooks, native } = options;
   if (skippedCommand(argv)) return { status: "skip" };
-  const isGlobal = hooks.isGlobalInstall ?? (() => isGlobalCliInstall());
-  if (!(await isGlobal())) return { status: "skip" };
+  if (!native) {
+    const isGlobal = hooks.isGlobalInstall ?? (() => isGlobalCliInstall());
+    if (!(await isGlobal())) return { status: "skip" };
+  }
   const settings = await readCliSettings(environment);
   if (!autoUpdateEnabled(environment, settings)) return { status: "skip" };
 
@@ -293,15 +603,16 @@ const selectUpdateCandidate = async (options: {
   readonly now: Date;
   readonly currentVersion: string;
   readonly fetchRegistry: (url: string) => Promise<NpmPackageView>;
+  readonly packageName: string;
 }): Promise<UpdateCandidate> => {
-  const { environment, io, now, currentVersion, fetchRegistry } = options;
+  const { environment, io, now, currentVersion, fetchRegistry, packageName } = options;
   try {
     await writeAutoUpdateState({ lastCheckedAt: now.toISOString() }, environment);
   } catch {}
 
   let view: NpmPackageView;
   try {
-    view = await fetchRegistry(npmPackageMetadataUrl(SDK_PACKAGE_NAME));
+    view = await fetchRegistry(npmPackageMetadataUrl(packageName));
   } catch (error) {
     io.stderr(
       `Auto-update check failed: ${error instanceof Error ? error.message : String(error)}.`,
@@ -316,7 +627,12 @@ const selectUpdateCandidate = async (options: {
   const published = publishedAt(view, version);
   if (published === undefined) return { status: "skip" };
   if (now.getTime() - published.getTime() < AUTO_UPDATE_QUARANTINE_MS) return { status: "skip" };
-  return { status: "update", currentVersion, version };
+  return {
+    status: "update",
+    currentVersion,
+    version,
+    distribution: view.versions?.[version]?.dist,
+  };
 };
 
 export async function maybeAutoUpdate(options: {
@@ -327,8 +643,16 @@ export async function maybeAutoUpdate(options: {
   readonly hooks?: AutoUpdateHooks;
 }): Promise<AutoUpdateResult> {
   const { argv, environment, io, hooks = {} } = options;
-  const check = await shouldCheckForUpdate({ argv, environment, hooks });
+  const native = hooks.isNative?.() ?? isNativeCli();
+  const check = await shouldCheckForUpdate({ argv, environment, hooks, native });
   if (check.status === "skip") return continueResult();
+  let packageName: string;
+  try {
+    packageName = native ? nativeCliPackageName() : SDK_PACKAGE_NAME;
+  } catch (error) {
+    io.stderr(`Auto-update check failed: ${formatCliError(error)}`);
+    return continueResult();
+  }
 
   const candidate = await selectUpdateCandidate({
     environment,
@@ -336,25 +660,40 @@ export async function maybeAutoUpdate(options: {
     now: check.now,
     currentVersion: options.currentVersion ?? SDK_VERSION,
     fetchRegistry: hooks.fetchRegistry ?? defaultFetchRegistry,
+    packageName,
   });
   if (candidate.status === "skip") return continueResult();
 
-  const install = hooks.install ?? defaultInstall;
   try {
-    await install({
-      packageName: SDK_PACKAGE_NAME,
+    const installOptions = {
+      packageName,
       version: candidate.version,
       registry: npmRegistryUrl(),
       environment,
-    });
+      ...(candidate.distribution === undefined ? {} : { distribution: candidate.distribution }),
+    };
+    if (hooks.install !== undefined) {
+      await hooks.install(installOptions);
+    } else if (native) {
+      if (candidate.distribution === undefined) {
+        throw new RegistryError("npm registry metadata has no native CLI distribution");
+      }
+      await defaultNativeInstall({
+        ...installOptions,
+        distribution: candidate.distribution,
+        executablePath: hooks.executablePath?.() ?? process.execPath,
+        fetchTarball: hooks.fetchTarball ?? fetch,
+        runtimeExecutable: hooks.runtimeExecutable ?? process.execPath,
+      });
+    } else {
+      await defaultJsInstall(installOptions);
+    }
   } catch (error) {
     io.stderr(`Auto-update to ${candidate.version} failed: ${formatCliError(error)}`);
     return continueResult();
   }
 
-  io.stderr(
-    `Updated ${SDK_PACKAGE_NAME} from ${candidate.currentVersion} to ${candidate.version}.`,
-  );
+  io.stderr(`Updated ${packageName} from ${candidate.currentVersion} to ${candidate.version}.`);
   const reexec = hooks.reexec ?? defaultReexec;
   return { status: "reexec", exitCode: await reexec({ argv, environment }) };
 }