create-windy 0.6.6 → 0.6.8

package/template/packages/delivery/src/ssh-process.ts

@@ -0,0 +1,349 @@
+import type {
+  ContentBlob,
+  DeliveryCapabilities,
+  DeliveryManifestV2,
+  Sha256Digest,
+} from "./types.js";
+import type { SshDeliveryTransport } from "./ssh-sync.js";
+import { redactSensitiveText } from "./redact.js";
+import { runProcessCommand } from "./process-command.js";
+import {
+  assertOwnerToken,
+  isExplicitLegacyCapabilityResponse,
+  parseCapabilities,
+  safeRemotePath,
+  safeSshTarget,
+} from "./ssh-capabilities.js";
+
+export interface SshProcessTransportOptions {
+  target: string;
+  remoteBinary: string;
+  remoteStore: string;
+  remoteInstallRoot: string;
+  remoteFallbackRoot: string;
+}
+
+export class SshProcessTransport implements SshDeliveryTransport {
+  readonly #target: string;
+  readonly #binary: string;
+  readonly #store: string;
+  readonly #installRoot: string;
+  readonly #fallbackRoot: string;
+
+  constructor(options: SshProcessTransportOptions) {
+    this.#target = safeSshTarget(options.target);
+    this.#binary = safeRemotePath(options.remoteBinary, "remoteBinary");
+    this.#store = safeRemotePath(options.remoteStore, "remoteStore");
+    this.#installRoot = safeRemotePath(
+      options.remoteInstallRoot,
+      "remoteInstallRoot",
+    );
+    this.#fallbackRoot = safeRemotePath(
+      options.remoteFallbackRoot,
+      "remoteFallbackRoot",
+    );
+  }
+
+  async capabilities(): Promise<DeliveryCapabilities | undefined> {
+    const result = await this.#run(
+      ["ssh", this.#target, this.#binary, "capabilities"],
+      true,
+    );
+    if (result.exitCode === 0) {
+      return parseCapabilities(result.stdout);
+    }
+    if (isExplicitLegacyCapabilityResponse(result)) return undefined;
+    throw new Error(
+      `SSH capability 探测失败 (${result.exitCode})：${redactSensitiveText(result.stderr)}`,
+    );
+  }
+
+  async lock(ownerToken: string): Promise<void> {
+    assertOwnerToken(ownerToken);
+    const lock = `${this.#installRoot}/.delivery-lock`;
+    await this.#run(["ssh", this.#target, "mkdir", "-p", this.#installRoot]);
+    const acquired = await this.#run(
+      ["ssh", this.#target, "mkdir", lock],
+      true,
+    );
+    if (acquired.exitCode !== 0) {
+      if (/File exists|文件已存在/i.test(acquired.stderr)) {
+        throw new Error("已有并发交付持有安装锁");
+      }
+      throw new Error(
+        `无法获取远端交付锁：${redactSensitiveText(acquired.stderr)}`,
+      );
+    }
+    try {
+      await this.#run(
+        ["ssh", this.#target, "tee", `${lock}/owner`],
+        false,
+        `${ownerToken}\n`,
+      );
+      await this.#run(["ssh", this.#target, "chmod", "600", `${lock}/owner`]);
+    } catch (error) {
+      await this.#run(["ssh", this.#target, "rm", "--", `${lock}/owner`], true);
+      await this.#run(["ssh", this.#target, "rmdir", "--", lock], true);
+      throw error;
+    }
+  }
+
+  async unlock(ownerToken: string): Promise<void> {
+    assertOwnerToken(ownerToken);
+    const lock = `${this.#installRoot}/.delivery-lock`;
+    const owner = await this.#run([
+      "ssh",
+      this.#target,
+      "cat",
+      `${lock}/owner`,
+    ]);
+    if (owner.stdout.trim() !== ownerToken) {
+      throw new Error("交付锁 owner 不匹配，拒绝释放");
+    }
+    await this.#run(["ssh", this.#target, "rm", "--", `${lock}/owner`]);
+    await this.#run(["ssh", this.#target, "rmdir", "--", lock]);
+  }
+
+  async inventory(blobs: readonly ContentBlob[]): Promise<Set<Sha256Digest>> {
+    const result = await this.#json<{ present: Sha256Digest[] }>(
+      ["inventory", "--store", this.#store],
+      JSON.stringify(blobs),
+    );
+    return new Set(result.present);
+  }
+
+  async preflight(
+    manifest: DeliveryManifestV2,
+    ownerToken: string,
+  ): Promise<void> {
+    await this.#json(
+      ["preflight", "--store", this.#store, ...this.#installArgs(ownerToken)],
+      JSON.stringify(manifest),
+    );
+  }
+
+  async preflightFallback(
+    fallback: NonNullable<DeliveryManifestV2["compatibility"]["fullBundle"]>,
+    ownerToken: string,
+  ): Promise<void> {
+    await this.#verifyProcessLock(ownerToken);
+    await this.#run(["ssh", this.#target, "mkdir", "-p", this.#fallbackRoot]);
+    const disk = await this.#run([
+      "ssh",
+      this.#target,
+      "df",
+      "-Pk",
+      this.#fallbackRoot,
+    ]);
+    const line = disk.stdout.trim().split(/\r?\n/).at(-1) ?? "";
+    const availableKiB = Number(line.trim().split(/\s+/)[3]);
+    const requiredBytes = fallback.size + 512 * 1024 * 1024;
+    if (
+      !Number.isSafeInteger(availableKiB) ||
+      availableKiB < 0 ||
+      !Number.isSafeInteger(availableKiB * 1024)
+    ) {
+      throw new Error("无法解析 fallback 文件系统空间");
+    }
+    const availableBytes = availableKiB * 1024;
+    if (availableBytes < requiredBytes) {
+      throw new Error(
+        `fallback 空间不足：需要 ${requiredBytes}，可用 ${availableBytes}`,
+      );
+    }
+  }
+
+  async partial(blob: ContentBlob): Promise<number> {
+    const result = await this.#json<{ offset: number }>([
+      "partial",
+      "--store",
+      this.#store,
+      ...blobArgs(blob),
+    ]);
+    return result.offset;
+  }
+
+  async upload(
+    blob: ContentBlob,
+    offset: number,
+    localPath: string,
+  ): Promise<number> {
+    const result = await this.#json<{
+      complete: boolean;
+      receivedBytes: number;
+    }>(
+      [
+        "receive",
+        "--store",
+        this.#store,
+        ...blobArgs(blob),
+        "--offset",
+        String(offset),
+      ],
+      Bun.file(localPath).slice(offset),
+    );
+    if (!result.complete || result.receivedBytes !== blob.size - offset) {
+      throw new Error(`远端未完整接收 blob：${blob.digest}`);
+    }
+    return result.receivedBytes;
+  }
+
+  async stage(
+    manifest: DeliveryManifestV2,
+    digest: Sha256Digest,
+    ownerToken: string,
+  ): Promise<string> {
+    const result = await this.#json<{ staged: boolean; directory: string }>(
+      [
+        "stage",
+        "--store",
+        this.#store,
+        "--manifest-digest",
+        digest,
+        ...this.#installArgs(ownerToken),
+      ],
+      JSON.stringify(manifest),
+    );
+    if (!result.staged) throw new Error("远端未确认 release stage");
+    return result.directory;
+  }
+
+  async uploadFallback(
+    fallback: NonNullable<DeliveryManifestV2["compatibility"]["fullBundle"]>,
+    localPath: string,
+    ownerToken: string,
+  ): Promise<{ remotePath: string; uploaded: boolean }> {
+    await this.#verifyProcessLock(ownerToken);
+    const remotePath = `${this.#fallbackRoot}/${fallback.file}`;
+    await this.#run(["ssh", this.#target, "mkdir", "-p", this.#fallbackRoot]);
+    const existing = await this.#run(
+      ["ssh", this.#target, "sha256sum", remotePath],
+      true,
+    );
+    if (existing.exitCode === 0) {
+      if (`sha256:${existing.stdout.split(/\s+/)[0]}` !== fallback.digest) {
+        throw new Error("远端已存在同名但不同 digest 的完整 bundle");
+      }
+      return { remotePath, uploaded: false };
+    }
+    if (!/No such file|not found/i.test(existing.stderr)) {
+      throw new Error(
+        `无法盘点远端完整 bundle：${redactSensitiveText(existing.stderr)}`,
+      );
+    }
+    const partial = `${remotePath}.${ownerToken}.partial`;
+    try {
+      await this.#run(["scp", localPath, `${this.#target}:${partial}`]);
+      const digest = await this.#run([
+        "ssh",
+        this.#target,
+        "sha256sum",
+        partial,
+      ]);
+      if (`sha256:${digest.stdout.split(/\s+/)[0]}` !== fallback.digest) {
+        throw new Error("远端完整 bundle 上传摘要不匹配");
+      }
+      await this.#run([
+        "ssh",
+        this.#target,
+        "mv",
+        "-n",
+        "--",
+        partial,
+        remotePath,
+      ]);
+      const committed = await this.#run([
+        "ssh",
+        this.#target,
+        "sha256sum",
+        remotePath,
+      ]);
+      if (`sha256:${committed.stdout.split(/\s+/)[0]}` !== fallback.digest) {
+        throw new Error("远端完整 bundle 原子提交后摘要不匹配");
+      }
+      return { remotePath, uploaded: true };
+    } finally {
+      await this.#run(["ssh", this.#target, "rm", "--", partial], true);
+    }
+  }
+
+  #installArgs(ownerToken: string): string[] {
+    assertOwnerToken(ownerToken);
+    return ["--install-root", this.#installRoot, "--owner-token", ownerToken];
+  }
+
+  async #json<T>(args: readonly string[], stdin?: string | Blob): Promise<T> {
+    const process = Bun.spawn(["ssh", this.#target, this.#binary, ...args], {
+      stdin: stdin === undefined ? "ignore" : "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    const feed =
+      stdin === undefined ? Promise.resolve() : feedStdin(process, stdin);
+    const [exitCode, stdout, stderr] = await Promise.all([
+      process.exited,
+      new Response(process.stdout).text(),
+      new Response(process.stderr).text(),
+      feed,
+    ]);
+    if (exitCode !== 0) {
+      throw new Error(
+        `SSH delivery 命令失败 (${exitCode})：${redactSensitiveText(stderr.trim())}`,
+      );
+    }
+    return JSON.parse(stdout) as T;
+  }
+
+  async #run(
+    command: readonly string[],
+    allowFailure = false,
+    stdin?: string,
+  ): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+    return runProcessCommand(command, {
+      allowFailure,
+      stdin,
+      errorLabel: "远端传输命令失败",
+    });
+  }
+
+  async #verifyProcessLock(ownerToken: string): Promise<void> {
+    assertOwnerToken(ownerToken);
+    const owner = await this.#run([
+      "ssh",
+      this.#target,
+      "cat",
+      `${this.#installRoot}/.delivery-lock/owner`,
+    ]);
+    if (owner.stdout.trim() !== ownerToken) {
+      throw new Error("交付锁 owner 不匹配");
+    }
+  }
+}
+
+async function feedStdin(
+  process: ReturnType<typeof Bun.spawn>,
+  input: string | Blob,
+): Promise<void> {
+  if (process.stdin === undefined || typeof process.stdin === "number") {
+    throw new Error("SSH 子进程 stdin 不可写");
+  }
+  if (typeof input === "string") {
+    process.stdin.write(input);
+  } else {
+    const reader = input.stream().getReader();
+    try {
+      while (true) {
+        const result = await reader.read();
+        if (result.done) break;
+        process.stdin.write(result.value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  await process.stdin.end();
+}
+
+function blobArgs(blob: ContentBlob): string[] {
+  return ["--digest", blob.digest, "--size", String(blob.size)];
+}

package/template/packages/delivery/src/ssh-sync.test.ts

@@ -0,0 +1,236 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ingestArchiveFiles, receiveBlobChunk } from "./blob-store.js";
+import { sha256Bytes } from "./digest.js";
+import { manifestDigest } from "./manifest.js";
+import { syncReleaseOverSsh, type SshDeliveryTransport } from "./ssh-sync.js";
+import type {
+  ContentBlob,
+  DeliveryCapabilities,
+  DeliveryManifestV2,
+  Sha256Digest,
+} from "./types.js";
+
+const roots: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+  );
+});
+
+describe("SSH 增量协调器", () => {
+  test("续传只统计本次实际上传 bytes，并在 stage 后释放锁", async () => {
+    const root = await temporaryRoot();
+    const archive = join(root, "archive");
+    const localStore = join(root, "local-store");
+    const remoteStore = join(root, "remote-store");
+    await mkdir(archive);
+    await writeFile(join(archive, "layer.tar"), "0123456789");
+    const graph = await ingestArchiveFiles(archive, ["layer.tar"], localStore);
+    const manifest = releaseManifest(graph.entries, graph.blobs);
+    const transport = new FilesystemTransport(remoteStore, manifest);
+    await receiveBlobChunk({
+      storeRoot: remoteStore,
+      blob: graph.blobs[0]!,
+      offset: 0,
+      bytes: new TextEncoder().encode("0123"),
+    });
+    const result = await syncReleaseOverSsh({
+      manifest,
+      localStoreRoot: localStore,
+      transport,
+      ownerToken: "a".repeat(48),
+      preference: "blob-delta",
+    });
+    expect(result).toEqual({
+      mode: "blob-delta",
+      stagedDirectory: "/remote/releases/release-1",
+      statistics: {
+        totalBytes: 10,
+        reusedBytes: 4,
+        uploadedBytes: 6,
+        totalBlobs: 1,
+        reusedBlobs: 0,
+        uploadedBlobs: 1,
+      },
+    });
+    expect(transport.locked).toBeFalse();
+  });
+
+  test("旧端没有 V2 capability 时显式返回完整 bundle", async () => {
+    const root = await temporaryRoot();
+    const archive = join(root, "archive");
+    const store = join(root, "store");
+    await mkdir(archive);
+    await writeFile(join(archive, "manifest.json"), "{}\n");
+    const graph = await ingestArchiveFiles(archive, ["manifest.json"], store);
+    const fallbackBytes = new Uint8Array(1024).fill(7);
+    const fallbackPath = join(root, "windy-full.tar.gz");
+    await writeFile(fallbackPath, fallbackBytes);
+    const manifest = releaseManifest(
+      graph.entries,
+      graph.blobs,
+      sha256Bytes(fallbackBytes),
+    );
+    const transport = new FilesystemTransport(store, manifest, null);
+    const result = await syncReleaseOverSsh({
+      manifest,
+      localStoreRoot: store,
+      transport,
+      ownerToken: "b".repeat(48),
+      fallbackPath,
+    });
+    expect(result).toEqual({
+      mode: "full-bundle-required",
+      selection: {
+        mode: "full-bundle",
+        schemaVersion: 1,
+        file: "windy-full.tar.gz",
+        digest: sha256Bytes(fallbackBytes),
+      },
+      remoteArchive: "/remote/windy-full.tar.gz",
+      statistics: {
+        totalBytes: 1024,
+        reusedBytes: 0,
+        uploadedBytes: 1024,
+        totalBlobs: 1,
+        reusedBlobs: 0,
+        uploadedBlobs: 1,
+      },
+    });
+    expect(transport.fallbackUploads).toBe(1);
+    expect(transport.locked).toBeFalse();
+  });
+});
+
+class FilesystemTransport implements SshDeliveryTransport {
+  locked = false;
+  fallbackUploads = 0;
+  readonly #store: string;
+  readonly #manifest: DeliveryManifestV2;
+  readonly #capabilities: DeliveryCapabilities | undefined;
+
+  constructor(
+    store: string,
+    manifest: DeliveryManifestV2,
+    capabilities: DeliveryCapabilities | null = modernCapabilities,
+  ) {
+    this.#store = store;
+    this.#manifest = manifest;
+    this.#capabilities = capabilities ?? undefined;
+  }
+
+  async capabilities() {
+    return this.#capabilities;
+  }
+
+  async lock() {
+    if (this.locked) throw new Error("locked");
+    this.locked = true;
+  }
+
+  async unlock() {
+    this.locked = false;
+  }
+
+  async inventory(blobs: readonly ContentBlob[]) {
+    const { inventoryBlobs } = await import("./blob-store.js");
+    return inventoryBlobs(this.#store, blobs);
+  }
+
+  async partial(blob: ContentBlob) {
+    const { partialBlobSize } = await import("./blob-store.js");
+    return partialBlobSize(this.#store, blob);
+  }
+
+  async preflight() {
+    if (!this.locked) throw new Error("preflight requires lock");
+  }
+
+  async preflightFallback() {
+    if (!this.locked) throw new Error("fallback preflight requires lock");
+  }
+
+  async upload(blob: ContentBlob, offset: number, localPath: string) {
+    const bytes = new Uint8Array(
+      await Bun.file(localPath).slice(offset).arrayBuffer(),
+    );
+    const result = await receiveBlobChunk({
+      storeRoot: this.#store,
+      blob,
+      offset,
+      bytes,
+    });
+    return result.receivedBytes;
+  }
+
+  async stage(
+    manifest: DeliveryManifestV2,
+    digest: Sha256Digest,
+    _ownerToken: string,
+  ) {
+    expect(manifest).toEqual(this.#manifest);
+    expect(digest).toBe(manifestDigest(this.#manifest));
+    return "/remote/releases/release-1";
+  }
+
+  async uploadFallback(
+    fallback: NonNullable<DeliveryManifestV2["compatibility"]["fullBundle"]>,
+    _localPath: string,
+  ) {
+    this.fallbackUploads += 1;
+    return { remotePath: `/remote/${fallback.file}`, uploaded: true };
+  }
+}
+
+const modernCapabilities: DeliveryCapabilities = {
+  schemaVersions: [1, 2],
+  registryByDigest: false,
+  blobDelta: true,
+  resumableBlobTransfer: true,
+};
+
+function releaseManifest(
+  entries: DeliveryManifestV2["dockerArchive"]["entries"],
+  blobs: ContentBlob[],
+  fallbackDigest?: Sha256Digest,
+): DeliveryManifestV2 {
+  return {
+    schemaVersion: 2,
+    minimumReaderVersion: 2,
+    product: "windy",
+    releaseId: "release-1",
+    gitSha: "a".repeat(40),
+    platform: "linux/amd64",
+    createdAt: "2026-08-08T00:00:00.000Z",
+    images: [
+      {
+        role: "server",
+        platform: "linux/amd64",
+        configDigest: blobs[0]!.digest,
+      },
+    ],
+    dockerArchive: { format: "oci-image-layout-v1", entries },
+    releaseFiles: [],
+    blobs,
+    compatibility: fallbackDigest
+      ? {
+          fullBundle: {
+            schemaVersion: 1,
+            file: "windy-full.tar.gz",
+            digest: fallbackDigest,
+            size: 1024,
+          },
+        }
+      : {},
+  };
+}
+
+async function temporaryRoot() {
+  const root = await mkdtemp(join(tmpdir(), "windy-ssh-sync-"));
+  roots.push(root);
+  return root;
+}