create-windy 0.6.6 → 0.6.8

package/template/packages/delivery/src/ssh-sync.ts

@@ -0,0 +1,167 @@
+import { blobPath, inventoryBlobs } from "./blob-store.js";
+import { sha256File } from "./digest.js";
+import { stat } from "node:fs/promises";
+import { manifestDigest, validateManifest } from "./manifest.js";
+import { negotiateDelivery, type DeliverySelection } from "./negotiation.js";
+import { planBlobTransfer } from "./transfer.js";
+import type {
+  ContentBlob,
+  DeliveryCapabilities,
+  DeliveryManifestV2,
+  Sha256Digest,
+  TransferStatistics,
+} from "./types.js";
+
+export interface SshDeliveryTransport {
+  capabilities(): Promise<DeliveryCapabilities | undefined>;
+  lock(ownerToken: string): Promise<void>;
+  unlock(ownerToken: string): Promise<void>;
+  inventory(blobs: readonly ContentBlob[]): Promise<Set<Sha256Digest>>;
+  preflight(manifest: DeliveryManifestV2, ownerToken: string): Promise<void>;
+  preflightFallback(
+    fallback: NonNullable<DeliveryManifestV2["compatibility"]["fullBundle"]>,
+    ownerToken: string,
+  ): Promise<void>;
+  partial(blob: ContentBlob): Promise<number>;
+  upload(blob: ContentBlob, offset: number, localPath: string): Promise<number>;
+  stage(
+    manifest: DeliveryManifestV2,
+    digest: Sha256Digest,
+    ownerToken: string,
+  ): Promise<string>;
+  uploadFallback(
+    fallback: NonNullable<DeliveryManifestV2["compatibility"]["fullBundle"]>,
+    localPath: string,
+    ownerToken: string,
+  ): Promise<{ remotePath: string; uploaded: boolean }>;
+}
+
+export type SshSyncResult =
+  | {
+      mode: "blob-delta";
+      stagedDirectory: string;
+      statistics: TransferStatistics;
+    }
+  | {
+      mode: "full-bundle-required";
+      selection: Extract<DeliverySelection, { mode: "full-bundle" }>;
+      remoteArchive?: string;
+      statistics?: TransferStatistics;
+    }
+  | {
+      mode: "registry-preferred";
+      selection: Extract<DeliverySelection, { mode: "registry" }>;
+    };
+
+export async function syncReleaseOverSsh(input: {
+  manifest: DeliveryManifestV2;
+  localStoreRoot: string;
+  transport: SshDeliveryTransport;
+  ownerToken: string;
+  preference?: "auto" | "blob-delta" | "full-bundle";
+  fallbackPath?: string;
+}): Promise<SshSyncResult> {
+  const manifest = validateManifest(input.manifest);
+  const capabilities = await input.transport.capabilities();
+  const selection = negotiateDelivery(
+    manifest,
+    capabilities ?? legacyCapabilities,
+    input.preference ?? "auto",
+  );
+  if (selection.mode === "registry") {
+    return { mode: "registry-preferred", selection };
+  }
+  if (selection.mode === "full-bundle") {
+    if (!input.fallbackPath || !manifest.compatibility.fullBundle) {
+      return { mode: "full-bundle-required", selection };
+    }
+    const fallback = manifest.compatibility.fullBundle;
+    const info = await stat(input.fallbackPath);
+    if (
+      info.size !== fallback.size ||
+      (await sha256File(input.fallbackPath)) !== fallback.digest
+    ) {
+      throw new Error("本地完整 bundle fallback 摘要或尺寸不匹配");
+    }
+    await input.transport.lock(input.ownerToken);
+    try {
+      await input.transport.preflightFallback(fallback, input.ownerToken);
+      const transferred = await input.transport.uploadFallback(
+        fallback,
+        input.fallbackPath,
+        input.ownerToken,
+      );
+      return {
+        mode: "full-bundle-required",
+        selection,
+        remoteArchive: transferred.remotePath,
+        statistics: {
+          totalBytes: fallback.size,
+          reusedBytes: transferred.uploaded ? 0 : fallback.size,
+          uploadedBytes: transferred.uploaded ? fallback.size : 0,
+          totalBlobs: 1,
+          reusedBlobs: transferred.uploaded ? 0 : 1,
+          uploadedBlobs: transferred.uploaded ? 1 : 0,
+        },
+      };
+    } finally {
+      await input.transport.unlock(input.ownerToken);
+    }
+  }
+  const localPresent = await inventoryBlobs(
+    input.localStoreRoot,
+    manifest.blobs,
+  );
+  if (localPresent.size !== manifest.blobs.length) {
+    throw new Error("本地 release blob store 不完整或损坏");
+  }
+  await input.transport.lock(input.ownerToken);
+  try {
+    const present = await input.transport.inventory(manifest.blobs);
+    await input.transport.preflight(manifest, input.ownerToken);
+    const plan = planBlobTransfer(manifest.blobs, present);
+    let uploadedBytes = 0;
+    let uploadedBlobs = 0;
+    for (const blob of plan.missing) {
+      const offset = await input.transport.partial(blob);
+      if (offset < 0 || offset > blob.size) {
+        throw new Error(`远端返回无效 blob offset：${blob.digest}`);
+      }
+      if (offset < blob.size) {
+        uploadedBytes += await input.transport.upload(
+          blob,
+          offset,
+          blobPath(input.localStoreRoot, blob.digest),
+        );
+        uploadedBlobs += 1;
+      }
+    }
+    const totalBytes = plan.statistics.totalBytes;
+    const stagedDirectory = await input.transport.stage(
+      manifest,
+      manifestDigest(manifest),
+      input.ownerToken,
+    );
+    return {
+      mode: "blob-delta",
+      stagedDirectory,
+      statistics: {
+        totalBytes,
+        reusedBytes: totalBytes - uploadedBytes,
+        uploadedBytes,
+        totalBlobs: plan.statistics.totalBlobs,
+        reusedBlobs: plan.statistics.totalBlobs - uploadedBlobs,
+        uploadedBlobs,
+      },
+    };
+  } finally {
+    await input.transport.unlock(input.ownerToken);
+  }
+}
+
+const legacyCapabilities: DeliveryCapabilities = {
+  schemaVersions: [1],
+  registryByDigest: false,
+  blobDelta: false,
+  resumableBlobTransfer: false,
+};

package/template/packages/delivery/src/staging.ts

@@ -0,0 +1,241 @@
+import { randomBytes } from "node:crypto";
+import {
+  mkdir,
+  readFile,
+  rename,
+  rm,
+  stat,
+  statfs,
+  writeFile,
+} from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { createDeterministicTar } from "./archive.js";
+import { blobPath, inventoryBlobs, materializeArchive } from "./blob-store.js";
+import { releaseChecksumFile, verifyReleaseChecksums } from "./checksums.js";
+import { canonicalManifestJson, manifestDigest } from "./manifest.js";
+import type { DeliveryManifestV2, Sha256Digest } from "./types.js";
+
+export interface DeliveryLock {
+  directory: string;
+  ownerToken: string;
+}
+
+export async function acquireDeliveryLock(
+  installRoot: string,
+  ownerToken = randomBytes(24).toString("hex"),
+): Promise<DeliveryLock> {
+  if (!/^[a-f0-9]{48}$/.test(ownerToken))
+    throw new Error("lock owner token 无效");
+  const directory = join(resolve(installRoot), ".delivery-lock");
+  await mkdir(dirname(directory), { recursive: true });
+  try {
+    await mkdir(directory);
+    await writeFile(join(directory, "owner"), `${ownerToken}\n`, {
+      flag: "wx",
+      mode: 0o600,
+    });
+    return { directory, ownerToken };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "EEXIST") {
+      throw new Error("已有并发交付持有安装锁", { cause: error });
+    }
+    await rm(directory, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+export async function releaseDeliveryLock(lock: DeliveryLock): Promise<void> {
+  const owner = (await readFile(join(lock.directory, "owner"), "utf8")).trim();
+  if (owner !== lock.ownerToken)
+    throw new Error("交付锁 owner 不匹配，拒绝释放");
+  await rm(lock.directory, { recursive: true, force: false });
+}
+
+export async function stageRelease(input: {
+  installRoot: string;
+  storeRoot: string;
+  manifest: DeliveryManifestV2;
+  expectedManifestDigest: Sha256Digest;
+  lock: DeliveryLock;
+  reserveBytes?: number;
+}): Promise<string> {
+  const installRoot = resolve(input.installRoot);
+  await verifyDeliveryLock(input.lock, installRoot);
+  if (manifestDigest(input.manifest) !== input.expectedManifestDigest) {
+    throw new Error("stage 前 manifest digest 校验失败");
+  }
+  const present = await inventoryBlobs(input.storeRoot, input.manifest.blobs);
+  if (present.size !== input.manifest.blobs.length) {
+    throw new Error("远端 blob 不完整或损坏，拒绝 stage release");
+  }
+  await preflightReleaseSpace(
+    installRoot,
+    input.storeRoot,
+    input.manifest,
+    present,
+    input.reserveBytes,
+  );
+  const stagingRoot = join(installRoot, ".staging");
+  const releasesRoot = join(installRoot, "releases");
+  const nonce = randomBytes(12).toString("hex");
+  const stage = join(stagingRoot, `${input.manifest.releaseId}.${nonce}`);
+  const final = join(releasesRoot, input.manifest.releaseId);
+  await mkdir(stagingRoot, { recursive: true });
+  await mkdir(releasesRoot, { recursive: true });
+  try {
+    await mkdir(stage);
+    const content = join(stage, "release");
+    if (input.manifest.releaseFiles.length > 0) {
+      await materializeArchive(
+        input.storeRoot,
+        content,
+        input.manifest.releaseFiles,
+      );
+    } else {
+      await mkdir(content);
+    }
+    await createDeterministicTar(
+      join(content, "images.tar"),
+      input.manifest.dockerArchive.entries.map((entry) => ({
+        source: blobPath(input.storeRoot, entry.digest),
+        path: entry.path,
+        mode: entry.mode,
+      })),
+    );
+    if (
+      input.manifest.releaseFiles.some(
+        ({ path }) => path === releaseChecksumFile,
+      )
+    ) {
+      await verifyReleaseChecksums(content, [
+        ...input.manifest.releaseFiles.map(({ path }) => path),
+        "images.tar",
+      ]);
+    }
+    await writeFile(
+      join(stage, "delivery-manifest.v2.json"),
+      canonicalManifestJson(input.manifest),
+      { flag: "wx", mode: 0o644 },
+    );
+    await writeFile(
+      join(stage, "DELIVERY-MANIFEST.sha256"),
+      `${input.expectedManifestDigest}\n`,
+      { flag: "wx", mode: 0o644 },
+    );
+    try {
+      await rename(stage, final);
+    } catch (error) {
+      if (
+        !new Set(["EEXIST", "ENOTEMPTY"]).has(
+          (error as NodeJS.ErrnoException).code ?? "",
+        )
+      ) {
+        throw error;
+      }
+      const digest = (
+        await readFile(join(final, "DELIVERY-MANIFEST.sha256"), "utf8")
+      ).trim();
+      if (digest !== input.expectedManifestDigest) {
+        throw new Error("同 releaseId 已存在不同 manifest，拒绝覆盖", {
+          cause: error,
+        });
+      }
+      await rm(stage, { recursive: true, force: true });
+    }
+    return join(final, "release");
+  } catch (error) {
+    await rm(stage, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+export async function cleanupStaging(installRoot: string, lock: DeliveryLock) {
+  const root = resolve(installRoot);
+  await verifyDeliveryLock(lock, root);
+  await rm(join(root, ".staging"), { recursive: true, force: true });
+}
+
+export async function preflightReleaseSpace(
+  installRoot: string,
+  storeRoot: string,
+  manifest: DeliveryManifestV2,
+  present: ReadonlySet<Sha256Digest>,
+  reserveBytes = 512 * 1024 * 1024,
+): Promise<{
+  storeAvailableBytes: number;
+  storeRequiredBytes: number;
+  installAvailableBytes: number;
+  installRequiredBytes: number;
+}> {
+  const missingBytes = manifest.blobs
+    .filter(({ digest }) => !present.has(digest))
+    .reduce((total, { size }) => total + size, 0);
+  const releaseBytes = manifest.releaseFiles.reduce(
+    (total, { size }) => total + size,
+    0,
+  );
+  const metadataBytes = Buffer.byteLength(canonicalManifestJson(manifest)) + 80;
+  const stageBytes = archiveSizeBudget(manifest) + releaseBytes + metadataBytes;
+  if (
+    !Number.isSafeInteger(missingBytes + stageBytes + reserveBytes) ||
+    reserveBytes < 0
+  ) {
+    throw new Error("交付空间预算超出安全范围");
+  }
+  const resolvedInstall = resolve(installRoot);
+  const resolvedStore = resolve(storeRoot);
+  await mkdir(resolvedStore, { recursive: true });
+  const [installFile, storeFile, installFs, storeFs] = await Promise.all([
+    stat(resolvedInstall),
+    stat(resolvedStore),
+    statfs(resolvedInstall),
+    statfs(resolvedStore),
+  ]);
+  const sameFilesystem = installFile.dev === storeFile.dev;
+  const installAvailableBytes = installFs.bavail * installFs.bsize;
+  const storeAvailableBytes = storeFs.bavail * storeFs.bsize;
+  const storeRequiredBytes = sameFilesystem
+    ? missingBytes + stageBytes + reserveBytes
+    : missingBytes + reserveBytes;
+  const installRequiredBytes = sameFilesystem
+    ? storeRequiredBytes
+    : stageBytes + reserveBytes;
+  if (storeAvailableBytes < storeRequiredBytes) {
+    throw new Error(
+      `blob store 空间不足：需要 ${storeRequiredBytes}，可用 ${storeAvailableBytes}`,
+    );
+  }
+  if (!sameFilesystem && installAvailableBytes < installRequiredBytes) {
+    throw new Error(
+      `release stage 空间不足：需要 ${installRequiredBytes}，可用 ${installAvailableBytes}`,
+    );
+  }
+  return {
+    storeAvailableBytes,
+    storeRequiredBytes,
+    installAvailableBytes,
+    installRequiredBytes,
+  };
+}
+
+function archiveSizeBudget(manifest: DeliveryManifestV2): number {
+  const bytes = manifest.dockerArchive.entries.reduce(
+    (total, entry) =>
+      total + 512 + entry.size + ((512 - (entry.size % 512)) % 512),
+    1024,
+  );
+  if (!Number.isSafeInteger(bytes))
+    throw new Error("archive 空间预算超出安全范围");
+  return bytes;
+}
+
+export async function verifyDeliveryLock(
+  lock: DeliveryLock,
+  installRoot: string,
+): Promise<void> {
+  if (resolve(lock.directory) !== join(installRoot, ".delivery-lock")) {
+    throw new Error("交付锁不属于目标安装目录");
+  }
+  const owner = (await readFile(join(lock.directory, "owner"), "utf8")).trim();
+  if (owner !== lock.ownerToken) throw new Error("交付锁 owner 不匹配");
+}

package/template/packages/delivery/src/transfer.ts

@@ -0,0 +1,54 @@
+import type { ContentBlob, Sha256Digest, TransferStatistics } from "./types.js";
+
+export interface BlobTransferPlan {
+  missing: ContentBlob[];
+  statistics: TransferStatistics;
+}
+
+export function planBlobTransfer(
+  blobs: readonly ContentBlob[],
+  present: ReadonlySet<Sha256Digest>,
+): BlobTransferPlan {
+  const unique = new Map<Sha256Digest, ContentBlob>();
+  for (const blob of blobs) {
+    const previous = unique.get(blob.digest);
+    if (previous && previous.size !== blob.size) {
+      throw new Error(`同一 blob digest 声明了不同尺寸：${blob.digest}`);
+    }
+    unique.set(blob.digest, blob);
+  }
+  const all = [...unique.values()].sort((left, right) =>
+    left.digest.localeCompare(right.digest),
+  );
+  const missing = all.filter(({ digest }) => !present.has(digest));
+  const totalBytes = sum(all.map(({ size }) => size));
+  const uploadedBytes = sum(missing.map(({ size }) => size));
+  return {
+    missing,
+    statistics: {
+      totalBytes,
+      reusedBytes: totalBytes - uploadedBytes,
+      uploadedBytes,
+      totalBlobs: all.length,
+      reusedBlobs: all.length - missing.length,
+      uploadedBlobs: missing.length,
+    },
+  };
+}
+
+export function formatTransferStatistics(stats: TransferStatistics): string {
+  return [
+    `总字节=${stats.totalBytes}`,
+    `复用字节=${stats.reusedBytes}`,
+    `实际上传字节=${stats.uploadedBytes}`,
+    `blob=${stats.totalBlobs}`,
+    `复用blob=${stats.reusedBlobs}`,
+    `上传blob=${stats.uploadedBlobs}`,
+  ].join(" ");
+}
+
+function sum(values: readonly number[]): number {
+  const total = values.reduce((result, value) => result + value, 0);
+  if (!Number.isSafeInteger(total)) throw new Error("传输字节统计超出安全范围");
+  return total;
+}

package/template/packages/delivery/src/types.ts

@@ -0,0 +1,78 @@
+export type ReleasePlatform = "linux/amd64" | "linux/arm64";
+export type Sha256Digest = `sha256:${string}`;
+
+export interface ContentBlob {
+  digest: Sha256Digest;
+  size: number;
+}
+
+export interface ArchiveEntry extends ContentBlob {
+  path: string;
+  mode: number;
+}
+
+export interface ReleaseImageV2 {
+  role: string;
+  platform: ReleasePlatform;
+  configDigest: Sha256Digest;
+  registry?: {
+    repository: string;
+    digest: Sha256Digest;
+    layers: ContentBlob[];
+  };
+}
+
+export interface FullBundleFallback {
+  schemaVersion: 1;
+  file: string;
+  digest: Sha256Digest;
+  size: number;
+}
+
+export interface DeliveryManifestV2 {
+  schemaVersion: 2;
+  minimumReaderVersion: 2;
+  product: string;
+  releaseId: string;
+  gitSha: string;
+  platform: ReleasePlatform;
+  createdAt: string;
+  images: ReleaseImageV2[];
+  dockerArchive: {
+    format: "oci-image-layout-v1";
+    entries: ArchiveEntry[];
+  };
+  releaseFiles: ArchiveEntry[];
+  blobs: ContentBlob[];
+  compatibility: {
+    fullBundle?: FullBundleFallback;
+  };
+}
+
+export interface DeliveryCapabilities {
+  schemaVersions: readonly number[];
+  registryByDigest: boolean;
+  blobDelta: boolean;
+  resumableBlobTransfer: boolean;
+}
+
+export interface TransferStatistics {
+  totalBytes: number;
+  reusedBytes: number;
+  uploadedBytes: number;
+  totalBlobs: number;
+  reusedBlobs: number;
+  uploadedBlobs: number;
+}
+
+export interface RegistryTransferStatistics {
+  totalBytes: number;
+  reusedBytes: number;
+  uploadedBytes: null;
+  descriptorPulledBytes: number;
+  totalBlobs: number;
+  reusedBlobs: number;
+  uploadedBlobs: null;
+  descriptorPulledBlobs: number;
+  measurement: "docker-layer-status";
+}

package/template/packages/delivery/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "lib": ["esnext"],
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "types": ["bun"]
+  },
+  "include": ["src/**/*", "index.ts"],
+  "exclude": ["node_modules", "dist"]
+}