create-windy 0.6.6 → 0.6.8

package/template/packages/delivery/src/archive.ts

@@ -0,0 +1,247 @@
+import { createReadStream, createWriteStream } from "node:fs";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { createGzip } from "node:zlib";
+import { dirname, resolve } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { assertRelativePortablePath } from "./paths.js";
+
+export interface DeterministicArchiveEntry {
+  source: string;
+  path: string;
+  mode?: number;
+}
+
+export async function createDeterministicTarGz(
+  output: string,
+  entries: readonly DeterministicArchiveEntry[],
+): Promise<void> {
+  const target = resolve(output);
+  const temporaryTar = `${target}.tar.tmp`;
+  const temporaryGzip = `${target}.tmp`;
+  await mkdir(dirname(target), { recursive: true });
+  await assertOutputAbsent(target);
+  const normalized = await normalizeEntries(entries);
+  try {
+    await writeTar(temporaryTar, normalized);
+    await pipeline(
+      createReadStream(temporaryTar),
+      createGzip({ level: 9 }),
+      createWriteStream(temporaryGzip, { flags: "wx", mode: 0o644 }),
+    );
+    await rename(temporaryGzip, target);
+  } finally {
+    await rm(temporaryTar, { force: true });
+    await rm(temporaryGzip, { force: true });
+  }
+}
+
+export async function createDeterministicTar(
+  output: string,
+  entries: readonly DeterministicArchiveEntry[],
+): Promise<void> {
+  const target = resolve(output);
+  const temporary = `${target}.${process.pid}.tmp`;
+  await mkdir(dirname(target), { recursive: true });
+  await assertOutputAbsent(target);
+  try {
+    await writeTar(temporary, await normalizeEntries(entries));
+    await rename(temporary, target);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+export async function listTarGzEntries(path: string): Promise<string[]> {
+  const process = Bun.spawn(["tar", "-tzf", resolve(path)], {
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...processEnvWithoutCopyfile(), COPYFILE_DISABLE: "1" },
+  });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    process.exited,
+    new Response(process.stdout).text(),
+    new Response(process.stderr).text(),
+  ]);
+  if (exitCode !== 0) throw new Error(`无法读取归档：${stderr.trim()}`);
+  return stdout
+    .split(/\r?\n/)
+    .filter(Boolean)
+    .map((entry) => entry.replace(/^\.\//, "").replace(/\/$/, ""))
+    .filter(Boolean)
+    .map(assertRelativePortablePath)
+    .sort();
+}
+
+export async function extractPortableTar(
+  archive: string,
+  destination: string,
+  compressed = false,
+): Promise<string[]> {
+  const source = resolve(archive);
+  const target = resolve(destination);
+  const flag = compressed ? "-tzf" : "-tf";
+  const verboseFlag = compressed ? "-tvzf" : "-tvf";
+  const [listing, verbose] = await Promise.all([
+    captureTar([flag, source]),
+    captureTar([verboseFlag, source]),
+  ]);
+  const paths = listing
+    .split(/\r?\n/)
+    .filter(Boolean)
+    .map((entry) => entry.replace(/^\.\//, "").replace(/\/$/, ""))
+    .filter(Boolean)
+    .map(assertRelativePortablePath);
+  for (const line of verbose.split(/\r?\n/).filter(Boolean)) {
+    if (!new Set(["-", "d"]).has(line[0] ?? "")) {
+      throw new Error("归档只允许普通文件与目录");
+    }
+  }
+  await mkdir(target, { recursive: false });
+  const extractFlag = compressed ? "-xzf" : "-xf";
+  await captureTar([extractFlag, source, "-C", target]);
+  return paths.sort();
+}
+
+interface NormalizedEntry {
+  source: string;
+  path: string;
+  size: number;
+  mode: number;
+}
+
+async function normalizeEntries(
+  entries: readonly DeterministicArchiveEntry[],
+): Promise<NormalizedEntry[]> {
+  const output: NormalizedEntry[] = [];
+  for (const entry of entries) {
+    const path = assertRelativePortablePath(entry.path);
+    const info = await lstat(entry.source);
+    if (!info.isFile() || info.isSymbolicLink()) {
+      throw new Error(`确定性归档只允许普通文件：${path}`);
+    }
+    const mode = entry.mode ?? (info.mode & 0o111 ? 0o755 : 0o644);
+    output.push({ source: resolve(entry.source), path, size: info.size, mode });
+  }
+  output.sort((left, right) => left.path.localeCompare(right.path));
+  if (new Set(output.map(({ path }) => path)).size !== output.length) {
+    throw new Error("确定性归档路径重复");
+  }
+  return output;
+}
+
+async function writeTar(path: string, entries: readonly NormalizedEntry[]) {
+  const file = await open(path, "wx", 0o644);
+  try {
+    let position = 0;
+    for (const entry of entries) {
+      const header = tarHeader(entry);
+      await file.write(header, 0, header.length, position);
+      position += header.length;
+      const content = await readFile(entry.source);
+      await file.write(content, 0, content.length, position);
+      position += content.length;
+      const padding = (512 - (content.length % 512)) % 512;
+      if (padding) {
+        await file.write(Buffer.alloc(padding), 0, padding, position);
+        position += padding;
+      }
+    }
+    await file.write(Buffer.alloc(1024), 0, 1024, position);
+    await file.sync();
+  } finally {
+    await file.close();
+  }
+}
+
+function tarHeader(entry: NormalizedEntry): Buffer {
+  const header = Buffer.alloc(512);
+  const { name, prefix } = splitTarPath(entry.path);
+  writeString(header, 0, 100, name);
+  writeOctal(header, 100, 8, entry.mode);
+  writeOctal(header, 108, 8, 0);
+  writeOctal(header, 116, 8, 0);
+  writeOctal(header, 124, 12, entry.size);
+  writeOctal(header, 136, 12, 0);
+  header.fill(0x20, 148, 156);
+  header[156] = "0".charCodeAt(0);
+  writeString(header, 257, 6, "ustar\0");
+  writeString(header, 263, 2, "00");
+  writeString(header, 265, 32, "root");
+  writeString(header, 297, 32, "root");
+  writeString(header, 345, 155, prefix);
+  const checksum = header.reduce((sum, byte) => sum + byte, 0);
+  writeChecksum(header, checksum);
+  return header;
+}
+
+function splitTarPath(path: string): { name: string; prefix: string } {
+  if (Buffer.byteLength(path) <= 100) return { name: path, prefix: "" };
+  for (
+    let index = path.lastIndexOf("/");
+    index > 0;
+    index = path.lastIndexOf("/", index - 1)
+  ) {
+    const prefix = path.slice(0, index);
+    const name = path.slice(index + 1);
+    if (Buffer.byteLength(prefix) <= 155 && Buffer.byteLength(name) <= 100) {
+      return { name, prefix };
+    }
+  }
+  throw new Error(`归档路径超过 ustar 限制：${path}`);
+}
+
+function writeString(
+  buffer: Buffer,
+  offset: number,
+  length: number,
+  value: string,
+) {
+  const bytes = Buffer.from(value);
+  if (bytes.length > length) throw new Error("tar header 字段超长");
+  bytes.copy(buffer, offset);
+}
+
+function writeOctal(
+  buffer: Buffer,
+  offset: number,
+  length: number,
+  value: number,
+) {
+  const text = value.toString(8).padStart(length - 1, "0");
+  writeString(buffer, offset, length, `${text}\0`);
+}
+
+function writeChecksum(buffer: Buffer, value: number) {
+  const text = value.toString(8).padStart(6, "0");
+  writeString(buffer, 148, 8, `${text}\0 `);
+}
+
+async function assertOutputAbsent(path: string): Promise<void> {
+  try {
+    await lstat(path);
+    throw new Error(`归档已存在，拒绝覆盖：${path}`);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+}
+
+function processEnvWithoutCopyfile(): NodeJS.ProcessEnv {
+  const env = { ...process.env };
+  delete env.COPYFILE_DISABLE;
+  return env;
+}
+
+async function captureTar(args: readonly string[]): Promise<string> {
+  const child = Bun.spawn(["tar", ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...processEnvWithoutCopyfile(), COPYFILE_DISABLE: "1" },
+  });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    child.exited,
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+  ]);
+  if (exitCode !== 0) throw new Error(`tar 命令失败：${stderr.trim()}`);
+  return stdout;
+}

package/template/packages/delivery/src/blob-store.ts

@@ -0,0 +1,242 @@
+import {
+  copyFile,
+  lstat,
+  link,
+  mkdir,
+  open,
+  rename,
+  rm,
+  stat,
+} from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { digestHex, sha256File } from "./digest.js";
+import { assertRelativePortablePath } from "./paths.js";
+import type { ArchiveEntry, ContentBlob, Sha256Digest } from "./types.js";
+
+export function blobPath(root: string, digest: Sha256Digest): string {
+  return join(resolve(root), "blobs", "sha256", digestHex(digest));
+}
+
+export async function ingestArchiveFiles(
+  archiveRoot: string,
+  paths: readonly string[],
+  storeRoot: string,
+): Promise<{ entries: ArchiveEntry[]; blobs: ContentBlob[] }> {
+  const entries: ArchiveEntry[] = [];
+  const blobs = new Map<Sha256Digest, ContentBlob>();
+  for (const rawPath of [...paths].sort()) {
+    const path = assertRelativePortablePath(rawPath);
+    const source = join(resolve(archiveRoot), path);
+    const info = await lstat(source);
+    if (!info.isFile() || info.isSymbolicLink()) {
+      throw new Error(`Docker archive 只允许普通文件：${path}`);
+    }
+    const digest = await sha256File(source);
+    const blob = { digest, size: info.size };
+    const existing = blobs.get(digest);
+    if (existing && existing.size !== info.size) {
+      throw new Error(`相同 digest 出现不同尺寸：${digest}`);
+    }
+    blobs.set(digest, blob);
+    await installBlob(source, storeRoot, blob);
+    entries.push({ ...blob, path, mode: info.mode & 0o111 ? 0o755 : 0o644 });
+  }
+  return {
+    entries,
+    blobs: [...blobs.values()].sort((left, right) =>
+      left.digest.localeCompare(right.digest),
+    ),
+  };
+}
+
+export async function inventoryBlobs(
+  storeRoot: string,
+  blobs: readonly ContentBlob[],
+): Promise<Set<Sha256Digest>> {
+  const present = new Set<Sha256Digest>();
+  for (const blob of blobs) {
+    const path = blobPath(storeRoot, blob.digest);
+    try {
+      const info = await stat(path);
+      if (
+        info.isFile() &&
+        info.size === blob.size &&
+        (await sha256File(path)) === blob.digest
+      ) {
+        present.add(blob.digest);
+      }
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+  return present;
+}
+
+export async function materializeArchive(
+  storeRoot: string,
+  destination: string,
+  entries: readonly ArchiveEntry[],
+): Promise<void> {
+  const target = resolve(destination);
+  await mkdir(target, { recursive: false });
+  try {
+    for (const entry of entries) {
+      const relativePath = assertRelativePortablePath(entry.path);
+      const source = blobPath(storeRoot, entry.digest);
+      await verifyBlob(source, entry);
+      const output = join(target, relativePath);
+      await mkdir(dirname(output), { recursive: true });
+      try {
+        await link(source, output);
+      } catch (error) {
+        if (
+          !new Set(["EXDEV", "EPERM", "EACCES"]).has(
+            (error as NodeJS.ErrnoException).code ?? "",
+          )
+        ) {
+          throw error;
+        }
+        await copyFile(source, output, 0);
+      }
+    }
+  } catch (error) {
+    await rm(target, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+export async function receiveBlobChunk(input: {
+  storeRoot: string;
+  blob: ContentBlob;
+  offset: number;
+  bytes: Uint8Array;
+}): Promise<{ complete: boolean; receivedBytes: number }> {
+  return receiveBlobStream({
+    storeRoot: input.storeRoot,
+    blob: input.blob,
+    offset: input.offset,
+    chunks: oneChunk(input.bytes),
+  });
+}
+
+export async function receiveBlobStream(input: {
+  storeRoot: string;
+  blob: ContentBlob;
+  offset: number;
+  chunks: AsyncIterable<Uint8Array>;
+}): Promise<{ complete: boolean; receivedBytes: number }> {
+  const { storeRoot, blob, offset, chunks } = input;
+  if (!Number.isSafeInteger(offset) || offset < 0 || offset > blob.size) {
+    throw new Error("blob resume offset 无效");
+  }
+  const finalPath = blobPath(storeRoot, blob.digest);
+  const partialPath = `${finalPath}.partial`;
+  await mkdir(dirname(finalPath), { recursive: true });
+  if (await validFinalBlob(finalPath, blob)) {
+    return { complete: true, receivedBytes: 0 };
+  }
+  const currentSize = await fileSize(partialPath);
+  if (currentSize !== offset) {
+    throw new Error(
+      `blob resume offset 不匹配：期望 ${currentSize}，收到 ${offset}`,
+    );
+  }
+  const file = await open(partialPath, currentSize === 0 ? "wx" : "r+");
+  let receivedBytes = 0;
+  try {
+    for await (const chunk of chunks) {
+      if (currentSize + receivedBytes + chunk.byteLength > blob.size) {
+        throw new Error("blob chunk 超出声明尺寸");
+      }
+      await file.write(chunk, 0, chunk.byteLength, currentSize + receivedBytes);
+      receivedBytes += chunk.byteLength;
+    }
+    await file.sync();
+  } catch (error) {
+    await file.truncate(currentSize);
+    await file.sync();
+    throw error;
+  } finally {
+    await file.close();
+  }
+  const size = currentSize + receivedBytes;
+  if (size !== blob.size) return { complete: false, receivedBytes };
+  if ((await sha256File(partialPath)) !== blob.digest) {
+    await rm(partialPath, { force: true });
+    throw new Error(`blob digest 不匹配：${blob.digest}`);
+  }
+  await rename(partialPath, finalPath);
+  return { complete: true, receivedBytes };
+}
+
+export async function partialBlobSize(
+  storeRoot: string,
+  blob: ContentBlob,
+): Promise<number> {
+  const finalPath = blobPath(storeRoot, blob.digest);
+  if (await validFinalBlob(finalPath, blob)) return blob.size;
+  const size = await fileSize(`${finalPath}.partial`);
+  if (size > blob.size)
+    throw new Error(`partial blob 尺寸超限：${blob.digest}`);
+  return size;
+}
+
+async function installBlob(source: string, root: string, blob: ContentBlob) {
+  const target = blobPath(root, blob.digest);
+  if (await validFinalBlob(target, blob)) return;
+  await mkdir(dirname(target), { recursive: true });
+  const temporary = `${target}.${process.pid}.tmp`;
+  try {
+    await copyFile(source, temporary, 0);
+    await verifyBlob(temporary, blob);
+    try {
+      await rename(temporary, target);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    }
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+async function verifyBlob(path: string, blob: ContentBlob): Promise<void> {
+  const info = await stat(path);
+  if (
+    !info.isFile() ||
+    info.size !== blob.size ||
+    (await sha256File(path)) !== blob.digest
+  ) {
+    throw new Error(`blob 完整性校验失败：${blob.digest}`);
+  }
+}
+
+async function validFinalBlob(
+  path: string,
+  blob: ContentBlob,
+): Promise<boolean> {
+  let info: Awaited<ReturnType<typeof stat>>;
+  try {
+    info = await stat(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw error;
+  }
+  return (
+    info.isFile() &&
+    info.size === blob.size &&
+    (await sha256File(path)) === blob.digest
+  );
+}
+
+async function fileSize(path: string): Promise<number> {
+  try {
+    return (await stat(path)).size;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0;
+    throw error;
+  }
+}
+
+async function* oneChunk(bytes: Uint8Array): AsyncIterable<Uint8Array> {
+  yield bytes;
+}

package/template/packages/delivery/src/builder.ts

@@ -0,0 +1,199 @@
+import {
+  chmod,
+  copyFile,
+  lstat,
+  mkdir,
+  readdir,
+  rm,
+  stat,
+} from "node:fs/promises";
+import { basename, join, relative, resolve, sep } from "node:path";
+import {
+  createDeterministicTar,
+  createDeterministicTarGz,
+  type DeterministicArchiveEntry,
+} from "./archive.js";
+import { ingestArchiveFiles } from "./blob-store.js";
+import { sha256File } from "./digest.js";
+import { releaseChecksumFile, writeReleaseChecksums } from "./checksums.js";
+import { validateManifest } from "./manifest.js";
+import { assertRelativePortablePath } from "./paths.js";
+import type {
+  DeliveryManifestV2,
+  FullBundleFallback,
+  ReleaseImageV2,
+  ReleasePlatform,
+} from "./types.js";
+
+export interface BuildDeliveryManifestInput {
+  archiveRoot: string;
+  releaseRoot: string;
+  storeRoot: string;
+  product: string;
+  releaseId: string;
+  gitSha: string;
+  platform: ReleasePlatform;
+  createdAt: string;
+  images: ReleaseImageV2[];
+  fullBundle?: FullBundleFallback;
+}
+
+export async function buildDeliveryManifest(
+  input: BuildDeliveryManifestInput,
+): Promise<DeliveryManifestV2> {
+  const paths = await listArchiveFiles(input.archiveRoot);
+  const graph = await ingestArchiveFiles(
+    input.archiveRoot,
+    paths,
+    input.storeRoot,
+  );
+  assertOciBlobIdentity(graph.entries);
+  const releasePaths = (await listArchiveFiles(input.releaseRoot)).filter(
+    (path) => path !== "images.tar",
+  );
+  const releaseGraph = await ingestArchiveFiles(
+    input.releaseRoot,
+    releasePaths,
+    input.storeRoot,
+  );
+  return validateManifest({
+    schemaVersion: 2,
+    minimumReaderVersion: 2,
+    product: input.product,
+    releaseId: input.releaseId,
+    gitSha: input.gitSha,
+    platform: input.platform,
+    createdAt: input.createdAt,
+    images: input.images,
+    dockerArchive: {
+      format: "oci-image-layout-v1",
+      entries: graph.entries,
+    },
+    releaseFiles: releaseGraph.entries,
+    blobs: mergeBlobs(graph.blobs, releaseGraph.blobs),
+    compatibility: {
+      ...(input.fullBundle ? { fullBundle: input.fullBundle } : {}),
+    },
+  });
+}
+
+export async function createFullBundleFallback(
+  releaseRoot: string,
+  output: string,
+): Promise<FullBundleFallback> {
+  const root = resolve(releaseRoot);
+  const target = resolve(output);
+  if (target === root || target.startsWith(`${root}${sep}`)) {
+    throw new Error("完整 bundle 输出不得位于 release 输入目录内");
+  }
+  const paths = await listArchiveFiles(root);
+  const entries: DeterministicArchiveEntry[] = paths.map((path) => ({
+    source: join(root, path),
+    path,
+  }));
+  await mkdir(resolve(target, ".."), { recursive: true });
+  await createDeterministicTarGz(target, entries);
+  const info = await stat(target);
+  return {
+    schemaVersion: 1,
+    file: basename(target),
+    digest: await sha256File(target),
+    size: info.size,
+  };
+}
+
+export async function prepareDeterministicRelease(
+  inputRoot: string,
+  ociRoot: string,
+  outputRoot: string,
+): Promise<void> {
+  const sourceRoot = resolve(inputRoot);
+  const targetRoot = resolve(outputRoot);
+  const releasePaths = (await listArchiveFiles(sourceRoot)).filter(
+    (path) => path !== "images.tar" && path !== releaseChecksumFile,
+  );
+  const ociPaths = await listArchiveFiles(ociRoot);
+  await mkdir(targetRoot);
+  try {
+    for (const path of releasePaths) {
+      const source = join(sourceRoot, path);
+      const target = join(targetRoot, path);
+      const info = await lstat(source);
+      await mkdir(resolve(target, ".."), { recursive: true });
+      await copyFile(source, target, 0);
+      await chmod(target, info.mode & 0o111 ? 0o755 : 0o644);
+    }
+    await createDeterministicTar(
+      join(targetRoot, "images.tar"),
+      ociPaths.map((path) => ({ source: join(ociRoot, path), path })),
+    );
+    await writeReleaseChecksums(targetRoot, [...releasePaths, "images.tar"]);
+  } catch (error) {
+    await rm(targetRoot, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+function mergeBlobs(
+  left: DeliveryManifestV2["blobs"],
+  right: DeliveryManifestV2["blobs"],
+): DeliveryManifestV2["blobs"] {
+  const merged = new Map(left.map((blob) => [blob.digest, blob]));
+  for (const blob of right) {
+    const current = merged.get(blob.digest);
+    if (current && current.size !== blob.size) {
+      throw new Error(`相同 digest 出现不同尺寸：${blob.digest}`);
+    }
+    merged.set(blob.digest, blob);
+  }
+  return [...merged.values()].sort((leftBlob, rightBlob) =>
+    leftBlob.digest.localeCompare(rightBlob.digest),
+  );
+}
+
+export async function listArchiveFiles(root: string): Promise<string[]> {
+  const resolvedRoot = resolve(root);
+  const output: string[] = [];
+  await walk(resolvedRoot, resolvedRoot, output);
+  if (output.length === 0) throw new Error("Docker archive 目录为空");
+  return output.sort();
+}
+
+async function walk(root: string, directory: string, output: string[]) {
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const absolute = join(directory, entry.name);
+    const info = await lstat(absolute);
+    if (info.isSymbolicLink()) throw new Error("Docker archive 不允许符号链接");
+    if (entry.isDirectory()) {
+      await walk(root, absolute, output);
+    } else if (entry.isFile()) {
+      output.push(
+        assertRelativePortablePath(
+          relative(root, absolute).split(sep).join("/"),
+        ),
+      );
+    } else {
+      throw new Error("Docker archive 包含不支持的文件类型");
+    }
+  }
+}
+
+function assertOciBlobIdentity(
+  entries: DeliveryManifestV2["dockerArchive"]["entries"],
+) {
+  const paths = new Set(entries.map(({ path }) => path));
+  for (const required of ["index.json", "oci-layout"]) {
+    if (!paths.has(required))
+      throw new Error(`Docker OCI archive 缺少 ${required}`);
+  }
+  let blobCount = 0;
+  for (const entry of entries) {
+    const match = /^blobs\/sha256\/([a-f0-9]{64})$/.exec(entry.path);
+    if (!match) continue;
+    blobCount += 1;
+    if (entry.digest !== `sha256:${match[1]}`) {
+      throw new Error(`OCI blob 路径与内容 digest 不一致：${entry.path}`);
+    }
+  }
+  if (blobCount === 0) throw new Error("Docker OCI archive 不含内容 blob");
+}