@procedures/snapshot 0.0.0-stage → 0.2.0

package/src/procedures/snapshot/create.ts

@@ -0,0 +1,354 @@
+/**
+ * snapshot.create procedure
+ *
+ * Create an environment snapshot and upload to S3.
+ */
+
+import { existsSync, mkdtempSync, rmSync, readFileSync, writeFileSync, statSync } from "node:fs";
+import { open } from "node:fs/promises";
+import { join, basename, dirname, resolve } from "node:path";
+import { tmpdir, hostname } from "node:os";
+import { randomUUID } from "node:crypto";
+import * as tar from "tar";
+import type { ProcedureContext } from "@procedures/client";
+import { assertSnapshotName, isExcluded } from "./names.js";
+import { readPart, sha256File } from "./archive-io.js";
+import { gitText, pnpmVersion } from "./commands.js";
+import type {
+  SnapshotCreateInput,
+  SnapshotCreateOutput,
+  SnapshotMetadata,
+  RepositoryInfo,
+  SnapshotPreset,
+} from "../../types.js";
+
+/** The size of each part of a multipart upload (the S3 minimum) */
+const PART_SIZE = 5 * 1024 * 1024;
+
+/**
+ * Create a snapshot of the environment
+ */
+export async function snapshotCreate(
+  input: SnapshotCreateInput,
+  ctx: ProcedureContext
+): Promise<SnapshotCreateOutput> {
+  // A plain name: it goes into S3 keys and the snapshot id (a "/" made ids that restore,
+  // diff and delete never found: deep dive DATA-1, DATA-16)
+  assertSnapshotName("name", input.name);
+  const id = `${input.name}-${Date.now()}-${randomUUID().slice(0, 8)}`;
+  const workDir = mkdtempSync(join(tmpdir(), "snapshot-"));
+
+  try {
+
+    // Get repository paths to snapshot
+    const repoPaths = input.paths || (await getEcosystemPaths(ctx));
+
+    // Gather repository info
+    const repositories = await Promise.all(
+      repoPaths.map((path) => getRepositoryInfo(path, ctx))
+    );
+
+    // Get environment info
+    const environment = await getEnvironmentInfo(ctx.signal);
+
+    // Create metadata
+    const metadata: SnapshotMetadata = {
+      id,
+      name: input.name,
+      preset: input.preset,
+      createdAt: new Date().toISOString(),
+      environment,
+      repositories,
+      checksum: "", // Will be calculated after archive creation
+      archiveSize: 0,
+      description: input.description,
+    };
+
+    // Write metadata to work directory
+    const metadataPath = join(workDir, "metadata.json");
+    writeFileSync(metadataPath, JSON.stringify(metadata, null, 2));
+
+    // Create archive based on preset
+    const archivePath = join(workDir, `${id}.tar.gz`);
+    await createArchive(archivePath, repoPaths, input.preset, workDir);
+
+    // Calculate checksum (a stream: the archive is never in memory as a whole)
+    const checksum = await sha256File(archivePath);
+    const archiveSize = statSync(archivePath).size;
+
+    // Update metadata with checksum
+    metadata.checksum = checksum;
+    metadata.archiveSize = archiveSize;
+    writeFileSync(metadataPath, JSON.stringify(metadata, null, 2));
+
+    // Upload to S3
+    const uploadStart = Date.now();
+    const s3Key = `snapshots/${input.name}/${id}.tar.gz`;
+
+    // Use multipart upload for large files (> 5MB). The parts are read from the file, one at a
+    // time (deep dive DATA-19).
+    if (archiveSize > PART_SIZE) {
+      await multipartUpload(ctx, input.bucket, s3Key, archivePath, archiveSize);
+    } else {
+      await ctx.client.call(["s3", "upload"], {
+        bucket: input.bucket,
+        key: s3Key,
+        body: readFileSync(archivePath).toString("base64"),
+        base64: true,
+        contentType: "application/gzip",
+        metadata: {
+          snapshotId: id,
+          snapshotName: input.name,
+          preset: input.preset,
+          checksum,
+        },
+      });
+    }
+
+    // Also upload metadata separately for quick listing
+    await ctx.client.call(["s3", "upload"], {
+      bucket: input.bucket,
+      key: `snapshots/${input.name}/${id}.metadata.json`,
+      body: JSON.stringify(metadata, null, 2),
+      contentType: "application/json",
+    });
+
+    const uploadDuration = Date.now() - uploadStart;
+
+    return {
+      id,
+      location: `s3://${input.bucket}/${s3Key}`,
+      metadata,
+      uploadDuration,
+    };
+  } finally {
+    // Cleanup work directory
+    if (existsSync(workDir)) {
+      rmSync(workDir, { recursive: true, force: true });
+    }
+  }
+}
+
+/**
+ * Get ecosystem repository paths from manifest
+ */
+async function getEcosystemPaths(_ctx: ProcedureContext): Promise<string[]> {
+  // TODO: Read from ecosystem manifest
+  // For now, return current working directory
+  return [process.cwd()];
+}
+
+/**
+ * Get repository info for a path
+ */
+async function getRepositoryInfo(
+  repoPath: string,
+  ctx: ProcedureContext
+): Promise<RepositoryInfo> {
+  try {
+    const result = await ctx.client.call<
+      { cwd?: string },
+      { branch: string; ahead: number; behind: number; files: unknown[]; clean: boolean }
+    >(["git", "status"], { cwd: repoPath });
+
+    // The commit hash ("" for a repository with no commit) and the remote URL (none without "origin")
+    const commit = (await gitText(["rev-parse", "HEAD"], repoPath, ctx.signal)) ?? "";
+    const remoteUrl = await gitText(["remote", "get-url", "origin"], repoPath, ctx.signal);
+
+    // Get stash count
+    let stashCount = 0;
+    try {
+      const stashList = await ctx.client.call<
+        { cwd?: string },
+        { stashes: unknown[]; count: number }
+      >(["git", "stash", "list"], { cwd: repoPath });
+      stashCount = stashList.count;
+    } catch {
+      // No stashes
+    }
+
+    // Get package name from package.json
+    let name = basename(repoPath);
+    try {
+      const pkgPath = join(repoPath, "package.json");
+      if (existsSync(pkgPath)) {
+        const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
+        name = pkg.name || name;
+      }
+    } catch {
+      // Use directory name
+    }
+
+    return {
+      path: repoPath,
+      name,
+      branch: result.branch,
+      commit,
+      dirty: !result.clean,
+      stashCount,
+      remoteUrl,
+      ahead: result.ahead,
+      behind: result.behind,
+    };
+  } catch (error) {
+    // Not a git repo or error
+    return {
+      path: repoPath,
+      name: basename(repoPath),
+      branch: "",
+      commit: "",
+      dirty: false,
+      stashCount: 0,
+      ahead: 0,
+      behind: 0,
+    };
+  }
+}
+
+/**
+ * Get environment info
+ */
+async function getEnvironmentInfo(signal?: AbortSignal): Promise<SnapshotMetadata["environment"]> {
+  return {
+    os: process.platform,
+    nodeVersion: process.version,
+    pnpmVersion: await pnpmVersion(signal),
+    username: process.env["USER"] || process.env["USERNAME"] || "unknown",
+    hostname: hostname(),
+  };
+}
+
+/**
+ * Create tar archive based on preset
+ */
+async function createArchive(
+  archivePath: string,
+  repoPaths: string[],
+  preset: SnapshotPreset,
+  workDir: string
+): Promise<void> {
+  // Only archive repositories that actually exist on disk.
+  const existingRepos = repoPaths.filter((p) => existsSync(p));
+
+  // tar.create accepts a single `cwd`, and each entry is stored relative to it.
+  // To store each repository as a top-level `<basename>/...` entry we must run
+  // with `cwd` set to the directory that contains the repositories (the
+  // ecosystem root), not the fresh temp `workDir` the repos are not inside.
+  const parents = new Set(existingRepos.map((p) => resolve(dirname(p))));
+  if (parents.size > 1) {
+    throw new Error(
+      `snapshot.create requires all repositories to share a parent directory; got multiple roots: ${[
+        ...parents,
+      ].join(", ")}`
+    );
+  }
+  const cwd = existingRepos.length > 0 ? resolve(dirname(existingRepos[0]!)) : workDir;
+
+  // Create gzip options based on preset
+  const gzipOpts = { level: preset === "heavy" ? 6 : 9 };
+
+  // Build exclude patterns based on preset
+  const excludePatterns: string[] = [];
+
+  if (preset === "light") {
+    // Exclude node_modules and pnpm store
+    excludePatterns.push("node_modules", ".pnpm-store", "dist", "*.log");
+  } else if (preset === "medium") {
+    // Include node_modules but exclude pnpm store
+    excludePatterns.push(".pnpm-store", "*.log");
+  }
+  // heavy preset includes everything
+
+  await tar.create(
+    {
+      gzip: gzipOpts,
+      file: archivePath,
+      cwd,
+      // Whole path segments, and "*.ext" for file endings. (Before, `path.includes("dist")`
+      // left out src/distance.ts, and "*.log" matched nothing: deep dive DATA-3.)
+      filter: (path) => !isExcluded(path, excludePatterns),
+    },
+    existingRepos.map((p) => basename(p))
+  );
+}
+
+/**
+ * Multipart upload for large files. Each part is read from the file when it goes up.
+ */
+async function multipartUpload(
+  ctx: ProcedureContext,
+  bucket: string,
+  key: string,
+  archivePath: string,
+  archiveSize: number
+): Promise<void> {
+  // Initialize multipart upload
+  const initResult = await ctx.client.call<
+    { bucket: string; key: string; contentType?: string },
+    { uploadId: string }
+  >(["s3", "multipart", "init"], {
+    bucket,
+    key,
+    contentType: "application/gzip",
+  });
+
+  const uploadId = initResult.uploadId;
+  const parts: Array<{ etag: string; partNumber: number }> = [];
+  const file = await open(archivePath, "r");
+
+  try {
+    // Upload parts
+    let partNumber = 1;
+    for (let offset = 0; offset < archiveSize; offset += PART_SIZE) {
+      const chunk = await readPart(file, offset, PART_SIZE);
+
+      const partResult = await ctx.client.call<
+        { bucket: string; key: string; uploadId: string; partNumber: number; body: string },
+        { etag: string; partNumber: number }
+      >(["s3", "multipart", "upload"], {
+        bucket,
+        key,
+        uploadId,
+        partNumber,
+        body: chunk.toString("base64"),
+      });
+
+      parts.push({
+        etag: partResult.etag,
+        partNumber: partResult.partNumber,
+      });
+
+      partNumber++;
+    }
+
+    // Complete multipart upload
+    await ctx.client.call(["s3", "multipart", "complete"], {
+      bucket,
+      key,
+      uploadId,
+      parts,
+    });
+  } catch (error) {
+    // Abort on failure. The error of the upload stays the error: before, a failed abort
+    // replaced it.
+    try {
+      await ctx.client.call(["s3", "multipart", "abort"], {
+        bucket,
+        key,
+        uploadId,
+      });
+    } catch (abortError) {
+      throw new Error(
+        `${messageOf(error)} (the abort of upload ${uploadId} also failed: ${messageOf(abortError)})`,
+        { cause: error }
+      );
+    }
+    throw error;
+  } finally {
+    await file.close();
+  }
+}
+
+function messageOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/procedures/snapshot/delete.ts

@@ -0,0 +1,40 @@
+/**
+ * snapshot.delete procedure
+ *
+ * Delete a snapshot from S3.
+ */
+
+import type { ProcedureContext } from "@procedures/client";
+import type { SnapshotDeleteInput, SnapshotDeleteOutput } from "../../types.js";
+import { listAllObjects, findSnapshotKeys } from "./s3-lookup.js";
+
+/**
+ * Delete a snapshot from S3
+ */
+export async function snapshotDelete(
+  input: SnapshotDeleteInput,
+  ctx: ProcedureContext
+): Promise<SnapshotDeleteOutput> {
+  // Find all files for this snapshot ID (paginated listing, exact basename
+  // match). Exact matching is critical here: a substring match could delete a
+  // different snapshot whose id contains this id as a substring.
+  const objects = await listAllObjects(ctx, input.bucket, "snapshots/");
+  const snapshotKeys = findSnapshotKeys(objects, input.id);
+
+  if (snapshotKeys.length === 0) {
+    throw new Error(`Snapshot not found: ${input.id}`);
+  }
+
+  // Delete each file
+  for (const key of snapshotKeys) {
+    await ctx.client.call(["s3", "delete"], {
+      bucket: input.bucket,
+      key,
+    });
+  }
+
+  return {
+    deleted: true,
+    id: input.id,
+  };
+}

package/src/procedures/snapshot/diff.ts

@@ -0,0 +1,147 @@
+/**
+ * snapshot.diff procedure
+ *
+ * Compare current state against a snapshot.
+ */
+
+import { existsSync } from "node:fs";
+import { basename } from "node:path";
+import { gitText } from "./commands.js";
+import type { ProcedureContext } from "@procedures/client";
+import type {
+  SnapshotDiffInput,
+  SnapshotDiffOutput,
+  SnapshotDiffRepository,
+  SnapshotMetadata,
+} from "../../types.js";
+import { listAllObjects, findSnapshotKey } from "./s3-lookup.js";
+
+/**
+ * Compare current state against a snapshot
+ */
+export async function snapshotDiff(
+  input: SnapshotDiffInput,
+  ctx: ProcedureContext
+): Promise<SnapshotDiffOutput> {
+  // Download snapshot metadata (paginated listing, exact basename match).
+  const objects = await listAllObjects(ctx, input.bucket, "snapshots/");
+  const metadataKey = findSnapshotKey(objects, input.id, ".metadata.json");
+
+  if (!metadataKey) {
+    throw new Error(`Snapshot not found: ${input.id}`);
+  }
+
+  const metadataResult = await ctx.client.call<
+    { bucket: string; key: string; encoding?: string },
+    { body: string }
+  >(["s3", "download"], {
+    bucket: input.bucket,
+    key: metadataKey,
+    encoding: "utf8",
+  });
+
+  const snapshotMetadata: SnapshotMetadata = JSON.parse(metadataResult.body);
+
+  // Compare each repository
+  const repositories: SnapshotDiffRepository[] = [];
+  let totalFilesChanged = 0;
+  let reposChanged = 0;
+
+  const pathsToCheck = input.paths || snapshotMetadata.repositories.map((r) => r.path);
+
+  for (const repoPath of pathsToCheck) {
+    const snapshotRepo = snapshotMetadata.repositories.find(
+      (r) => r.path === repoPath || r.name === basename(repoPath)
+    );
+
+    if (!snapshotRepo) continue;
+
+    const diff: SnapshotDiffRepository = {
+      path: repoPath,
+      filesChanged: 0,
+      newStashes: 0,
+    };
+
+    if (!existsSync(repoPath)) {
+      // Repository was deleted
+      diff.filesChanged = -1; // Indicates missing
+      repositories.push(diff);
+      reposChanged++;
+      continue;
+    }
+
+    try {
+      // Get current branch (a failure is an error of this repository: the catch below)
+      const currentBranch = await gitText(["rev-parse", "--abbrev-ref", "HEAD"], repoPath, ctx.signal);
+      if (currentBranch === undefined) throw new Error(`git cannot read the branch of ${repoPath}`);
+
+      if (currentBranch !== snapshotRepo.branch) {
+        diff.branchDiff = {
+          current: currentBranch,
+          snapshot: snapshotRepo.branch,
+        };
+      }
+
+      // Get current commit
+      const currentCommit = await gitText(["rev-parse", "HEAD"], repoPath, ctx.signal);
+      if (currentCommit === undefined) throw new Error(`git cannot read the commit of ${repoPath}`);
+
+      if (currentCommit !== snapshotRepo.commit) {
+        diff.commitDiff = {
+          current: currentCommit,
+          snapshot: snapshotRepo.commit,
+        };
+
+        // Count files changed since snapshot commit.
+        // The commit comes from snapshot metadata downloaded from S3: accept only a hex
+        // object name, and pass it as an argument (no shell), so it cannot inject a command.
+        try {
+          if (!/^[0-9a-f]{7,64}$/i.test(snapshotRepo.commit)) {
+            throw new Error(`Not a commit hash: ${snapshotRepo.commit}`);
+          }
+          const diffStat = await gitText(["diff", "--stat", `${snapshotRepo.commit}..HEAD`], repoPath, ctx.signal);
+          if (diffStat === undefined) throw new Error(`git cannot compare ${snapshotRepo.commit} with HEAD`);
+          // Count lines that represent files (contain |)
+          diff.filesChanged = diffStat.split("\n").filter((l) => l.includes("|")).length;
+          totalFilesChanged += diff.filesChanged;
+        } catch {
+          // Snapshot commit may not exist locally
+          diff.filesChanged = -1;
+        }
+      }
+
+      // Check for new stashes
+      const stashResult = await ctx.client.call<
+        { cwd?: string },
+        { count: number }
+      >(["git", "stash", "list"], { cwd: repoPath });
+
+      diff.newStashes = stashResult.count - snapshotRepo.stashCount;
+
+      if (
+        diff.branchDiff ||
+        diff.commitDiff ||
+        diff.filesChanged > 0 ||
+        diff.newStashes > 0
+      ) {
+        reposChanged++;
+      }
+    } catch {
+      // Not a git repo or error
+      diff.filesChanged = -1;
+      reposChanged++;
+    }
+
+    repositories.push(diff);
+  }
+
+  return {
+    snapshotMetadata,
+    repositories,
+    summary: {
+      reposChanged,
+      totalFilesChanged,
+      isMatch: reposChanged === 0,
+    },
+  };
+}

package/src/procedures/snapshot/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Snapshot Procedures Index
+ */
+
+export { snapshotCreate } from "./create.js";
+export { snapshotList } from "./list.js";
+export { snapshotRestore } from "./restore.js";
+export { snapshotDiff } from "./diff.js";
+export { snapshotDelete } from "./delete.js";

package/src/procedures/snapshot/list.ts

@@ -0,0 +1,80 @@
+/**
+ * snapshot.list procedure
+ *
+ * List available snapshots in S3 bucket.
+ */
+
+import type { ProcedureContext } from "@procedures/client";
+import type {
+  SnapshotListInput,
+  SnapshotListOutput,
+  SnapshotListEntry,
+  SnapshotMetadata,
+} from "../../types.js";
+import { listAllObjects } from "./s3-lookup.js";
+
+/**
+ * List snapshots in S3 bucket
+ */
+export async function snapshotList(
+  input: SnapshotListInput,
+  ctx: ProcedureContext
+): Promise<SnapshotListOutput> {
+  const prefix = input.prefix
+    ? `snapshots/${input.prefix}`
+    : "snapshots/";
+
+  // List the metadata files of all pages. Sort them by the time of the write (newest first),
+  // then cut. Before, the procedure read one page (S3 lists by key, not by time) and cut it
+  // before the sort, so it returned the oldest snapshots (deep dive DATA-16).
+  const objects = await listAllObjects(ctx, input.bucket, prefix);
+  const metadataKeys = objects
+    .filter((obj) => obj.key.endsWith(".metadata.json"))
+    .sort((a, b) => timeOf(b.lastModified) - timeOf(a.lastModified))
+    .slice(0, input.maxResults);
+
+  // Fetch metadata for each snapshot
+  const snapshots: SnapshotListEntry[] = [];
+
+  for (const obj of metadataKeys) {
+    try {
+      const downloadResult = await ctx.client.call<
+        { bucket: string; key: string; encoding?: string },
+        { body: string }
+      >(["s3", "download"], {
+        bucket: input.bucket,
+        key: obj.key,
+        encoding: "utf8",
+      });
+
+      const metadata: SnapshotMetadata = JSON.parse(downloadResult.body);
+
+      snapshots.push({
+        id: metadata.id,
+        name: metadata.name,
+        preset: metadata.preset,
+        createdAt: metadata.createdAt,
+        size: metadata.archiveSize,
+        os: metadata.environment.os,
+      });
+    } catch {
+      // Skip invalid metadata
+    }
+  }
+
+  // Sort by creation date (newest first)
+  snapshots.sort((a, b) =>
+    new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
+  );
+
+  return {
+    snapshots,
+    count: snapshots.length,
+  };
+}
+
+/** A time in milliseconds. A missing or invalid time is 0, so it sorts last. */
+function timeOf(iso: string | undefined): number {
+  const time = iso ? Date.parse(iso) : NaN;
+  return Number.isNaN(time) ? 0 : time;
+}

package/src/procedures/snapshot/names.ts

@@ -0,0 +1,24 @@
+/**
+ * Names and exclusion rules of snapshots.
+ */
+
+/** A snapshot name or id: letters, digits, ".", "_" and "-", and not ".." (it goes into paths and S3 keys). */
+const SNAPSHOT_NAME = /^[A-Za-z0-9._-]+$/;
+
+export function assertSnapshotName(field: string, value: string): void {
+  if (!SNAPSHOT_NAME.test(value) || value.includes("..")) {
+    throw new Error(`Invalid snapshot ${field}: ${JSON.stringify(value)} (use letters, digits, ".", "_" and "-")`);
+  }
+}
+
+/**
+ * True when an archive entry is excluded: a pattern matches a whole path segment ("dist"
+ * excludes `dist/` but not `src/distance.ts`), or, for "*.ext", the end of the file name.
+ */
+export function isExcluded(entryPath: string, patterns: readonly string[]): boolean {
+  const segments = entryPath.split(/[\\/]/).filter((segment) => segment.length > 0);
+  const last = segments[segments.length - 1] ?? "";
+  return patterns.some((pattern) =>
+    pattern.startsWith("*.") ? last.endsWith(pattern.slice(1)) : segments.includes(pattern),
+  );
+}