aethel 0.4.0 → 1.2.0

package/src/core/pack-manifest.js

@@ -0,0 +1,163 @@
+/**
+ * Pack manifest CRUD operations.
+ * Manages the pack-manifest.json data structure.
+ */
+
+import crypto from "node:crypto";
+
+const MANIFEST_VERSION = 1;
+
+/**
+ * Create a new empty manifest.
+ * @returns {{ version: number, packs: {} }}
+ */
+export function createManifest() {
+  return {
+    version: MANIFEST_VERSION,
+    packs: {},
+  };
+}
+
+/**
+ * Get pack information for a specific path.
+ * @param {object} manifest - Manifest object
+ * @param {string} packPath - Directory path (e.g., "node_modules")
+ * @returns {object|null} Pack info or null if not found
+ */
+export function getPack(manifest, packPath) {
+  const normalized = normalizePath(packPath);
+  return manifest.packs[normalized] ?? null;
+}
+
+/**
+ * Set or update pack information.
+ * @param {object} manifest - Manifest object
+ * @param {string} packPath - Directory path
+ * @param {object} data - Pack data to set/merge
+ * @returns {object} Updated manifest
+ */
+export function setPack(manifest, packPath, data) {
+  const normalized = normalizePath(packPath);
+  const existing = manifest.packs[normalized] ?? {};
+
+  manifest.packs[normalized] = {
+    ...existing,
+    ...data,
+    // Always update lastModified when setting
+    lastModified: new Date().toISOString(),
+  };
+
+  return manifest;
+}
+
+/**
+ * Remove a pack from the manifest.
+ * @param {object} manifest - Manifest object
+ * @param {string} packPath - Directory path
+ * @returns {object} Updated manifest
+ */
+export function removePack(manifest, packPath) {
+  const normalized = normalizePath(packPath);
+  delete manifest.packs[normalized];
+  return manifest;
+}
+
+/**
+ * List all packs in the manifest.
+ * @param {object} manifest - Manifest object
+ * @returns {Array<{ path: string, info: object }>}
+ */
+export function listPacks(manifest) {
+  return Object.entries(manifest.packs).map(([path, info]) => ({
+    path,
+    info,
+  }));
+}
+
+/**
+ * Check if a path is covered by any pack rule.
+ * @param {object} manifest - Manifest object
+ * @param {string} filePath - File path to check
+ * @returns {{ isPacked: boolean, packPath: string|null }}
+ */
+export function isPathPacked(manifest, filePath) {
+  const normalized = normalizePath(filePath);
+
+  for (const packPath of Object.keys(manifest.packs)) {
+    // Check if filePath starts with packPath
+    if (normalized === packPath || normalized.startsWith(packPath + "/")) {
+      return { isPacked: true, packPath };
+    }
+  }
+
+  return { isPacked: false, packPath: null };
+}
+
+/**
+ * Validate manifest structure.
+ * @param {object} manifest - Manifest to validate
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+export function validateManifest(manifest) {
+  const errors = [];
+
+  if (!manifest || typeof manifest !== "object") {
+    errors.push("Manifest must be an object");
+    return { valid: false, errors };
+  }
+
+  if (typeof manifest.version !== "number") {
+    errors.push("Manifest version must be a number");
+  }
+
+  if (manifest.version !== MANIFEST_VERSION) {
+    errors.push(`Unsupported manifest version: ${manifest.version} (expected ${MANIFEST_VERSION})`);
+  }
+
+  if (!manifest.packs || typeof manifest.packs !== "object") {
+    errors.push("Manifest packs must be an object");
+  } else {
+    for (const [path, info] of Object.entries(manifest.packs)) {
+      if (typeof path !== "string" || path.length === 0) {
+        errors.push(`Invalid pack path: ${path}`);
+      }
+      if (!info || typeof info !== "object") {
+        errors.push(`Pack info for "${path}" must be an object`);
+      } else {
+        if (!info.packId || typeof info.packId !== "string") {
+          errors.push(`Pack "${path}" missing valid packId`);
+        }
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}
+
+/**
+ * Generate a unique pack ID.
+ * @param {string} packPath - Directory path
+ * @returns {string} Pack ID like "pack-node_modules-a1b2c3d4"
+ */
+export function generatePackId(packPath) {
+  const normalized = normalizePath(packPath);
+  // Sanitize path for ID: replace / with _ and remove special chars
+  const sanitized = normalized
+    .replace(/\//g, "_")
+    .replace(/[^a-zA-Z0-9_-]/g, "");
+  const shortHash = crypto.randomBytes(4).toString("hex");
+  return `pack-${sanitized}-${shortHash}`;
+}
+
+/**
+ * Normalize a path for consistent manifest keys.
+ * @param {string} p - Path to normalize
+ * @returns {string}
+ */
+function normalizePath(p) {
+  // Remove leading/trailing slashes and normalize to forward slashes
+  return p
+    .replace(/\\/g, "/")
+    .replace(/^\/+/, "")
+    .replace(/\/+$/, "");
+}

package/src/core/pack.js

@@ -0,0 +1,355 @@
+/**
+ * Directory packing operations: tar archive creation/extraction and tree hash.
+ */
+
+import * as tar from "tar";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { pipeline } from "node:stream/promises";
+import {
+  Algorithm,
+  EXTENSIONS,
+  createCompressStream,
+  createDecompressStream,
+  detectAlgorithm,
+  resolveAlgorithm,
+} from "./compress.js";
+
+/**
+ * Calculate tree hash for a directory using mtime + size.
+ * This is the key optimization: ~30x faster than MD5 hashing all files.
+ * @param {string} dirPath - Directory to hash
+ * @returns {Promise<string>} Hash in format "sha256:..."
+ */
+export async function getTreeHash(dirPath) {
+  const entries = [];
+
+  async function walk(currentPath) {
+    let items;
+    try {
+      items = await fsp.readdir(currentPath, { withFileTypes: true });
+    } catch {
+      return; // Skip directories we can't read
+    }
+
+    for (const item of items) {
+      const fullPath = path.join(currentPath, item.name);
+
+      if (item.isDirectory()) {
+        await walk(fullPath);
+      } else if (item.isFile()) {
+        try {
+          const stat = await fsp.stat(fullPath);
+          const relativePath = path.relative(dirPath, fullPath);
+          // Use forward slashes for consistency across platforms
+          const normalizedPath = relativePath.replace(/\\/g, "/");
+          entries.push(`${normalizedPath}:${stat.mtimeMs}:${stat.size}`);
+        } catch {
+          // Skip files we can't stat
+        }
+      }
+    }
+  }
+
+  await walk(dirPath);
+
+  // Sort for deterministic hash
+  entries.sort();
+
+  const fingerprint = entries.join("\n");
+  const hash = crypto.createHash("sha256").update(fingerprint).digest("hex");
+  return `sha256:${hash}`;
+}
+
+/**
+ * Create a pack (tar archive) from a directory.
+ * @param {string} sourcePath - Source directory path
+ * @param {string} destPath - Destination file path (without extension)
+ * @param {{ algorithm?: string, level?: number }} options
+ * @returns {Promise<{
+ *   packPath: string,
+ *   fileCount: number,
+ *   originalSize: number,
+ *   packedSize: number,
+ *   treeHash: string,
+ *   compression: { algorithm: string, level: number }
+ * }>}
+ */
+export async function createPack(sourcePath, destPath, options = {}) {
+  const preferredAlgorithm = options.algorithm ?? Algorithm.GZIP;
+  const level = options.level ?? 6;
+
+  // Resolve to available algorithm
+  const algorithm = await resolveAlgorithm(preferredAlgorithm);
+  const extension = EXTENSIONS[algorithm];
+  const packPath = destPath + extension;
+
+  // Calculate tree hash before packing
+  const treeHash = await getTreeHash(sourcePath);
+
+  // Count files and total size
+  let fileCount = 0;
+  let originalSize = 0;
+
+  async function countFiles(dir) {
+    const items = await fsp.readdir(dir, { withFileTypes: true });
+    for (const item of items) {
+      const fullPath = path.join(dir, item.name);
+      if (item.isDirectory()) {
+        await countFiles(fullPath);
+      } else if (item.isFile()) {
+        fileCount++;
+        const stat = await fsp.stat(fullPath);
+        originalSize += stat.size;
+      }
+    }
+  }
+  await countFiles(sourcePath);
+
+  // Create tar archive with compression
+  const tempTarPath = path.join(os.tmpdir(), `aethel-tar-${Date.now()}.tar`);
+
+  try {
+    // Create tar archive
+    await tar.create(
+      {
+        file: tempTarPath,
+        cwd: path.dirname(sourcePath),
+        gzip: false,
+      },
+      [path.basename(sourcePath)]
+    );
+
+    // Apply compression
+    if (algorithm === Algorithm.NONE) {
+      // No compression, just rename
+      await fsp.rename(tempTarPath, packPath);
+    } else {
+      const readStream = fs.createReadStream(tempTarPath);
+      const writeStream = fs.createWriteStream(packPath);
+      const compressStream = await createCompressStream(algorithm, { level });
+      await pipeline(readStream, compressStream, writeStream);
+      await fsp.unlink(tempTarPath);
+    }
+
+    const packStat = await fsp.stat(packPath);
+
+    return {
+      packPath,
+      fileCount,
+      originalSize,
+      packedSize: packStat.size,
+      treeHash,
+      compression: { algorithm, level },
+    };
+  } catch (err) {
+    // Cleanup on error
+    try {
+      await fsp.unlink(tempTarPath);
+    } catch {}
+    try {
+      await fsp.unlink(packPath);
+    } catch {}
+    throw err;
+  }
+}
+
+/**
+ * Extract a pack to a destination directory.
+ * @param {string} packPath - Pack file path
+ * @param {string} destPath - Destination directory path
+ * @param {{ algorithm?: string }} options - Algorithm override (auto-detected if not provided)
+ * @returns {Promise<{ fileCount: number, extractedSize: number }>}
+ */
+export async function extractPack(packPath, destPath, options = {}) {
+  const algorithm = options.algorithm ?? detectAlgorithm(packPath) ?? Algorithm.GZIP;
+
+  // Ensure destination exists
+  await fsp.mkdir(destPath, { recursive: true });
+
+  const tempTarPath = path.join(os.tmpdir(), `aethel-extract-${Date.now()}.tar`);
+
+  try {
+    // Decompress if needed
+    if (algorithm === Algorithm.NONE) {
+      // No decompression needed
+      await tar.extract({
+        file: packPath,
+        cwd: destPath,
+        strip: 1, // Remove top-level directory
+      });
+    } else {
+      // Decompress first
+      const readStream = fs.createReadStream(packPath);
+      const writeStream = fs.createWriteStream(tempTarPath);
+      const decompressStream = await createDecompressStream(algorithm);
+      await pipeline(readStream, decompressStream, writeStream);
+
+      // Then extract
+      await tar.extract({
+        file: tempTarPath,
+        cwd: destPath,
+        strip: 1,
+      });
+
+      await fsp.unlink(tempTarPath);
+    }
+
+    // Count extracted files
+    let fileCount = 0;
+    let extractedSize = 0;
+
+    async function countFiles(dir) {
+      const items = await fsp.readdir(dir, { withFileTypes: true });
+      for (const item of items) {
+        const fullPath = path.join(dir, item.name);
+        if (item.isDirectory()) {
+          await countFiles(fullPath);
+        } else if (item.isFile()) {
+          fileCount++;
+          const stat = await fsp.stat(fullPath);
+          extractedSize += stat.size;
+        }
+      }
+    }
+    await countFiles(destPath);
+
+    return { fileCount, extractedSize };
+  } catch (err) {
+    try {
+      await fsp.unlink(tempTarPath);
+    } catch {}
+    throw err;
+  }
+}
+
+/**
+ * Check if a pack is stale (local directory has changed).
+ * @param {string} currentHash - Current tree hash of directory
+ * @param {string|null} manifestHash - Hash stored in manifest
+ * @returns {boolean} True if pack needs to be recreated
+ */
+export function isPackStale(currentHash, manifestHash) {
+  if (!manifestHash) return true;
+  return currentHash !== manifestHash;
+}
+
+/**
+ * List contents of a pack without extracting.
+ * @param {string} packPath - Pack file path
+ * @returns {Promise<Array<{ path: string, size: number, mtime: Date }>>}
+ */
+export async function listPackContents(packPath) {
+  const algorithm = detectAlgorithm(packPath) ?? Algorithm.GZIP;
+  const entries = [];
+
+  if (algorithm === Algorithm.NONE) {
+    // Direct tar list
+    await tar.list({
+      file: packPath,
+      onentry: (entry) => {
+        entries.push({
+          path: entry.path,
+          size: entry.size,
+          mtime: entry.mtime,
+        });
+      },
+    });
+  } else {
+    // Need to decompress first
+    const tempTarPath = path.join(os.tmpdir(), `aethel-list-${Date.now()}.tar`);
+    try {
+      const readStream = fs.createReadStream(packPath);
+      const writeStream = fs.createWriteStream(tempTarPath);
+      const decompressStream = await createDecompressStream(algorithm);
+      await pipeline(readStream, decompressStream, writeStream);
+
+      await tar.list({
+        file: tempTarPath,
+        onentry: (entry) => {
+          entries.push({
+            path: entry.path,
+            size: entry.size,
+            mtime: entry.mtime,
+          });
+        },
+      });
+
+      await fsp.unlink(tempTarPath);
+    } catch (err) {
+      try {
+        await fsp.unlink(tempTarPath);
+      } catch {}
+      throw err;
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Extract a single file from a pack.
+ * @param {string} packPath - Pack file path
+ * @param {string} filePath - Relative path within the pack
+ * @param {string} destPath - Destination path for extracted file
+ * @returns {Promise<void>}
+ */
+export async function extractSingleFile(packPath, filePath, destPath) {
+  const algorithm = detectAlgorithm(packPath) ?? Algorithm.GZIP;
+
+  // Ensure destination directory exists
+  await fsp.mkdir(path.dirname(destPath), { recursive: true });
+
+  const tempDir = path.join(os.tmpdir(), `aethel-single-${Date.now()}`);
+  await fsp.mkdir(tempDir, { recursive: true });
+
+  try {
+    if (algorithm === Algorithm.NONE) {
+      await tar.extract({
+        file: packPath,
+        cwd: tempDir,
+        filter: (p) => p === filePath || p.endsWith("/" + filePath),
+      });
+    } else {
+      // Decompress first
+      const tempTarPath = path.join(tempDir, "archive.tar");
+      const readStream = fs.createReadStream(packPath);
+      const writeStream = fs.createWriteStream(tempTarPath);
+      const decompressStream = await createDecompressStream(algorithm);
+      await pipeline(readStream, decompressStream, writeStream);
+
+      await tar.extract({
+        file: tempTarPath,
+        cwd: tempDir,
+        filter: (p) => p === filePath || p.endsWith("/" + filePath),
+      });
+    }
+
+    // Find the extracted file and move it
+    async function findFile(dir) {
+      const items = await fsp.readdir(dir, { withFileTypes: true });
+      for (const item of items) {
+        const fullPath = path.join(dir, item.name);
+        if (item.isDirectory()) {
+          const found = await findFile(fullPath);
+          if (found) return found;
+        } else if (item.name === path.basename(filePath)) {
+          return fullPath;
+        }
+      }
+      return null;
+    }
+
+    const extractedPath = await findFile(tempDir);
+    if (!extractedPath) {
+      throw new Error(`File not found in pack: ${filePath}`);
+    }
+
+    await fsp.copyFile(extractedPath, destPath);
+  } finally {
+    await fsp.rm(tempDir, { recursive: true, force: true });
+  }
+}

package/src/core/snapshot.js

@@ -1,8 +1,9 @@
 import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { AETHEL_DIR } from "./config.js";
+import { AETHEL_DIR, loadPackConfig, getPackRule } from "./config.js";
 import { loadIgnoreRules } from "./ignore.js";
+import { getTreeHash } from "./pack.js";
 
 const HASH_CACHE_FILE = ".hash-cache.json";
 
@@ -86,14 +87,17 @@ function saveHashCache(root, cache) {
 
 const PARALLEL_HASH_LIMIT = 128;
 
-export async function scanLocal(root, { respectIgnore = true } = {}) {
+export async function scanLocal(root, { respectIgnore = true, respectPacking = true } = {}) {
   const resolvedRoot = path.resolve(root);
   const ignoreRules = respectIgnore ? loadIgnoreRules(resolvedRoot) : null;
+  const packConfig = respectPacking ? loadPackConfig(resolvedRoot) : null;
+  const packingEnabled = packConfig?.packing?.enabled === true;
   const hashCache = loadHashCache(resolvedRoot);
   const nextCache = new Map();
 
   // Phase 1: collect all file stats (fast — no hashing yet)
   const filesToHash = [];
+  const packedDirs = {};
   // Track directories and their child counts to detect empty folders
   const dirChildCount = new Map();
   // Map relative dir path → absolute path (for deferred stat on empty dirs only)
@@ -130,16 +134,46 @@ export async function scanLocal(root, { respectIgnore = true } = {}) {
         .split(path.sep)
         .join("/");
 
-      if (ignoreRules?.ignores(relativePath)) {
-        continue;
-      }
-
       if (entry.isDirectory()) {
+        // Check if this directory is a pack target BEFORE checking ignore rules
+        // Pack targets should be processed even if they match ignore patterns
+        if (packingEnabled) {
+          const packRule = getPackRule(packConfig, relativePath);
+          if (packRule && packRule.path === relativePath) {
+            // This directory should be packed - compute tree hash instead of scanning
+            try {
+              const treeHash = await getTreeHash(fullPath);
+              packedDirs[relativePath] = {
+                path: relativePath,
+                isPacked: true,
+                treeHash,
+                packRule: packRule.strategy || "full",
+              };
+            } catch {
+              // If we can't compute tree hash, fall back to normal scanning
+              if (!ignoreRules?.ignores(relativePath)) {
+                trackedChildren++;
+                subdirs.push(fullPath);
+              }
+            }
+            continue; // Don't descend into packed directory
+          }
+        }
+
+        // Apply ignore rules for non-pack directories
+        if (ignoreRules?.ignores(relativePath)) {
+          continue;
+        }
         trackedChildren++;
         subdirs.push(fullPath);
         continue;
       }
 
+      // Apply ignore rules for files
+      if (ignoreRules?.ignores(relativePath)) {
+        continue;
+      }
+
       if (!entry.isFile()) {
         continue;
       }
@@ -230,7 +264,14 @@ export async function scanLocal(root, { respectIgnore = true } = {}) {
 
   // Persist updated cache
   saveHashCache(resolvedRoot, nextCache);
-  return result;
+
+  // Return both files and packed directories
+  return {
+    files: result,
+    packedDirs,
+    // For backward compatibility, also expose files at top level
+    ...result,
+  };
 }
 
 async function getMd5Cached(oldCache, newCache, fullPath, relativePath, stat) {
@@ -252,7 +293,7 @@ async function getMd5Cached(oldCache, newCache, fullPath, relativePath, stat) {
 
 // ── Snapshot building ────────────────────────────────────────────────
 
-export function buildSnapshot(remoteFiles, localFiles, message = "") {
+export function buildSnapshot(remoteFiles, localFiles, message = "", packedDirs = {}) {
   const files = {};
 
   for (const file of remoteFiles) {
@@ -269,11 +310,16 @@ export function buildSnapshot(remoteFiles, localFiles, message = "") {
     };
   }
 
+  // Handle localFiles which may be the new format with .files property
+  const localFilesData = localFiles?.files ?? localFiles;
+  const localPackedDirs = localFiles?.packedDirs ?? packedDirs;
+
   const snapshot = {
     timestamp: new Date().toISOString(),
     message,
     files,
-    localFiles: { ...localFiles },
+    localFiles: { ...localFilesData },
+    packedDirs: { ...localPackedDirs },
   };
 
   // Embed integrity checksum