aethel 0.4.0 → 1.2.0

package/src/core/compress.js

@@ -0,0 +1,285 @@
+/**
+ * Multi-algorithm compression/decompression abstraction.
+ * Supports gzip, brotli (built-in), and optionally zstd, xz.
+ */
+
+import zlib from "node:zlib";
+import { pipeline } from "node:stream/promises";
+import fs from "node:fs";
+import path from "node:path";
+
+// Algorithm enumeration
+export const Algorithm = Object.freeze({
+  NONE: "none",
+  GZIP: "gzip",
+  ZSTD: "zstd",
+  BROTLI: "brotli",
+  XZ: "xz",
+});
+
+// File extension mapping
+export const EXTENSIONS = {
+  [Algorithm.NONE]: ".tar",
+  [Algorithm.GZIP]: ".tar.gz",
+  [Algorithm.ZSTD]: ".tar.zst",
+  [Algorithm.BROTLI]: ".tar.br",
+  [Algorithm.XZ]: ".tar.xz",
+};
+
+// Compression profiles for easy configuration
+export const PROFILES = {
+  fast: { algorithm: Algorithm.ZSTD, level: 1 },
+  balanced: { algorithm: Algorithm.ZSTD, level: 6 },
+  maximum: { algorithm: Algorithm.ZSTD, level: 19 },
+  extreme: { algorithm: Algorithm.XZ, level: 6 },
+};
+
+// Cache for optional dependency availability
+const availabilityCache = new Map();
+
+/**
+ * Try to load an optional dependency.
+ * @param {string} moduleName
+ * @returns {Promise<any|null>}
+ */
+async function tryLoadModule(moduleName) {
+  if (availabilityCache.has(moduleName)) {
+    return availabilityCache.get(moduleName);
+  }
+  try {
+    const mod = await import(moduleName);
+    availabilityCache.set(moduleName, mod.default || mod);
+    return availabilityCache.get(moduleName);
+  } catch {
+    availabilityCache.set(moduleName, null);
+    return null;
+  }
+}
+
+/**
+ * Check if an algorithm is available in the current environment.
+ * @param {string} algorithm - Algorithm name from Algorithm enum
+ * @returns {Promise<boolean>}
+ */
+export async function isAlgorithmAvailable(algorithm) {
+  switch (algorithm) {
+    case Algorithm.NONE:
+    case Algorithm.GZIP:
+    case Algorithm.BROTLI:
+      return true; // Built-in Node.js
+    case Algorithm.ZSTD:
+      return (await tryLoadModule("@bokuweb/zstd-wasm")) !== null;
+    case Algorithm.XZ:
+      return (await tryLoadModule("lzma-native")) !== null;
+    default:
+      return false;
+  }
+}
+
+/**
+ * Get the best available algorithm for compression.
+ * Falls back to gzip if preferred is unavailable.
+ * @param {string} preferred - Preferred algorithm
+ * @returns {Promise<string>} Available algorithm
+ */
+export async function resolveAlgorithm(preferred) {
+  if (await isAlgorithmAvailable(preferred)) {
+    return preferred;
+  }
+  // Fallback chain: zstd -> gzip
+  if (preferred !== Algorithm.ZSTD && (await isAlgorithmAvailable(Algorithm.ZSTD))) {
+    return Algorithm.ZSTD;
+  }
+  return Algorithm.GZIP;
+}
+
+/**
+ * Create a compression stream for the given algorithm.
+ * @param {string} algorithm - Algorithm name
+ * @param {{ level?: number }} options - Compression options
+ * @returns {Promise<import("node:stream").Transform>} Compression stream
+ */
+export async function createCompressStream(algorithm, options = {}) {
+  const level = options.level ?? 6;
+
+  switch (algorithm) {
+    case Algorithm.NONE:
+      // Pass-through stream
+      const { PassThrough } = await import("node:stream");
+      return new PassThrough();
+
+    case Algorithm.GZIP:
+      return zlib.createGzip({ level });
+
+    case Algorithm.BROTLI:
+      return zlib.createBrotliCompress({
+        params: {
+          [zlib.constants.BROTLI_PARAM_QUALITY]: Math.min(level, 11),
+        },
+      });
+
+    case Algorithm.ZSTD: {
+      const zstd = await tryLoadModule("@bokuweb/zstd-wasm");
+      if (!zstd) {
+        throw new Error("zstd not available. Install @bokuweb/zstd-wasm");
+      }
+      // zstd-wasm provides compress/decompress functions, not streams
+      // We need to wrap it in a transform stream
+      const { Transform } = await import("node:stream");
+      const chunks = [];
+      return new Transform({
+        transform(chunk, encoding, callback) {
+          chunks.push(chunk);
+          callback();
+        },
+        async flush(callback) {
+          try {
+            const input = Buffer.concat(chunks);
+            const compressed = await zstd.compress(input, level);
+            this.push(Buffer.from(compressed));
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        },
+      });
+    }
+
+    case Algorithm.XZ: {
+      const lzma = await tryLoadModule("lzma-native");
+      if (!lzma) {
+        throw new Error("xz not available. Install lzma-native");
+      }
+      return lzma.createCompressor({ preset: level });
+    }
+
+    default:
+      throw new Error(`Unknown compression algorithm: ${algorithm}`);
+  }
+}
+
+/**
+ * Create a decompression stream for the given algorithm.
+ * @param {string} algorithm - Algorithm name
+ * @returns {Promise<import("node:stream").Transform>} Decompression stream
+ */
+export async function createDecompressStream(algorithm) {
+  switch (algorithm) {
+    case Algorithm.NONE: {
+      const { PassThrough } = await import("node:stream");
+      return new PassThrough();
+    }
+
+    case Algorithm.GZIP:
+      return zlib.createGunzip();
+
+    case Algorithm.BROTLI:
+      return zlib.createBrotliDecompress();
+
+    case Algorithm.ZSTD: {
+      const zstd = await tryLoadModule("@bokuweb/zstd-wasm");
+      if (!zstd) {
+        throw new Error("zstd not available. Install @bokuweb/zstd-wasm");
+      }
+      const { Transform } = await import("node:stream");
+      const chunks = [];
+      return new Transform({
+        transform(chunk, encoding, callback) {
+          chunks.push(chunk);
+          callback();
+        },
+        async flush(callback) {
+          try {
+            const input = Buffer.concat(chunks);
+            const decompressed = await zstd.decompress(input);
+            this.push(Buffer.from(decompressed));
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        },
+      });
+    }
+
+    case Algorithm.XZ: {
+      const lzma = await tryLoadModule("lzma-native");
+      if (!lzma) {
+        throw new Error("xz not available. Install lzma-native");
+      }
+      return lzma.createDecompressor();
+    }
+
+    default:
+      throw new Error(`Unknown decompression algorithm: ${algorithm}`);
+  }
+}
+
+/**
+ * Compress a file to destination.
+ * @param {string} inputPath - Source file path
+ * @param {string} outputPath - Destination file path
+ * @param {{ algorithm?: string, level?: number }} options
+ * @returns {Promise<{ originalSize: number, compressedSize: number, ratio: number }>}
+ */
+export async function compressFile(inputPath, outputPath, options = {}) {
+  const algorithm = options.algorithm ?? Algorithm.GZIP;
+  const level = options.level ?? 6;
+
+  const inputStat = fs.statSync(inputPath);
+  const originalSize = inputStat.size;
+
+  const readStream = fs.createReadStream(inputPath);
+  const writeStream = fs.createWriteStream(outputPath);
+  const compressStream = await createCompressStream(algorithm, { level });
+
+  await pipeline(readStream, compressStream, writeStream);
+
+  const outputStat = fs.statSync(outputPath);
+  const compressedSize = outputStat.size;
+  const ratio = originalSize > 0 ? 1 - compressedSize / originalSize : 0;
+
+  return { originalSize, compressedSize, ratio };
+}
+
+/**
+ * Decompress a file to destination.
+ * @param {string} inputPath - Compressed file path
+ * @param {string} outputPath - Destination file path
+ * @param {string} algorithm - Algorithm used for compression
+ * @returns {Promise<void>}
+ */
+export async function decompressFile(inputPath, outputPath, algorithm) {
+  const readStream = fs.createReadStream(inputPath);
+  const writeStream = fs.createWriteStream(outputPath);
+  const decompressStream = await createDecompressStream(algorithm);
+
+  await pipeline(readStream, decompressStream, writeStream);
+}
+
+/**
+ * Detect algorithm from file extension.
+ * @param {string} filePath - File path with extension
+ * @returns {string|null} Algorithm name or null if unknown
+ */
+export function detectAlgorithm(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  const fullExt = filePath.toLowerCase();
+
+  if (fullExt.endsWith(".tar.gz") || fullExt.endsWith(".tgz")) {
+    return Algorithm.GZIP;
+  }
+  if (fullExt.endsWith(".tar.zst")) {
+    return Algorithm.ZSTD;
+  }
+  if (fullExt.endsWith(".tar.br")) {
+    return Algorithm.BROTLI;
+  }
+  if (fullExt.endsWith(".tar.xz")) {
+    return Algorithm.XZ;
+  }
+  if (ext === ".tar") {
+    return Algorithm.NONE;
+  }
+
+  return null;
+}

package/src/core/config.js

@@ -5,6 +5,8 @@
 import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import YAML from "yaml";
+import { createManifest } from "./pack-manifest.js";
 
 export const AETHEL_DIR = ".aethel";
 export const CONFIG_FILE = "config.json";
@@ -12,6 +14,8 @@ export const INDEX_FILE = "index.json";
 export const SNAPSHOTS_DIR = "snapshots";
 export const HISTORY_DIR = "history";
 export const LATEST_SNAPSHOT = "latest.json";
+export const PACK_MANIFEST_FILE = "pack-manifest.json";
+export const PACK_CONFIG_FILE = ".aethelconfig";
 
 /** Walk up from `start` looking for a .aethel/ directory. */
 export function findRoot(start = process.cwd()) {
@@ -135,3 +139,118 @@ export function writeSnapshot(root, snapshot) {
   // Compact JSON — snapshots can be large, pretty-printing is slow + wastes disk
   fs.writeFileSync(latest, JSON.stringify(snapshot) + "\n");
 }
+
+// ── pack config helpers ───────────────────────────────────────────────
+
+const DEFAULT_PACK_CONFIG = {
+  packing: {
+    enabled: false,
+    compression: {
+      default: {
+        algorithm: "zstd",
+        level: 6,
+      },
+      overrides: [],
+    },
+    rules: [],
+  },
+};
+
+/**
+ * Load pack configuration from .aethelconfig (YAML).
+ * Returns default config if file doesn't exist.
+ * @param {string} root - Workspace root
+ * @returns {object} Pack configuration
+ */
+export function loadPackConfig(root) {
+  const p = path.join(root, PACK_CONFIG_FILE);
+  if (!fs.existsSync(p)) {
+    return structuredClone(DEFAULT_PACK_CONFIG);
+  }
+  try {
+    const content = fs.readFileSync(p, "utf-8");
+    const parsed = YAML.parse(content);
+    // Merge with defaults for missing keys
+    return {
+      packing: {
+        ...DEFAULT_PACK_CONFIG.packing,
+        ...parsed?.packing,
+        compression: {
+          ...DEFAULT_PACK_CONFIG.packing.compression,
+          ...parsed?.packing?.compression,
+        },
+      },
+    };
+  } catch {
+    return structuredClone(DEFAULT_PACK_CONFIG);
+  }
+}
+
+/**
+ * Save pack configuration to .aethelconfig.
+ * @param {string} root - Workspace root
+ * @param {object} config - Configuration to save
+ */
+export function savePackConfig(root, config) {
+  const p = path.join(root, PACK_CONFIG_FILE);
+  const content = YAML.stringify(config);
+  fs.writeFileSync(p, content);
+}
+
+/**
+ * Read pack manifest from .aethel/pack-manifest.json.
+ * Returns empty manifest if file doesn't exist.
+ * @param {string} root - Workspace root
+ * @returns {object} Pack manifest
+ */
+export function loadPackManifest(root) {
+  const p = path.join(dot(root), PACK_MANIFEST_FILE);
+  if (!fs.existsSync(p)) {
+    return createManifest();
+  }
+  try {
+    return JSON.parse(fs.readFileSync(p, "utf-8"));
+  } catch {
+    return createManifest();
+  }
+}
+
+/**
+ * Save pack manifest to .aethel/pack-manifest.json.
+ * @param {string} root - Workspace root
+ * @param {object} manifest - Manifest to save
+ */
+export function savePackManifest(root, manifest) {
+  const p = path.join(dot(root), PACK_MANIFEST_FILE);
+  fs.writeFileSync(p, JSON.stringify(manifest, null, 2) + "\n");
+}
+
+/**
+ * Check if packing feature is enabled.
+ * @param {string} root - Workspace root
+ * @returns {boolean}
+ */
+export function isPackingEnabled(root) {
+  const config = loadPackConfig(root);
+  return config.packing?.enabled === true;
+}
+
+/**
+ * Get packing rule for a specific path.
+ * @param {object} packConfig - Pack configuration
+ * @param {string} relativePath - Path to check
+ * @returns {object|null} Matching rule or null
+ */
+export function getPackRule(packConfig, relativePath) {
+  const rules = packConfig.packing?.rules ?? [];
+  const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+$/, "");
+
+  for (const rule of rules) {
+    const rulePath = rule.path.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+$/, "");
+    if (normalized === rulePath || normalized.startsWith(rulePath + "/")) {
+      return rule;
+    }
+  }
+
+  return null;
+}

package/src/core/diff.js

@@ -1,5 +1,6 @@
 import { isWorkspaceType } from "./drive-api.js";
 import { loadIgnoreRules } from "./ignore.js";
+import { loadPackManifest } from "./config.js";
 
 export const ChangeType = Object.freeze({
   REMOTE_ADDED: "remote_added",
@@ -9,6 +10,12 @@ export const ChangeType = Object.freeze({
   LOCAL_MODIFIED: "local_modified",
   LOCAL_DELETED: "local_deleted",
   CONFLICT: "conflict",
+  // Pack-specific change types
+  PACK_LOCAL_MODIFIED: "pack_local_modified",
+  PACK_REMOTE_MODIFIED: "pack_remote_modified",
+  PACK_SYNCED: "pack_synced",
+  PACK_CONFLICT: "pack_conflict",
+  PACK_NEW: "pack_new",
 });
 
 const SHORT_STATUS = {
@@ -19,6 +26,11 @@ const SHORT_STATUS = {
   [ChangeType.LOCAL_MODIFIED]: "ML",
   [ChangeType.LOCAL_DELETED]: "-L",
   [ChangeType.CONFLICT]: "!!",
+  [ChangeType.PACK_LOCAL_MODIFIED]: "PL",
+  [ChangeType.PACK_REMOTE_MODIFIED]: "PR",
+  [ChangeType.PACK_SYNCED]: "P=",
+  [ChangeType.PACK_CONFLICT]: "P!",
+  [ChangeType.PACK_NEW]: "P+",
 };
 
 const DESCRIPTION = {
@@ -29,6 +41,11 @@ const DESCRIPTION = {
   [ChangeType.LOCAL_MODIFIED]: "modified locally",
   [ChangeType.LOCAL_DELETED]: "deleted locally",
   [ChangeType.CONFLICT]: "both sides changed",
+  [ChangeType.PACK_LOCAL_MODIFIED]: "pack changed locally",
+  [ChangeType.PACK_REMOTE_MODIFIED]: "pack changed on Drive",
+  [ChangeType.PACK_SYNCED]: "pack up to date",
+  [ChangeType.PACK_CONFLICT]: "pack conflict",
+  [ChangeType.PACK_NEW]: "new pack",
 };
 
 const SUGGESTED_ACTION = {
@@ -39,6 +56,11 @@ const SUGGESTED_ACTION = {
   [ChangeType.LOCAL_MODIFIED]: "upload",
   [ChangeType.LOCAL_DELETED]: "delete_remote",
   [ChangeType.CONFLICT]: "conflict",
+  [ChangeType.PACK_LOCAL_MODIFIED]: "push_pack",
+  [ChangeType.PACK_REMOTE_MODIFIED]: "pull_pack",
+  [ChangeType.PACK_SYNCED]: "none",
+  [ChangeType.PACK_CONFLICT]: "resolve_pack",
+  [ChangeType.PACK_NEW]: "push_pack",
 };
 
 function createChange({
@@ -62,9 +84,10 @@ function createChange({
   };
 }
 
-function buildDiffResult(changes) {
+function buildDiffResult(changes, packChanges = []) {
   return {
     changes,
+    packChanges,
     get remoteChanges() {
       return this.changes.filter((change) =>
         change.changeType.startsWith("remote")
@@ -80,8 +103,29 @@ function buildDiffResult(changes) {
         (change) => change.changeType === ChangeType.CONFLICT
       );
     },
+    get packConflicts() {
+      return this.packChanges.filter(
+        (change) => change.changeType === ChangeType.PACK_CONFLICT
+      );
+    },
+    get pendingPackChanges() {
+      return this.packChanges.filter(
+        (change) =>
+          change.changeType === ChangeType.PACK_LOCAL_MODIFIED ||
+          change.changeType === ChangeType.PACK_REMOTE_MODIFIED ||
+          change.changeType === ChangeType.PACK_NEW
+      );
+    },
+    get syncedPacks() {
+      return this.packChanges.filter(
+        (change) => change.changeType === ChangeType.PACK_SYNCED
+      );
+    },
+    get hasPackChanges() {
+      return this.pendingPackChanges.length > 0 || this.packConflicts.length > 0;
+    },
     get isClean() {
-      return this.changes.length === 0;
+      return this.changes.length === 0 && this.pendingPackChanges.length === 0;
     },
   };
 }
@@ -154,6 +198,93 @@ function promoteConflicts(changes) {
   return filtered;
 }
 
+/**
+ * Compute pack-level changes by comparing local packedDirs against manifest.
+ * @param {string|null} root - Workspace root for loading manifest
+ * @param {object} packedDirs - Local packed directories from scanLocal
+ * @param {object|null} snapshot - Previous snapshot (may contain packedDirs)
+ * @returns {object[]} Array of pack change objects
+ */
+function computePackChanges(root, packedDirs, snapshot) {
+  if (!root || !packedDirs || Object.keys(packedDirs).length === 0) {
+    return [];
+  }
+
+  const manifest = loadPackManifest(root);
+  const snapshotPackedDirs = snapshot?.packedDirs || {};
+  const changes = [];
+
+  for (const [packPath, packInfo] of Object.entries(packedDirs)) {
+    const manifestEntry = manifest.packs?.[packPath];
+    const snapshotEntry = snapshotPackedDirs[packPath];
+    const localTreeHash = packInfo.treeHash;
+
+    if (!manifestEntry) {
+      // Pack not in manifest = new pack
+      changes.push(
+        createChange({
+          changeType: ChangeType.PACK_NEW,
+          path: packPath,
+          localMeta: packInfo,
+        })
+      );
+      continue;
+    }
+
+    const { localTreeHash: manifestLocalHash, remoteTreeHash: manifestRemoteHash } = manifestEntry;
+
+    // Check for local modification
+    const localChanged = localTreeHash !== manifestLocalHash;
+    // Check for remote modification (comparing against what we last synced)
+    const remoteChanged = manifestRemoteHash && manifestRemoteHash !== manifestLocalHash;
+
+    if (localChanged && remoteChanged) {
+      // Both sides changed = conflict
+      changes.push(
+        createChange({
+          changeType: ChangeType.PACK_CONFLICT,
+          path: packPath,
+          localMeta: { ...packInfo, treeHash: localTreeHash },
+          snapshotMeta: { treeHash: manifestLocalHash },
+          remoteMeta: { treeHash: manifestRemoteHash },
+        })
+      );
+    } else if (localChanged) {
+      // Only local changed
+      changes.push(
+        createChange({
+          changeType: ChangeType.PACK_LOCAL_MODIFIED,
+          path: packPath,
+          localMeta: { ...packInfo, treeHash: localTreeHash },
+          snapshotMeta: { treeHash: manifestLocalHash },
+        })
+      );
+    } else if (remoteChanged) {
+      // Only remote changed
+      changes.push(
+        createChange({
+          changeType: ChangeType.PACK_REMOTE_MODIFIED,
+          path: packPath,
+          localMeta: packInfo,
+          remoteMeta: { treeHash: manifestRemoteHash },
+          snapshotMeta: { treeHash: manifestLocalHash },
+        })
+      );
+    } else {
+      // No changes = synced
+      changes.push(
+        createChange({
+          changeType: ChangeType.PACK_SYNCED,
+          path: packPath,
+          localMeta: packInfo,
+        })
+      );
+    }
+  }
+
+  return changes;
+}
+
 /**
  * @param {object|null} snapshot
  * @param {object[]} remoteFiles
@@ -182,6 +313,11 @@ export function computeDiff(snapshot, remoteFiles, localFiles, { root, respectIg
   if (ignoreRules) {
     remoteFiles = remoteFiles.filter((f) => !ignoreRules.ignores(f.path));
   }
+
+  // Handle new localFiles format with .files and .packedDirs
+  const localFilesData = localFiles?.files ?? localFiles;
+  const packedDirs = localFiles?.packedDirs ?? {};
+
   const changes = [];
   const snapshotFiles = snapshot?.files || {};
   const snapshotLocalFiles = snapshot?.localFiles || {};
@@ -189,13 +325,13 @@ export function computeDiff(snapshot, remoteFiles, localFiles, { root, respectIg
   // Build sets of all folder paths that implicitly exist on each side
   // (from parent directories of files), so we can skip redundant folder additions.
   const remoteFolderPaths = collectFolderPaths(remoteFiles.map((f) => f.path));
-  const localFolderPaths = collectFolderPaths(Object.keys(localFiles));
+  const localFolderPaths = collectFolderPaths(Object.keys(localFilesData));
 
   // Also include explicit folder entries
   for (const f of remoteFiles) {
     if (f.isFolder) remoteFolderPaths.add(f.path);
   }
-  for (const [p, meta] of Object.entries(localFiles)) {
+  for (const [p, meta] of Object.entries(localFilesData)) {
     if (meta.isFolder) localFolderPaths.add(p);
   }
 
@@ -254,7 +390,7 @@ export function computeDiff(snapshot, remoteFiles, localFiles, { root, respectIg
     }
   }
 
-  for (const [relativePath, localMeta] of Object.entries(localFiles)) {
+  for (const [relativePath, localMeta] of Object.entries(localFilesData)) {
     const snapshotEntry = snapshotLocalFiles[relativePath];
 
     if (!snapshotEntry) {
@@ -285,7 +421,7 @@ export function computeDiff(snapshot, remoteFiles, localFiles, { root, respectIg
   }
 
   for (const [relativePath, snapshotEntry] of Object.entries(snapshotLocalFiles)) {
-    if (!(relativePath in localFiles)) {
+    if (!(relativePath in localFilesData)) {
       // Skip folder deletion if the folder still implicitly exists locally
       if (snapshotEntry.isFolder && localFolderPaths.has(relativePath)) {
         continue;
@@ -300,5 +436,8 @@ export function computeDiff(snapshot, remoteFiles, localFiles, { root, respectIg
     }
   }
 
-  return buildDiffResult(promoteConflicts(changes));
+  // Compute pack changes
+  const packChanges = computePackChanges(root, packedDirs, snapshot);
+
+  return buildDiffResult(promoteConflicts(changes), packChanges);
 }