agentel 0.2.0 → 0.2.3

package/src/sync.js

@@ -72,6 +72,56 @@ async function snapshotArchive(options = {}, env = process.env) {
   return syncArchive({ ...options, snapshot: true, force: true }, env);
 }
 
+async function replaceRemoteArchive(options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const mode = normalizeSyncMode(options["sync-mode"] || options.syncMode || options.mode || cfg.sync?.mode || "upload");
+  if (mode !== "upload") {
+    throw new Error(`remote sync mode "${mode}" is planned but not implemented yet; use upload mode to replace this device namespace`);
+  }
+  const lock = acquireSyncLock(env);
+  try {
+    const target = remoteTargetFromFlags(options, cfg, env);
+    if (target.protocol === "file") return replaceRemoteDirectory(target, options, env);
+    return replaceRemoteS3(target, options, env);
+  } finally {
+    releaseSyncLock(lock);
+  }
+}
+
+async function wipeRemoteArchive(options = {}, env = process.env) {
+  const lock = acquireSyncLock(env);
+  try {
+    const cfg = loadConfig(env);
+    const target = remoteTargetFromFlags(options, cfg, env);
+    if (target.protocol === "file") return wipeRemoteDirectory(target, options, env);
+    return wipeRemoteS3(target, options, env);
+  } finally {
+    releaseSyncLock(lock);
+  }
+}
+
+async function listRemoteSnapshots(options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const target = remoteTargetFromFlags(options, cfg, env);
+  const prefix = remoteSnapshotsPrefix(target);
+  if (target.protocol === "file") {
+    const root = path.join(target.root, prefix);
+    const objects = listDirectoryObjects(root).map((file) => {
+      const stat = safeStat(file);
+      const relative = path.relative(root, file).split(path.sep).join("/");
+      return {
+        key: [prefix, relative].filter(Boolean).join("/"),
+        size: stat?.size || 0,
+        lastModified: stat ? stat.mtime.toISOString() : ""
+      };
+    });
+    return snapshotListSummary(target, prefix, objects);
+  }
+  const listPrefix = prefix ? `${prefix}/` : "";
+  const objects = await listS3Objects(target, listPrefix, options);
+  return snapshotListSummary(target, prefix, objects);
+}
+
 async function syncArchiveIfConfigured(env = process.env, options = {}) {
   const cfg = loadConfig(env);
   if (!hasRemoteTarget(cfg, env)) return { configured: false, scanned: 0, uploaded: 0, skipped: 0, errors: [] };
@@ -150,12 +200,19 @@ async function syncArchiveToS3(target, options = {}, env = process.env) {
   const objectPrefix = remoteObjectPrefix(target, cfg, options);
   const localObjects = listLocalArchiveObjects(env, objectPrefix);
   const state = loadSyncState(env);
-  const remoteObjects = await listS3Objects(target, objectPrefix);
+  reportSyncEvent(options, {
+    current: 0,
+    total: 0,
+    message: "listing remote objects",
+    path: objectPrefix
+  });
+  const remoteObjects = await listS3Objects(target, objectPrefix, options);
   const remoteByKey = new Map(remoteObjects.map((item) => [item.key, item]));
   const summary = {
     target: `${target.type}:${target.bucket}/${objectPrefix}`,
     mode: "upload",
     device: deviceLabel(cfg),
+    prefix: objectPrefix,
     snapshot: Boolean(options.snapshot),
     snapshotName: options.snapshot ? snapshotName(options) : "",
     scanned: localObjects.length,
@@ -172,7 +229,7 @@ async function syncArchiveToS3(target, options = {}, env = process.env) {
     const stateEntry = state.objects[object.key];
     const unchangedRemote = remote && Number(remote.size) === object.size;
     const unchangedState = stateEntry && stateEntry.fingerprint === object.fingerprint && unchangedRemote;
-    if (unchangedState || (unchangedRemote && !options.force)) {
+    if (!options.force && (unchangedState || unchangedRemote)) {
       summary.skipped++;
       reportSyncProgress(options, summary, index + 1, localObjects.length, object.key);
       continue;
@@ -183,7 +240,7 @@ async function syncArchiveToS3(target, options = {}, env = process.env) {
       continue;
     }
     try {
-      const upload = await putS3ObjectWithRetry(target, object.key, fs.readFileSync(object.file), options);
+      const upload = await putS3FileObjectWithRetry(target, object, options);
       summary.retried += upload.retried;
       state.objects[object.key] = {
         fingerprint: object.fingerprint,
@@ -210,6 +267,7 @@ function syncArchiveToDirectory(target, options = {}, env = process.env) {
     target: pathToFileURL(root).href,
     mode: "upload",
     device: deviceLabel(cfg),
+    prefix: objectPrefix,
     snapshot: Boolean(options.snapshot),
     snapshotName: options.snapshot ? snapshotName(options) : "",
     scanned: localObjects.length,
@@ -246,6 +304,221 @@ function syncArchiveToDirectory(target, options = {}, env = process.env) {
   return summary;
 }
 
+async function replaceRemoteS3(target, options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const objectPrefix = remoteObjectPrefix(target, cfg, options);
+  const localObjects = listLocalArchiveObjects(env, objectPrefix);
+  reportSyncEvent(options, {
+    current: 0,
+    total: 0,
+    message: "listing remote objects to replace",
+    path: objectPrefix
+  });
+  const remoteObjects = await listS3Objects(target, objectPrefix, options);
+  const summary = replaceSummary(target, cfg, options, objectPrefix, remoteObjects.length, localObjects.length);
+  if (options["dry-run"] || options.dryRun) {
+    reportSyncEvent(options, {
+      current: remoteObjects.length,
+      total: remoteObjects.length,
+      message: `would delete=${remoteObjects.length} upload=${localObjects.length}`,
+      path: objectPrefix
+    });
+    return summary;
+  }
+
+  reportSyncEvent(options, {
+    current: 0,
+    total: remoteObjects.length,
+    message: "deleting remote objects",
+    path: objectPrefix
+  });
+  for (let index = 0; index < remoteObjects.length; index++) {
+    const object = remoteObjects[index];
+    try {
+      const deleted = await deleteS3ObjectWithRetry(target, object.key, options);
+      summary.retried += deleted.retried;
+      summary.deleted++;
+    } catch (error) {
+      summary.errors.push(`${object.key}: ${error.message}`);
+    }
+    reportSyncEvent(options, {
+      current: index + 1,
+      total: remoteObjects.length,
+      message: `deleted=${summary.deleted} errors=${summary.errors.length}`,
+      path: object.key
+    });
+  }
+  if (summary.errors.length) return summary;
+
+  const uploaded = await syncArchiveToS3(target, { ...options, force: true }, env);
+  mergeReplaceUploadSummary(summary, uploaded);
+  return summary;
+}
+
+async function wipeRemoteS3(target, options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const scope = normalizeWipeScope(options.scope || options["wipe-scope"]);
+  const deletePrefix = remoteWipePrefix(target, cfg, scope, options);
+  reportSyncEvent(options, {
+    current: 0,
+    total: 0,
+    message: `listing remote objects to wipe (${scope})`,
+    path: deletePrefix || target.bucket
+  });
+  const remoteObjects = await listS3Objects(target, deletePrefix, options);
+  const summary = wipeSummary(target, cfg, options, scope, deletePrefix, remoteObjects.length);
+  if (options["dry-run"] || options.dryRun) {
+    reportSyncEvent(options, {
+      current: remoteObjects.length,
+      total: remoteObjects.length,
+      message: `would delete=${remoteObjects.length}`,
+      path: deletePrefix || target.bucket
+    });
+    return summary;
+  }
+
+  reportSyncEvent(options, {
+    current: 0,
+    total: remoteObjects.length,
+    message: `deleting remote objects (${scope})`,
+    path: deletePrefix || target.bucket
+  });
+  for (let index = 0; index < remoteObjects.length; index++) {
+    const object = remoteObjects[index];
+    try {
+      const deleted = await deleteS3ObjectWithRetry(target, object.key, options);
+      summary.retried += deleted.retried;
+      summary.deleted++;
+    } catch (error) {
+      summary.errors.push(`${object.key}: ${error.message}`);
+    }
+    reportSyncEvent(options, {
+      current: index + 1,
+      total: remoteObjects.length,
+      message: `deleted=${summary.deleted} errors=${summary.errors.length}`,
+      path: object.key
+    });
+  }
+  return summary;
+}
+
+function replaceRemoteDirectory(target, options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const objectPrefix = remoteObjectPrefix(target, cfg, options);
+  const localObjects = listLocalArchiveObjects(env, objectPrefix);
+  const root = path.join(target.root, objectPrefix);
+  const remoteObjects = listDirectoryObjects(root);
+  const summary = replaceSummary(target, cfg, options, objectPrefix, remoteObjects.length, localObjects.length);
+  summary.target = pathToFileURL(root).href;
+  if (options["dry-run"] || options.dryRun) {
+    reportSyncEvent(options, {
+      current: remoteObjects.length,
+      total: remoteObjects.length,
+      message: `would delete=${remoteObjects.length} upload=${localObjects.length}`,
+      path: objectPrefix
+    });
+    return summary;
+  }
+
+  reportSyncEvent(options, {
+    current: 0,
+    total: remoteObjects.length,
+    message: "deleting remote objects",
+    path: objectPrefix
+  });
+  fs.rmSync(root, { recursive: true, force: true });
+  summary.deleted = remoteObjects.length;
+  reportSyncEvent(options, {
+    current: remoteObjects.length,
+    total: remoteObjects.length,
+    message: `deleted=${summary.deleted} errors=${summary.errors.length}`,
+    path: objectPrefix
+  });
+  const uploaded = syncArchiveToDirectory(target, { ...options, force: true }, env);
+  mergeReplaceUploadSummary(summary, uploaded);
+  return summary;
+}
+
+function wipeRemoteDirectory(target, options = {}, env = process.env) {
+  const cfg = loadConfig(env);
+  const scope = normalizeWipeScope(options.scope || options["wipe-scope"]);
+  const deletePrefix = remoteWipePrefix(target, cfg, scope, options);
+  const root = deletePrefix ? path.join(target.root, deletePrefix) : target.root;
+  const remoteObjects = listDirectoryObjects(root);
+  const summary = wipeSummary(target, cfg, options, scope, deletePrefix, remoteObjects.length);
+  summary.target = pathToFileURL(root).href;
+  if (options["dry-run"] || options.dryRun) {
+    reportSyncEvent(options, {
+      current: remoteObjects.length,
+      total: remoteObjects.length,
+      message: `would delete=${remoteObjects.length}`,
+      path: deletePrefix || target.root
+    });
+    return summary;
+  }
+
+  reportSyncEvent(options, {
+    current: 0,
+    total: remoteObjects.length,
+    message: `deleting remote objects (${scope})`,
+    path: deletePrefix || target.root
+  });
+  fs.rmSync(root, { recursive: true, force: true });
+  summary.deleted = remoteObjects.length;
+  reportSyncEvent(options, {
+    current: remoteObjects.length,
+    total: remoteObjects.length,
+    message: `deleted=${summary.deleted} errors=${summary.errors.length}`,
+    path: deletePrefix || target.root
+  });
+  return summary;
+}
+
+function replaceSummary(target, cfg, options, objectPrefix, deleteCount, uploadCount) {
+  return {
+    target: target.protocol === "file" ? target.endpoint : `${target.type}:${target.bucket}/${objectPrefix}`,
+    mode: "replace",
+    scope: "device",
+    device: deviceLabel(cfg),
+    prefix: objectPrefix,
+    dryRun: Boolean(options["dry-run"] || options.dryRun),
+    remoteScanned: deleteCount,
+    deleted: options["dry-run"] || options.dryRun ? deleteCount : 0,
+    scanned: uploadCount,
+    uploaded: options["dry-run"] || options.dryRun ? uploadCount : 0,
+    skipped: 0,
+    retried: 0,
+    errors: []
+  };
+}
+
+function wipeSummary(target, cfg, options, scope, deletePrefix, deleteCount) {
+  return {
+    target: target.protocol === "file" ? target.endpoint : `${target.type}:${target.bucket}/${deletePrefix || ""}`,
+    mode: "wipe",
+    scope,
+    snapshotName: scope === "snapshot" ? requestedSnapshotName(options) : "",
+    device: deviceLabel(cfg),
+    prefix: deletePrefix,
+    dryRun: Boolean(options["dry-run"] || options.dryRun),
+    remoteScanned: deleteCount,
+    deleted: options["dry-run"] || options.dryRun ? deleteCount : 0,
+    scanned: 0,
+    uploaded: 0,
+    skipped: 0,
+    retried: 0,
+    errors: []
+  };
+}
+
+function mergeReplaceUploadSummary(summary, uploaded) {
+  summary.scanned = uploaded.scanned;
+  summary.uploaded = uploaded.uploaded;
+  summary.skipped = uploaded.skipped;
+  summary.retried += uploaded.retried || 0;
+  summary.errors.push(...(uploaded.errors || []));
+}
+
 function listLocalArchiveObjects(env = process.env, prefix = "agentlog") {
   const root = archiveRoot(env);
   const objects = [];
@@ -265,7 +538,17 @@ function listLocalArchiveObjects(env = process.env, prefix = "agentlog") {
   return objects.sort((a, b) => a.key.localeCompare(b.key));
 }
 
-async function listS3Objects(target, prefix) {
+function listDirectoryObjects(root) {
+  const objects = [];
+  walk(root, (file) => {
+    const stat = safeStat(file);
+    if (!stat || !stat.isFile()) return;
+    objects.push(file);
+  });
+  return objects;
+}
+
+async function listS3Objects(target, prefix, options = {}) {
   const objects = [];
   let continuationToken = "";
   for (;;) {
@@ -278,6 +561,12 @@ async function listS3Objects(target, prefix) {
     if (response.statusCode >= 300) throw new Error(`list remote objects failed (${response.statusCode}): ${response.body.toString("utf8")}`);
     const parsed = parseListBucketResult(response.body.toString("utf8"));
     objects.push(...parsed.objects);
+    reportSyncEvent(options, {
+      current: objects.length,
+      total: 0,
+      message: `listed=${objects.length} remote`,
+      path: prefix
+    });
     if (!parsed.isTruncated || !parsed.nextContinuationToken) break;
     continuationToken = parsed.nextContinuationToken;
   }
@@ -297,6 +586,30 @@ async function putS3Object(target, key, body) {
   return response;
 }
 
+async function putS3FileObject(target, object, bodySha256) {
+  const response = await s3Request(target, {
+    method: "PUT",
+    key: object.key,
+    bodyFile: object.file,
+    bodyLength: object.size,
+    bodySha256,
+    headers: {
+      "content-type": contentTypeForKey(object.key)
+    }
+  });
+  if (response.statusCode >= 300) throw new Error(`upload failed (${response.statusCode}): ${response.body.toString("utf8")}`);
+  return response;
+}
+
+async function deleteS3Object(target, key) {
+  const response = await s3Request(target, {
+    method: "DELETE",
+    key
+  });
+  if (response.statusCode >= 300) throw new Error(`delete failed (${response.statusCode}): ${response.body.toString("utf8")}`);
+  return response;
+}
+
 async function putS3ObjectWithRetry(target, key, body, options = {}) {
   const maxAttempts = Math.max(1, Number(options.retries || options["retries"] || 3));
   let lastError;
@@ -313,8 +626,42 @@ async function putS3ObjectWithRetry(target, key, body, options = {}) {
   throw lastError;
 }
 
+async function putS3FileObjectWithRetry(target, object, options = {}) {
+  const maxAttempts = Math.max(1, Number(options.retries || options["retries"] || 3));
+  const bodySha256 = await fileSha256Hex(object.file);
+  let lastError;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      await putS3FileObject(target, object, bodySha256);
+      return { retried: attempt - 1 };
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxAttempts || !isRetryableUploadError(error)) break;
+      await delay(Math.min(250 * 2 ** (attempt - 1), 2000));
+    }
+  }
+  throw lastError;
+}
+
+async function deleteS3ObjectWithRetry(target, key, options = {}) {
+  const maxAttempts = Math.max(1, Number(options.retries || options["retries"] || 3));
+  let lastError;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      await deleteS3Object(target, key);
+      return { retried: attempt - 1 };
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxAttempts || !isRetryableUploadError(error)) break;
+      await delay(Math.min(250 * 2 ** (attempt - 1), 2000));
+    }
+  }
+  throw lastError;
+}
+
 function s3Request(target, request) {
-  const body = Buffer.isBuffer(request.body) ? request.body : Buffer.from(request.body || "");
+  const hasBodyFile = Boolean(request.bodyFile);
+  const body = hasBodyFile ? null : Buffer.isBuffer(request.body) ? request.body : Buffer.from(request.body || "");
   const method = request.method || "GET";
   const key = request.key || "";
   const query = request.query || {};
@@ -322,11 +669,12 @@ function s3Request(target, request) {
   const now = new Date();
   const headers = {
     host: target.url.host,
-    "x-amz-content-sha256": sha256Hex(body),
+    "x-amz-content-sha256": hasBodyFile ? request.bodySha256 : sha256Hex(body),
     "x-amz-date": amzDate(now),
     ...(request.headers || {})
   };
-  if (body.length) headers["content-length"] = String(body.length);
+  const bodyLength = hasBodyFile ? Number(request.bodyLength || 0) : body.length;
+  if (bodyLength) headers["content-length"] = String(bodyLength);
   headers.authorization = authorizationHeader({ target, method, pathName, query, headers, now });
 
   return new Promise((resolve, reject) => {
@@ -347,6 +695,14 @@ function s3Request(target, request) {
       }
     );
     req.on("error", reject);
+    if (hasBodyFile) {
+      const stream = fs.createReadStream(request.bodyFile);
+      stream.on("error", (error) => {
+        req.destroy(error);
+      });
+      stream.pipe(req);
+      return;
+    }
     if (body.length) req.write(body);
     req.end();
   });
@@ -434,10 +790,7 @@ function decodeXml(value) {
 }
 
 function reportSyncProgress(options, summary, current, total, key = "") {
-  if (typeof options.onProgress !== "function") return;
-  options.onProgress({
-    kind: "sync",
-    provider: "Remote Sync",
+  reportSyncEvent(options, {
     current,
     total,
     uploaded: summary.uploaded,
@@ -449,6 +802,22 @@ function reportSyncProgress(options, summary, current, total, key = "") {
   });
 }
 
+function reportSyncEvent(options, event) {
+  if (typeof options.onProgress !== "function") return;
+  options.onProgress({
+    kind: "sync",
+    provider: "Remote Sync",
+    current: event.current || 0,
+    total: event.total || 0,
+    uploaded: event.uploaded || 0,
+    skipped: event.skipped || 0,
+    retried: event.retried || 0,
+    errors: event.errors || 0,
+    message: event.message || "working",
+    path: event.path || ""
+  });
+}
+
 function loadSyncState(env = process.env) {
   return readJson(path.join(paths(env).state, "sync.json"), { objects: {} });
 }
@@ -582,6 +951,50 @@ function remoteObjectPrefix(target, cfg = {}, options = {}) {
   return [prefix, "devices", deviceSlug(cfg)].filter(Boolean).join("/");
 }
 
+function remoteSnapshotsPrefix(target) {
+  return [normalizePrefix(target.prefix || "agentlog"), "snapshots"].filter(Boolean).join("/");
+}
+
+function remoteWipePrefix(target, cfg = {}, scope = "device", options = {}) {
+  if (scope === "bucket") return "";
+  const prefix = normalizePrefix(target.prefix || "agentlog");
+  if (scope === "prefix") return prefix;
+  if (scope === "snapshots") return `${remoteSnapshotsPrefix(target)}/`;
+  if (scope === "snapshot") return `${remoteSnapshotsPrefix(target)}/${requestedSnapshotName(options)}/`;
+  return remoteObjectPrefix(target, cfg, {});
+}
+
+function normalizeWipeScope(scope) {
+  const value = String(scope || "device").trim().toLowerCase();
+  const normalized = {
+    current: "device",
+    "current-device": "device",
+    device: "device",
+    namespace: "device",
+    prefix: "prefix",
+    archive: "prefix",
+    agentlog: "prefix",
+    snapshot: "snapshot",
+    "one-snapshot": "snapshot",
+    "single-snapshot": "snapshot",
+    snapshots: "snapshots",
+    "all-snapshots": "snapshots",
+    "snapshot-prefix": "snapshots",
+    all: "bucket",
+    bucket: "bucket"
+  }[value] || value;
+  if (!["device", "snapshot", "snapshots", "prefix", "bucket"].includes(normalized)) {
+    throw new Error("sync wipe --scope must be device, snapshot, snapshots, prefix, or bucket");
+  }
+  return normalized;
+}
+
+function requestedSnapshotName(options = {}) {
+  const explicit = value(options["snapshot-name"] || options.snapshotName || options.name || options.snapshot);
+  if (!explicit) throw new Error("sync wipe --scope snapshot requires --snapshot-name <name>");
+  return slugifyRemoteSegment(explicit);
+}
+
 function snapshotName(options = {}) {
   const explicit = value(options["snapshot-name"] || options.snapshotName || options.name);
   if (explicit) return slugifyRemoteSegment(explicit);
@@ -591,6 +1004,27 @@ function snapshotName(options = {}) {
   return options._snapshotName;
 }
 
+function snapshotListSummary(target, prefix, objects = []) {
+  const snapshots = new Map();
+  for (const object of objects) {
+    const relative = stripPrefix(object.key, prefix).replace(/^\/+/, "");
+    const name = relative.split("/").filter(Boolean)[0];
+    if (!name) continue;
+    const current = snapshots.get(name) || { name, objects: 0, bytes: 0, lastModified: "" };
+    current.objects++;
+    current.bytes += Number(object.size || 0);
+    if (object.lastModified && (!current.lastModified || String(object.lastModified) > current.lastModified)) {
+      current.lastModified = String(object.lastModified);
+    }
+    snapshots.set(name, current);
+  }
+  return {
+    target: target.protocol === "file" ? pathToFileURL(path.join(target.root, prefix)).href : `${target.type}:${target.bucket}/${prefix}`,
+    prefix,
+    snapshots: Array.from(snapshots.values()).sort((a, b) => b.name.localeCompare(a.name))
+  };
+}
+
 function deviceSlug(cfg = {}) {
   return slugifyRemoteSegment(cfg.device?.slug || cfg.device?.name || cfg.device?.id || "this-device");
 }
@@ -652,6 +1086,16 @@ function sha256Hex(value) {
   return crypto.createHash("sha256").update(value).digest("hex");
 }
 
+function fileSha256Hex(file) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash("sha256");
+    const stream = fs.createReadStream(file);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("error", reject);
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
 function hmac(key, value) {
   return crypto.createHmac("sha256", key).update(value).digest();
 }
@@ -668,10 +1112,13 @@ module.exports = {
   configureRemoteFromFlags,
   hasRemoteTarget,
   listLocalArchiveObjects,
+  listRemoteSnapshots,
   parseListBucketResult,
   remoteTargetFromFlags,
   remoteObjectPrefix,
+  replaceRemoteArchive,
   snapshotArchive,
   syncArchive,
-  syncArchiveIfConfigured
+  syncArchiveIfConfigured,
+  wipeRemoteArchive
 };