agentel 0.2.6 → 0.3.0

package/src/unavailable-sources.js

@@ -0,0 +1,358 @@
+"use strict";
+
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { isWebChatProvider, listSessions } = require("./archive");
+const { ensureDir, readJson } = require("./paths");
+const { HISTORY_PROVIDER_OPTIONS, IMPORT_SOURCE_ORDER } = require("./sources");
+
+const CLAUDE_CODE_REPAIR_COMMAND = "agentlog repair claude-code-backups";
+const CLAUDE_CODE_REPAIR_PREVIEW_COMMAND = `${CLAUDE_CODE_REPAIR_COMMAND} --dry-run`;
+
+function preservedArchiveSession(session) {
+  return preservedWebChatSession(session) || sessionHasUnavailableSource(session);
+}
+
+function preservedWebChatSession(session) {
+  if (isWebChatProvider(session?.provider, session?.sourceType)) return true;
+  return /^\[(chatgpt|claude)\]conversations\//i.test(String(session?.scopeCanonical || ""));
+}
+
+function unavailableSourceSessions(env = process.env) {
+  return listSessions(env).filter((session) => !preservedWebChatSession(session) && sessionHasUnavailableSource(session));
+}
+
+function sessionHasUnavailableSource(session) {
+  return unavailableSourcePaths(session).length > 0;
+}
+
+function unavailableSourcePaths(session) {
+  return requiredSourcePaths(session).filter((sourcePath) => !safeSourceStat(sourcePath));
+}
+
+function requiredSourcePaths(session) {
+  const values = [filesystemSourcePath(session?.sourcePath)];
+  if (sessionMayNeedAuxiliaryTranscripts(session)) {
+    for (const record of sessionRawRecords(session)) {
+      const sourcePath = filesystemSourcePath(record?.originalPath || record?.sourcePath);
+      if (auxiliaryTranscriptSourcePath(sourcePath)) values.push(sourcePath);
+    }
+  }
+  return [...new Set(values.filter(Boolean).map((value) => path.resolve(value)))];
+}
+
+function sessionMayNeedAuxiliaryTranscripts(session) {
+  const provider = String(session?.provider || "");
+  const sourceType = String(session?.sourceType || "");
+  return provider === "claude_code" || provider === "claude_desktop" || provider === "claude_sdk" || sourceType.startsWith("claude-");
+}
+
+function auxiliaryTranscriptSourcePath(sourcePath) {
+  if (!sourcePath) return false;
+  return /(?:^|[\\/])audit\.jsonl$/i.test(sourcePath) || /\.jsonl(?:\.zst)?$/i.test(sourcePath);
+}
+
+function filesystemSourcePath(value) {
+  const text = String(value || "").trim();
+  if (!text) return "";
+  const hashIndex = text.indexOf("#");
+  const base = hashIndex > 0 ? text.slice(0, hashIndex) : text;
+  if (!base) return "";
+  if (path.isAbsolute(base)) return path.resolve(base);
+  if (path.win32.isAbsolute(base)) return base;
+  return "";
+}
+
+function sessionRawRecords(session) {
+  const records = Array.isArray(session?.rawFiles) ? [...session.rawFiles] : [];
+  if (session?.rawPath) {
+    const manifest = readJson(path.join(session.rawPath, "manifest.json"), null);
+    for (const record of Array.isArray(manifest?.files) ? manifest.files : []) {
+      records.push(record);
+    }
+  }
+  return records;
+}
+
+function summarizeUnavailableSourceSessions(sessions, options = {}) {
+  const selected = (Array.isArray(sessions) ? sessions : []).filter((session) =>
+    !preservedWebChatSession(session) && (options.includeAvailable || sessionHasUnavailableSource(session))
+  );
+  const breakdown = unavailableSourceBreakdown(selected);
+  const totalMissingSources = breakdown.reduce((sum, row) => sum + row.missingSources, 0);
+  const claudeCodeRepair = summarizeClaudeCodeBackupRepair(selected, options.env || process.env);
+  return {
+    totalSessions: selected.length,
+    totalMissingSources,
+    breakdown,
+    claudeCodeRepair
+  };
+}
+
+function unavailableSourceBreakdown(sessions) {
+  const byKey = new Map();
+  for (const session of sessions || []) {
+    const source = sourceForSession(session);
+    const provider = String(session?.provider || "unknown");
+    const sourceType = String(session?.sourceType || "unknown");
+    const key = `${source}\0${provider}\0${sourceType}`;
+    const existing = byKey.get(key) || {
+      source,
+      provider,
+      sourceType,
+      sessions: 0,
+      missingSources: 0
+    };
+    existing.sessions++;
+    existing.missingSources += unavailableSourcePaths(session).length;
+    byKey.set(key, existing);
+  }
+  return [...byKey.values()].sort(compareSourceBreakdownRows);
+}
+
+function sourceForSession(session) {
+  const provider = String(session?.provider || "");
+  const sourceType = String(session?.sourceType || "");
+  const exact = SOURCE_BY_PROVIDER_AND_TYPE.get(`${provider}\0${sourceType}`);
+  if (exact) return exact;
+  const sourceOption = HISTORY_PROVIDER_OPTIONS.find((item) => item.provider === provider && item.sourceType === sourceType);
+  if (sourceOption?.source) return sourceOption.source;
+  return SOURCE_BY_PROVIDER.get(provider) || sourceType || provider || "unknown";
+}
+
+function compareSourceBreakdownRows(a, b) {
+  const left = sourceOrder(a.source);
+  const right = sourceOrder(b.source);
+  return left - right
+    || a.source.localeCompare(b.source)
+    || a.provider.localeCompare(b.provider)
+    || a.sourceType.localeCompare(b.sourceType);
+}
+
+function sourceOrder(source) {
+  const index = IMPORT_SOURCE_ORDER.indexOf(source);
+  return index === -1 ? 999 : index;
+}
+
+function summarizeClaudeCodeBackupRepair(sessions, env = process.env) {
+  const plan = planClaudeCodeBackupRepair(env, { sessions, dryRun: true });
+  return {
+    candidateSessions: plan.candidateSessions,
+    restoreSessions: plan.restoreSessions,
+    restoreCount: plan.restoreCount,
+    existingCount: plan.existingCount,
+    missingBackupCount: plan.missingBackupCount,
+    command: CLAUDE_CODE_REPAIR_COMMAND,
+    previewCommand: CLAUDE_CODE_REPAIR_PREVIEW_COMMAND
+  };
+}
+
+function planClaudeCodeBackupRepair(env = process.env, options = {}) {
+  const roots = claudeCodeSourceRoots(env);
+  const sessions = (Array.isArray(options.sessions) ? options.sessions : unavailableSourceSessions(env))
+    .filter(claudeCodeBackupRepairSession);
+  const seenTargets = new Set();
+  const actions = [];
+  const skipped = [];
+  for (const session of sessions) {
+    let sessionActionCount = 0;
+    for (const record of sessionRawRecords(session)) {
+      const targetPath = filesystemSourcePath(record?.originalPath || record?.sourcePath);
+      if (!targetPath || !isInsideAnyRoot(targetPath, roots)) continue;
+      const targetKey = path.resolve(targetPath);
+      if (seenTargets.has(targetKey)) {
+        skipped.push(skipRecord(session, record, targetPath, "duplicate-target"));
+        continue;
+      }
+      seenTargets.add(targetKey);
+      const backupPath = backupPathForRawRecord(record);
+      if (!backupPath || !safeSourceStat(backupPath)?.isFile()) {
+        skipped.push(skipRecord(session, record, targetPath, "missing-backup"));
+        continue;
+      }
+      const targetStat = safeSourceStat(targetPath);
+      if (targetStat && !options.overwrite) {
+        skipped.push(skipRecord(session, record, targetPath, "target-exists", backupPath));
+        continue;
+      }
+      actions.push({
+        sessionId: session.sessionId || "",
+        title: session.title || "",
+        sourceType: session.sourceType || "",
+        provider: session.provider || "",
+        targetPath,
+        backupPath,
+        size: Number(record?.size || 0) || safeSourceStat(backupPath)?.size || 0,
+        mtime: record?.mtime || ""
+      });
+      sessionActionCount++;
+    }
+    if (!sessionActionCount) {
+      const sourcePath = filesystemSourcePath(session?.sourcePath);
+      if (sourcePath && isInsideAnyRoot(sourcePath, roots)) {
+        skipped.push({
+          sessionId: session.sessionId || "",
+          targetPath: sourcePath,
+          reason: "no-restorable-raw-backup"
+        });
+      }
+    }
+  }
+  const restoreSessionIds = new Set(actions.map((item) => item.sessionId).filter(Boolean));
+  return {
+    dryRun: Boolean(options.dryRun),
+    overwrite: Boolean(options.overwrite),
+    sourceRoots: roots,
+    candidateSessions: sessions.length,
+    restoreSessions: restoreSessionIds.size,
+    restoreCount: actions.length,
+    existingCount: skipped.filter((item) => item.reason === "target-exists").length,
+    missingBackupCount: skipped.filter((item) => item.reason === "missing-backup" || item.reason === "no-restorable-raw-backup").length,
+    duplicateCount: skipped.filter((item) => item.reason === "duplicate-target").length,
+    actions,
+    skipped
+  };
+}
+
+function repairClaudeCodeBackups(env = process.env, options = {}) {
+  const plan = planClaudeCodeBackupRepair(env, options);
+  if (options.dryRun) return { ...plan, restored: 0 };
+  let restored = 0;
+  for (const action of plan.actions) {
+    if (!isInsideAnyRoot(action.targetPath, plan.sourceRoots)) continue;
+    if (safeSourceStat(action.targetPath) && !options.overwrite) continue;
+    ensureDir(path.dirname(action.targetPath));
+    fs.copyFileSync(action.backupPath, action.targetPath);
+    try {
+      fs.chmodSync(action.targetPath, 0o600);
+    } catch {
+      // Best-effort permissions normalization.
+    }
+    if (action.mtime) {
+      const mtime = new Date(action.mtime);
+      if (Number.isFinite(mtime.getTime())) {
+        try {
+          fs.utimesSync(action.targetPath, mtime, mtime);
+        } catch {
+          // Best-effort timestamp preservation.
+        }
+      }
+    }
+    restored++;
+  }
+  return { ...plan, restored };
+}
+
+function claudeCodeBackupRepairSession(session) {
+  return String(session?.provider || "") === "claude_code";
+}
+
+function claudeCodeSourceRoots(env = process.env) {
+  const home = env && env.HOME ? env.HOME : os.homedir();
+  return [
+    path.join(home, ".claude", "projects"),
+    path.join(home, ".claude", "file-history")
+  ];
+}
+
+function backupPathForRawRecord(record) {
+  const candidates = [record?.archivedPath, record?.sharedRawPath]
+    .map(filesystemSourcePath)
+    .filter(Boolean);
+  return candidates.find((candidate) => safeSourceStat(candidate)?.isFile()) || "";
+}
+
+function skipRecord(session, record, targetPath, reason, backupPath = "") {
+  return {
+    sessionId: session?.sessionId || "",
+    title: session?.title || "",
+    sourceType: session?.sourceType || "",
+    provider: session?.provider || "",
+    targetPath,
+    backupPath: backupPath || backupPathForRawRecord(record),
+    reason
+  };
+}
+
+function isInsideAnyRoot(value, roots) {
+  return roots.some((root) => isSameOrInside(value, root));
+}
+
+function isSameOrInside(value, root) {
+  const relative = path.relative(path.resolve(root), path.resolve(value));
+  if (relative === "") return true;
+  // Entry names may legitimately start with ".." (e.g. "..foo"), so only an
+  // exact ".." segment marks the path as outside the root.
+  return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+}
+
+function safeSourceStat(target) {
+  try {
+    return fs.statSync(target);
+  } catch (error) {
+    if (["ENOENT", "ENOTDIR", "EACCES", "EPERM"].includes(error.code)) return null;
+    throw error;
+  }
+}
+
+const SOURCE_BY_PROVIDER_AND_TYPE = new Map([
+  ["codex\0cli-history", "codex-cli"],
+  ["claude_code\0cli-history", "claude"],
+  ["claude_sdk\0claude-sdk-history", "claude-sdk"],
+  ["claude_desktop\0claude-code-desktop-metadata", "claude-code-desktop"],
+  ["claude_desktop\0claude-workspace-desktop", "claude-cowork"],
+  ["cursor\0cursor-workspace-sqlite", "cursor"],
+  ["cursor\0cursor-global-sqlite", "cursor"],
+  ["cursor\0cursor-raw-sqlite-salvage", "cursor"],
+  ["cursor\0cursor-agent-transcripts", "cursor"],
+  ["opencode\0opencode-history", "opencode-cli"],
+  ["opencode\0opencode-sqlite-history", "opencode-cli"],
+  ["opencode\0opencode-cli-sqlite-history", "opencode-cli"],
+  ["opencode\0opencode-desktop-sqlite-history", "opencode-desktop"],
+  ["opencode\0opencode-web-sqlite-history", "opencode-web"],
+  ["antigravity_cli\0antigravity-cli-brain", "antigravity-cli"],
+  ["antigravity_ide\0antigravity-ide-brain", "antigravity-ide"],
+  ...HISTORY_PROVIDER_OPTIONS
+    .filter((item) => item.provider && item.sourceType)
+    .map((item) => [`${item.provider}\0${item.sourceType}`, item.source])
+]);
+
+const SOURCE_BY_PROVIDER = {
+  aider: "aider",
+  antigravity: "antigravity",
+  antigravity_cli: "antigravity-cli",
+  antigravity_ide: "antigravity-ide",
+  chatgpt: "chatgpt",
+  claude_code: "claude",
+  claude_sdk: "claude-sdk",
+  claude_web: "claude-web",
+  cline: "cline",
+  copilot: "copilot-cli",
+  cursor: "cursor",
+  devin: "devin-cli",
+  factory: "factory",
+  gemini_cli: "gemini-cli",
+  grok: "grok-build",
+  pi: "pi",
+  windsurf: "windsurf"
+};
+
+module.exports = {
+  CLAUDE_CODE_REPAIR_COMMAND,
+  CLAUDE_CODE_REPAIR_PREVIEW_COMMAND,
+  claudeCodeSourceRoots,
+  filesystemSourcePath,
+  planClaudeCodeBackupRepair,
+  preservedArchiveSession,
+  preservedWebChatSession,
+  repairClaudeCodeBackups,
+  requiredSourcePaths,
+  sessionHasUnavailableSource,
+  sessionRawRecords,
+  sourceForSession,
+  summarizeUnavailableSourceSessions,
+  unavailableSourceBreakdown,
+  unavailableSourcePaths,
+  unavailableSourceSessions
+};

package/src/web-export-instructions.js

@@ -7,17 +7,19 @@ const WEB_EXPORT_INSTRUCTIONS = {
     label: "ChatGPT",
     requestUrl: "https://privacy.openai.com/policies/en/",
     helpUrl: "https://help.openai.com/en/articles/7260999-how-do-i-export-my-chatgpt-history-and-data",
-    fileDescription: "official ChatGPT data export ZIP or extracted folder",
-    importCommand: "agentlog import chatgpt <path-to-export.zip> --username <email> --scope local",
+    fileDescription: "official OpenAI export ZIP, then unzip it to the OpenAI-export/User Online Activity folder",
+    importCommand: "agentlog import chatgpt",
     steps: [
       "Open the OpenAI Privacy Portal, choose Make a Privacy Request, select I have a consumer ChatGPT account, then select Download my data.",
       "Or in ChatGPT, open your profile menu, choose Settings, open Data Controls, then use Export Data.",
       "Complete account verification and wait for the export email.",
-      "Download the ZIP before the email link expires, then pass the ZIP or extracted folder to agentlog."
+      "Download the ZIP before the email link expires.",
+      "For privacy-portal exports named OpenAI-export, unzip the outer download and pass the parent User Online Activity folder so split Conversations__...part-0001/0002 ZIPs or folders, manifests, and attached files are imported together."
     ],
     notes: [
       "OpenAI says ChatGPT Business and Enterprise chat exports are not available through the ChatGPT account export flow.",
-      "OpenAI export emails can take time to arrive, and the download link expires after 24 hours."
+      "OpenAI export emails can take time to arrive, and the download link expires after 24 hours.",
+      "If your export has multiple Conversations__...chatgpt...part ZIPs or folders, import their parent folder rather than one part at a time."
     ]
   },
   "claude-web": {