@empirical-gauzza/cli 1.0.9

package/lib/skillSync.js

@@ -0,0 +1,508 @@
+'use strict';
+
+// US-011: core sync logic shared by `empirical kit use`, `empirical kit sync`, and
+// `empirical --install`. Fetches the device-scoped resolved Skill set from the server and
+// writes each Skill's SKILL.md into every detected agent's skill directory, tracking a local
+// manifest for the local-edit guard and additive/removal diffing.
+const crypto = require('crypto');
+const fs = require('fs');
+const matter = require('gray-matter');
+const os = require('os');
+const path = require('path');
+const { httpJson } = require('./http');
+const { log, warn, createPrompter } = require('./ui');
+const { detectInstallTargets } = require('./installers/detect');
+const {
+  readDeviceConfig,
+  writeDeviceConfig,
+  ensureDeviceId,
+  readSyncManifest,
+  writeSyncManifest,
+  readPendingReview,
+  writePendingReview
+} = require('./deviceConfig');
+
+const SYNC_READ_MAX_ATTEMPTS = 2;
+const SYNC_READ_RETRY_DELAY_MS = 250;
+
+function isRetryableSyncReadError(error) {
+  if (error?.statusCode) return false;
+  return /socket hang up|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|ENOTFOUND|network/i.test(error?.message || '');
+}
+
+async function fetchSyncRead(url, headers, label) {
+  let lastError;
+  for (let attempt = 1; attempt <= SYNC_READ_MAX_ATTEMPTS; attempt += 1) {
+    try {
+      return await httpJson(url, 'GET', undefined, headers);
+    } catch (error) {
+      lastError = error;
+      if (attempt >= SYNC_READ_MAX_ATTEMPTS || !isRetryableSyncReadError(error)) break;
+      await new Promise((resolve) => setTimeout(resolve, SYNC_READ_RETRY_DELAY_MS));
+    }
+  }
+
+  const detail = lastError?.message || 'unknown request error';
+  const wrapped = new Error(`${label} request failed: ${detail}`);
+  if (lastError?.code) wrapped.code = lastError.code;
+  if (lastError?.statusCode) wrapped.statusCode = lastError.statusCode;
+  wrapped.cause = lastError;
+  throw wrapped;
+}
+
+// Per-agent skill file path conventions, mirrored exactly from the hardcoded paths already
+// used by installers/claude.js, installers/codex.js, installers/copilot.js (for the single
+// empirical-memory skill) — just parameterized by skill name instead of hardcoded to it.
+function skillFilePath(targetId, skillName) {
+  const home = os.homedir();
+  if (targetId === 'claude') return path.join(home, '.claude', 'skills', skillName, 'SKILL.md');
+  if (targetId === 'codex') return path.join(home, '.agents', 'skills', skillName, 'SKILL.md');
+  if (targetId === 'copilot') return path.join(home, '.copilot', 'skills', skillName, 'SKILL.md');
+  return null;
+}
+
+function hashContent(content) {
+  return crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+}
+
+// Builds the SKILL.md file content for a resolved Skill. The server stores name/description
+// as structured fields and bodyMarkdown as frontmatter-stripped content (see gitResolver in
+// sourceResolver.service.ts, which parses the original SKILL.md's YAML frontmatter via
+// gray-matter and keeps only parsed.content) — so the frontmatter block must be reconstructed
+// here, or every agent that requires a well-formed SKILL.md (Codex CLI validates this strictly)
+// fails to load the skill at all.
+// Skills resolved from a git_custom Source carry license/authorName/authorUrl/upstreamRepoUrl —
+// append a short attribution footer for those. Skills with none of those fields set (authored
+// Skills, or the builtin ones this module never writes — see the CRITICAL exclusion below) get
+// no fabricated attribution block, just the frontmatter + bodyMarkdown as-is.
+function buildSkillFileContent(skill) {
+  const body = String(skill.bodyMarkdown || '');
+  const attributionParts = [];
+  if (skill.upstreamRepoUrl) attributionParts.push(`Source: ${skill.upstreamRepoUrl}.`);
+  if (skill.license) attributionParts.push(`License: ${skill.license}.`);
+  if (skill.authorName && skill.authorUrl) {
+    attributionParts.push(`Author: ${skill.authorName} (${skill.authorUrl}).`);
+  } else if (skill.authorName) {
+    attributionParts.push(`Author: ${skill.authorName}.`);
+  } else if (skill.authorUrl) {
+    attributionParts.push(`Author URL: ${skill.authorUrl}.`);
+  }
+
+  const withAttribution = attributionParts.length === 0
+    ? body
+    : `${body.replace(/\s+$/, '')}\n\n---\n\n*${attributionParts.join(' ')}*\n`;
+
+  // js-yaml's dump throws on `undefined` values, so default rather than pass them through raw.
+  return matter.stringify(withAttribution, {
+    name: skill.name || '',
+    description: skill.description || ''
+  });
+}
+
+// ────────────────────────────────────────────────────────────────────────────────────────
+// CRITICAL TRAP TO AVOID (see PRD US-011): the server's resolveBuiltin() (see
+// packages/server/src/services/sourceResolver.service.ts) currently returns ONE hardcoded
+// PLACEHOLDER Skill named `empirical-memory`, whose bodyMarkdown is literally
+// "Placeholder body — real content wiring is a later story." — it is NOT the native
+// empirical-memory plugin content. The plugin repository owns the real Skill, and native
+// agent plugin installers are responsible for installing it.
+//
+// If this generalized writer were allowed to write a resolved Skill named `empirical-memory`,
+// it would silently overwrite that real plugin file with the server's placeholder stub text on
+// every sync — a real regression for every existing user. So: any
+// resolved Skill named exactly `empirical-memory` is filtered out here, unconditionally,
+// before any writing/tracking/removal-consideration happens. A future story that finally
+// wires up real builtin content server-side (not this one) needs to revisit this filter.
+function filterOutEmpiricalMemory(skills) {
+  return (skills || []).filter((skill) => skill && skill.name !== 'empirical-memory');
+}
+// ────────────────────────────────────────────────────────────────────────────────────────
+
+function readOnDiskHash(filePath) {
+  if (!fs.existsSync(filePath)) return null;
+  try {
+    return hashContent(fs.readFileSync(filePath, 'utf8'));
+  } catch (_error) {
+    return null;
+  }
+}
+
+function removeFileAndMaybeDir(filePath) {
+  try {
+    fs.rmSync(filePath, { force: true });
+  } catch (_error) {
+    // best-effort
+  }
+  try {
+    const dir = path.dirname(filePath);
+    if (fs.existsSync(dir) && fs.readdirSync(dir).length === 0) {
+      fs.rmdirSync(dir);
+    }
+  } catch (_error) {
+    // best-effort cleanup only, never fatal
+  }
+}
+
+// Picks the four attribution fields off a resolved Skill, omitting any that aren't set.
+// Returns null if none of the fields are present (nothing to attribute — e.g. authored Skills).
+function attributionFieldsFor(skill) {
+  const fields = {};
+  if (skill.license) fields.license = skill.license;
+  if (skill.authorName) fields.authorName = skill.authorName;
+  if (skill.authorUrl) fields.authorUrl = skill.authorUrl;
+  if (skill.upstreamRepoUrl) fields.upstreamRepoUrl = skill.upstreamRepoUrl;
+  return Object.keys(fields).length > 0 ? fields : null;
+}
+
+// Best-effort identifying label for a git_custom-sourced skill, used both in the human-facing
+// review prompt and as the `source` field persisted to pending-review.json.
+function sourceLabelFor(attribution) {
+  if (!attribution) return null;
+  return attribution.upstreamRepoUrl || attribution.authorUrl || attribution.authorName || 'git_custom source';
+}
+
+// US-016: review-gate prompts. Both print a short human-facing summary line and, when
+// interactive, block on askYesNo. Never called when quiet/dryRun — callers gate that.
+async function confirmFirstSync(prompter, skill, attribution) {
+  const kind = attribution ? 'git_custom' : 'authored';
+  warn(`  new: ${skill.name} (${kind})`);
+  return prompter.askYesNo(`  Install new skill "${skill.name}"?`, true);
+}
+
+async function confirmContentChange(prompter, skill, attribution) {
+  warn(`  changed: ${skill.name} (git_custom, source: ${sourceLabelFor(attribution)})`);
+  return prompter.askYesNo(`  Apply upstream content change to "${skill.name}"?`, true);
+}
+
+// Computes (and, unless dryRun, applies) the sync diff for a single agent target.
+// Returns { added, updated, removed, skippedLocalEdit, declinedNew, declinedChange } (arrays of
+// { name, filePath }) and, via the `nextEntries` out-param behavior, the manifest entries to
+// persist for this target.
+//
+// US-016 review gates (see PRD US-016):
+//   - First-ever sync of ANY skill (authored or git_custom): interactive runs require an
+//     explicit y/n via askYesNo before writing. Declining leaves it unwritten/untracked (it's
+//     simply offered again next sync, since there's no manifest entry either way).
+//   - Content change (resolved hash differs from the manifest, and the file was NOT locally
+//     edited) to a git_custom-sourced skill (attributionFieldsFor() != null): interactive runs
+//     require y/n before overwriting; declining leaves the OLD file untouched and the manifest
+//     entry unchanged. Non-interactive (--quiet) runs NEVER block — they apply the change
+//     immediately and instead record it in pending-review.json via the `flagPendingReview`
+//     callback (so the device never silently falls behind), to be reviewed later through an
+//     interactive `kit sync`.
+//   - Authored skills are exempt from the content-change gate entirely (only the first-sync
+//     gate applies to them) — the push confirmation in `skill push` already is the review.
+//   - The existing local-edit guard (US-011) is checked FIRST and wins outright: a locally
+//     edited file is skipped before either review gate is ever considered, so a local edit
+//     always takes precedence over an upstream content change being applied/flagged.
+async function syncTarget({ targetId, skills, previousEntries, dryRun, force, quiet, prompter, clearPendingReview, flagPendingReview }) {
+  const added = [];
+  const updated = [];
+  const removed = [];
+  const skippedLocalEdit = [];
+  const declinedNew = [];
+  const declinedChange = [];
+  const nextEntries = {};
+
+  const newNames = new Set(skills.map((s) => s.name));
+
+  for (const skill of skills) {
+    const filePath = skillFilePath(targetId, skill.name);
+    if (!filePath) continue;
+    const content = buildSkillFileContent(skill);
+    const newHash = hashContent(content);
+    const prevEntry = previousEntries[skill.name];
+    const attribution = attributionFieldsFor(skill);
+
+    if (!prevEntry) {
+      // AC1: first-ever sync of this skill name — gate interactively, never in quiet/dry-run.
+      if (!dryRun && !quiet) {
+        const accepted = await confirmFirstSync(prompter, skill, attribution);
+        if (!accepted) {
+          declinedNew.push({ name: skill.name, action: 'declined_new', filePath });
+          // Left unwritten and untracked — offered again next sync.
+          continue;
+        }
+      }
+      added.push({ name: skill.name, action: 'added', filePath, ...attribution });
+      if (!dryRun) {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, content, 'utf8');
+        nextEntries[skill.name] = { filePath, contentHash: newHash, resolvedVersion: skill.resolvedVersion };
+      }
+      continue;
+    }
+
+    const onDiskHash = readOnDiskHash(filePath);
+    const locallyEdited = onDiskHash !== null && onDiskHash !== prevEntry.contentHash;
+
+    if (locallyEdited && !force) {
+      // Local-edit guard (US-011) wins outright over either review gate below.
+      skippedLocalEdit.push({ name: skill.name, action: 'skipped_local_edit', filePath });
+      if (!dryRun) {
+        nextEntries[skill.name] = prevEntry;
+      }
+      continue;
+    }
+
+    if (onDiskHash === newHash) {
+      // Nothing changed — carry the entry forward untouched, no write needed.
+      if (!dryRun) {
+        nextEntries[skill.name] = prevEntry;
+      }
+      continue;
+    }
+
+    // AC2/AC3: content changed upstream (and not a local edit — that was ruled out above).
+    // Only git_custom-sourced skills (attribution present) are gated; authored skills apply
+    // like any other update, no prompt, no pending-review tracking.
+    if (attribution && !dryRun) {
+      if (quiet) {
+        // Never block under --quiet: apply immediately, but flag for later human review.
+        if (flagPendingReview) {
+          flagPendingReview(skill.name, { from: prevEntry.contentHash, source: sourceLabelFor(attribution) });
+        }
+      } else {
+        const accepted = await confirmContentChange(prompter, skill, attribution);
+        if (!accepted) {
+          declinedChange.push({ name: skill.name, action: 'declined_change', filePath });
+          // Leave the old file content in place, untouched; manifest entry unchanged.
+          nextEntries[skill.name] = prevEntry;
+          continue;
+        }
+        // Accepted: applying now IS the review — clear any stale pending-review flag.
+        if (clearPendingReview) clearPendingReview(skill.name);
+      }
+    }
+
+    updated.push({ name: skill.name, action: 'updated', filePath, ...attribution });
+    if (!dryRun) {
+      fs.mkdirSync(path.dirname(filePath), { recursive: true });
+      fs.writeFileSync(filePath, content, 'utf8');
+      nextEntries[skill.name] = { filePath, contentHash: newHash, resolvedVersion: skill.resolvedVersion };
+    }
+  }
+
+  for (const name of Object.keys(previousEntries)) {
+    if (newNames.has(name)) continue;
+    const prevEntry = previousEntries[name];
+    removed.push({ name, action: 'removed', filePath: prevEntry.filePath });
+    if (!dryRun) {
+      removeFileAndMaybeDir(prevEntry.filePath);
+      // omitted from nextEntries -> removed from manifest
+    }
+  }
+
+  return { added, updated, removed, skippedLocalEdit, declinedNew, declinedChange, nextEntries };
+}
+
+// US-016 AC6: a dedicated review pass for skills that were already applied unattended (during a
+// prior `--quiet` sync) and are still sitting in pending-review.json. This is deliberately
+// separate from syncTarget's own content-change gate, because by the time this runs again
+// interactively the on-disk content already matches the resolved content (the quiet run already
+// wrote it) — so syncTarget's hash-diff would see "no change" and never re-surface it. Mutates
+// `pendingReview` in place (deleting resolved entries) and returns the list of names reviewed.
+//
+// Design choice (see AC "declining an already-applied change" ambiguity): on decline we
+// ACKNOWLEDGE AND CLEAR the flag WITHOUT reverting the file. Two reasons: (1) pending-review.json
+// intentionally stores only a `from` hash/description, not the full prior file content, so an
+// exact revert isn't reliably possible from the data this story's shape defines; and (2) the
+// device may have already been running with the new content for a while (that's the whole
+// premise of --quiet applying immediately) — silently flipping it back on a later, unrelated
+// `kit sync` would itself be an unattended content change, the exact failure mode this story
+// exists to prevent. Declining here is recorded as "reviewed, not re-approved for next time"
+// rather than "undo".
+async function reviewPendingEntries(pendingReview, quiet, dryRun, prompter) {
+  const names = Object.keys(pendingReview);
+  if (names.length === 0 || quiet || dryRun) return [];
+
+  warn('The following skill(s) were updated by an unattended sync and are awaiting your review:');
+  const reviewed = [];
+  for (const name of names) {
+    const entry = pendingReview[name];
+    warn(`  ${name} (source: ${entry.source || 'unknown'}, applied ${entry.firstDetectedAt || 'unknown time'})`);
+    const acknowledged = await prompter.askYesNo(`  Mark "${name}" as reviewed? (content already applied; declining just clears the flag, it does not revert)`, true);
+    warn(acknowledged ? `  ${name}: reviewed, keeping applied content.` : `  ${name}: acknowledged without re-approval; content left as-is.`);
+    delete pendingReview[name];
+    reviewed.push(name);
+  }
+  return reviewed;
+}
+
+// Core sync entrypoint. Never calls process.exit — callers (kitCommands.js, installWizard.js)
+// own exit-code/printing behavior. Throws on HTTP failure (callers use printApiError/warn).
+async function performSync({ accessToken, baseUrl, dryRun = false, force = false, quiet = false }) {
+  const deviceId = ensureDeviceId();
+  const config = readDeviceConfig();
+
+  // US-016: a SINGLE shared prompter (readline interface) for the whole sync, only opened when
+  // prompting is even possible (interactive, not dry-run) and closed exactly once at the very
+  // end — see createPrompter()'s doc comment in ui.js for why per-question interfaces break
+  // against piped/non-TTY stdin.
+  const prompter = (!quiet && !dryRun) ? createPrompter(process.stderr) : null;
+
+  try {
+    // US-016: surface + clear any previously-flagged unattended content changes before anything
+    // else. Runs regardless of whether a new version is available, since pending-review is local
+    // state independent of the server's version counter. No-ops under --quiet/--dry-run.
+    const pendingReview = readPendingReview();
+    const reviewed = await reviewPendingEntries(pendingReview, quiet, dryRun, prompter);
+    if (reviewed.length > 0 && !dryRun) {
+      writePendingReview(pendingReview);
+    }
+
+    const versionUrl = new URL(`/api/v1/devices/${encodeURIComponent(deviceId)}/version`, baseUrl).toString();
+    const versionResponse = await fetchSyncRead(versionUrl, {
+      Authorization: `Bearer ${accessToken}`,
+      'x-empirical-surface': 'cli'
+    }, 'Kit version');
+    const versionData = (versionResponse && versionResponse.data) ? versionResponse.data : versionResponse;
+    const version = versionData.version;
+
+    if (!force && config.lastSyncedVersion && config.lastSyncedVersion === version) {
+      return { upToDate: true, dryRun, version, deviceId, targets: {}, tierSkipped: [], reviewed };
+    }
+
+    const resolvedUrl = new URL(`/api/v1/devices/${encodeURIComponent(deviceId)}/resolved`, baseUrl).toString();
+    const resolvedResponse = await fetchSyncRead(resolvedUrl, {
+      Authorization: `Bearer ${accessToken}`,
+      'x-empirical-surface': 'cli'
+    }, 'Resolved Kit skills');
+    const resolvedData = (resolvedResponse && resolvedResponse.data) ? resolvedResponse.data : resolvedResponse;
+    const skills = filterOutEmpiricalMemory(resolvedData.skills);
+    const tierSkipped = resolvedData.skipped || [];
+
+    const detectedTargets = detectInstallTargets();
+    const manifest = readSyncManifest();
+    const targets = {};
+    const nextManifest = { ...manifest };
+
+    // US-016: pending-review mutations collected across all targets in this sync, written once
+    // at the end (mirrors nextManifest's accumulate-then-write pattern). A skill written for
+    // multiple detected targets (e.g. claude + codex) is only asked about once per performSync
+    // call — the flag/clear closures are keyed by skill name, not by target.
+    const nextPendingReview = { ...pendingReview };
+    const flagPendingReview = (name, info) => {
+      const existing = nextPendingReview[name];
+      nextPendingReview[name] = {
+        from: info.from,
+        firstDetectedAt: existing ? existing.firstDetectedAt : new Date().toISOString(),
+        source: info.source,
+        status: 'content_changed_unreviewed'
+      };
+    };
+    const clearPendingReview = (name) => {
+      delete nextPendingReview[name];
+    };
+
+    for (const target of detectedTargets) {
+      const previousEntries = manifest[target.id] || {};
+      const result = await syncTarget({
+        targetId: target.id,
+        skills,
+        previousEntries,
+        dryRun,
+        force,
+        quiet,
+        prompter,
+        flagPendingReview,
+        clearPendingReview
+      });
+      targets[target.id] = {
+        added: result.added,
+        updated: result.updated,
+        removed: result.removed,
+        skippedLocalEdit: result.skippedLocalEdit,
+        declinedNew: result.declinedNew,
+        declinedChange: result.declinedChange
+      };
+      if (!dryRun) {
+        nextManifest[target.id] = result.nextEntries;
+      }
+    }
+
+    if (!dryRun) {
+      writeSyncManifest(nextManifest);
+      writeDeviceConfig({ ...config, lastSyncedVersion: version });
+      writePendingReview(nextPendingReview);
+    }
+
+    return {
+      upToDate: false,
+      dryRun,
+      version,
+      deviceId,
+      detectedTargetIds: detectedTargets.map((t) => t.id),
+      targets,
+      tierSkipped,
+      reviewed
+    };
+  } finally {
+    if (prompter) prompter.close();
+  }
+}
+
+// Shared stdout/stderr reporting for a performSync() result, used by `kit sync`, `kit use`,
+// and `empirical --install`. JSON summary (added/updated/removed/skippedLocalEdit per target,
+// with counts) goes to stdout via log() for machine parsing; the /resolved response's
+// cross-tier `skipped` collisions go to stderr via warn() as human-readable warnings, matching
+// the existing CLI convention (structured JSON results on stdout, human warnings on stderr).
+// Fully suppressed when quiet is true (still writes files/manifest — this only controls
+// printing).
+function printSyncSummary(result, quiet) {
+  if (quiet) return;
+
+  if (result.upToDate) {
+    log(JSON.stringify({ status: 'up_to_date', version: result.version }));
+    return;
+  }
+
+  const targetsSummary = {};
+  const attributionByName = new Map();
+  for (const [targetId, t] of Object.entries(result.targets)) {
+    targetsSummary[targetId] = {
+      added: t.added,
+      updated: t.updated,
+      removed: t.removed,
+      skippedLocalEdit: t.skippedLocalEdit,
+      declinedNew: t.declinedNew || [],
+      declinedChange: t.declinedChange || [],
+      counts: {
+        added: t.added.length,
+        updated: t.updated.length,
+        removed: t.removed.length,
+        skippedLocalEdit: t.skippedLocalEdit.length,
+        declinedNew: (t.declinedNew || []).length,
+        declinedChange: (t.declinedChange || []).length
+      }
+    };
+    for (const entry of [...t.added, ...t.updated]) {
+      const { license, authorName, authorUrl, upstreamRepoUrl } = entry;
+      if (!license && !authorName && !authorUrl && !upstreamRepoUrl) continue;
+      if (attributionByName.has(entry.name)) continue;
+      attributionByName.set(entry.name, { name: entry.name, license: license || null, authorName: authorName || null, authorUrl: authorUrl || null, upstreamRepoUrl: upstreamRepoUrl || null });
+    }
+  }
+
+  log(JSON.stringify({
+    status: result.dryRun ? 'dry_run' : 'synced',
+    version: result.version,
+    deviceId: result.deviceId,
+    targets: targetsSummary,
+    attribution: Array.from(attributionByName.values()),
+    reviewed: result.reviewed || []
+  }));
+
+  for (const skip of result.tierSkipped || []) {
+    warn(`Skipped "${skip.name}" (from ${skip.from}, lost to ${skip.lostTo})`);
+  }
+}
+
+module.exports = {
+  skillFilePath,
+  hashContent,
+  buildSkillFileContent,
+  filterOutEmpiricalMemory,
+  performSync,
+  printSyncSummary
+};