agentsys 5.13.4 → 6.0.0

package/lib/repo-intel/embed/binary.js

@@ -0,0 +1,242 @@
+'use strict';
+
+/**
+ * Binary resolver for `agent-analyzer-embed`.
+ *
+ * Mirrors the structure of `lib/binary/index.js` but for the separate
+ * embedder binary. Kept as its own module rather than parameterizing
+ * the existing resolver — the current single-binary helper is heavily
+ * specialized and a refactor would touch every call site for one
+ * use case.
+ *
+ * Both binaries share:
+ *   - the same install dir (`~/.agent-sh/bin/`)
+ *   - the same release-tag-aware download (latest tag with TTL cache)
+ *   - the same platform map
+ *
+ * They differ in:
+ *   - binary name (`agent-analyzer-embed` here)
+ *   - GitHub release asset path uses the embed binary name
+ *
+ * @module lib/embed/binary
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const https = require('https');
+const cp = require('child_process');
+
+// Reuse PLATFORM_MAP from the main resolver to avoid drift if a new
+// platform is added.
+const mainBinary = require('../../binary');
+// Reuse HTTP + archive helpers so a single bug fix to e.g. timeout
+// behavior or redirect handling lands once and applies to both
+// binaries (`agent-analyzer` and `agent-analyzer-embed`).
+const sharedHelpers = require('../../binary/shared-helpers');
+
+const EMBED_BINARY_NAME = 'agent-analyzer-embed';
+const EMBED_GITHUB_REPO = 'agent-sh/agent-analyzer';
+const LATEST_VERSION_TTL_MS = 60 * 60 * 1000;
+const PLATFORM_MAP = mainBinary.PLATFORM_MAP;
+
+function getBinaryPath() {
+  const ext = process.platform === 'win32' ? '.exe' : '';
+  return path.join(os.homedir(), '.agent-sh', 'bin', EMBED_BINARY_NAME + ext);
+}
+
+// Platform-specific ONNX Runtime dylib bundled beside the embed binary in the
+// release tarball (see release.yml). The Rust binary's runtime resolver
+// (resolve_and_preflight_ort) looks for exactly this name next to the
+// executable. Mirrored here so the JS side can tell whether an existing
+// install predates the bundling and needs a re-download.
+function getBundledOrtName() {
+  if (process.platform === 'win32') return 'onnxruntime.dll';
+  if (process.platform === 'darwin') return 'libonnxruntime.dylib';
+  return 'libonnxruntime.so';
+}
+
+function getBundledOrtPath() {
+  return path.join(path.dirname(getBinaryPath()), getBundledOrtName());
+}
+
+// musl has no bundled ORT (no MS musl build; a glibc dylib cannot dlopen under
+// musl). On musl we never expect a sibling lib, so its absence must not trigger
+// a perpetual re-download loop.
+function platformBundlesOrt() {
+  const key = getPlatformKey();
+  return !!key && !key.includes('musl');
+}
+
+function getPlatformKey() {
+  const key = process.platform + '-' + process.arch;
+  return PLATFORM_MAP[key] || null;
+}
+
+function getVersion() {
+  const binPath = getBinaryPath();
+  if (!fs.existsSync(binPath)) return null;
+  try {
+    const out = cp.execFileSync(binPath, ['--version'], {
+      timeout: 5000,
+      encoding: 'utf8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+      windowsHide: true
+    });
+    const match = out.trim().match(/(\d+\.\d+\.\d+)/);
+    return match ? match[1] : out.trim();
+  } catch (e) {
+    return null;
+  }
+}
+
+function isAvailable() {
+  return fs.existsSync(getBinaryPath());
+}
+
+let _latestVersionCache = null;
+
+async function getLatestReleaseVersion() {
+  if (
+    _latestVersionCache &&
+    Date.now() - _latestVersionCache.fetchedAt < LATEST_VERSION_TTL_MS
+  ) {
+    return _latestVersionCache.version;
+  }
+  return new Promise(function (resolve, reject) {
+    const ghToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+    const headers = {
+      'User-Agent': 'agent-sh/embed-resolver',
+      'Accept': 'application/vnd.github+json'
+    };
+    if (ghToken) headers['Authorization'] = 'Bearer ' + ghToken;
+
+    const url = 'https://api.github.com/repos/' + EMBED_GITHUB_REPO + '/releases/latest';
+    const fail = function (msg) {
+      reject(new Error(msg + ' fetching ' + url));
+    };
+    const req = https.get(url, { headers: headers, timeout: 5000 }, function (res) {
+      if (res.statusCode !== 200) {
+        res.resume();
+        fail('HTTP ' + res.statusCode);
+        return;
+      }
+      const chunks = [];
+      res.on('data', function (chunk) { chunks.push(chunk); });
+      res.on('end', function () {
+        try {
+          const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+          const tag = (body && body.tag_name) || '';
+          const version = tag.replace(/^v/, '');
+          if (/^\d+\.\d+\.\d+/.test(version)) {
+            _latestVersionCache = { version: version, fetchedAt: Date.now() };
+            resolve(version);
+          } else {
+            fail('No valid release tag');
+          }
+        } catch (e) {
+          fail('Failed to parse release JSON: ' + e.message);
+        }
+      });
+      res.on('error', function (e) { fail(e.message); });
+    });
+    req.on('error', function (e) { fail(e.message); });
+    req.on('timeout', function () { req.destroy(); fail('Timeout'); });
+  });
+}
+
+function buildDownloadUrl(ver, platformKey) {
+  const ext = process.platform === 'win32' ? '.zip' : '.tar.gz';
+  return (
+    'https://github.com/' +
+    EMBED_GITHUB_REPO +
+    '/releases/download/v' +
+    ver +
+    '/' +
+    EMBED_BINARY_NAME +
+    '-' +
+    platformKey +
+    ext
+  );
+}
+
+// Per-call shim so the embed resolver passes its own User-Agent
+// header without duplicating the rest of the HTTP plumbing.
+function downloadToBuffer(url) {
+  return sharedHelpers.downloadToBuffer(url, { userAgent: 'agent-sh/embed-resolver' });
+}
+
+const extractTarGz = sharedHelpers.extractTarGz;
+const extractZip = sharedHelpers.extractZip;
+
+async function downloadBinary(ver) {
+  const platformKey = getPlatformKey();
+  if (!platformKey) {
+    throw new Error(
+      'Unsupported platform: ' + process.platform + '-' + process.arch + '. ' +
+      'Supported: ' + Object.keys(PLATFORM_MAP).join(', ')
+    );
+  }
+  const url = buildDownloadUrl(ver, platformKey);
+  process.stderr.write('Downloading ' + EMBED_BINARY_NAME + ' v' + ver + ' for ' + platformKey + '...\n');
+
+  const binPath = getBinaryPath();
+  const binDir = path.dirname(binPath);
+  fs.mkdirSync(binDir, { recursive: true });
+
+  let buf;
+  try {
+    buf = await downloadToBuffer(url);
+  } catch (err) {
+    throw new Error(
+      'Failed to download ' + EMBED_BINARY_NAME + ':\n' +
+      '  URL: ' + url + '\n' +
+      '  Error: ' + err.message + '\n\n' +
+      'To install manually:\n' +
+      '  1. Download: ' + url + '\n' +
+      '  2. Extract the binary to: ' + binDir + '\n' +
+      '  3. Ensure it is named: ' + path.basename(binPath)
+    );
+  }
+
+  if (process.platform === 'win32') {
+    await extractZip(buf, binDir, path.basename(binPath));
+  } else {
+    await extractTarGz(buf, binDir);
+  }
+  if (process.platform !== 'win32') {
+    fs.chmodSync(binPath, 0o755);
+  }
+  return binPath;
+}
+
+async function ensureBinary(options) {
+  const opts = options || {};
+  const binPath = getBinaryPath();
+  if (fs.existsSync(binPath)) {
+    // An install from before ORT bundling has the binary but no sibling
+    // dylib; without it the embedder fails at runtime. Re-download once to
+    // pick up the bundled lib. Skip on musl (never bundled) so this can't loop.
+    if (platformBundlesOrt() && !fs.existsSync(getBundledOrtPath())) {
+      const targetVer = opts.version || (await getLatestReleaseVersion());
+      return downloadBinary(targetVer);
+    }
+    return binPath;
+  }
+  const targetVer = opts.version || (await getLatestReleaseVersion());
+  return downloadBinary(targetVer);
+}
+
+module.exports = {
+  EMBED_BINARY_NAME,
+  getBinaryPath,
+  getBundledOrtName,
+  getBundledOrtPath,
+  platformBundlesOrt,
+  getVersion,
+  getPlatformKey,
+  getLatestReleaseVersion,
+  isAvailable,
+  ensureBinary,
+  buildDownloadUrl
+};

package/lib/repo-intel/embed/index.js

@@ -0,0 +1,26 @@
+'use strict';
+
+/**
+ * Public surface for the embed module — preference cache, binary
+ * resolver, and high-level orchestrator (scan / update / status).
+ *
+ * Skill code should import from here, not the internal modules,
+ * so the wiring stays swappable.
+ *
+ * @module lib/embed
+ */
+
+const preference = require('./preference');
+const binary = require('./binary');
+const orchestrator = require('./orchestrator');
+
+module.exports = {
+  preference,
+  binary,
+  orchestrator,
+  // Convenience re-exports for the common cases.
+  isEnabled: orchestrator.isEnabled,
+  runScan: orchestrator.runScan,
+  runUpdate: orchestrator.runUpdate,
+  status: orchestrator.status
+};

package/lib/repo-intel/embed/orchestrator.js

@@ -0,0 +1,239 @@
+'use strict';
+
+/**
+ * High-level embed orchestration. Glues together:
+ *
+ *   user preference  →  binary download  →  scan/update  →  set-embeddings
+ *
+ * Called from the `/repo-intel enrich` command after the existing
+ * weighter and summarizer Haiku agents finish; degrades to a no-op
+ * when the user has chosen `embedder: "none"`.
+ *
+ * Also exposes `runUpdate()` for the standalone `/repo-intel embed
+ * update` action group (and the `npx ... embed update` CI hook).
+ *
+ * @module lib/embed/orchestrator
+ */
+
+const fs = require('fs');
+const path = require('path');
+const cp = require('child_process');
+
+const preference = require('./preference');
+const embedBinary = require('./binary');
+const mainBinary = require('../../binary');
+const cache = require('../cache');
+
+/**
+ * Should the orchestrator run for this repo? Returns false when the
+ * user has not opted in (preference unset or 'none'), which lets
+ * callers safely no-op without wrapping every call site in a guard.
+ *
+ * @param {string} cwd
+ * @returns {boolean}
+ */
+function isEnabled(cwd) {
+  const pref = preference.read(cwd);
+  return pref.embedder === 'small' || pref.embedder === 'big';
+}
+
+/**
+ * Run a full scan: ensures the embed binary is downloaded, runs the
+ * scan subcommand, pipes the JSON document into `agent-analyzer
+ * repo-intel set-embeddings`. Returns a small status object so callers
+ * can report what happened.
+ *
+ * @param {string} cwd
+ * @returns {Promise<{ran: boolean, files?: number, durationMs?: number, reason?: string}>}
+ */
+async function runScan(cwd) {
+  if (!isEnabled(cwd)) {
+    return { ran: false, reason: 'embedder preference is "none" or unset' };
+  }
+  const pref = preference.read(cwd);
+  const detail = preference.detailToCliArg(pref.embedderDetail || 'balanced');
+
+  const mapFile = cache.getPath(cwd);
+  if (!fs.existsSync(mapFile)) {
+    return { ran: false, reason: 'no repo-intel map found; run `/repo-intel init` first' };
+  }
+
+  const start = Date.now();
+  const embedBin = await embedBinary.ensureBinary();
+  const mainBin = await mainBinary.ensureBinary();
+
+  const result = await streamEmbedToSetEmbeddings(
+    embedBin,
+    ['scan', cwd, '--variant', pref.embedder, '--detail', detail],
+    mainBin,
+    mapFile
+  );
+  return Object.assign({ ran: true, durationMs: Date.now() - start }, result);
+}
+
+/**
+ * Run a delta update: only re-embeds files whose content hash differs
+ * from the existing sidecar. Falls back to a full scan when no
+ * sidecar exists yet.
+ *
+ * @param {string} cwd
+ * @returns {Promise<{ran: boolean, files?: number, durationMs?: number, reason?: string}>}
+ */
+async function runUpdate(cwd) {
+  if (!isEnabled(cwd)) {
+    return { ran: false, reason: 'embedder preference is "none" or unset' };
+  }
+  const pref = preference.read(cwd);
+  const detail = preference.detailToCliArg(pref.embedderDetail || 'balanced');
+
+  const mapFile = cache.getPath(cwd);
+  if (!fs.existsSync(mapFile)) {
+    return { ran: false, reason: 'no repo-intel map; run `/repo-intel init` then `enrich`' };
+  }
+
+  const start = Date.now();
+  const embedBin = await embedBinary.ensureBinary();
+  const mainBin = await mainBinary.ensureBinary();
+
+  const result = await streamEmbedToSetEmbeddings(
+    embedBin,
+    ['update', cwd, '--map-file', mapFile, '--variant', pref.embedder, '--detail', detail],
+    mainBin,
+    mapFile
+  );
+  return Object.assign({ ran: true, durationMs: Date.now() - start }, result);
+}
+
+/**
+ * Status snapshot: which preference is set, whether the binary is
+ * installed, whether the bundled ONNX Runtime is present, whether the
+ * sidecar exists.
+ *
+ * `ortBundled` surfaces the prerequisite the embed binary needs at runtime:
+ * the binary loads libonnxruntime from beside itself (shipped in the release
+ * tarball). On a platform that bundles ORT, a present binary with a missing
+ * lib means an upgrade re-download is pending - ensureBinary handles it, but
+ * status reports it so callers can explain a one-time re-fetch.
+ *
+ * @param {string} cwd
+ * @returns {{enabled: boolean, embedder?: string, embedderDetail?: string, binaryInstalled: boolean, ortBundled: boolean, sidecarExists: boolean, sidecarPath?: string}}
+ */
+function status(cwd) {
+  const pref = preference.read(cwd);
+  const mapFile = cache.getPath(cwd);
+  const sidecarPath = deriveSidecarPath(mapFile);
+  return {
+    enabled: isEnabled(cwd),
+    embedder: pref.embedder,
+    embedderDetail: pref.embedderDetail,
+    binaryInstalled: embedBinary.isAvailable(),
+    ortBundled: !embedBinary.platformBundlesOrt() || fs.existsSync(embedBinary.getBundledOrtPath()),
+    sidecarExists: fs.existsSync(sidecarPath),
+    sidecarPath: sidecarPath
+  };
+}
+
+/**
+ * Stream the embed binary's stdout directly into the main binary's
+ * `set-embeddings --input -` stdin. The intermediate JSON document
+ * can run into the megabytes for big repos at high detail; piping
+ * keeps memory flat instead of buffering the whole document.
+ *
+ * The promise resolves with `{ files }` when both children exit cleanly;
+ * rejects with the failing child's exit code + captured stderr otherwise.
+ *
+ * Hardened against the failure modes a bare `pipe()` leaves open: a stream
+ * error (e.g. set-embeddings dies mid-write) used to leave the promise
+ * unsettled forever and leak the surviving process. Now any failure path
+ * — spawn error, stream error, or non-zero exit — kills the sibling and
+ * rejects once. Embed stderr is captured (not inherited) so the error
+ * message carries the diagnostic instead of just an exit code.
+ */
+function streamEmbedToSetEmbeddings(embedBinPath, embedArgs, mainBinPath, mapFile) {
+  return new Promise(function (resolve, reject) {
+    const embedChild = cp.spawn(embedBinPath, embedArgs, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      windowsHide: true
+    });
+    const setChild = cp.spawn(
+      mainBinPath,
+      ['repo-intel', 'set-embeddings', '--map-file', mapFile, '--input', '-'],
+      { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
+    );
+
+    let embedExit = null;
+    let setExit = null;
+    let settled = false;
+    let setStdout = '';
+    let embedStderr = '';
+    let setStderr = '';
+
+    // Single exit path. Kills the sibling on any failure so neither process
+    // is left running against a closed pipe (FD leak / zombie).
+    function done(err, value) {
+      if (settled) return;
+      settled = true;
+      if (err) {
+        try { embedChild.kill('SIGTERM'); } catch (e) { /* already gone */ }
+        try { setChild.kill('SIGTERM'); } catch (e) { /* already gone */ }
+        reject(err);
+      } else {
+        resolve(value);
+      }
+    }
+
+    function maybeFinish() {
+      if (settled || embedExit === null || setExit === null) return;
+      if (embedExit !== 0) {
+        return done(new Error(
+          embedBinary.EMBED_BINARY_NAME + ' exited ' + embedExit +
+          (embedStderr.trim() ? ': ' + embedStderr.trim().slice(0, 500) : '')
+        ));
+      }
+      if (setExit !== 0) {
+        return done(new Error(
+          'agent-analyzer set-embeddings exited ' + setExit +
+          (setStderr.trim() ? ': ' + setStderr.trim().slice(0, 500) : '')
+        ));
+      }
+      const m = setStdout.match(/(\d+)\s+files?/);
+      done(null, { files: m ? parseInt(m[1], 10) : undefined });
+    }
+
+    // stderr piped (not inherited) so failures carry a message.
+    embedChild.stderr.on('data', function (d) { embedStderr += d.toString('utf8'); });
+    setChild.stderr.on('data', function (d) { setStderr += d.toString('utf8'); });
+    setChild.stdout.on('data', function (d) { setStdout += d.toString('utf8'); });
+
+    // Stream wiring with error handling on BOTH ends of the pipe — a bare
+    // .pipe() swallows these and hangs.
+    embedChild.stdout.on('error', function (e) { done(e); });
+    setChild.stdin.on('error', function (e) {
+      // EPIPE when set-embeddings has already exited is benign — its close
+      // handler reports the real cause. Only surface other stdin errors.
+      if (e && e.code !== 'EPIPE') done(e);
+    });
+    embedChild.stdout.pipe(setChild.stdin);
+
+    embedChild.on('error', function (e) { done(e); });
+    setChild.on('error', function (e) { done(e); });
+    embedChild.on('close', function (code) { embedExit = code; maybeFinish(); });
+    setChild.on('close', function (code) { setExit = code; maybeFinish(); });
+  });
+}
+
+function deriveSidecarPath(mapFile) {
+  if (!mapFile) return '';
+  const dir = path.dirname(mapFile);
+  const stem = path.basename(mapFile, path.extname(mapFile));
+  return path.join(dir, stem + '.embeddings.bin');
+}
+
+module.exports = {
+  isEnabled,
+  runScan,
+  runUpdate,
+  status,
+  // exported for testing the dual-process pipe in isolation
+  streamEmbedToSetEmbeddings
+};

package/lib/repo-intel/embed/preference.js

@@ -0,0 +1,136 @@
+'use strict';
+
+/**
+ * User preference for the embedder. Persists the answer to two
+ * one-time prompts so subsequent enrich runs don't re-ask:
+ *
+ *   embedder       : "none" | "small" | "big"
+ *   embedderDetail : "compact" | "balanced" | "maximum"
+ *
+ * Stored in `<stateDir>/sources/preference.json` alongside the existing
+ * task source preference (so the user has a single place to clear all
+ * cached choices via `/repo-intel embed reset`).
+ *
+ * @module lib/embed/preference
+ */
+
+const fs = require('fs');
+const path = require('path');
+const cache = require('../cache');
+
+const VALID_EMBEDDER = ['none', 'small', 'big'];
+const VALID_DETAIL = ['compact', 'balanced', 'maximum'];
+
+// State-directory resolution delegates to `cache.getStateDirPath` so
+// the embedder reads/writes the same directory the artifact loader
+// uses. Reviewer-flagged split-brain risk: an inline copy here would
+// drift on candidate order, env-var support, or other policy and
+// silently land preferences in a different place than the map file.
+function preferencePath(cwd) {
+  return path.join(cache.getStateDirPath(cwd), 'sources', 'preference.json');
+}
+
+/**
+ * Load preference from disk. Returns an empty object when absent or
+ * unreadable so callers can treat "unset" uniformly.
+ *
+ * @param {string} cwd
+ * @returns {{embedder?: string, embedderDetail?: string}}
+ */
+function read(cwd) {
+  const p = preferencePath(cwd);
+  if (!fs.existsSync(p)) return {};
+  try {
+    const raw = JSON.parse(fs.readFileSync(p, 'utf8'));
+    return raw && typeof raw === 'object' ? raw : {};
+  } catch (e) {
+    return {};
+  }
+}
+
+/**
+ * Merge updates into the on-disk preference file. Creates the
+ * containing directory if needed.
+ *
+ * @param {string} cwd
+ * @param {Object} patch  fields to set/overwrite
+ * @returns {Object} the merged preference
+ */
+function update(cwd, patch) {
+  const current = read(cwd);
+  const next = Object.assign({}, current, patch || {});
+  const p = preferencePath(cwd);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  fs.writeFileSync(p, JSON.stringify(next, null, 2));
+  return next;
+}
+
+/**
+ * Strip embedder fields from preference. Used by `embed reset` to
+ * trigger fresh prompts on the next enrich.
+ *
+ * @param {string} cwd
+ */
+function reset(cwd) {
+  const current = read(cwd);
+  delete current.embedder;
+  delete current.embedderDetail;
+  const p = preferencePath(cwd);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  fs.writeFileSync(p, JSON.stringify(current, null, 2));
+}
+
+/**
+ * Did the user already answer the embedder install prompt?
+ *
+ * @param {string} cwd
+ * @returns {boolean}
+ */
+function hasEmbedderChoice(cwd) {
+  const pref = read(cwd);
+  return VALID_EMBEDDER.includes(pref.embedder);
+}
+
+/**
+ * Did the user already answer the detail prompt? Only meaningful when
+ * embedder !== 'none'.
+ *
+ * @param {string} cwd
+ * @returns {boolean}
+ */
+function hasDetailChoice(cwd) {
+  const pref = read(cwd);
+  return VALID_DETAIL.includes(pref.embedderDetail);
+}
+
+/**
+ * Translate the user-facing detail label into the analyzer-embed CLI
+ * value. Centralizes the mapping so the CLI flag never appears as a
+ * string literal scattered across modules.
+ *
+ * @param {string} detail
+ * @returns {string}
+ */
+function detailToCliArg(detail) {
+  switch (detail) {
+    case 'compact':
+      return 'compact';
+    case 'maximum':
+      return 'maximum';
+    case 'balanced':
+    default:
+      return 'balanced';
+  }
+}
+
+module.exports = {
+  read,
+  update,
+  reset,
+  hasEmbedderChoice,
+  hasDetailChoice,
+  detailToCliArg,
+  preferencePath,
+  VALID_EMBEDDER,
+  VALID_DETAIL
+};