@myvillage/cli 1.89.2 → 1.90.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myvillage/cli",
-  "version": "1.89.2",
+  "version": "1.90.0",
   "description": "MyVillageOS CLI for community developers",
   "type": "module",
   "bin": {

package/src/agent-runtime/log-shipper.js

@@ -0,0 +1,350 @@
+// ── Agent log shipper ────────────────────────────────────────────────────────
+// Ships a rolling tail of each leased agent's logs to the control plane, so an
+// agent running on this machine can be read from Studio on another one without
+// SSH.
+//
+// Design notes worth knowing before editing:
+//
+//   * Recent, not complete. Only the newest lines are shipped and the server
+//     keeps a bounded window per agent. Full history stays here on disk and is
+//     read with `myvillage agent logs`. This is what makes the whole thing scale
+//     with fleet size instead of uptime.
+//
+//   * Batched per machine. One request carries every leased agent's delta, so
+//     request rate tracks machine count, not agent count.
+//
+//   * Never on the heartbeat. Lease renewal is correctness-critical; log volume
+//     must not be able to slow it down or fail it.
+//
+//   * Secrets are masked HERE, before anything leaves the box. Tool arguments
+//     and results do ship (they are the point), but .env values and the OAuth
+//     token never do.
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { getAgentDir } from '../utils/local-agent.js';
+import { getAccessToken } from '../utils/auth.js';
+
+const CURSOR_FILE = path.join(os.homedir(), '.myvillage', 'log-cursors.json');
+
+/** Per-stream ship caps. Must not exceed the server's retention, or we would be
+ *  uploading lines it is about to delete. */
+const SHIP_CAP = { ACTIVITY: 300, STDERR: 100 };
+
+/** Hard ceiling on one request across all agents; mirrors the server's cap. */
+const MAX_LINES_PER_PUSH = 2000;
+
+/** Shortest .env value we will mask. Below this you get false positives that
+ *  black out ordinary words — a var set to "true" or a family name. */
+const MIN_SECRET_LEN = 8;
+
+const DATE_LOG_RE = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
+
+// ── Cursor persistence ───────────────────────────────────────────────────────
+// Keyed `${agentProfileId}:${stream}` → { file, offset }. Persisted so a runner
+// restart resumes mid-file instead of re-shipping a tail we already sent.
+
+function readCursors() {
+  try {
+    return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+function writeCursors(cursors) {
+  try {
+    fs.mkdirSync(path.dirname(CURSOR_FILE), { recursive: true });
+    fs.writeFileSync(CURSOR_FILE, JSON.stringify(cursors, null, 2), { mode: 0o600 });
+  } catch {
+    /* best-effort: a lost cursor costs one duplicated tail, not correctness */
+  }
+}
+
+const key = (agentProfileId, stream) => `${agentProfileId}:${stream}`;
+
+// ── Redaction ────────────────────────────────────────────────────────────────
+
+/**
+ * Literal secrets for one agent: its own .env values plus this machine's OAuth
+ * token. Values only — keys stay readable, so `GITHUB_TOKEN=••••` still tells
+ * you which credential is in play.
+ */
+function secretsForAgent(agentName) {
+  const out = [];
+  try {
+    const envPath = path.join(getAgentDir(agentName), '.env');
+    const text = fs.readFileSync(envPath, 'utf-8');
+    for (const line of text.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      const eq = trimmed.indexOf('=');
+      if (eq <= 0) continue;
+      let value = trimmed.slice(eq + 1).trim();
+      // Strip matching surrounding quotes so we mask the value, not the quoting.
+      if (value.length >= 2 && /^(".*"|'.*')$/s.test(value)) value = value.slice(1, -1);
+      if (value.length >= MIN_SECRET_LEN) out.push(value);
+    }
+  } catch {
+    /* no .env, or unreadable — nothing agent-specific to mask */
+  }
+  try {
+    const token = getAccessToken();
+    if (token && token.length >= MIN_SECRET_LEN) out.push(token);
+  } catch {
+    /* not signed in */
+  }
+  // Longest first, so a value containing another is masked whole.
+  return out.sort((a, b) => b.length - a.length);
+}
+
+/** Shapes that look like credentials even when we have never seen the value —
+ *  a token echoed back by a third-party API, for instance. */
+const SECRET_PATTERNS = [
+  /\b(sk|pk|rk)-[A-Za-z0-9_-]{16,}\b/g,
+  /\bgh[pousr]_[A-Za-z0-9]{16,}\b/g,
+  /\bAKIA[0-9A-Z]{12,}\b/g,
+  /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g,
+  /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, // JWT
+  /\b[Bb]earer\s+[A-Za-z0-9._~+/-]{20,}=*/g,
+];
+
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Mask every known secret and credential-shaped token in a line.
+ *
+ * Runs on every shipped line, so it must be cheap and must never throw — a
+ * redaction bug that crashes the shipper would take log visibility down, and one
+ * that silently returns the input would leak. On failure we drop the line rather
+ * than ship it unmasked.
+ */
+export function redactSecrets(line, secrets) {
+  let out = line;
+  for (const secret of secrets) {
+    if (!secret) continue;
+    out = out.split(secret).join('••••');
+  }
+  for (const re of SECRET_PATTERNS) out = out.replace(re, '••••');
+  return out;
+}
+
+// ── Tailing ──────────────────────────────────────────────────────────────────
+
+/** The daemon writes `logs/<YYYY-MM-DD>.jsonl` and rolls over at UTC midnight.
+ *  ISO names sort chronologically, so lexical max is the current file. */
+function latestActivityFile(logsDir) {
+  try {
+    const names = fs.readdirSync(logsDir).filter((n) => DATE_LOG_RE.test(n));
+    if (names.length === 0) return null;
+    names.sort();
+    return names[names.length - 1];
+  } catch {
+    return null;
+  }
+}
+
+function sourceFileFor(agentName, stream) {
+  const logsDir = path.join(getAgentDir(agentName), 'logs');
+  if (stream === 'STDERR') {
+    const f = path.join(logsDir, 'daemon-stderr.log');
+    return fs.existsSync(f) ? { dir: logsDir, name: 'daemon-stderr.log', full: f } : null;
+  }
+  const name = latestActivityFile(logsDir);
+  return name ? { dir: logsDir, name, full: path.join(logsDir, name) } : null;
+}
+
+/**
+ * Read the lines this agent has produced since our cursor.
+ *
+ * Resets to the tail (not the top) whenever continuity is broken: a new day's
+ * file, a truncated file, or no cursor at all. That is the backlog cap — a
+ * machine that was offline for a day ships its recent tail, never a day of
+ * backlog, so a fleet reconnecting after an outage cannot stampede the endpoint.
+ */
+function readNewLines(agentName, stream, cursor) {
+  const src = sourceFileFor(agentName, stream);
+  if (!src) return null;
+
+  let size;
+  try {
+    size = fs.statSync(src.full).size;
+  } catch {
+    return null;
+  }
+
+  const continuous =
+    cursor && cursor.file === src.name && typeof cursor.offset === 'number' && cursor.offset <= size;
+  const start = continuous ? cursor.offset : 0;
+  if (continuous && start === size) return null; // nothing new
+
+  let text = '';
+  const len = size - start;
+  if (len > 0) {
+    let fd;
+    try {
+      fd = fs.openSync(src.full, 'r');
+      const buf = Buffer.alloc(len);
+      fs.readSync(fd, buf, 0, len, start);
+      text = buf.toString('utf8');
+    } catch {
+      return null;
+    } finally {
+      if (fd !== undefined) {
+        try { fs.closeSync(fd); } catch { /* ignore */ }
+      }
+    }
+  }
+
+  let lines = text.split('\n').filter((l) => l.trim().length > 0);
+  const cap = SHIP_CAP[stream];
+  // On a discontinuity we may be looking at a whole file; keep only the tail.
+  if (lines.length > cap) lines = lines.slice(-cap);
+
+  return { lines, file: src.name, offset: size };
+}
+
+/** Pull the line's own timestamp out of the JSONL, so a backlog shipped after a
+ *  reconnect keeps its real times instead of collapsing onto arrival. */
+function timestampOf(payload) {
+  if (payload.charCodeAt(0) !== 123 /* { */) return null;
+  try {
+    const obj = JSON.parse(payload);
+    const ts = obj.ts ?? obj.timestamp ?? obj.time;
+    return typeof ts === 'string' ? ts : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Shipper ──────────────────────────────────────────────────────────────────
+
+/**
+ * Ship one round for every agent this runner holds a lease on.
+ *
+ * `leases` is the runner's Map of agentProfileId → { name }. Only lease holders
+ * may ship, which the server enforces too — the same lease that guarantees
+ * single-active execution guarantees a single writer per agent's log.
+ *
+ * Bootstrap: on the first round after start we send a zero-line batch to learn
+ * the server's cursor. If it says this machine wrote it, we adopt the offset and
+ * carry on mid-file; otherwise we start from the tail. That is what keeps a
+ * runner restart from re-shipping (and thus duplicating) the last few hundred
+ * lines it had already sent.
+ */
+export async function shipLogsOnce(ctx) {
+  if (!ctx.leases || ctx.leases.size === 0) return { pushed: 0 };
+
+  const cursors = readCursors();
+  const bootstrap = !ctx.logShipperBootstrapped;
+  const batches = [];
+  let budget = MAX_LINES_PER_PUSH;
+
+  for (const [agentProfileId, lease] of ctx.leases) {
+    let secrets = null;
+    for (const stream of ['ACTIVITY', 'STDERR']) {
+      if (budget <= 0) break;
+      const k = key(agentProfileId, stream);
+
+      // First round: ask for the cursor without shipping anything.
+      if (bootstrap && !cursors[k]) {
+        batches.push({ agentProfileId, stream, lines: [] });
+        continue;
+      }
+
+      const found = readNewLines(lease.name, stream, cursors[k]);
+      if (!found || found.lines.length === 0) continue;
+
+      if (secrets === null) secrets = secretsForAgent(lease.name);
+
+      const lines = [];
+      for (const raw of found.lines) {
+        if (budget <= 0) break;
+        let payload;
+        try {
+          payload = redactSecrets(raw, secrets);
+        } catch {
+          continue; // never ship a line we failed to mask
+        }
+        lines.push({ payload, ts: timestampOf(raw) });
+        budget--;
+      }
+      if (lines.length === 0) continue;
+
+      batches.push({
+        agentProfileId,
+        stream,
+        lines,
+        sourceFile: found.file,
+        sourceOffset: found.offset,
+      });
+    }
+  }
+
+  if (batches.length === 0) {
+    ctx.logShipperBootstrapped = true;
+    return { pushed: 0 };
+  }
+
+  const res = await ctx.cp(`/machines/${ctx.machineId}/logs`, {
+    method: 'POST',
+    body: { agents: batches },
+  });
+
+  // Leave cursors untouched on failure so the next round retries the same
+  // range. Re-sending is safe; losing lines is not.
+  if (!res.ok) return { pushed: 0, error: res.json?.error || res.status };
+
+  // Advance our cursors only for batches the server accepted.
+  const rejected = new Set(
+    (res.json?.data?.cursors || [])
+      .filter((c) => c && c.rejected)
+      .map((c) => key(c.agentProfileId, c.stream)),
+  );
+
+  let pushed = 0;
+  for (const b of batches) {
+    const k = key(b.agentProfileId, b.stream);
+    if (rejected.has(k)) continue;
+    if (b.sourceFile) {
+      cursors[k] = { file: b.sourceFile, offset: b.sourceOffset };
+      pushed += b.lines.length;
+    }
+  }
+
+  // Adopt the server's position for bootstrap batches, but only where this
+  // machine is the one that wrote it — another machine's byte offset means
+  // nothing against our filesystem.
+  if (bootstrap) {
+    for (const c of res.json?.data?.cursors || []) {
+      if (!c || c.rejected) continue;
+      const k = key(c.agentProfileId, c.stream);
+      if (cursors[k]) continue;
+      if (c.lastMachineId === ctx.machineId && c.lastFile) {
+        cursors[k] = { file: c.lastFile, offset: c.lastOffset ?? 0 };
+      }
+    }
+    ctx.logShipperBootstrapped = true;
+  }
+
+  writeCursors(cursors);
+  return { pushed };
+}
+
+/** Forget an agent's cursors — used when a lease is released, so if the agent
+ *  comes back here later we re-establish position rather than trusting an offset
+ *  into a file another machine has since been writing. */
+export function forgetLogCursor(agentProfileId) {
+  const cursors = readCursors();
+  let changed = false;
+  for (const stream of ['ACTIVITY', 'STDERR']) {
+    const k = key(agentProfileId, stream);
+    if (cursors[k]) {
+      delete cursors[k];
+      changed = true;
+    }
+  }
+  if (changed) writeCursors(cursors);
+}

package/src/agent-runtime/runner.js

@@ -26,11 +26,18 @@ import { getAccessToken, isTokenExpired } from '../utils/auth.js';
 import { refreshAccessToken } from '../utils/api.js';
 import { listLocalAgents, readAgentConfig, getAgentDir, writeAgentConfig } from '../utils/local-agent.js';
 import { startDaemon, stopDaemon, isDaemonRunning, getDaemonInfo } from './daemon.js';
+import { shipLogsOnce, forgetLogCursor } from './log-shipper.js';
 
 const RUNNER_STATE_FILE = path.join(os.homedir(), '.myvillage', 'runner.json');
 const DEFAULT_HEARTBEAT_MS = 20_000;
 const LEASE_TTL_SECONDS = 90;
 
+// Log shipping runs on its own clock, faster than reconcile so a remote tail
+// feels live, and in its own loop so a slow or failing log push can never delay
+// a heartbeat. Lease renewal rides the heartbeat, and single-active correctness
+// depends on it — logs must never be able to hold it up.
+const LOG_SHIP_MS = 5_000;
+
 function log(msg, extra) {
   const line = `[runner] ${new Date().toISOString()} ${msg}`;
   if (extra !== undefined) console.log(line, extra);
@@ -402,6 +409,10 @@ async function reconcileOnce(ctx) {
     try { stopDaemon(lease.name); } catch { /* best-effort */ }
     await ctx.cp(`/agents/${profileId}/lease`, { method: 'POST', body: { action: 'release' } });
     ctx.leases.delete(profileId);
+    // Drop our log position too: if this agent is placed here again later,
+    // another machine will have been writing its tail in the meantime, so our
+    // old byte offset would be meaningless.
+    forgetLogCursor(profileId);
     log(`stopped + released ${lease.name} (no longer assigned here)`);
   }
 }
@@ -449,6 +460,23 @@ export async function runRunner(opts = {}) {
 
   const intervalMs = opts.intervalMs || DEFAULT_HEARTBEAT_MS;
   log(`reconcile loop every ${Math.round(intervalMs / 1000)}s (lease TTL ${LEASE_TTL_SECONDS}s)`);
+
+  // Independent loop, deliberately not awaited by the reconcile loop below.
+  // A log push that hangs or errors must not delay lease renewal.
+  const shipLoop = (async () => {
+    while (!controller.signal.aborted) {
+      await sleep(LOG_SHIP_MS, controller.signal);
+      if (controller.signal.aborted) break;
+      try {
+        await shipLogsOnce(ctx);
+      } catch (err) {
+        // Logs are best-effort. Cursors only advance on a confirmed write, so a
+        // failed round is retried next tick rather than losing lines.
+        log('log ship error', err?.message || err);
+      }
+    }
+  })();
+
   while (!controller.signal.aborted) {
     try {
       await reconcileOnce(ctx);
@@ -457,5 +485,6 @@ export async function runRunner(opts = {}) {
     }
     if (!controller.signal.aborted) await sleep(intervalMs, controller.signal);
   }
+  await shipLoop;
   log('runner stopped');
 }

package/src/agent-runtime/scheduler.js

@@ -46,8 +46,23 @@ export function isWithinActiveHours(schedule) {
   return currentTime >= startTime || currentTime <= endTime;
 }
 
+/**
+ * Floor on how often an agent may loop, in minutes.
+ *
+ * These agents act on human-paced events — a knowledge submission, a activity
+ * completion, an email — so anything under a minute is burning tokens and log
+ * volume against a world that has not changed. It also protects the fleet: log
+ * shipping and MVT spend both scale with agents × loops-per-hour, so a single
+ * misconfigured 15-second agent multiplied across a large fleet is what turns a
+ * comfortable load into an expensive one.
+ *
+ * Not an error, deliberately. An existing agent set faster than this keeps
+ * running, just at the floor, rather than failing to start.
+ */
+export const MIN_CHECK_IN_MINUTES = 1;
+
 export function getNextCheckInMs(schedule) {
   const interval = schedule?.check_in_interval;
   if (!interval || interval <= 0) return 0; // manual only
-  return interval * 60 * 1000;
+  return Math.max(interval, MIN_CHECK_IN_MINUTES) * 60 * 1000;
 }