@myvillage/cli 1.89.1 → 1.90.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myvillage/cli",
-  "version": "1.89.1",
+  "version": "1.90.0",
   "description": "MyVillageOS CLI for community developers",
   "type": "module",
   "bin": {

package/src/agent-runtime/log-shipper.js

@@ -0,0 +1,350 @@
+// ── Agent log shipper ────────────────────────────────────────────────────────
+// Ships a rolling tail of each leased agent's logs to the control plane, so an
+// agent running on this machine can be read from Studio on another one without
+// SSH.
+//
+// Design notes worth knowing before editing:
+//
+//   * Recent, not complete. Only the newest lines are shipped and the server
+//     keeps a bounded window per agent. Full history stays here on disk and is
+//     read with `myvillage agent logs`. This is what makes the whole thing scale
+//     with fleet size instead of uptime.
+//
+//   * Batched per machine. One request carries every leased agent's delta, so
+//     request rate tracks machine count, not agent count.
+//
+//   * Never on the heartbeat. Lease renewal is correctness-critical; log volume
+//     must not be able to slow it down or fail it.
+//
+//   * Secrets are masked HERE, before anything leaves the box. Tool arguments
+//     and results do ship (they are the point), but .env values and the OAuth
+//     token never do.
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { getAgentDir } from '../utils/local-agent.js';
+import { getAccessToken } from '../utils/auth.js';
+
+const CURSOR_FILE = path.join(os.homedir(), '.myvillage', 'log-cursors.json');
+
+/** Per-stream ship caps. Must not exceed the server's retention, or we would be
+ *  uploading lines it is about to delete. */
+const SHIP_CAP = { ACTIVITY: 300, STDERR: 100 };
+
+/** Hard ceiling on one request across all agents; mirrors the server's cap. */
+const MAX_LINES_PER_PUSH = 2000;
+
+/** Shortest .env value we will mask. Below this you get false positives that
+ *  black out ordinary words — a var set to "true" or a family name. */
+const MIN_SECRET_LEN = 8;
+
+const DATE_LOG_RE = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
+
+// ── Cursor persistence ───────────────────────────────────────────────────────
+// Keyed `${agentProfileId}:${stream}` → { file, offset }. Persisted so a runner
+// restart resumes mid-file instead of re-shipping a tail we already sent.
+
+function readCursors() {
+  try {
+    return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+function writeCursors(cursors) {
+  try {
+    fs.mkdirSync(path.dirname(CURSOR_FILE), { recursive: true });
+    fs.writeFileSync(CURSOR_FILE, JSON.stringify(cursors, null, 2), { mode: 0o600 });
+  } catch {
+    /* best-effort: a lost cursor costs one duplicated tail, not correctness */
+  }
+}
+
+const key = (agentProfileId, stream) => `${agentProfileId}:${stream}`;
+
+// ── Redaction ────────────────────────────────────────────────────────────────
+
+/**
+ * Literal secrets for one agent: its own .env values plus this machine's OAuth
+ * token. Values only — keys stay readable, so `GITHUB_TOKEN=••••` still tells
+ * you which credential is in play.
+ */
+function secretsForAgent(agentName) {
+  const out = [];
+  try {
+    const envPath = path.join(getAgentDir(agentName), '.env');
+    const text = fs.readFileSync(envPath, 'utf-8');
+    for (const line of text.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      const eq = trimmed.indexOf('=');
+      if (eq <= 0) continue;
+      let value = trimmed.slice(eq + 1).trim();
+      // Strip matching surrounding quotes so we mask the value, not the quoting.
+      if (value.length >= 2 && /^(".*"|'.*')$/s.test(value)) value = value.slice(1, -1);
+      if (value.length >= MIN_SECRET_LEN) out.push(value);
+    }
+  } catch {
+    /* no .env, or unreadable — nothing agent-specific to mask */
+  }
+  try {
+    const token = getAccessToken();
+    if (token && token.length >= MIN_SECRET_LEN) out.push(token);
+  } catch {
+    /* not signed in */
+  }
+  // Longest first, so a value containing another is masked whole.
+  return out.sort((a, b) => b.length - a.length);
+}
+
+/** Shapes that look like credentials even when we have never seen the value —
+ *  a token echoed back by a third-party API, for instance. */
+const SECRET_PATTERNS = [
+  /\b(sk|pk|rk)-[A-Za-z0-9_-]{16,}\b/g,
+  /\bgh[pousr]_[A-Za-z0-9]{16,}\b/g,
+  /\bAKIA[0-9A-Z]{12,}\b/g,
+  /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g,
+  /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, // JWT
+  /\b[Bb]earer\s+[A-Za-z0-9._~+/-]{20,}=*/g,
+];
+
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Mask every known secret and credential-shaped token in a line.
+ *
+ * Runs on every shipped line, so it must be cheap and must never throw — a
+ * redaction bug that crashes the shipper would take log visibility down, and one
+ * that silently returns the input would leak. On failure we drop the line rather
+ * than ship it unmasked.
+ */
+export function redactSecrets(line, secrets) {
+  let out = line;
+  for (const secret of secrets) {
+    if (!secret) continue;
+    out = out.split(secret).join('••••');
+  }
+  for (const re of SECRET_PATTERNS) out = out.replace(re, '••••');
+  return out;
+}
+
+// ── Tailing ──────────────────────────────────────────────────────────────────
+
+/** The daemon writes `logs/<YYYY-MM-DD>.jsonl` and rolls over at UTC midnight.
+ *  ISO names sort chronologically, so lexical max is the current file. */
+function latestActivityFile(logsDir) {
+  try {
+    const names = fs.readdirSync(logsDir).filter((n) => DATE_LOG_RE.test(n));
+    if (names.length === 0) return null;
+    names.sort();
+    return names[names.length - 1];
+  } catch {
+    return null;
+  }
+}
+
+function sourceFileFor(agentName, stream) {
+  const logsDir = path.join(getAgentDir(agentName), 'logs');
+  if (stream === 'STDERR') {
+    const f = path.join(logsDir, 'daemon-stderr.log');
+    return fs.existsSync(f) ? { dir: logsDir, name: 'daemon-stderr.log', full: f } : null;
+  }
+  const name = latestActivityFile(logsDir);
+  return name ? { dir: logsDir, name, full: path.join(logsDir, name) } : null;
+}
+
+/**
+ * Read the lines this agent has produced since our cursor.
+ *
+ * Resets to the tail (not the top) whenever continuity is broken: a new day's
+ * file, a truncated file, or no cursor at all. That is the backlog cap — a
+ * machine that was offline for a day ships its recent tail, never a day of
+ * backlog, so a fleet reconnecting after an outage cannot stampede the endpoint.
+ */
+function readNewLines(agentName, stream, cursor) {
+  const src = sourceFileFor(agentName, stream);
+  if (!src) return null;
+
+  let size;
+  try {
+    size = fs.statSync(src.full).size;
+  } catch {
+    return null;
+  }
+
+  const continuous =
+    cursor && cursor.file === src.name && typeof cursor.offset === 'number' && cursor.offset <= size;
+  const start = continuous ? cursor.offset : 0;
+  if (continuous && start === size) return null; // nothing new
+
+  let text = '';
+  const len = size - start;
+  if (len > 0) {
+    let fd;
+    try {
+      fd = fs.openSync(src.full, 'r');
+      const buf = Buffer.alloc(len);
+      fs.readSync(fd, buf, 0, len, start);
+      text = buf.toString('utf8');
+    } catch {
+      return null;
+    } finally {
+      if (fd !== undefined) {
+        try { fs.closeSync(fd); } catch { /* ignore */ }
+      }
+    }
+  }
+
+  let lines = text.split('\n').filter((l) => l.trim().length > 0);
+  const cap = SHIP_CAP[stream];
+  // On a discontinuity we may be looking at a whole file; keep only the tail.
+  if (lines.length > cap) lines = lines.slice(-cap);
+
+  return { lines, file: src.name, offset: size };
+}
+
+/** Pull the line's own timestamp out of the JSONL, so a backlog shipped after a
+ *  reconnect keeps its real times instead of collapsing onto arrival. */
+function timestampOf(payload) {
+  if (payload.charCodeAt(0) !== 123 /* { */) return null;
+  try {
+    const obj = JSON.parse(payload);
+    const ts = obj.ts ?? obj.timestamp ?? obj.time;
+    return typeof ts === 'string' ? ts : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Shipper ──────────────────────────────────────────────────────────────────
+
+/**
+ * Ship one round for every agent this runner holds a lease on.
+ *
+ * `leases` is the runner's Map of agentProfileId → { name }. Only lease holders
+ * may ship, which the server enforces too — the same lease that guarantees
+ * single-active execution guarantees a single writer per agent's log.
+ *
+ * Bootstrap: on the first round after start we send a zero-line batch to learn
+ * the server's cursor. If it says this machine wrote it, we adopt the offset and
+ * carry on mid-file; otherwise we start from the tail. That is what keeps a
+ * runner restart from re-shipping (and thus duplicating) the last few hundred
+ * lines it had already sent.
+ */
+export async function shipLogsOnce(ctx) {
+  if (!ctx.leases || ctx.leases.size === 0) return { pushed: 0 };
+
+  const cursors = readCursors();
+  const bootstrap = !ctx.logShipperBootstrapped;
+  const batches = [];
+  let budget = MAX_LINES_PER_PUSH;
+
+  for (const [agentProfileId, lease] of ctx.leases) {
+    let secrets = null;
+    for (const stream of ['ACTIVITY', 'STDERR']) {
+      if (budget <= 0) break;
+      const k = key(agentProfileId, stream);
+
+      // First round: ask for the cursor without shipping anything.
+      if (bootstrap && !cursors[k]) {
+        batches.push({ agentProfileId, stream, lines: [] });
+        continue;
+      }
+
+      const found = readNewLines(lease.name, stream, cursors[k]);
+      if (!found || found.lines.length === 0) continue;
+
+      if (secrets === null) secrets = secretsForAgent(lease.name);
+
+      const lines = [];
+      for (const raw of found.lines) {
+        if (budget <= 0) break;
+        let payload;
+        try {
+          payload = redactSecrets(raw, secrets);
+        } catch {
+          continue; // never ship a line we failed to mask
+        }
+        lines.push({ payload, ts: timestampOf(raw) });
+        budget--;
+      }
+      if (lines.length === 0) continue;
+
+      batches.push({
+        agentProfileId,
+        stream,
+        lines,
+        sourceFile: found.file,
+        sourceOffset: found.offset,
+      });
+    }
+  }
+
+  if (batches.length === 0) {
+    ctx.logShipperBootstrapped = true;
+    return { pushed: 0 };
+  }
+
+  const res = await ctx.cp(`/machines/${ctx.machineId}/logs`, {
+    method: 'POST',
+    body: { agents: batches },
+  });
+
+  // Leave cursors untouched on failure so the next round retries the same
+  // range. Re-sending is safe; losing lines is not.
+  if (!res.ok) return { pushed: 0, error: res.json?.error || res.status };
+
+  // Advance our cursors only for batches the server accepted.
+  const rejected = new Set(
+    (res.json?.data?.cursors || [])
+      .filter((c) => c && c.rejected)
+      .map((c) => key(c.agentProfileId, c.stream)),
+  );
+
+  let pushed = 0;
+  for (const b of batches) {
+    const k = key(b.agentProfileId, b.stream);
+    if (rejected.has(k)) continue;
+    if (b.sourceFile) {
+      cursors[k] = { file: b.sourceFile, offset: b.sourceOffset };
+      pushed += b.lines.length;
+    }
+  }
+
+  // Adopt the server's position for bootstrap batches, but only where this
+  // machine is the one that wrote it — another machine's byte offset means
+  // nothing against our filesystem.
+  if (bootstrap) {
+    for (const c of res.json?.data?.cursors || []) {
+      if (!c || c.rejected) continue;
+      const k = key(c.agentProfileId, c.stream);
+      if (cursors[k]) continue;
+      if (c.lastMachineId === ctx.machineId && c.lastFile) {
+        cursors[k] = { file: c.lastFile, offset: c.lastOffset ?? 0 };
+      }
+    }
+    ctx.logShipperBootstrapped = true;
+  }
+
+  writeCursors(cursors);
+  return { pushed };
+}
+
+/** Forget an agent's cursors — used when a lease is released, so if the agent
+ *  comes back here later we re-establish position rather than trusting an offset
+ *  into a file another machine has since been writing. */
+export function forgetLogCursor(agentProfileId) {
+  const cursors = readCursors();
+  let changed = false;
+  for (const stream of ['ACTIVITY', 'STDERR']) {
+    const k = key(agentProfileId, stream);
+    if (cursors[k]) {
+      delete cursors[k];
+      changed = true;
+    }
+  }
+  if (changed) writeCursors(cursors);
+}

package/src/agent-runtime/loop.js

@@ -121,6 +121,19 @@ export async function agentLoop(agentName, { signal }) {
   // max_tokens_per_loop and the rate limiter regardless of this ceiling.
   const maxSteps = config.brain?.max_steps_per_loop || 25;
 
+  // Refresh BEFORE the first MCP connect. Whoever spawned us put a snapshot of
+  // MYVILLAGE_ACCESS_TOKEN in our env, and the MCP client bakes it into its
+  // Authorization header at construction with no refresh path of its own. The
+  // per-iteration self-heal below only runs once we're already in the loop, so
+  // without this the very first connect uses a possibly-dead token and comes up
+  // with zero tools. Doing it here covers every start path (fleet runner,
+  // Studio's local Start, `myvillage agent start`) in one place.
+  try {
+    if (isTokenExpired()) await refreshAccessToken();
+    const startupToken = getAccessToken();
+    if (startupToken) process.env.MYVILLAGE_ACCESS_TOKEN = startupToken;
+  } catch { /* fall through with whatever we were handed */ }
+
   // Initialize MCP tools
   let tools;
   try {
@@ -183,8 +196,20 @@ export async function agentLoop(agentName, { signal }) {
     // 401s until the daemon is restarted. Re-read credentials each loop and
     // swap the MCP client when the token changes.
     try {
+      // refreshAccessToken() RETURNS false on failure, it does not throw, so the
+      // catch below never fires for a dead refresh token. Check the return value
+      // or the whole failure is silent: no mcp_reconnected, no error, just an
+      // agent that 401s on every call for the rest of its life.
       if (isTokenExpired()) {
-        await refreshAccessToken();
+        const refreshed = await refreshAccessToken();
+        if (!refreshed) {
+          logActivity(agentDir, {
+            type: 'error',
+            error:
+              'Could not refresh the MyVillageOS sign-in on this machine. Tools and the model will keep ' +
+              'returning 401 until you run `myvillage login` here.',
+          });
+        }
       }
       const currentToken = getAccessToken();
       if (currentToken && currentToken !== process.env.MYVILLAGE_ACCESS_TOKEN) {

package/src/agent-runtime/runner.js

@@ -22,14 +22,22 @@ import os from 'os';
 import fs from 'fs';
 import path from 'path';
 import { getConfig } from '../utils/config.js';
-import { getAccessToken } from '../utils/auth.js';
+import { getAccessToken, isTokenExpired } from '../utils/auth.js';
+import { refreshAccessToken } from '../utils/api.js';
 import { listLocalAgents, readAgentConfig, getAgentDir, writeAgentConfig } from '../utils/local-agent.js';
 import { startDaemon, stopDaemon, isDaemonRunning, getDaemonInfo } from './daemon.js';
+import { shipLogsOnce, forgetLogCursor } from './log-shipper.js';
 
 const RUNNER_STATE_FILE = path.join(os.homedir(), '.myvillage', 'runner.json');
 const DEFAULT_HEARTBEAT_MS = 20_000;
 const LEASE_TTL_SECONDS = 90;
 
+// Log shipping runs on its own clock, faster than reconcile so a remote tail
+// feels live, and in its own loop so a slow or failing log push can never delay
+// a heartbeat. Lease renewal rides the heartbeat, and single-active correctness
+// depends on it — logs must never be able to hold it up.
+const LOG_SHIP_MS = 5_000;
+
 function log(msg, extra) {
   const line = `[runner] ${new Date().toISOString()} ${msg}`;
   if (extra !== undefined) console.log(line, extra);
@@ -75,13 +83,29 @@ function makeClient(machineKey, machineId) {
   };
 }
 
+/**
+ * Read the signed-in user's OAuth bearer, refreshing it first when it is stale.
+ *
+ * Every headless machine needs this. On a desktop, Studio's own API traffic
+ * keeps the stored token fresh as a side effect; on a Pi nothing else uses the
+ * OAuth credential (the runner itself authenticates with its machine API key),
+ * so the token on disk simply expires and stays expired. Handing that dead
+ * token to a registration call or to a spawned daemon is a guaranteed 401.
+ *
+ * Returns null when there is no usable credential at all.
+ */
+async function currentUserToken() {
+  if (isTokenExpired()) await refreshAccessToken();
+  return getAccessToken();
+}
+
 /**
  * One-shot client authenticated as the signed-in *user* (OAuth bearer). Used
  * only to register a machine and receive its minted credential.
  */
-function makeUserClient() {
+async function makeUserClient() {
   const { networkBaseUrl } = getConfig();
-  const token = getAccessToken();
+  const token = await currentUserToken();
   if (!token) return null;
   return async function cp(pathname, { method = 'GET', body } = {}) {
     const headers = { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` };
@@ -198,9 +222,13 @@ function indexLocalAgentsByProfileId() {
   return map;
 }
 
-function startAgent(name, config, machineId, leaseEpoch) {
-  const token = getAccessToken();
+async function startAgent(name, config, machineId, leaseEpoch) {
+  // Refresh before snapshotting. The daemon inherits this env and bakes the
+  // token into its MCP Authorization header at construction, so a stale value
+  // here means the agent comes up with zero tools and no model access.
+  const token = await currentUserToken();
   if (token) process.env.MYVILLAGE_ACCESS_TOKEN = token;
+  else log(`no usable sign-in on this machine — ${name} will start unauthenticated; run \`myvillage login\``);
   if (config.man?.agent_id) process.env.MYVILLAGE_AGENT_ID = config.man.agent_id;
   if (config.man?.village_agent_id) process.env.MYVILLAGE_VILLAGE_AGENT_ID = config.man.village_agent_id;
   process.env.MYVILLAGE_MACHINE_ID = machineId;
@@ -262,7 +290,7 @@ async function resolveMachineIdentity(opts) {
     return { machineId, machineKey: saved.machineKey };
   }
 
-  const userClient = makeUserClient();
+  const userClient = await makeUserClient();
   if (!userClient) {
     throw new Error(
       'Not signed in. Run `myvillage login` on this machine (or pass --key with a machine:runner API key).',
@@ -370,7 +398,7 @@ async function reconcileOnce(ctx) {
       continue;
     }
 
-    const pid = startAgent(entry.name, entry.config, ctx.machineId, epoch);
+    const pid = await startAgent(entry.name, entry.config, ctx.machineId, epoch);
     ctx.leases.set(a.agentProfileId, { name: entry.name, epoch });
     log(`started ${entry.name} (pid ${pid}, epoch ${epoch})`);
   }
@@ -381,6 +409,10 @@ async function reconcileOnce(ctx) {
     try { stopDaemon(lease.name); } catch { /* best-effort */ }
     await ctx.cp(`/agents/${profileId}/lease`, { method: 'POST', body: { action: 'release' } });
     ctx.leases.delete(profileId);
+    // Drop our log position too: if this agent is placed here again later,
+    // another machine will have been writing its tail in the meantime, so our
+    // old byte offset would be meaningless.
+    forgetLogCursor(profileId);
     log(`stopped + released ${lease.name} (no longer assigned here)`);
   }
 }
@@ -428,6 +460,23 @@ export async function runRunner(opts = {}) {
 
   const intervalMs = opts.intervalMs || DEFAULT_HEARTBEAT_MS;
   log(`reconcile loop every ${Math.round(intervalMs / 1000)}s (lease TTL ${LEASE_TTL_SECONDS}s)`);
+
+  // Independent loop, deliberately not awaited by the reconcile loop below.
+  // A log push that hangs or errors must not delay lease renewal.
+  const shipLoop = (async () => {
+    while (!controller.signal.aborted) {
+      await sleep(LOG_SHIP_MS, controller.signal);
+      if (controller.signal.aborted) break;
+      try {
+        await shipLogsOnce(ctx);
+      } catch (err) {
+        // Logs are best-effort. Cursors only advance on a confirmed write, so a
+        // failed round is retried next tick rather than losing lines.
+        log('log ship error', err?.message || err);
+      }
+    }
+  })();
+
   while (!controller.signal.aborted) {
     try {
       await reconcileOnce(ctx);
@@ -436,5 +485,6 @@ export async function runRunner(opts = {}) {
     }
     if (!controller.signal.aborted) await sleep(intervalMs, controller.signal);
   }
+  await shipLoop;
   log('runner stopped');
 }

package/src/agent-runtime/scheduler.js

@@ -46,8 +46,23 @@ export function isWithinActiveHours(schedule) {
   return currentTime >= startTime || currentTime <= endTime;
 }
 
+/**
+ * Floor on how often an agent may loop, in minutes.
+ *
+ * These agents act on human-paced events — a knowledge submission, a activity
+ * completion, an email — so anything under a minute is burning tokens and log
+ * volume against a world that has not changed. It also protects the fleet: log
+ * shipping and MVT spend both scale with agents × loops-per-hour, so a single
+ * misconfigured 15-second agent multiplied across a large fleet is what turns a
+ * comfortable load into an expensive one.
+ *
+ * Not an error, deliberately. An existing agent set faster than this keeps
+ * running, just at the floor, rather than failing to start.
+ */
+export const MIN_CHECK_IN_MINUTES = 1;
+
 export function getNextCheckInMs(schedule) {
   const interval = schedule?.check_in_interval;
   if (!interval || interval <= 0) return 0; // manual only
-  return interval * 60 * 1000;
+  return Math.max(interval, MIN_CHECK_IN_MINUTES) * 60 * 1000;
 }

package/src/commands/agent-local.js

@@ -771,7 +771,29 @@ export async function agentLogsCommand(name, options = {}) {
   console.log('');
 }
 
+// Calendar day of the last entry printed, so we only emit a date header when
+// the day actually changes. readAgentLogs walks the per-day files newest-first
+// and keeps accumulating until it hits `limit`, so one invocation routinely
+// spans midnight. Without a header, yesterday's 10:21 and today's 10:21 render
+// identically and there is no way to tell a live agent from a stale log.
+let lastPrintedDay = null;
+
+function printDayHeader(ts) {
+  const d = new Date(ts);
+  if (Number.isNaN(d.getTime())) return;
+  const day = d.toLocaleDateString(undefined, {
+    weekday: 'short',
+    year: 'numeric',
+    month: 'short',
+    day: 'numeric',
+  });
+  if (day === lastPrintedDay) return;
+  lastPrintedDay = day;
+  console.log(`  ${chalk.dim(`── ${day} ──`)}`);
+}
+
 function printLogEntry(entry) {
+  printDayHeader(entry.ts);
   const time = brand.teal(new Date(entry.ts).toLocaleTimeString());
   const type = entry.type || 'info';
 

package/src/utils/auth.js

@@ -100,7 +100,15 @@ export function getAccessToken() {
 
 export function isTokenExpired() {
   const creds = loadCredentials();
-  if (!creds?.expires_at) return false;
+  // Nothing stored (or nothing decryptable): not "expired", just signed out.
+  // Callers treat a null token as unauthenticated on their own.
+  if (!creds?.access_token) return false;
+  // Fail closed. A credential with no recorded expiry cannot be proven valid,
+  // so treat it as expired and let the refresh path replace it — which also
+  // writes a real expires_at, so this self-corrects after one call. Returning
+  // false here meant such a token was considered valid forever: a long-lived
+  // agent daemon would 401 on every request and never once try to refresh.
+  if (!creds.expires_at) return true;
   // Consider expired if within 60 seconds of expiry
   return Date.now() >= (creds.expires_at - 60000);
 }