mixdog 1.2.5 → 1.3.0

package/src/runtime/agent/orchestrator/tools/builtin/git-repo-rw-lock.mjs

@@ -1,3 +1,40 @@
+// In-process write lock for git work on one repository.
+//
+// Why it exists: Mixdog runs several sessions and sub-agents against one
+// repository at once, and some of its own git work is a multi-step transaction
+// that git's lock files cannot make atomic:
+//   - a managed worktree mutation (worktrees/lock.mjs: create, checkout,
+//     merge, remove, reconcile) inspects state, verifies it is clean or free,
+//     then mutates refs, worktrees and the registry; an agent commit, checkout
+//     or `worktree add` landing between the check and the mutation would be
+//     lost or contradict it;
+//   - `git action:stage` re-runs the diff, verifies it against the snapshot,
+//     then applies the selection to the index; another write to that index in
+//     between would stage a patch built from a stale view;
+//   - two writes to one worktree (checkout + commit) must not interleave.
+// The previous repository-wide read/write lock covered these by making every
+// git tool call, reads included, queue on one lock, so one long read
+// (`git log -S`) held back a writer and that queued writer held back every
+// later read in every worktree of the repository.
+//
+// Policy:
+//   - Reads take no lock. Git publishes the index and refs by atomic rename,
+//     so a read always sees a consistent index and ref state; the git tool runs
+//     reads with optional locks off, so `git status` never takes index.lock
+//     beside a write (`git diff` may still refresh it briefly, which the write
+//     retry absorbs). A read during a same-worktree write sees git's state at
+//     that instant, as any second terminal (or the shell tool) does.
+//   - A write names the worktrees it touches (the cwd's worktree, or the
+//     target paths of `worktree add/remove/move`). Writes conflict only when
+//     those sets intersect; a whole-repository write (managed mutations,
+//     object pruning, init/clone, an unresolved worktree target) conflicts
+//     with every write of the repository.
+//   - Fair FIFO without convoys: a waiting write is granted once it conflicts
+//     with no running write and no earlier waiting write. A later write never
+//     overtakes an earlier one it conflicts with, so no write starves; a later
+//     write that conflicts with nothing ahead of it runs at once.
+//   - Git's own lock files still guard index, refs and config against other
+//     processes; the git tool retries a write that git refused on one of them.
 import { pathKey } from '../../../../shared/worktrees/paths.mjs';
 
 const states = new Map();
@@ -10,52 +47,45 @@ export function gitRepoLockKey(repo) {
   return pathKey(String(repo || '.'));
 }
 
-function keyFor(repo) {
-  return gitRepoLockKey(repo);
-}
-
 function stateFor(key) {
   let state = states.get(key);
   if (!state) {
-    state = { readers: 0, writer: false, queue: [] };
+    state = { running: new Set(), queue: [] };
     states.set(key, state);
   }
   return state;
 }
 
-function drain(key, state) {
-  if (state.writer) return;
-  if (state.readers > 0 && state.queue[0]?.mode === 'write') return;
-  if (state.queue.length === 0) {
-    if (state.readers === 0) states.delete(key);
-    return;
-  }
-  if (state.readers === 0 && state.queue[0].mode === 'write') {
-    const entry = state.queue.shift();
-    state.writer = true;
-    entry.granted = true;
-    entry.signal?.removeEventListener?.('abort', entry.onAbort);
-    entry.resolve(releaseFor(key, state, 'write'));
-    return;
-  }
-  while (state.queue[0]?.mode === 'read' && !state.writer) {
-    const entry = state.queue.shift();
-    state.readers++;
-    entry.granted = true;
-    entry.signal?.removeEventListener?.('abort', entry.onAbort);
-    entry.resolve(releaseFor(key, state, 'read'));
-  }
+// `scopes` null is the whole repository.
+function conflicts(left, right) {
+  if (!left.scopes || !right.scopes) return true;
+  return left.scopes.some((scope) => right.scopes.includes(scope));
 }
 
-function releaseFor(key, state, mode) {
+function grant(key, state, entry) {
+  entry.granted = true;
+  entry.signal?.removeEventListener?.('abort', entry.onAbort);
+  state.running.add(entry);
   let released = false;
-  return () => {
+  entry.resolve(() => {
     if (released) return;
     released = true;
-    if (mode === 'write') state.writer = false;
-    else state.readers = Math.max(0, state.readers - 1);
+    state.running.delete(entry);
     drain(key, state);
-  };
+  });
+}
+
+function drain(key, state) {
+  const waiting = [];
+  for (const entry of state.queue) {
+    const blocked =
+      [...state.running].some((running) => conflicts(running, entry)) ||
+      waiting.some((earlier) => conflicts(earlier, entry));
+    if (blocked) waiting.push(entry);
+    else grant(key, state, entry);
+  }
+  state.queue = waiting;
+  if (state.running.size === 0 && waiting.length === 0) states.delete(key);
 }
 
 function abortError(signal) {
@@ -63,12 +93,12 @@ function abortError(signal) {
   return Object.assign(new Error('git repository lock aborted'), { name: 'AbortError', code: 'ABORT_ERR' });
 }
 
-function acquire(repo, mode, signal) {
+function acquire(repo, scopes, signal) {
   if (signal?.aborted) return Promise.reject(abortError(signal));
-  const key = keyFor(repo);
+  const key = gitRepoLockKey(repo);
   const state = stateFor(key);
   return new Promise((resolveLock, rejectLock) => {
-    const entry = { mode, resolve: resolveLock, reject: rejectLock, signal, onAbort: null, granted: false };
+    const entry = { scopes, resolve: resolveLock, signal, onAbort: null, granted: false };
     entry.onAbort = () => {
       if (entry.granted) return;
       const index = state.queue.indexOf(entry);
@@ -83,9 +113,9 @@ function acquire(repo, mode, signal) {
   });
 }
 
-async function withLock(repo, mode, fn, options = {}) {
+async function withLock(repo, scopes, fn, options = {}) {
   const signal = options?.signal || null;
-  const release = await acquire(repo, mode, signal);
+  const release = await acquire(repo, scopes, signal);
   try {
     if (signal?.aborted) throw abortError(signal);
     return await fn();
@@ -94,10 +124,29 @@ async function withLock(repo, mode, fn, options = {}) {
   }
 }
 
-export function withGitRepoReadLock(repo, fn, options = {}) {
-  return withLock(repo, 'read', fn, options);
+/**
+ * Run `fn` holding the write lock of the whole repository.
+ * @template T
+ * @param {string} repo canonical common git dir (or the path being created)
+ * @param {() => Promise<T>|T} fn
+ * @param {{ signal?: AbortSignal }} [options]
+ * @returns {Promise<T>}
+ */
+export function withGitRepoWriteLock(repo, fn, options = {}) {
+  return withLock(repo, null, fn, options);
 }
 
-export function withGitRepoWriteLock(repo, fn, options = {}) {
-  return withLock(repo, 'write', fn, options);
+/**
+ * Run `fn` holding the write lock of the named worktrees only; no (or no
+ * non-empty) worktree list holds the whole repository.
+ * @template T
+ * @param {string} repo canonical common git dir
+ * @param {string[]|null} worktrees worktree roots (or target paths) the write touches
+ * @param {() => Promise<T>|T} fn
+ * @param {{ signal?: AbortSignal }} [options]
+ * @returns {Promise<T>}
+ */
+export function withGitWorktreeWriteLock(repo, worktrees, fn, options = {}) {
+  const scopes = [...new Set((worktrees || []).filter(Boolean).map((path) => gitRepoLockKey(path)))];
+  return withLock(repo, scopes.length ? scopes : null, fn, options);
 }

package/src/runtime/agent/orchestrator/tools/builtin/lib/list-helpers.mjs

@@ -4,6 +4,7 @@ import { normalizeErrorMessage } from '../path-diagnostics.mjs';
 import { isBinaryBuffer } from '../binary-file.mjs';
 import { isUncPath, isWindowsDevicePath, hasUnsafeWin32Component } from '../device-paths.mjs';
 import { normalizeOutputPath, resolveAgainstCwd } from '../path-utils.mjs';
+import { windowMarker } from '../tool-output-limit.mjs';
 
 /** Report the requested path's result; suggestions never retarget the operation. */
 export async function readFamilyPathEnoentOrError(workDir, fullPath, inputPath, err) {
@@ -52,8 +53,23 @@ export function guardedWalkRoot(inputPath, workDir) {
   return { fullPath };
 }
 
+// Code-unit path order: the tiebreak that makes a size/mtime sort total, so
+// equal keys cannot reorder with the walk (and shift across offset pages).
+export function comparePathsStable(a, b) {
+  const x = String(a);
+  const y = String(b);
+  if (x === y) return 0;
+  return x < y ? -1 : 1;
+}
+
 export function pageContinuationLine(offset, shown, total) {
-  return `... [entries ${offset + 1}-${offset + shown} of ${total}; pass offset:${offset + shown} to continue]`;
+  return windowMarker({
+    unit: 'entries',
+    from: offset + 1,
+    to: offset + shown,
+    total,
+    nextOffset: offset + shown,
+  });
 }
 
 const LINE_COUNT_MAX_BYTES = 2 * 1024 * 1024;

package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-delivery.mjs

@@ -0,0 +1,199 @@
+// One shape for every piece of shell output handed to an agent — a foreground
+// result, a background move, task wait/read and a completion notice:
+//   1. ANSI stripped, then `\r` progress frames folded to their last frame
+//      (before anything is counted or cut);
+//   2. a line longer than DELIVERY_LINE_MAX_CHARS is cut: `… [N chars cut]`;
+//   3. DELIVERY_FULL_CHARS or less goes whole; otherwise a head of at most
+//      DELIVERY_HEAD_LINES lines (≤ DELIVERY_HEAD_MAX_CHARS), the marker, and
+//      the last ~DELIVERY_TAIL_CHARS;
+//   4. the marker is the same everywhere and always ends with the full log:
+//      `[… N lines, X KB omitted — read with offset/limit or grep, full log: <path>]`;
+//   5. anything shaped like a harness/system tag is escaped.
+// MIXDOG_SHELL_DELIVERY_MAX_CHARS is the one knob (the whole-delivery cap); the
+// head and tail sizes are fixed. It is separate from the 50 KB
+// MIXDOG_TOOL_OUTPUT_MAX_BYTES that the other tools use.
+import { randomBytes } from 'node:crypto';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { getPluginData } from '../../../config.mjs';
+import { stripAnsi } from '../../shell-exec-output.mjs';
+import { createCarriageReturnFolder } from '../shell-lossless-compact.mjs';
+import { positiveEnvInt } from '../../../../../shared/tool-output-limit.mjs';
+import { escapeHarnessTags } from '../../../../../shared/shell-output-summary.mjs';
+
+export const DELIVERY_FULL_CHARS = positiveEnvInt('MIXDOG_SHELL_DELIVERY_MAX_CHARS', 6_000);
+const DELIVERY_HEAD_LINES = 10;
+const DELIVERY_HEAD_MAX_CHARS = 1_000;
+const DELIVERY_TAIL_CHARS = 2_000;
+export const DELIVERY_LINE_MAX_CHARS = 2_000;
+
+export { createCarriageReturnFolder };
+
+const newlines = (text) => {
+  let count = 0;
+  for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) count += 1;
+  return count;
+};
+
+export function formatKb(bytes) {
+  return `${(Math.max(0, bytes) / 1024).toFixed(1)} KB`;
+}
+
+/** The omission marker — identical on every delivery path, path last. */
+export function omissionMarker({ lines, bytes, path }) {
+  return `[… ${lines} lines, ${formatKb(bytes)} omitted — read with offset/limit or grep, full log: ${path}]`;
+}
+
+// `cut` is the number of chars removed from over-long lines.
+function cutLongLines(text) {
+  if (text.length <= DELIVERY_LINE_MAX_CHARS) return { text, cut: 0 };
+  let cut = 0;
+  const cutText = text
+    .split('\n')
+    .map((line) => {
+      if (line.length <= DELIVERY_LINE_MAX_CHARS) return line;
+      cut += line.length - DELIVERY_LINE_MAX_CHARS;
+      return `${line.slice(0, DELIVERY_LINE_MAX_CHARS)} … [${line.length - DELIVERY_LINE_MAX_CHARS} chars cut]`;
+    })
+    .join('\n');
+  return { text: cutText, cut };
+}
+
+/** Step 1 for one chunk (the cut of step 2 is made where the text is
+ *  delivered, so the uncut text can be archived). `fold` carries `\r` state
+ *  across chunks of the same stream; omitted, the chunk is folded on its own. */
+export function prepareShellText(text, fold = createCarriageReturnFolder()) {
+  return fold(stripAnsi(String(text ?? '')));
+}
+
+// `scale` is the share of the delivery budget a stream was given.
+function headOf(text, scale = 1) {
+  let end = 0;
+  for (let line = 0; line < DELIVERY_HEAD_LINES && end < text.length; line += 1) {
+    const newline = text.indexOf('\n', end);
+    end = newline < 0 ? text.length : newline + 1;
+  }
+  return text.slice(0, Math.min(end, Math.floor(DELIVERY_HEAD_MAX_CHARS * scale)));
+}
+
+function tailOf(text, floor = 0, scale = 1) {
+  const start = Math.max(floor, text.length - Math.floor(DELIVERY_TAIL_CHARS * scale));
+  if (start === floor) return text.slice(start);
+  // Start on a line boundary when one is near.
+  const newline = text.indexOf('\n', start);
+  return newline >= 0 && newline - start < 200 && newline + 1 < text.length
+    ? text.slice(newline + 1)
+    : text.slice(start);
+}
+
+/** head + marker + tail; `omitted` counts what lies between them. */
+function joinHeadTail(head, tail, omitted) {
+  const gap = head && !head.endsWith('\n') ? '\n' : '';
+  return `${head}${gap}${omissionMarker(omitted)}\n${tail}`;
+}
+
+// One stream's delivery: `natural` is the length it would have whole (escaped,
+// markers included; Infinity when it can only be shown at its ends) and
+// `render(budget)` the text within `budget` chars. Streams share one budget.
+export const EMPTY_PIECE = { natural: 0, render: () => '' };
+
+/** Step 3 when only the two ends of a long range were read: `total` is the
+ *  whole range's bytes and lines. */
+export function edgesPiece(headText, tailText, { totalBytes, totalLines, path }) {
+  return {
+    natural: Number.POSITIVE_INFINITY,
+    render: (budget) => {
+      const scale = budget / DELIVERY_FULL_CHARS;
+      const head = headOf(escapeHarnessTags(headText), scale);
+      const tail = tailOf(escapeHarnessTags(tailText), 0, scale);
+      return joinHeadTail(head, tail, {
+        lines: Math.max(0, totalLines - newlines(head) - newlines(tail)),
+        bytes: Math.max(0, totalBytes - Buffer.byteLength(head) - Buffer.byteLength(tail)),
+        path,
+      });
+    },
+  };
+}
+
+export { newlines as countNewlines };
+
+/** Steps 2, 3 and 5 on prepared text. `path` names where the whole (uncut)
+ *  text can be read; it is resolved only when something is removed. */
+export function textPiece(text, { path }) {
+  let archive;
+  const where = () => (archive ??= typeof path === 'function' ? path() : path);
+  const { text: cutText, cut } = cutLongLines(text);
+  const shown = escapeHarnessTags(cutText);
+  // Cut long lines are removed content too: the whole text carries the marker.
+  const whole = cut
+    ? `${shown}${shown.endsWith('\n') ? '' : '\n'}${omissionMarker({ lines: 0, bytes: cut, path: where() })}\n`
+    : shown;
+  return {
+    natural: whole.length,
+    render: (budget) => {
+      if (whole.length <= budget) return whole;
+      const scale = budget / DELIVERY_FULL_CHARS;
+      const head = headOf(shown, scale);
+      const tail = tailOf(shown, head.length, scale);
+      const middle = shown.slice(head.length, shown.length - tail.length);
+      return joinHeadTail(head, tail, { lines: newlines(middle), bytes: Buffer.byteLength(middle), path: where() });
+    },
+  };
+}
+
+/** Max-min fair split of `budget` over the streams' natural lengths: a stream
+ *  that fits its fair share is given exactly what it needs, the rest share
+ *  what remains. */
+function shareBudget(lengths, budget) {
+  const shares = lengths.map(() => 0);
+  let open = lengths.map((_, index) => index);
+  let left = Math.max(0, budget);
+  while (open.length) {
+    const fair = left / open.length;
+    const fits = open.filter((index) => lengths[index] <= fair);
+    if (!fits.length) {
+      for (const index of open) shares[index] = fair;
+      break;
+    }
+    for (const index of fits) {
+      shares[index] = lengths[index];
+      left -= lengths[index];
+    }
+    open = open.filter((index) => !fits.includes(index));
+  }
+  return shares;
+}
+
+/** Deliver streams within one shared budget (default: the whole-delivery cap),
+ *  each escaped; the caller deducts what it adds around them. */
+export function fitPieces(pieces, budget = DELIVERY_FULL_CHARS) {
+  const shares = shareBudget(
+    pieces.map((piece) => piece.natural),
+    budget
+  );
+  return pieces.map((piece, index) => escapeHarnessTags(piece.render(shares[index])));
+}
+
+/** Text that has no log of its own is kept where the marker can name it:
+ *  the shell-output folder, swept with the spill files. */
+export function persistDeliveryLog(text, label = 'output') {
+  const dir = join(getPluginData(), 'shell-output');
+  mkdirSync(dir, { recursive: true });
+  const file = join(dir, `delivered_${Date.now().toString(36)}_${randomBytes(4).toString('hex')}.${label}`);
+  writeFileSync(file, text);
+  return file;
+}
+
+/** Steps 1–5 for a whole string, as a piece. Without `path`, the original
+ *  (uncut) text is saved when anything is removed, so the marker can name it. */
+export function shellTextPiece(text, { path = null, label = 'output', fold } = {}) {
+  const prepared = prepareShellText(text, fold);
+  return textPiece(prepared, { path: path || (() => persistDeliveryLog(prepared, label)) });
+}
+
+export function deliverShellText(text, options) {
+  return fitPieces([shellTextPiece(text, options)])[0];
+}
+
+/** Step 5 alone, for text the caller already capped. */
+export { escapeHarnessTags };

package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-job-records.mjs

@@ -339,11 +339,23 @@ export function releaseShellJobWorkspace(jobId) {
   if (workspaceLeases.get(key) === leave) workspaceLeases.delete(key);
 }
 
+/** True once `sessionId` closed in this runtime. */
+export function isShellOwnerSessionClosed(sessionId) {
+  return closedOwnerSessions.has(String(sessionId || '').trim());
+}
+
 /** Flag a still-open job record as "termination unconfirmed" for the readouts. */
 export function markShellJobTerminationUnconfirmed(jobId) {
   return patchRunningShellJobRecord(jobId, { terminationUnconfirmed: true });
 }
 
+/** Record the pids a held job's termination watch last found alive, so a
+ *  reader in another process can tell a live hold from a stale one. */
+export function noteShellJobSurvivors(jobId, pids) {
+  const survivorPids = (pids || []).map(Number).filter((pid) => Number.isInteger(pid) && pid > 0);
+  return patchRunningShellJobRecord(jobId, { survivorPids });
+}
+
 function patchRunningShellJobRecord(jobId, patch) {
   const key = validJobId(jobId);
   if (!key) return Promise.resolve(false);