@cordfuse/crosstalk 5.0.0-alpha.2

package/src/dispatch.ts

@@ -0,0 +1,519 @@
+import { resolve, join } from 'path';
+import { spawn } from 'child_process';
+import {
+  mkdirSync,
+  writeFileSync,
+  readFileSync,
+  existsSync,
+  appendFileSync,
+  openSync,
+  closeSync,
+} from 'fs';
+import { watch } from 'fs/promises';
+import {
+  findHostFile,
+  loadActorProfile,
+  pickTier,
+  tokenizeCli,
+  type HostActorTiers,
+  type HostFile,
+} from './actor.js';
+import {
+  discoverChannels,
+  listChannelMessages,
+  gitPull,
+  gitCommitAndPush,
+  writeErrorLog,
+  sweepStaleReadReceipts,
+  type ChannelMessage,
+} from './transport.js';
+import { readCursor, writeCursor } from './cursor.js';
+import { now, messageFilename } from './filenames.js';
+import { serializeFrontmatter } from './frontmatter.js';
+import { withLock } from './turnq.js';
+import { writeDlqEntry, isQuarantined, isActorQuarantined } from './dlq.js';
+
+const transportRoot = resolve(process.cwd());
+const argv = process.argv.slice(2);
+
+function flag(name: string): string | undefined {
+  const i = argv.indexOf(name);
+  if (i === -1 || i === argv.length - 1) return undefined;
+  return argv[i + 1];
+}
+
+const onceMode = argv.includes('--once');
+const jsonMode = argv.includes('--json');
+const hostOverride = flag('--host');
+const pollSeconds = Number(flag('--poll')) || 30;
+const logFile = flag('--log-file');
+
+// Backoff config — persistent infra failures (git pull/push) trigger
+// exponential delay. Reset on any successful pull+push cycle.
+const MAX_BACKOFF_MULTIPLIER = 10; // cap: pollSeconds * 10
+const BACKOFF_GRACE = 2; // first N failures don't trigger backoff
+
+// Stale-read-receipt sweep config — runs at most every SWEEP_INTERVAL_MS
+// of wall-clock to surface read receipts that never produced a reply
+// (indicates dispatch crashed mid-tick or CLI hung silently).
+const SWEEP_INTERVAL_MS = 5 * 60_000;
+const STALE_RECEIPT_THRESHOLD_MS = 5 * 60_000;
+let lastSweepAt = 0;
+
+function log(event: string, fields: Record<string, unknown> = {}): void {
+  let line: string;
+  if (jsonMode) {
+    line = JSON.stringify({ ts: new Date().toISOString(), event, ...fields });
+  } else {
+    const tail = Object.entries(fields)
+      .map(([k, v]) => `${k}=${typeof v === 'string' ? v : JSON.stringify(v)}`)
+      .join(' ');
+    line = `[${new Date().toISOString()}] ${event}${tail ? ' ' + tail : ''}`;
+  }
+  console.log(line);
+  if (logFile) {
+    try { appendFileSync(logFile, line + '\n'); } catch { /* best-effort */ }
+  }
+}
+
+function writeHeartbeat(): void {
+  try {
+    const dir = join(transportRoot, '.turnq');
+    mkdirSync(dir, { recursive: true });
+    const data = { ts: new Date().toISOString(), pid: process.pid, version: '5.0.0-alpha.1' };
+    writeFileSync(join(dir, 'heartbeat'), JSON.stringify(data) + '\n');
+  } catch { /* best-effort */ }
+}
+
+function loadProtocolPrompt(): string {
+  const p = join(transportRoot, 'upstream', 'PROTOCOL.md');
+  if (!existsSync(p)) return '';
+  return readFileSync(p, 'utf-8').trim();
+}
+
+const protocolPrompt = loadProtocolPrompt();
+
+function recipients(toField: unknown): string[] {
+  if (Array.isArray(toField)) return toField.map(String);
+  if (typeof toField === 'string') return [toField];
+  return [];
+}
+
+function composeSystemPrompt(actorPrompt: string): string {
+  return [protocolPrompt, actorPrompt]
+    .filter((p) => p.length > 0)
+    .join('\n\n---\n\n');
+}
+
+function actorConcurrency(tiers: HostActorTiers): number {
+  for (const value of Object.values(tiers)) {
+    if (typeof value === 'object' && typeof value.count === 'number' && value.count > 0) {
+      return value.count;
+    }
+  }
+  return 1;
+}
+
+interface CliResult {
+  status: number;
+  stdout: string;
+  stderr: string;
+}
+
+function invokeCli(cli: string, systemPrompt: string, userMessage: string): Promise<CliResult> {
+  return new Promise((res) => {
+    const fullPrompt = `${systemPrompt}\n\n---\n\n${userMessage}`;
+    const parts = tokenizeCli(cli);
+    if (parts.length === 0) {
+      res({ status: 1, stdout: '', stderr: 'tokenized cli is empty' });
+      return;
+    }
+    const child = spawn(parts[0], parts.slice(1), { stdio: ['pipe', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    let resolved = false;
+    const timeout = setTimeout(() => {
+      if (resolved) return;
+      resolved = true;
+      child.kill('SIGKILL');
+      res({ status: 124, stdout, stderr: stderr + '\n[timeout]' });
+    }, 5 * 60_000);
+    child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('close', (code) => {
+      if (resolved) return;
+      resolved = true;
+      clearTimeout(timeout);
+      res({ status: code ?? 1, stdout, stderr });
+    });
+    child.on('error', (err) => {
+      if (resolved) return;
+      resolved = true;
+      clearTimeout(timeout);
+      res({ status: 1, stdout, stderr: stderr + '\n' + err.message });
+    });
+    // The child may exit before reading stdin (e.g. cli=`false`). Attach
+    // an error handler so EPIPE is swallowed instead of crashing dispatch,
+    // and guard the write itself.
+    child.stdin.on('error', () => { /* EPIPE/etc. — child closed stdin */ });
+    try {
+      child.stdin.write(fullPrompt);
+    } catch { /* same: child closed stdin before we could write */ }
+    try {
+      child.stdin.end();
+    } catch { /* ignore */ }
+  });
+}
+
+function writeReply(
+  channelUuid: string,
+  fromActor: string,
+  toActor: string,
+  body: string,
+): void {
+  const ts = now();
+  const dir = join(transportRoot, 'data', 'channels', channelUuid, ts.pathDate);
+  mkdirSync(dir, { recursive: true });
+  const content = serializeFrontmatter(
+    { from: fromActor, to: toActor, type: 'text', timestamp: ts.iso },
+    body,
+  );
+  writeFileSync(join(dir, messageFilename(ts)), content);
+}
+
+function writeReadReceipt(
+  channelUuid: string,
+  fromActor: string,
+  toActor: string,
+  ref: string,
+): void {
+  const ts = now();
+  const dir = join(transportRoot, 'data', 'channels', channelUuid, ts.pathDate);
+  mkdirSync(dir, { recursive: true });
+  const content = serializeFrontmatter(
+    { from: fromActor, to: toActor, type: 'read', ref, timestamp: ts.iso },
+    '',
+  );
+  writeFileSync(join(dir, messageFilename(ts)), content);
+}
+
+interface PendingDispatch {
+  actorName: string;
+  channelUuid: string;
+  msg: ChannelMessage;
+  from: string;
+  tiers: HostActorTiers;
+}
+
+async function dispatchOne(p: PendingDispatch): Promise<boolean> {
+  // Resolve CLI per-message — message frontmatter may request a specific
+  // tier via `tier: <name>`. Falls back to first declared tier.
+  const preferredTier = typeof p.msg.data['tier'] === 'string'
+    ? (p.msg.data['tier'] as string)
+    : undefined;
+  let resolved;
+  try {
+    resolved = pickTier(p.tiers, preferredTier);
+  } catch (err) {
+    const r = writeDlqEntry(
+      transportRoot,
+      'config',
+      p.actorName,
+      '(config)',
+      '(config)',
+      `tier selection failed: ${(err as Error).message}`,
+    );
+    log('actor_config_error', {
+      actor: p.actorName,
+      dlq_id: r.id,
+      attempts: r.attempts,
+      quarantined: r.quarantined,
+    });
+    return false;
+  }
+  const cli = resolved.cli;
+  if (isQuarantined(transportRoot, 'dispatch', p.actorName, p.channelUuid, p.msg.relPath)) {
+    log('dispatch_skipped_quarantined', {
+      actor: p.actorName,
+      channel: p.channelUuid.slice(0, 8),
+      msg: p.msg.relPath,
+    });
+    return false;
+  }
+
+  log('dispatch', {
+    actor: p.actorName,
+    channel: p.channelUuid.slice(0, 8),
+    msg: p.msg.relPath,
+  });
+
+  writeReadReceipt(p.channelUuid, p.actorName, p.from, p.msg.relPath);
+
+  let profile;
+  try {
+    profile = loadActorProfile(transportRoot, p.actorName);
+  } catch (err) {
+    const r = writeDlqEntry(
+      transportRoot,
+      'config',
+      p.actorName,
+      '(config)',
+      '(config)',
+      `actor profile load failed: ${(err as Error).message}`,
+    );
+    log('dispatch_config_error', {
+      actor: p.actorName,
+      dlq_id: r.id,
+      attempts: r.attempts,
+      quarantined: r.quarantined,
+    });
+    return false;
+  }
+
+  const systemPrompt = composeSystemPrompt(profile.systemPrompt);
+  const result = await invokeCli(cli, systemPrompt, p.msg.body);
+
+  if (result.status !== 0) {
+    const r = writeDlqEntry(
+      transportRoot,
+      'dispatch',
+      p.actorName,
+      p.channelUuid,
+      p.msg.relPath,
+      `cli exit=${result.status}\n${result.stderr.slice(0, 1000)}`,
+    );
+    log('dispatch_failed', {
+      actor: p.actorName,
+      channel: p.channelUuid.slice(0, 8),
+      dlq_id: r.id,
+      attempts: r.attempts,
+      quarantined: r.quarantined,
+      exit: result.status,
+    });
+    return false;
+  }
+
+  const reply = result.stdout.trim();
+  if (reply.length === 0) {
+    const r = writeDlqEntry(
+      transportRoot,
+      'dispatch',
+      p.actorName,
+      p.channelUuid,
+      p.msg.relPath,
+      'cli returned empty reply',
+    );
+    log('dispatch_empty_reply', {
+      actor: p.actorName,
+      channel: p.channelUuid.slice(0, 8),
+      dlq_id: r.id,
+      attempts: r.attempts,
+      quarantined: r.quarantined,
+    });
+    return false;
+  }
+
+  writeReply(p.channelUuid, p.actorName, p.from, reply);
+  return true;
+}
+
+interface TickResult {
+  didWork: boolean;
+  infraOk: boolean;
+}
+
+async function dispatchTick(): Promise<TickResult> {
+  writeHeartbeat();
+
+  return withLock('dispatch', async () => {
+    let infraOk = true;
+
+    const pullResult = gitPull(transportRoot);
+    if (!pullResult.ok && pullResult.error) {
+      const errId = writeErrorLog(transportRoot, 'git_pull', pullResult.error);
+      log('git_pull_failed', { error_id: errId, error: pullResult.error.slice(0, 120) });
+      infraOk = false;
+    }
+
+    let host: HostFile;
+    try {
+      host = findHostFile(transportRoot, hostOverride);
+    } catch (err) {
+      const r = writeDlqEntry(
+        transportRoot,
+        'config',
+        '(host)',
+        '(config)',
+        '(config)',
+        `host file load failed: ${(err as Error).message}`,
+      );
+      log('tick_config_error', {
+        scope: 'host',
+        dlq_id: r.id,
+        attempts: r.attempts,
+        quarantined: r.quarantined,
+      });
+      return { didWork: false, infraOk };
+    }
+
+    let didWork = false;
+
+    for (const actorName of Object.keys(host.actors)) {
+      if (isActorQuarantined(transportRoot, actorName)) {
+        log('actor_skipped_quarantined', { actor: actorName });
+        continue;
+      }
+
+      const tiers = host.actors[actorName];
+      const concurrency = actorConcurrency(tiers);
+
+      const pending: PendingDispatch[] = [];
+      const channels = discoverChannels(transportRoot);
+      for (const channelUuid of channels) {
+        const cursor = readCursor(transportRoot, actorName, channelUuid);
+        const messages = listChannelMessages(transportRoot, channelUuid);
+        const post = cursor ? messages.filter((m) => m.relPath > cursor) : messages;
+
+        log('tick_scan', {
+          actor: actorName,
+          channel: channelUuid.slice(0, 8),
+          cursor: cursor ?? '(none)',
+          total_msgs: messages.length,
+          post_cursor_msgs: post.length,
+        });
+
+        for (const msg of post) {
+          const to = recipients(msg.data['to']);
+          const from = typeof msg.data['from'] === 'string' ? msg.data['from'] : 'unknown';
+          if (!to.includes(actorName) || from === actorName) {
+            writeCursor(transportRoot, actorName, channelUuid, msg.relPath);
+            continue;
+          }
+          pending.push({ actorName, channelUuid, msg, from, tiers });
+        }
+      }
+
+      for (let i = 0; i < pending.length; i += concurrency) {
+        const batch = pending.slice(i, i + concurrency);
+        const results = await Promise.all(batch.map((p) => dispatchOne(p)));
+        for (let j = 0; j < batch.length; j++) {
+          writeCursor(
+            transportRoot,
+            batch[j].actorName,
+            batch[j].channelUuid,
+            batch[j].msg.relPath,
+          );
+          if (results[j]) didWork = true;
+        }
+      }
+    }
+
+    // Always attempt commit+push at end of tick — gitCommitAndPush
+    // short-circuits if the working tree is clean. This is required
+    // even when no replies were produced, because cursors advance for
+    // messages addressed to other actors (the actor's own replies and
+    // read receipts appear on the next pull and need to be skipped past).
+    // Without this commit, the orphan cursor change blocks the next
+    // git pull --rebase and dispatch dead-ends in backoff.
+    const commitMsg = didWork
+      ? `dispatch: replies + cursor advance ${new Date().toISOString()}`
+      : `dispatch: cursor advance ${new Date().toISOString()}`;
+    const pushResult = gitCommitAndPush(transportRoot, commitMsg);
+    if (!pushResult.ok && pushResult.error) {
+      const kind = pushResult.committed ? 'git_push' : 'git_commit';
+      const errId = writeErrorLog(transportRoot, kind, pushResult.error);
+      log('git_push_failed', {
+        error_id: errId,
+        committed_locally: pushResult.committed,
+        error: pushResult.error.slice(0, 120),
+      });
+      infraOk = false;
+    }
+
+    // Periodic stale-read-receipt sweep
+    if (Date.now() - lastSweepAt > SWEEP_INTERVAL_MS) {
+      const surfaced = sweepStaleReadReceipts(transportRoot, STALE_RECEIPT_THRESHOLD_MS);
+      lastSweepAt = Date.now();
+      if (surfaced > 0) {
+        log('stale_receipts_surfaced', { count: surfaced });
+      }
+    }
+
+    return { didWork, infraOk };
+  });
+}
+
+async function waitForWakeOrTimeout(ms: number): Promise<'wake' | 'timeout'> {
+  const wakeDir = join(transportRoot, '.turnq');
+  mkdirSync(wakeDir, { recursive: true });
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), ms);
+  try {
+    const watcher = watch(wakeDir, { signal: ac.signal });
+    for await (const ev of watcher) {
+      if (ev.filename === 'wake.signal') {
+        clearTimeout(timer);
+        return 'wake';
+      }
+    }
+    return 'timeout';
+  } catch {
+    return 'timeout';
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function main(): Promise<void> {
+  log('dispatch_start', {
+    transport: transportRoot,
+    version: '5.0.0-alpha.1',
+    log_file: logFile ?? null,
+  });
+  if (onceMode) {
+    await dispatchTick();
+    return;
+  }
+  log('coordinator_running', { quiet_poll_s: pollSeconds, active_poll_s: 1 });
+
+  let consecutiveInfraFailures = 0;
+
+  while (true) {
+    try {
+      const r = await dispatchTick();
+      if (r.infraOk) {
+        if (consecutiveInfraFailures > 0) {
+          log('backoff_cleared', { previous_consecutive_failures: consecutiveInfraFailures });
+        }
+        consecutiveInfraFailures = 0;
+      } else {
+        consecutiveInfraFailures++;
+      }
+
+      // Backoff kicks in only after a grace period of failures.
+      const beyondGrace = Math.max(0, consecutiveInfraFailures - BACKOFF_GRACE);
+      const backoffFactor = Math.min(MAX_BACKOFF_MULTIPLIER, 2 ** beyondGrace);
+
+      if (consecutiveInfraFailures > BACKOFF_GRACE) {
+        log('backoff_active', {
+          consecutive_failures: consecutiveInfraFailures,
+          factor: backoffFactor,
+        });
+      }
+
+      if (r.didWork) {
+        await new Promise((res) => setTimeout(res, 1_000 * backoffFactor));
+      } else {
+        await waitForWakeOrTimeout(pollSeconds * 1_000 * backoffFactor);
+      }
+    } catch (err) {
+      const msg = (err as Error).message;
+      writeErrorLog(transportRoot, 'other', `tick error: ${msg}`);
+      log('tick_error', { message: msg });
+      consecutiveInfraFailures++;
+      await new Promise((res) => setTimeout(res, pollSeconds * 1_000));
+    }
+  }
+}
+
+main();