@cordfuse/crosstalkd 7.0.0-alpha.1

package/src/dispatchers.ts

@@ -0,0 +1,91 @@
+// dispatchers.ts — the shared dispatcher registry.
+//
+// Each running dispatcher publishes its identity + claimed models to
+// `data/dispatchers/<alias>.yaml` so workflow.ts can scope fanout
+// sub-primitives across the actually-running dispatcher set instead of
+// addressing every fanout to bare `<model>` and watching every claimant
+// race for it.
+//
+// Lifecycle:
+//   - dispatcher startup → writeRegistryEntry (committed alongside other
+//     state changes by the next tick's gitCommitAndPush)
+//   - dispatcher SIGTERM/SIGINT → removeRegistryEntry (committed same way)
+//   - hard crash → entry stays; operator runs cleanup, or restarts
+//
+// Liveness is best-effort. v7.0.0-alpha.1 does NOT track per-tick
+// heartbeats in the registry — the timestamp would force a commit every
+// tick on every machine, which is far more push traffic than the system
+// otherwise generates. Heartbeats live in the machine-local state dir
+// (state.ts) for "is THIS dispatcher alive" checks; the registry answers
+// "which dispatchers exist on the bus." A stale registry entry from a
+// crashed dispatcher is an operator-cleanup concern, not a runtime concern.
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+
+export interface RegistryEntry {
+  alias: string;
+  claims: string[];        // model names this dispatcher has on PATH
+  version: string;
+}
+
+export function registryDir(transportRoot: string): string {
+  return join(transportRoot, 'data', 'dispatchers');
+}
+
+export function registryFile(transportRoot: string, alias: string): string {
+  return join(registryDir(transportRoot), `${alias}.yaml`);
+}
+
+export function writeRegistryEntry(
+  transportRoot: string,
+  alias: string,
+  claims: string[],
+  version: string,
+): void {
+  mkdirSync(registryDir(transportRoot), { recursive: true });
+  const entry: RegistryEntry = { alias, claims, version };
+  writeFileSync(registryFile(transportRoot, alias), stringifyYaml(entry));
+}
+
+export function removeRegistryEntry(transportRoot: string, alias: string): void {
+  try {
+    unlinkSync(registryFile(transportRoot, alias));
+  } catch { /* already gone */ }
+}
+
+export function readAllRegistry(transportRoot: string): RegistryEntry[] {
+  const dir = registryDir(transportRoot);
+  if (!existsSync(dir)) return [];
+  const out: RegistryEntry[] = [];
+  for (const entry of readdirSync(dir)) {
+    if (!entry.endsWith('.yaml')) continue;
+    try {
+      const raw = readFileSync(join(dir, entry), 'utf-8');
+      const parsed = parseYaml(raw) as Partial<RegistryEntry> | null;
+      if (
+        !parsed ||
+        typeof parsed.alias !== 'string' ||
+        !Array.isArray(parsed.claims) ||
+        typeof parsed.version !== 'string'
+      ) continue;
+      out.push({
+        alias: parsed.alias,
+        claims: parsed.claims.map(String),
+        version: parsed.version,
+      });
+    } catch { /* skip malformed */ }
+  }
+  return out;
+}
+
+// Aliases of dispatchers currently claiming the given model, sorted for
+// deterministic ordering — round-robin assignment becomes reproducible,
+// which makes the test path predictable.
+export function dispatchersForModel(transportRoot: string, modelName: string): string[] {
+  return readAllRegistry(transportRoot)
+    .filter((e) => e.claims.includes(modelName))
+    .map((e) => e.alias)
+    .sort();
+}

package/src/filenames.ts

@@ -0,0 +1,28 @@
+import { randomBytes } from 'crypto';
+
+export interface Timestamp {
+  iso: string;
+  pathDate: string;
+  fileTime: string;
+  hex: string;
+}
+
+export function now(): Timestamp {
+  const d = new Date();
+  const iso = d.toISOString();
+  const yyyy = iso.slice(0, 4);
+  const mm = iso.slice(5, 7);
+  const dd = iso.slice(8, 10);
+  const time = iso.slice(11, 13) + iso.slice(14, 16) + iso.slice(17, 19);
+  const ms = iso.slice(20, 23);
+  return {
+    iso,
+    pathDate: `${yyyy}/${mm}/${dd}`,
+    fileTime: `${time}${ms}Z`,
+    hex: randomBytes(4).toString('hex'),
+  };
+}
+
+export function messageFilename(ts: Timestamp = now()): string {
+  return `${ts.fileTime}-${ts.hex}.md`;
+}

package/src/frontmatter.ts

@@ -0,0 +1,26 @@
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+
+export interface ParsedDocument<T = Record<string, unknown>> {
+  data: T;
+  body: string;
+}
+
+export function parseFrontmatter<T = Record<string, unknown>>(raw: string): ParsedDocument<T> {
+  if (!raw.startsWith('---')) return { data: {} as T, body: raw };
+  const lines = raw.split('\n');
+  let end = -1;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i].trim() === '---') { end = i; break; }
+  }
+  if (end === -1) return { data: {} as T, body: raw };
+  const yamlBlock = lines.slice(1, end).join('\n');
+  const body = lines.slice(end + 1).join('\n').replace(/^\n/, '');
+  const data = (parseYaml(yamlBlock) as T) ?? ({} as T);
+  return { data, body };
+}
+
+export function serializeFrontmatter(data: Record<string, unknown>, body: string): string {
+  const yaml = stringifyYaml(data).trimEnd();
+  const trailing = body.endsWith('\n') ? '' : '\n';
+  return `---\n${yaml}\n---\n\n${body}${trailing}`;
+}

package/src/init.ts

@@ -0,0 +1,108 @@
+// crosstalkd init <directory> — scaffold a new transport.
+//
+// Copies the bundled flat template (CROSSTALK-VERSION, PROTOCOL.md,
+// CROSSTALK.md, data/models.yaml, local/actors/orchestrator.md, etc.)
+// into the target directory and renames `gitignore` → `.gitignore`.
+//
+// v7 vs v6 init differences:
+//  - No host file is generated. v7 has no host files at all; machine
+//    identity comes from the --alias flag at dispatch boot.
+//  - No first channel is auto-created. Operator runs
+//    `crosstalkd channel <name>` themselves.
+//  - Template layout is flat: no upstream/ prefix on CROSSTALK-VERSION
+//    or PROTOCOL.md.
+//
+// Template lookup order:
+//   1. <runtime_root>/template/      — bundled at publish time (production)
+//   2. <runtime_root>/../transport/  — monorepo layout (local dev)
+
+import { existsSync, mkdirSync, writeFileSync, cpSync, renameSync } from 'fs';
+import { resolve, join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const argv = process.argv.slice(2);
+const force = argv.includes('--force');
+
+const positional = argv.filter((a) => !a.startsWith('--'));
+if (positional.length === 0) {
+  console.error('Usage: crosstalkd init <directory> [--force]');
+  console.error('       crosstalkd init .                  (scaffold into current dir)');
+  process.exit(1);
+}
+
+const targetDir = resolve(positional[0]!);
+
+if (existsSync(join(targetDir, 'CROSSTALK-VERSION')) && !force) {
+  console.error(`crosstalkd init: ${targetDir} already contains a transport.`);
+  console.error('Pass --force to overwrite.');
+  process.exit(1);
+}
+
+const runtimeRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const candidates = [
+  join(runtimeRoot, 'template'),
+  join(runtimeRoot, '..', 'transport'),
+];
+const templateDir = candidates.find((c) => existsSync(join(c, 'CROSSTALK-VERSION')));
+
+if (!templateDir) {
+  console.error('crosstalkd init: cannot find the transport template.');
+  console.error('Looked in:');
+  for (const c of candidates) console.error(`  ${c}`);
+  console.error('The @cordfuse/crosstalkd installation may be corrupted; try reinstalling.');
+  process.exit(1);
+}
+
+mkdirSync(targetDir, { recursive: true });
+// Skip the template directory's own README — that's the meta-readme
+// documenting the seed template itself, not content the operator wants
+// inside their transport. (We write a fresh operator-facing README below.)
+const templateReadme = join(templateDir, 'README.md');
+cpSync(templateDir, targetDir, {
+  recursive: true,
+  force,
+  filter: (src) => src !== templateReadme,
+});
+// npm strips .gitignore from published packages; the template ships it as
+// `gitignore` and we rename it here.
+const gitignoreSrc = join(targetDir, 'gitignore');
+const gitignoreDst = join(targetDir, '.gitignore');
+if (existsSync(gitignoreSrc)) renameSync(gitignoreSrc, gitignoreDst);
+
+const readmePath = join(targetDir, 'README.md');
+if (!existsSync(readmePath) || force) {
+  writeFileSync(
+    readmePath,
+    `# Transport
+
+A Crosstalk transport created by \`crosstalk init\`.
+
+- Protocol spec:           [\`CROSSTALK.md\`](./CROSSTALK.md)
+- Agent orientation:       [\`PROTOCOL.md\`](./PROTOCOL.md)
+- Model registry:          [\`data/models.yaml\`](./data/models.yaml)
+- Actor persona files:     [\`local/actors/\`](./local/actors/)
+
+## Next steps
+
+From the transport directory, on a host with the \`crosstalk\` client installed:
+
+\`\`\`sh
+git init && git add -A && git commit -m "initial transport"
+crosstalk up                                       # start the engine container
+crosstalk channel general                          # create your first channel
+crosstalk run --type primitive --to sonnet "hello" # send your first message
+crosstalk status                                   # check state
+\`\`\`
+`,
+  );
+}
+
+console.log('');
+console.log(`Transport initialized at ${targetDir}`);
+console.log('');
+console.log('Next steps (from a host with the `crosstalk` client installed):');
+console.log(`  cd ${targetDir}`);
+console.log('  git init && git add -A && git commit -m "initial transport"');
+console.log('  crosstalk up                                       # start the engine container');
+console.log('  crosstalk channel general                          # create your first channel');
+console.log('  crosstalk run --type primitive --to sonnet "hello" # send your first message');

package/src/invoke.ts

@@ -0,0 +1,148 @@
+// invoke.ts — model invocation, persona loading, reply writing.
+//
+// Called from dispatch.ts when a message wakes a claimed model. Composes
+// the system prompt (PROTOCOL.md + optional actor persona), invokes the
+// model CLI from data/models.yaml, captures stdout, writes a reply
+// message back into the same channel (success or failed:true).
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { spawn } from 'child_process';
+import { now, messageFilename } from './filenames.js';
+import { serializeFrontmatter } from './frontmatter.js';
+import type { ModelEntry } from './models.js';
+import type { ChannelMessage } from './transport.js';
+
+export interface InvokeResult {
+  status: number;
+  stdout: string;
+  stderr: string;
+}
+
+// 10 minutes — orchestrator-persona workflows can take 3-5 minutes
+// of model thinking before the first tool call. v6's 5-minute timeout
+// was tight for any orchestrator pattern.
+const CLI_TIMEOUT_MS = 10 * 60_000;
+const ARGV_PROMPT_LIMIT = 64 * 1024;
+
+export function loadProtocolPrompt(transportRoot: string): string {
+  const p = join(transportRoot, 'PROTOCOL.md');
+  return existsSync(p) ? readFileSync(p, 'utf-8').trim() : '';
+}
+
+export function loadActorPersona(transportRoot: string, actorName: string | undefined): string {
+  if (!actorName) return '';
+  const p = join(transportRoot, 'local', 'actors', `${actorName}.md`);
+  if (!existsSync(p)) return '';
+  return readFileSync(p, 'utf-8').trim();
+}
+
+export function composeSystemPrompt(parts: string[]): string {
+  return parts.filter((p) => p && p.length > 0).join('\n\n---\n\n');
+}
+
+export function messageSender(msg: ChannelMessage): string {
+  return typeof msg.data['from'] === 'string' ? (msg.data['from'] as string) : 'unknown';
+}
+
+// Pass the prompt as the CLI's last argv entry. Every modern agent CLI
+// (Claude --print, codex exec, gemini -p, qwen --yolo, opencode -p,
+// agy -p) reads its prompt from the trailing positional, so appending
+// works universally. Fallback to stdin when the prompt would exceed a
+// safe argv size — ARG_MAX is ~128 KB on Linux and ~256 KB on macOS.
+export function invokeModelCli(
+  model: ModelEntry,
+  systemPrompt: string,
+  userMessage: string,
+  env: Record<string, string>,
+): Promise<InvokeResult> {
+  return new Promise((res) => {
+    const fullPrompt = systemPrompt.length > 0
+      ? `${systemPrompt}\n\n---\n\n${userMessage}`
+      : userMessage;
+    const useStdin = Buffer.byteLength(fullPrompt, 'utf-8') > ARGV_PROMPT_LIMIT;
+    const argv = useStdin ? [...model.args] : [...model.args, fullPrompt];
+    // detached: new process group, so the timeout SIGKILL takes the model's
+    // children with it — orphans writing to the transport after a timeout
+    // was an observed v5/v6 hazard.
+    const child = spawn(model.cli, argv, {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      detached: true,
+      env: { ...process.env, ...env },
+    });
+    let stdout = '';
+    let stderr = '';
+    let resolved = false;
+    const timeout = setTimeout(() => {
+      if (resolved) return;
+      resolved = true;
+      try {
+        if (typeof child.pid === 'number') process.kill(-child.pid, 'SIGKILL');
+        else child.kill('SIGKILL');
+      } catch {
+        try { child.kill('SIGKILL'); } catch { /* already dead */ }
+      }
+      res({ status: 124, stdout, stderr: stderr + '\n[timeout]' });
+    }, CLI_TIMEOUT_MS);
+    child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('close', (code) => {
+      if (resolved) return;
+      resolved = true;
+      clearTimeout(timeout);
+      res({ status: code ?? 1, stdout, stderr });
+    });
+    child.on('error', (err) => {
+      if (resolved) return;
+      resolved = true;
+      clearTimeout(timeout);
+      res({ status: 1, stdout, stderr: stderr + '\n' + err.message });
+    });
+    child.stdin.on('error', () => { /* child closed stdin */ });
+    if (useStdin) {
+      try { child.stdin.write(fullPrompt); } catch { /* same */ }
+    }
+    try { child.stdin.end(); } catch { /* ignore */ }
+  });
+}
+
+export function formatBatchedUserMessage(msgs: ChannelMessage[]): string {
+  if (msgs.length === 1) return msgs[0]!.body;
+  const parts = [`You have ${msgs.length} new messages in this channel. Process them collectively and reply once.`];
+  for (let i = 0; i < msgs.length; i++) {
+    const m = msgs[i]!;
+    const ts = typeof m.data['timestamp'] === 'string' ? `, ts: ${m.data['timestamp']}` : '';
+    parts.push(`--- Message ${i + 1} of ${msgs.length} (from: ${messageSender(m)}, ref: ${m.relPath}${ts}) ---`);
+    parts.push(m.body);
+  }
+  return parts.join('\n\n');
+}
+
+export interface ReplyOpts {
+  transportRoot: string;
+  channelUuid: string;
+  fromModel: string;     // e.g. "sonnet@cachy"
+  to: string;            // the requester
+  re: string | string[];
+  body: string;
+  failed?: { error: string };
+}
+
+export function writeReply(opts: ReplyOpts): string {
+  const ts = now();
+  const dir = join(opts.transportRoot, 'data', 'channels', opts.channelUuid, ts.pathDate);
+  mkdirSync(dir, { recursive: true });
+  const fm: Record<string, unknown> = {
+    from: opts.fromModel,
+    to: opts.to,
+    timestamp: ts.iso,
+    re: opts.re,
+  };
+  if (opts.failed) {
+    fm['failed'] = true;
+    fm['error'] = opts.failed.error.slice(0, 2000);
+  }
+  const filename = messageFilename(ts);
+  writeFileSync(join(dir, filename), serializeFrontmatter(fm, opts.body));
+  return join(ts.pathDate, filename);
+}

package/src/models.ts

@@ -0,0 +1,86 @@
+// models.ts — model registry parsing and PATH-based self-selection.
+//
+// Reads data/models.yaml from the transport root, returns the parsed
+// registry plus the subset whose CLI is on PATH (the "claimed" models for
+// the current dispatcher).
+
+import { existsSync, readFileSync, statSync } from 'fs';
+import { join, delimiter } from 'path';
+import { parse as parseYaml } from 'yaml';
+
+export interface ModelEntry {
+  name: string;
+  command: string;       // raw command-template string
+  cli: string;           // first token — the binary the dispatcher checks PATH for
+  args: string[];        // remaining tokens (the body is appended at invocation time)
+}
+
+export interface ModelsRegistry {
+  all: Map<string, ModelEntry>;
+  claimed: Map<string, ModelEntry>;
+}
+
+export function modelsYamlPath(transportRoot: string): string {
+  return join(transportRoot, 'data', 'models.yaml');
+}
+
+export function readModels(transportRoot: string): Map<string, ModelEntry> {
+  const path = modelsYamlPath(transportRoot);
+  if (!existsSync(path)) {
+    throw new Error(
+      `crosstalkd: data/models.yaml not found at ${path}. ` +
+      `Run from a v7 transport root, or regenerate via 'crosstalkd init'.`,
+    );
+  }
+  const raw = readFileSync(path, 'utf8');
+  const parsed = parseYaml(raw);
+  if (parsed == null || typeof parsed !== 'object') {
+    throw new Error(`crosstalkd: data/models.yaml parsed to non-object (got ${typeof parsed})`);
+  }
+  const out = new Map<string, ModelEntry>();
+  for (const [name, value] of Object.entries(parsed as Record<string, unknown>)) {
+    if (typeof value !== 'string') {
+      throw new Error(
+        `crosstalkd: data/models.yaml entry '${name}' has non-string value ` +
+        `(expected a command-template string, got ${typeof value})`,
+      );
+    }
+    const tokens = value.trim().split(/\s+/);
+    if (tokens.length === 0 || tokens[0] === '') {
+      throw new Error(`crosstalkd: data/models.yaml entry '${name}' is empty`);
+    }
+    out.set(name, { name, command: value, cli: tokens[0], args: tokens.slice(1) });
+  }
+  return out;
+}
+
+export function isOnPath(binary: string, env: NodeJS.ProcessEnv = process.env): boolean {
+  const pathVar = env.PATH ?? '';
+  for (const dir of pathVar.split(delimiter)) {
+    if (!dir) continue;
+    const candidate = join(dir, binary);
+    try {
+      if (existsSync(candidate) && statSync(candidate).isFile()) return true;
+    } catch {
+      // unreadable entry, skip
+    }
+  }
+  return false;
+}
+
+export function claimModels(
+  all: Map<string, ModelEntry>,
+  env: NodeJS.ProcessEnv = process.env,
+): Map<string, ModelEntry> {
+  const claimed = new Map<string, ModelEntry>();
+  for (const [name, entry] of all) {
+    if (isOnPath(entry.cli, env)) claimed.set(name, entry);
+  }
+  return claimed;
+}
+
+export function loadRegistry(transportRoot: string): ModelsRegistry {
+  const all = readModels(transportRoot);
+  const claimed = claimModels(all);
+  return { all, claimed };
+}

package/src/replies.ts

@@ -0,0 +1,73 @@
+// crosstalkd replies — have my messages been answered yet?
+//
+// Checks each given relPath for a message whose `re:` points back at it.
+// Exit 0 when everything is answered, 2 while anything is pending — so
+// agents can poll cheaply in a loop.
+//
+// v7 surface: positional relPaths (space-separated), no --re flag.
+// `crosstalkd replies <relPath> [<relPath>...]`
+
+import { resolve } from 'path';
+import { gitPull, discoverChannels, listChannelMessages } from './transport.js';
+import { reList } from './activation.js';
+
+const transportRoot = resolve(process.cwd());
+const argv = process.argv.slice(2);
+
+function flag(name: string): string | undefined {
+  const i = argv.indexOf(name);
+  if (i === -1 || i === argv.length - 1) return undefined;
+  return argv[i + 1];
+}
+
+const channelArg = flag('--channel') ?? process.env['CROSSTALK_DISPATCH_CHANNEL'];
+
+// All non-flag args are positional relPaths. Skip flag values too.
+const flagsTakingValue = new Set(['--channel']);
+const targets: string[] = [];
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i]!;
+  if (a.startsWith('--')) {
+    if (flagsTakingValue.has(a)) i++;
+    continue;
+  }
+  targets.push(a);
+}
+
+if (targets.length === 0) {
+  console.error('Usage: crosstalkd replies <relPath> [<relPath>...] [--channel <uuid>]');
+  process.exit(1);
+}
+
+gitPull(transportRoot); // best-effort freshness
+
+const channels = channelArg ? [channelArg] : discoverChannels(transportRoot);
+
+const found = new Map<string, { from: string; relPath: string; failed: boolean }>();
+for (const channel of channels) {
+  for (const msg of listChannelMessages(transportRoot, channel)) {
+    for (const entry of reList(msg.data['re'])) {
+      if (targets.includes(entry) && !found.has(entry)) {
+        found.set(entry, {
+          from: String(msg.data['from']),
+          relPath: msg.relPath,
+          failed: msg.data['failed'] === true,
+        });
+      }
+    }
+  }
+}
+
+let pending = 0;
+for (const target of targets) {
+  const reply = found.get(target);
+  if (reply) {
+    const tag = reply.failed ? 'FAILED  ' : 'REPLIED ';
+    console.log(`${tag} ${target}  <- ${reply.from} (${reply.relPath})`);
+  } else {
+    console.log(`PENDING  ${target}`);
+    pending++;
+  }
+}
+
+process.exit(pending > 0 ? 2 : 0);