@cordfuse/crosstalkd 7.0.0-alpha.1

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@cordfuse/crosstalkd",
+  "version": "7.0.0-alpha.1",
+  "description": "Crosstalk daemon — the engine that runs inside the crosstalk-server container. Operators interact via @cordfuse/crosstalk on the host.",
+  "type": "module",
+  "license": "MIT",
+  "homepage": "https://github.com/cordfuse/crosstalk",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cordfuse/crosstalk.git"
+  },
+  "bin": {
+    "crosstalkd": "./bin/crosstalkd.js"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "template/",
+    "GUIDE-CLI.md",
+    "GUIDE-PROMPTS.md"
+  ],
+  "scripts": {
+    "build": "tsc --noEmit",
+    "lint": "tsc --noEmit",
+    "test": "bun test",
+    "prepack": "cp -r ../transport template && cp ../GUIDE-CLI.md ../GUIDE-PROMPTS.md .",
+    "postpack": "rm -rf template GUIDE-CLI.md GUIDE-PROMPTS.md"
+  },
+  "dependencies": {
+    "tsx": "^4.20.0",
+    "yaml": "^2.8.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "typescript": "^5.7.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "ai",
+    "agents",
+    "messaging",
+    "git",
+    "crosstalk",
+    "cordfuse"
+  ]
+}

package/src/activation.ts

@@ -0,0 +1,104 @@
+// The activation rule — pure functions, no I/O, exhaustively unit-tested.
+//
+// One rule (CROSSTALK.md "Activation"):
+//
+//   A message wakes its addressee if it has no `re:` (a new task), or its
+//   `re:` points at a message the addressee sent.
+//
+// `re:` is written by the runtime at send time, never inferred at read
+// time. It is a string or a list: a reply that answers a batch of N
+// messages records ALL N relPaths, so no answered message is ever lost to
+// batching.
+
+export interface ActivationMessage {
+  from: string;
+  to: string[];
+  re?: string | string[];
+}
+
+export function recipients(toField: unknown): string[] {
+  if (Array.isArray(toField)) return toField.map(String);
+  if (typeof toField === 'string') return [toField];
+  return [];
+}
+
+export function reList(reField: unknown): string[] {
+  if (Array.isArray(reField)) return reField.map(String);
+  if (typeof reField === 'string') return [reField];
+  return [];
+}
+
+// A recipient is `actor` or `actor@host`.
+export function extractActor(recipient: string): string {
+  const at = recipient.indexOf('@');
+  return at === -1 ? recipient : recipient.slice(0, at);
+}
+
+export function targetHost(recipient: string): string | null {
+  const at = recipient.indexOf('@');
+  return at === -1 ? null : recipient.slice(at + 1);
+}
+
+export interface RoutingResult {
+  addressed: boolean;
+  // Actor was named, but every instance targeted a different host — logged
+  // by the dispatcher so wrong-host routes are visible, never silent.
+  wrongHost: boolean;
+}
+
+export function matchRouting(
+  recipientList: string[],
+  actorName: string,
+  thisHost: string,
+): RoutingResult {
+  let actorNamedAtAll = false;
+  for (const r of recipientList) {
+    if (r === 'all') return { addressed: true, wrongHost: false };
+    if (extractActor(r) !== actorName) continue;
+    actorNamedAtAll = true;
+    const host = targetHost(r);
+    if (host === null || host === thisHost) return { addressed: true, wrongHost: false };
+  }
+  return { addressed: false, wrongHost: actorNamedAtAll };
+}
+
+export type WakeDecision = 'wake' | 'skip' | 'wrong-host';
+
+// `senderOf` resolves a channel relPath to the `from:` of the message there,
+// or undefined if no such message exists. A dangling `re:` entry (target
+// missing) wakes the addressee — fail open so a message is never silently
+// dropped; no loop is possible because the reply to it carries a
+// resolvable `re:`.
+export function decideWake(
+  msg: ActivationMessage,
+  actorName: string,
+  thisHost: string,
+  senderOf: (relPath: string) => string | undefined,
+): WakeDecision {
+  const routing = matchRouting(msg.to, actorName, thisHost);
+  if (!routing.addressed) return routing.wrongHost ? 'wrong-host' : 'skip';
+  if (msg.from === actorName) return 'skip';
+  const targets = reList(msg.re);
+  if (targets.length === 0) return 'wake';
+  for (const target of targets) {
+    const asker = senderOf(target);
+    if (asker === undefined || asker === actorName) return 'wake';
+  }
+  return 'skip';
+}
+
+// Split a channel's pending messages (already sorted by relPath) into
+// contiguous batches sized for the actor's concurrency. Contiguous so each
+// batch's highest relPath is monotone across batches — the cursor advances
+// safely per batch. When pending fits within concurrency, every batch is a
+// single message (parallel fan-out); when it exceeds, batches collapse into
+// ~concurrency invocations (fan-in stays O(1) per actor).
+export function splitForConcurrency<T>(msgs: T[], concurrency: number): T[][] {
+  if (concurrency <= 1 || msgs.length <= 1) return [msgs];
+  const chunkSize = Math.max(1, Math.ceil(msgs.length / concurrency));
+  const out: T[][] = [];
+  for (let i = 0; i < msgs.length; i += chunkSize) {
+    out.push(msgs.slice(i, i + chunkSize));
+  }
+  return out;
+}

package/src/api.ts

@@ -0,0 +1,430 @@
+// api.ts — local HTTP API the host-side client (@cordfuse/crosstalk)
+// connects to. Binds 127.0.0.1 only; never exposes to the network.
+//
+// Design choices (see conversation 2026-06-12):
+//   - No auth. Localhost dev tool, same model as ollama / postgres-on-
+//     localhost / jupyter-without-token. If you're on the host, you're
+//     trusted. The actual security boundary is the bind address.
+//   - Bind hardcoded to 127.0.0.1. Operators can't accidentally expose
+//     to the network even by misconfiguring docker-compose port mapping.
+//   - Plain Node `http` module, no Express/Fastify dep. v7 ships with
+//     two runtime deps (tsx + yaml); HTTP framework would break that.
+//   - JSON in, JSON out. Error responses carry { error: <message> }.
+//
+// Endpoint surface (v7.0.0-alpha.1):
+//   GET  /healthz                       — liveness probe (no body needed)
+//   GET  /version                       — engine version + alias
+//   GET  /status                        — status summary
+//   GET  /channels                      — channel listing (name → uuid)
+//   POST /channels                      — create a channel
+//   POST /messages                      — write primitive or workflow marker
+//   GET  /messages/:relPath/replies     — reply status for given relPath(s)
+//
+// Deferred to follow-up: SSE streaming endpoint for `replies --watch`,
+// /transport/template (export template tarball for `crosstalk init`),
+// /shutdown (graceful engine stop).
+
+import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'http';
+import { randomUUID } from 'crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { now, messageFilename } from './filenames.js';
+import { parseFrontmatter, serializeFrontmatter } from './frontmatter.js';
+import {
+  discoverChannels,
+  gitCommitAndPush,
+  listChannelMessages,
+} from './transport.js';
+import { sendWakeSignal, readHeartbeat, readCursor, countErrors } from './state.js';
+import { reList } from './activation.js';
+import type { ModelEntry } from './models.js';
+
+export interface ApiContext {
+  transportRoot: string;
+  alias: string;
+  version: string;
+  claimed: Map<string, ModelEntry>;
+}
+
+interface JsonRes {
+  status: number;
+  body: unknown;
+}
+
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+function readBody(req: IncomingMessage): Promise<string> {
+  return new Promise((resolveBody, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (c) => chunks.push(Buffer.from(c)));
+    req.on('end', () => resolveBody(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+async function readJson<T = unknown>(req: IncomingMessage): Promise<T> {
+  const raw = await readBody(req);
+  if (raw.length === 0) return {} as T;
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    throw new HttpError(400, 'invalid JSON body');
+  }
+}
+
+class HttpError extends Error {
+  constructor(public status: number, message: string) {
+    super(message);
+  }
+}
+
+function writeJson(res: ServerResponse, status: number, body: unknown): void {
+  const json = JSON.stringify(body);
+  res.writeHead(status, {
+    'Content-Type': 'application/json; charset=utf-8',
+    'Content-Length': Buffer.byteLength(json).toString(),
+  });
+  res.end(json);
+}
+
+// ── channel discovery helpers ────────────────────────────────────────
+
+interface ChannelMeta {
+  uuid: string;
+  name: string | null;
+  parent: string | null;
+}
+
+function readChannelMeta(transportRoot: string, uuid: string): ChannelMeta {
+  const chPath = join(transportRoot, 'data', 'channels', uuid, 'CHANNEL.md');
+  if (!existsSync(chPath)) return { uuid, name: null, parent: null };
+  const raw = readFileSync(chPath, 'utf-8');
+  const { data } = parseFrontmatter<{ name?: unknown; parent?: unknown }>(raw);
+  return {
+    uuid,
+    name: typeof data.name === 'string' ? data.name : null,
+    parent: typeof data.parent === 'string' ? data.parent : null,
+  };
+}
+
+function allChannelMeta(transportRoot: string): ChannelMeta[] {
+  return discoverChannels(transportRoot).map((u) => readChannelMeta(transportRoot, u));
+}
+
+function resolveChannelHandle(transportRoot: string, handle: string): ChannelMeta | null {
+  if (UUID_RE.test(handle)) {
+    const path = join(transportRoot, 'data', 'channels', handle);
+    if (!existsSync(path)) return null;
+    return readChannelMeta(transportRoot, handle);
+  }
+  for (const meta of allChannelMeta(transportRoot)) {
+    if (meta.name === handle) return meta;
+  }
+  return null;
+}
+
+// ── route handlers ───────────────────────────────────────────────────
+
+function handleHealthz(ctx: ApiContext): JsonRes {
+  return { status: 200, body: { ok: true, alias: ctx.alias, version: ctx.version } };
+}
+
+function handleVersion(ctx: ApiContext): JsonRes {
+  return {
+    status: 200,
+    body: { engine: 'crosstalkd', version: ctx.version, alias: ctx.alias },
+  };
+}
+
+function handleStatus(ctx: ApiContext): JsonRes {
+  const { transportRoot } = ctx;
+  const hb = readHeartbeat(transportRoot);
+  const cursor = readCursor(transportRoot);
+  const channels = allChannelMeta(transportRoot);
+  return {
+    status: 200,
+    body: {
+      transport_root: transportRoot,
+      alias: ctx.alias,
+      version: ctx.version,
+      claimed_models: [...ctx.claimed.keys()],
+      heartbeat: hb
+        ? { ts: hb.ts, pid: hb.pid, version: hb.version, alias: hb.alias }
+        : null,
+      cursor: cursor ?? null,
+      channels: channels.map((c) => ({ uuid: c.uuid, name: c.name, parent: c.parent })),
+      errors_logged: countErrors(transportRoot),
+    },
+  };
+}
+
+function handleListChannels(ctx: ApiContext): JsonRes {
+  return {
+    status: 200,
+    body: { channels: allChannelMeta(ctx.transportRoot) },
+  };
+}
+
+interface CreateChannelReq { name: string }
+
+function handleCreateChannel(ctx: ApiContext, body: CreateChannelReq): JsonRes {
+  const { transportRoot } = ctx;
+  if (typeof body.name !== 'string' || body.name.length === 0) {
+    throw new HttpError(400, 'channel name is required');
+  }
+  if (UUID_RE.test(body.name)) {
+    throw new HttpError(400, 'channel name cannot be UUID-shaped');
+  }
+  if (allChannelMeta(transportRoot).some((c) => c.name === body.name)) {
+    throw new HttpError(409, `a channel named '${body.name}' already exists`);
+  }
+  const uuid = randomUUID();
+  const dir = join(transportRoot, 'data', 'channels', uuid);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, 'CHANNEL.md'), serializeFrontmatter({ name: body.name }, ''));
+  const push = gitCommitAndPush(transportRoot, `channel(create): ${body.name} (${uuid.slice(0, 8)})`);
+  if (!push.ok && push.error) {
+    throw new HttpError(500, `commit failed: ${push.error.slice(0, 300)}`);
+  }
+  return { status: 201, body: { uuid, name: body.name } };
+}
+
+interface PostMessageReq {
+  type: 'primitive' | 'workflow';
+  channel?: string;        // name or UUID; defaults to single channel if exactly one exists
+  from?: string;
+  to?: string;             // required for primitive
+  as?: string;             // optional persona for primitive
+  fanout?: number;         // optional, default 1
+  body: string;
+  re?: string[];           // optional explicit re: targets
+}
+
+function resolveChannelUuid(transportRoot: string, handle?: string): string {
+  if (handle) {
+    const meta = resolveChannelHandle(transportRoot, handle);
+    if (!meta) throw new HttpError(404, `channel '${handle}' not found`);
+    return meta.uuid;
+  }
+  const all = discoverChannels(transportRoot);
+  if (all.length === 1) return all[0]!;
+  if (all.length === 0) {
+    throw new HttpError(409, "no channels exist — create one via POST /channels first");
+  }
+  throw new HttpError(409, 'multiple channels exist — specify "channel" in request body');
+}
+
+function writeMessageFile(opts: {
+  transportRoot: string;
+  channelUuid: string;
+  from: string;
+  to: string;
+  as?: string;
+  body: string;
+  re: string[];
+  extraFrontmatter?: Record<string, unknown>;
+  workflow?: boolean;
+}): string {
+  const ts = now();
+  const dir = join(opts.transportRoot, 'data', 'channels', opts.channelUuid, ts.pathDate);
+  mkdirSync(dir, { recursive: true });
+  const fm: Record<string, unknown> = {
+    from: opts.from,
+    to: opts.to,
+    timestamp: ts.iso,
+  };
+  if (opts.as) fm['as'] = opts.as;
+  if (opts.workflow) fm['type'] = 'workflow';
+  if (opts.re.length === 1) fm['re'] = opts.re[0];
+  else if (opts.re.length > 1) fm['re'] = opts.re;
+  if (opts.extraFrontmatter) Object.assign(fm, opts.extraFrontmatter);
+  const filename = messageFilename(ts);
+  writeFileSync(join(dir, filename), serializeFrontmatter(fm, opts.body));
+  return join(ts.pathDate, filename);
+}
+
+function handlePostMessage(ctx: ApiContext, body: PostMessageReq): JsonRes {
+  const { transportRoot } = ctx;
+  if (typeof body.body !== 'string') {
+    throw new HttpError(400, 'message body (`body` field) is required and must be a string');
+  }
+  const re = Array.isArray(body.re) ? body.re.filter((s) => typeof s === 'string') : [];
+  const from = typeof body.from === 'string' && body.from.length > 0 ? body.from : 'operator';
+
+  if (body.type === 'primitive') {
+    if (typeof body.to !== 'string' || body.to.length === 0) {
+      throw new HttpError(400, 'primitive requires `to` (model name)');
+    }
+    const channelUuid = resolveChannelUuid(transportRoot, body.channel);
+    const fanout = Math.max(1, Math.min(50, body.fanout ?? 1));
+    const relPaths: string[] = [];
+    for (let i = 0; i < fanout; i++) {
+      relPaths.push(
+        writeMessageFile({
+          transportRoot,
+          channelUuid,
+          from,
+          to: body.to,
+          as: body.as,
+          body: body.body,
+          re,
+        }),
+      );
+    }
+    const commitMsg =
+      fanout === 1
+        ? `run: ${from} -> ${body.to} in ${channelUuid.slice(0, 8)}`
+        : `run: ${from} -> ${body.to} x${fanout} in ${channelUuid.slice(0, 8)}`;
+    const push = gitCommitAndPush(transportRoot, commitMsg);
+    sendWakeSignal(transportRoot);
+    if (!push.ok && push.error) {
+      throw new HttpError(500, `commit failed: ${push.error.slice(0, 300)}`);
+    }
+    return { status: 201, body: { relPaths, channel: channelUuid, type: 'primitive' } };
+  }
+
+  if (body.type === 'workflow') {
+    const parsed = parseFrontmatter<Record<string, unknown>>(body.body);
+    if (parsed.data['type'] !== 'workflow') {
+      throw new HttpError(400, "workflow body must declare 'type: workflow' in frontmatter");
+    }
+    const parentUuid = resolveChannelUuid(transportRoot, body.channel);
+    const childUuid = randomUUID();
+    const childDir = join(transportRoot, 'data', 'channels', childUuid);
+    mkdirSync(childDir, { recursive: true });
+    writeFileSync(
+      join(childDir, 'CHANNEL.md'),
+      serializeFrontmatter({ name: `workflow-${Date.now()}`, parent: parentUuid }, ''),
+    );
+    const heartbeat = readHeartbeat(transportRoot);
+    const dispatchHost = heartbeat?.alias;
+    const extraFrontmatter: Record<string, unknown> = { child_channel: childUuid };
+    if (dispatchHost) extraFrontmatter['dispatch_host'] = dispatchHost;
+    const relPath = writeMessageFile({
+      transportRoot,
+      channelUuid: parentUuid,
+      from,
+      to: 'workflow',
+      body: parsed.body,
+      re,
+      extraFrontmatter,
+      workflow: true,
+    });
+    const push = gitCommitAndPush(
+      transportRoot,
+      `run(workflow): ${from} child ${childUuid.slice(0, 8)}`,
+    );
+    sendWakeSignal(transportRoot);
+    if (!push.ok && push.error) {
+      throw new HttpError(500, `commit failed: ${push.error.slice(0, 300)}`);
+    }
+    return {
+      status: 201,
+      body: { relPath, channel: parentUuid, childChannel: childUuid, type: 'workflow' },
+    };
+  }
+
+  throw new HttpError(400, "`type` must be 'primitive' or 'workflow'");
+}
+
+function handleReplies(ctx: ApiContext, relPathQuery: string): JsonRes {
+  const targets = relPathQuery.split(',').map((s) => s.trim()).filter(Boolean);
+  if (targets.length === 0) {
+    throw new HttpError(400, 'at least one relPath query argument required');
+  }
+  const found = new Map<string, { from: string; relPath: string; failed: boolean }>();
+  for (const channel of discoverChannels(ctx.transportRoot)) {
+    for (const msg of listChannelMessages(ctx.transportRoot, channel)) {
+      for (const entry of reList(msg.data['re'])) {
+        if (targets.includes(entry) && !found.has(entry)) {
+          found.set(entry, {
+            from: String(msg.data['from']),
+            relPath: msg.relPath,
+            failed: msg.data['failed'] === true,
+          });
+        }
+      }
+    }
+  }
+  const results = targets.map((t) => {
+    const reply = found.get(t);
+    if (!reply) return { target: t, status: 'PENDING' };
+    return {
+      target: t,
+      status: reply.failed ? 'FAILED' : 'REPLIED',
+      from: reply.from,
+      replyRelPath: reply.relPath,
+    };
+  });
+  return { status: 200, body: { replies: results } };
+}
+
+// ── router ───────────────────────────────────────────────────────────
+
+async function route(ctx: ApiContext, req: IncomingMessage): Promise<JsonRes> {
+  const { method = 'GET', url = '/' } = req;
+  const u = new URL(url, 'http://localhost');
+  const path = u.pathname;
+
+  if (method === 'GET' && path === '/healthz') return handleHealthz(ctx);
+  if (method === 'GET' && path === '/version') return handleVersion(ctx);
+  if (method === 'GET' && path === '/status') return handleStatus(ctx);
+  if (method === 'GET' && path === '/channels') return handleListChannels(ctx);
+  if (method === 'POST' && path === '/channels') {
+    const body = await readJson<CreateChannelReq>(req);
+    return handleCreateChannel(ctx, body);
+  }
+  if (method === 'POST' && path === '/messages') {
+    const body = await readJson<PostMessageReq>(req);
+    return handlePostMessage(ctx, body);
+  }
+  if (method === 'GET' && path === '/replies') {
+    const targets = u.searchParams.get('relPaths') ?? '';
+    return handleReplies(ctx, targets);
+  }
+  return { status: 404, body: { error: `no route for ${method} ${path}` } };
+}
+
+// ── server lifecycle ─────────────────────────────────────────────────
+
+export const DEFAULT_API_PORT = 7000;
+
+export interface StartApiOptions {
+  port?: number;
+  log: (event: string, fields?: Record<string, unknown>) => void;
+}
+
+export function startApi(ctx: ApiContext, opts: StartApiOptions): Server {
+  const port = opts.port ?? (Number(process.env['CROSSTALKD_API_PORT']) || DEFAULT_API_PORT);
+  // Bind address resolution:
+  //   - Native (default):  127.0.0.1 — engine running as a host process
+  //     should not expose the API to the network. Localhost only.
+  //   - Container:         0.0.0.0 (set via ENV CROSSTALKD_API_BIND in
+  //     the Dockerfile) — Docker's bridge networking means "127.0.0.1
+  //     inside the container" can't be reached via the daemon's port
+  //     forwarder. The actual security boundary moves up to compose's
+  //     port mapping (`127.0.0.1:HOST:CONTAINER`), which restricts
+  //     reachability to host loopback only.
+  //
+  // The engine bind address is therefore env-controlled, BUT operators
+  // never need to set it manually — the container image does it.
+  const bind = process.env['CROSSTALKD_API_BIND'] ?? '127.0.0.1';
+  const server = createServer(async (req, res) => {
+    try {
+      const result = await route(ctx, req);
+      writeJson(res, result.status, result.body);
+    } catch (err) {
+      if (err instanceof HttpError) {
+        writeJson(res, err.status, { error: err.message });
+      } else {
+        const msg = (err as Error).message;
+        opts.log('api_handler_crash', { error: msg.slice(0, 300) });
+        writeJson(res, 500, { error: `internal error: ${msg.slice(0, 200)}` });
+      }
+    }
+  });
+  server.listen(port, bind, () => {
+    opts.log('api_listening', { host: bind, port });
+  });
+  return server;
+}