@cordfuse/crosstalk 7.0.0-alpha.8 → 7.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GUIDE-CLI.md +322 -0
- package/GUIDE-PROMPTS.md +118 -0
- package/LICENSE +21 -0
- package/README.md +402 -61
- package/bin/crosstalk.js +88 -54
- package/commands/agent.js +69 -0
- package/commands/auth.js +273 -0
- package/commands/channel.js +54 -44
- package/commands/chat.js +107 -71
- package/commands/daemon.js +120 -0
- package/commands/logs.js +108 -19
- package/commands/message.js +125 -0
- package/commands/server.js +153 -0
- package/commands/settings.js +49 -0
- package/commands/status.js +37 -13
- package/commands/token.js +136 -0
- package/commands/transport.js +270 -0
- package/commands/version.js +3 -3
- package/commands/workflow.js +234 -0
- package/deploy/crosstalk@.service +62 -0
- package/deploy/install.sh +82 -0
- package/lib/api-client.js +77 -22
- package/lib/credentials.js +207 -0
- package/lib/nativeServer.js +173 -0
- package/lib/resolve.js +101 -34
- package/package.json +27 -4
- package/src/activation.ts +104 -0
- package/src/api.ts +1716 -0
- package/src/auth/enforce.ts +68 -0
- package/src/auth/handlers.ts +266 -0
- package/src/auth/middleware.ts +132 -0
- package/src/auth/setup.ts +263 -0
- package/src/auth/tokens.ts +285 -0
- package/src/auth/users.ts +267 -0
- package/src/dispatch.ts +492 -0
- package/src/dispatchers.ts +91 -0
- package/src/filenames.ts +28 -0
- package/src/frontmatter.ts +26 -0
- package/src/init.ts +116 -0
- package/src/invoke.ts +201 -0
- package/src/log-buffer.ts +67 -0
- package/src/models.ts +283 -0
- package/src/resolve.ts +100 -0
- package/src/state.ts +190 -0
- package/src/stop.ts +37 -0
- package/src/transport.ts +243 -0
- package/src/web/auth-pages.ts +160 -0
- package/src/web/channels.ts +395 -0
- package/src/web/chat-page.ts +636 -0
- package/src/web/chat-pty.ts +254 -0
- package/src/web/dashboard.ts +129 -0
- package/src/web/layout.ts +237 -0
- package/src/web/stubs.ts +510 -0
- package/src/web/workflows.ts +490 -0
- package/src/workflow.ts +470 -0
- package/template/CLAUDE.md +10 -0
- package/template/CROSSTALK-VERSION +1 -0
- package/template/CROSSTALK.md +262 -0
- package/template/PROTOCOL.md +70 -0
- package/template/README.md +64 -0
- package/template/auth/.gitkeep +0 -0
- package/template/auth/README.md +224 -0
- package/template/data/crosstalk.yaml +196 -0
- package/template/gitignore +4 -0
- package/commands/down.js +0 -40
- package/commands/init.js +0 -243
- package/commands/pull.js +0 -22
- package/commands/replies.js +0 -40
- package/commands/restart.js +0 -29
- package/commands/rm.js +0 -109
- package/commands/run.js +0 -122
- package/commands/up.js +0 -135
package/src/init.ts
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// crosstalk daemon init <directory> — scaffold a new transport.
|
|
2
|
+
//
|
|
3
|
+
// Copies the bundled flat template (CROSSTALK-VERSION, PROTOCOL.md,
|
|
4
|
+
// CROSSTALK.md, data/crosstalk.yaml, auth/README.md, local/actors/...)
|
|
5
|
+
// into the target directory and renames `gitignore` → `.gitignore`.
|
|
6
|
+
//
|
|
7
|
+
// v7 vs v6 init differences:
|
|
8
|
+
// - No host file is generated. v7 has no host files at all; machine
|
|
9
|
+
// identity comes from the --alias flag at dispatch boot.
|
|
10
|
+
// - No first channel is auto-created. Operator runs
|
|
11
|
+
// `crosstalk channel create <name>` themselves.
|
|
12
|
+
// - Template layout is flat: no upstream/ prefix on CROSSTALK-VERSION
|
|
13
|
+
// or PROTOCOL.md.
|
|
14
|
+
//
|
|
15
|
+
// Template lookup order:
|
|
16
|
+
// 1. <runtime_root>/template/ — bundled at publish time (production —
|
|
17
|
+
// package.json prepack copies transport/ → template/)
|
|
18
|
+
// 2. <runtime_root>/transport/ — monorepo layout (local dev: working
|
|
19
|
+
// from a checked-out clone where transport/ is the canonical source)
|
|
20
|
+
|
|
21
|
+
import { existsSync, mkdirSync, writeFileSync, cpSync, renameSync } from 'fs';
|
|
22
|
+
import { resolve, join, dirname } from 'path';
|
|
23
|
+
import { fileURLToPath } from 'url';
|
|
24
|
+
|
|
25
|
+
const argv = process.argv.slice(2);
|
|
26
|
+
const force = argv.includes('--force');
|
|
27
|
+
|
|
28
|
+
const positional = argv.filter((a) => !a.startsWith('--'));
|
|
29
|
+
if (positional.length === 0) {
|
|
30
|
+
console.error('Usage: crosstalk daemon init <directory> [--force]');
|
|
31
|
+
console.error(' crosstalk daemon init . (scaffold into current dir)');
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const targetDir = resolve(positional[0]!);
|
|
36
|
+
|
|
37
|
+
if (existsSync(join(targetDir, 'CROSSTALK-VERSION')) && !force) {
|
|
38
|
+
console.error(`crosstalk daemon init: ${targetDir} already contains a transport.`);
|
|
39
|
+
console.error('Pass --force to overwrite.');
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const runtimeRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
44
|
+
const candidates = [
|
|
45
|
+
join(runtimeRoot, 'template'),
|
|
46
|
+
join(runtimeRoot, 'transport'),
|
|
47
|
+
];
|
|
48
|
+
const templateDir = candidates.find((c) => existsSync(join(c, 'CROSSTALK-VERSION')));
|
|
49
|
+
|
|
50
|
+
if (!templateDir) {
|
|
51
|
+
console.error('crosstalk daemon init: cannot find the transport template.');
|
|
52
|
+
console.error('Looked in:');
|
|
53
|
+
for (const c of candidates) console.error(` ${c}`);
|
|
54
|
+
console.error('The @cordfuse/crosstalk installation may be corrupted; try reinstalling.');
|
|
55
|
+
process.exit(1);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
mkdirSync(targetDir, { recursive: true });
|
|
59
|
+
// Skip the template directory's own README — that's the meta-readme
|
|
60
|
+
// documenting the seed template itself, not content the operator wants
|
|
61
|
+
// inside their transport. (We write a fresh operator-facing README below.)
|
|
62
|
+
const templateReadme = join(templateDir, 'README.md');
|
|
63
|
+
cpSync(templateDir, targetDir, {
|
|
64
|
+
recursive: true,
|
|
65
|
+
force,
|
|
66
|
+
filter: (src) => src !== templateReadme,
|
|
67
|
+
});
|
|
68
|
+
// npm strips .gitignore from published packages; the template ships it as
|
|
69
|
+
// `gitignore` and we rename it here.
|
|
70
|
+
const gitignoreSrc = join(targetDir, 'gitignore');
|
|
71
|
+
const gitignoreDst = join(targetDir, '.gitignore');
|
|
72
|
+
if (existsSync(gitignoreSrc)) renameSync(gitignoreSrc, gitignoreDst);
|
|
73
|
+
|
|
74
|
+
const readmePath = join(targetDir, 'README.md');
|
|
75
|
+
if (!existsSync(readmePath) || force) {
|
|
76
|
+
writeFileSync(
|
|
77
|
+
readmePath,
|
|
78
|
+
`# Transport
|
|
79
|
+
|
|
80
|
+
A Crosstalk transport created by \`crosstalk transport init\`.
|
|
81
|
+
|
|
82
|
+
- Protocol spec: [\`CROSSTALK.md\`](./CROSSTALK.md)
|
|
83
|
+
- Agent orientation: [\`PROTOCOL.md\`](./PROTOCOL.md)
|
|
84
|
+
- Operator config: [\`data/crosstalk.yaml\`](./data/crosstalk.yaml) — providers + models
|
|
85
|
+
- Auth recipes: [\`auth/README.md\`](./auth/README.md) — per-agent env vars
|
|
86
|
+
- Actor persona files: [\`local/actors/\`](./local/actors/)
|
|
87
|
+
|
|
88
|
+
## First-run setup
|
|
89
|
+
|
|
90
|
+
1. Edit \`data/crosstalk.yaml\` — uncomment at least one provider + model block.
|
|
91
|
+
2. Drop your auth secrets into \`auth/<provider>.env\` per \`auth/README.md\`.
|
|
92
|
+
3. Then:
|
|
93
|
+
|
|
94
|
+
\`\`\`sh
|
|
95
|
+
crosstalk up # start the engine container
|
|
96
|
+
crosstalk channel general # create your first channel
|
|
97
|
+
crosstalk run --type primitive --to anthropic-personal/sonnet "hi" # send your first message
|
|
98
|
+
crosstalk status # check state
|
|
99
|
+
\`\`\`
|
|
100
|
+
|
|
101
|
+
Bare model names also work when unambiguous: \`--to sonnet\` resolves cleanly if only one provider has a \`sonnet\` model.
|
|
102
|
+
`,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
console.log('');
|
|
107
|
+
console.log(`Transport initialized at ${targetDir}`);
|
|
108
|
+
console.log('');
|
|
109
|
+
console.log('First-run setup:');
|
|
110
|
+
console.log(' 1. Edit data/crosstalk.yaml — uncomment at least one provider + model.');
|
|
111
|
+
console.log(' 2. Drop auth secrets into auth/<provider>.env per auth/README.md.');
|
|
112
|
+
console.log(' 3. Then (from a host with the `crosstalk` client installed):');
|
|
113
|
+
console.log(` cd ${targetDir}`);
|
|
114
|
+
console.log(' crosstalk up # start the engine container');
|
|
115
|
+
console.log(' crosstalk channel general # create your first channel');
|
|
116
|
+
console.log(' crosstalk run --type primitive --to anthropic-personal/sonnet "hi" # send your first message');
|
package/src/invoke.ts
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// invoke.ts — model invocation, persona loading, reply writing.
|
|
2
|
+
//
|
|
3
|
+
// Called from dispatch.ts when a message wakes a claimed model. Composes
|
|
4
|
+
// the system prompt (PROTOCOL.md + optional actor persona), invokes the
|
|
5
|
+
// model CLI from data/crosstalk.yaml, captures stdout, writes a reply
|
|
6
|
+
// message back into the same channel (success or failed:true).
|
|
7
|
+
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
9
|
+
import { join } from 'path';
|
|
10
|
+
import { spawn } from 'child_process';
|
|
11
|
+
import { now, messageFilename } from './filenames.js';
|
|
12
|
+
import { serializeFrontmatter } from './frontmatter.js';
|
|
13
|
+
import type { ModelEntry } from './models.js';
|
|
14
|
+
import type { ChannelMessage } from './transport.js';
|
|
15
|
+
|
|
16
|
+
// Parse a .env-style file: KEY=value per line, # comments ignored, blank
|
|
17
|
+
// lines ignored. No shell expansion, no escaping. First `=` splits the
|
|
18
|
+
// key from the value; trailing/leading whitespace trimmed on both.
|
|
19
|
+
export function readEnvFile(path: string): Record<string, string> {
|
|
20
|
+
if (!existsSync(path)) return {};
|
|
21
|
+
const out: Record<string, string> = {};
|
|
22
|
+
const raw = readFileSync(path, 'utf-8');
|
|
23
|
+
for (const line of raw.split('\n')) {
|
|
24
|
+
const trimmed = line.trim();
|
|
25
|
+
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
26
|
+
const eq = trimmed.indexOf('=');
|
|
27
|
+
if (eq < 1) continue; // no key, or starts with `=`
|
|
28
|
+
const key = trimmed.slice(0, eq).trim();
|
|
29
|
+
const value = trimmed.slice(eq + 1).trim();
|
|
30
|
+
if (!key) continue;
|
|
31
|
+
out[key] = value;
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface InvokeResult {
|
|
37
|
+
status: number;
|
|
38
|
+
stdout: string;
|
|
39
|
+
stderr: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// 10 minutes — orchestrator-persona workflows can take 3-5 minutes
|
|
43
|
+
// of model thinking before the first tool call. v6's 5-minute timeout
|
|
44
|
+
// was tight for any orchestrator pattern.
|
|
45
|
+
const CLI_TIMEOUT_MS = 10 * 60_000;
|
|
46
|
+
const ARGV_PROMPT_LIMIT = 64 * 1024;
|
|
47
|
+
|
|
48
|
+
export function loadProtocolPrompt(transportRoot: string): string {
|
|
49
|
+
const p = join(transportRoot, 'PROTOCOL.md');
|
|
50
|
+
return existsSync(p) ? readFileSync(p, 'utf-8').trim() : '';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function loadActorPersona(transportRoot: string, actorName: string | undefined): string {
|
|
54
|
+
if (!actorName) return '';
|
|
55
|
+
const p = join(transportRoot, 'local', 'actors', `${actorName}.md`);
|
|
56
|
+
if (!existsSync(p)) return '';
|
|
57
|
+
return readFileSync(p, 'utf-8').trim();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function composeSystemPrompt(parts: string[]): string {
|
|
61
|
+
return parts.filter((p) => p && p.length > 0).join('\n\n---\n\n');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function messageSender(msg: ChannelMessage): string {
|
|
65
|
+
return typeof msg.data['from'] === 'string' ? (msg.data['from'] as string) : 'unknown';
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Resolve a provider's inline `env:` block against the host process env.
|
|
69
|
+
// Each value is a `${VAR}` reference (validated at parse time in models.ts).
|
|
70
|
+
// Undefined host vars are dropped silently — agent CLI surfaces its own
|
|
71
|
+
// auth error and the failure path captures both streams (alpha.9 F1 fix).
|
|
72
|
+
// This matches env_file semantics where a missing file contributes nothing
|
|
73
|
+
// rather than crashing the spawn.
|
|
74
|
+
function resolveInlineEnv(
|
|
75
|
+
inlineEnv: Record<string, string> | null,
|
|
76
|
+
hostEnv: NodeJS.ProcessEnv,
|
|
77
|
+
): Record<string, string> {
|
|
78
|
+
if (inlineEnv == null) return {};
|
|
79
|
+
const out: Record<string, string> = {};
|
|
80
|
+
for (const [key, ref] of Object.entries(inlineEnv)) {
|
|
81
|
+
const varName = ref.slice(2, -1); // strip `${` and `}` — parse-time validated
|
|
82
|
+
const value = hostEnv[varName];
|
|
83
|
+
if (typeof value === 'string') out[key] = value;
|
|
84
|
+
}
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Pass the prompt as the CLI's last argv entry. Every modern agent CLI
|
|
89
|
+
// (Claude --print, codex exec, gemini -p, qwen --yolo, opencode -p,
|
|
90
|
+
// agy -p) reads its prompt from the trailing positional, so appending
|
|
91
|
+
// works universally. Fallback to stdin when the prompt would exceed a
|
|
92
|
+
// safe argv size — ARG_MAX is ~128 KB on Linux and ~256 KB on macOS.
|
|
93
|
+
//
|
|
94
|
+
// Env merge precedence at spawn (last-wins via spread order):
|
|
95
|
+
// 1. process.env — container baseline (PATH, IS_SANDBOX, etc.)
|
|
96
|
+
// 2. provider env_file — model's provider auth from a dotenv file
|
|
97
|
+
// 3. provider env — model's provider auth from inline ${VAR} refs
|
|
98
|
+
// (alpha.10 — overrides env_file per-key when both
|
|
99
|
+
// set, so an operator can bulk-load from a file
|
|
100
|
+
// and override one key inline)
|
|
101
|
+
// 4. dispatchEnv — per-spawn dispatch metadata (CROSSTALK_DISPATCH_*)
|
|
102
|
+
// Dispatch wins because per-spawn truth must override per-provider config;
|
|
103
|
+
// provider wins over baseline because auth varies per model.
|
|
104
|
+
export function invokeModelCli(
|
|
105
|
+
model: ModelEntry,
|
|
106
|
+
systemPrompt: string,
|
|
107
|
+
userMessage: string,
|
|
108
|
+
dispatchEnv: Record<string, string>,
|
|
109
|
+
): Promise<InvokeResult> {
|
|
110
|
+
return new Promise((res) => {
|
|
111
|
+
const fullPrompt = systemPrompt.length > 0
|
|
112
|
+
? `${systemPrompt}\n\n---\n\n${userMessage}`
|
|
113
|
+
: userMessage;
|
|
114
|
+
const useStdin = Buffer.byteLength(fullPrompt, 'utf-8') > ARGV_PROMPT_LIMIT;
|
|
115
|
+
const argv = useStdin ? [...model.args] : [...model.args, fullPrompt];
|
|
116
|
+
const envFileEnv = model.envFile ? readEnvFile(model.envFile) : {};
|
|
117
|
+
const inlineProviderEnv = resolveInlineEnv(model.inlineEnv, process.env);
|
|
118
|
+
// detached: new process group, so the timeout SIGKILL takes the model's
|
|
119
|
+
// children with it — orphans writing to the transport after a timeout
|
|
120
|
+
// was an observed v5/v6 hazard.
|
|
121
|
+
const child = spawn(model.cli, argv, {
|
|
122
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
123
|
+
detached: true,
|
|
124
|
+
env: { ...process.env, ...envFileEnv, ...inlineProviderEnv, ...dispatchEnv },
|
|
125
|
+
});
|
|
126
|
+
let stdout = '';
|
|
127
|
+
let stderr = '';
|
|
128
|
+
let resolved = false;
|
|
129
|
+
const timeout = setTimeout(() => {
|
|
130
|
+
if (resolved) return;
|
|
131
|
+
resolved = true;
|
|
132
|
+
try {
|
|
133
|
+
if (typeof child.pid === 'number') process.kill(-child.pid, 'SIGKILL');
|
|
134
|
+
else child.kill('SIGKILL');
|
|
135
|
+
} catch {
|
|
136
|
+
try { child.kill('SIGKILL'); } catch { /* already dead */ }
|
|
137
|
+
}
|
|
138
|
+
res({ status: 124, stdout, stderr: stderr + '\n[timeout]' });
|
|
139
|
+
}, CLI_TIMEOUT_MS);
|
|
140
|
+
child.stdout.on('data', (d) => { stdout += d.toString(); });
|
|
141
|
+
child.stderr.on('data', (d) => { stderr += d.toString(); });
|
|
142
|
+
child.on('close', (code) => {
|
|
143
|
+
if (resolved) return;
|
|
144
|
+
resolved = true;
|
|
145
|
+
clearTimeout(timeout);
|
|
146
|
+
res({ status: code ?? 1, stdout, stderr });
|
|
147
|
+
});
|
|
148
|
+
child.on('error', (err) => {
|
|
149
|
+
if (resolved) return;
|
|
150
|
+
resolved = true;
|
|
151
|
+
clearTimeout(timeout);
|
|
152
|
+
res({ status: 1, stdout, stderr: stderr + '\n' + err.message });
|
|
153
|
+
});
|
|
154
|
+
child.stdin.on('error', () => { /* child closed stdin */ });
|
|
155
|
+
if (useStdin) {
|
|
156
|
+
try { child.stdin.write(fullPrompt); } catch { /* same */ }
|
|
157
|
+
}
|
|
158
|
+
try { child.stdin.end(); } catch { /* ignore */ }
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function formatBatchedUserMessage(msgs: ChannelMessage[]): string {
|
|
163
|
+
if (msgs.length === 1) return msgs[0]!.body;
|
|
164
|
+
const parts = [`You have ${msgs.length} new messages in this channel. Process them collectively and reply once.`];
|
|
165
|
+
for (let i = 0; i < msgs.length; i++) {
|
|
166
|
+
const m = msgs[i]!;
|
|
167
|
+
const ts = typeof m.data['timestamp'] === 'string' ? `, ts: ${m.data['timestamp']}` : '';
|
|
168
|
+
parts.push(`--- Message ${i + 1} of ${msgs.length} (from: ${messageSender(m)}, ref: ${m.relPath}${ts}) ---`);
|
|
169
|
+
parts.push(m.body);
|
|
170
|
+
}
|
|
171
|
+
return parts.join('\n\n');
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export interface ReplyOpts {
|
|
175
|
+
transportRoot: string;
|
|
176
|
+
channelUuid: string;
|
|
177
|
+
fromModel: string; // e.g. "sonnet@cachy"
|
|
178
|
+
to: string; // the requester
|
|
179
|
+
re: string | string[];
|
|
180
|
+
body: string;
|
|
181
|
+
failed?: { error: string };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export function writeReply(opts: ReplyOpts): string {
|
|
185
|
+
const ts = now();
|
|
186
|
+
const dir = join(opts.transportRoot, 'data', 'channels', opts.channelUuid, ts.pathDate);
|
|
187
|
+
mkdirSync(dir, { recursive: true });
|
|
188
|
+
const fm: Record<string, unknown> = {
|
|
189
|
+
from: opts.fromModel,
|
|
190
|
+
to: opts.to,
|
|
191
|
+
timestamp: ts.iso,
|
|
192
|
+
re: opts.re,
|
|
193
|
+
};
|
|
194
|
+
if (opts.failed) {
|
|
195
|
+
fm['failed'] = true;
|
|
196
|
+
fm['error'] = opts.failed.error.slice(0, 2000);
|
|
197
|
+
}
|
|
198
|
+
const filename = messageFilename(ts);
|
|
199
|
+
writeFileSync(join(dir, filename), serializeFrontmatter(fm, opts.body));
|
|
200
|
+
return join(ts.pathDate, filename);
|
|
201
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// log-buffer.ts — in-memory ring buffer for engine log events.
|
|
2
|
+
//
|
|
3
|
+
// The engine's structured-JSON log lines (emitted via dispatch.ts's
|
|
4
|
+
// log()) are tee'd here in addition to stdout, so the /logs web page
|
|
5
|
+
// and /api/logs/stream SSE endpoint can serve a live tail without
|
|
6
|
+
// reading stdout from another process.
|
|
7
|
+
//
|
|
8
|
+
// Design choices:
|
|
9
|
+
// - Single global singleton; the engine is one process per transport.
|
|
10
|
+
// - Bounded ring buffer (LOG_BUFFER_CAPACITY entries) so memory
|
|
11
|
+
// stays predictable on long-running engines.
|
|
12
|
+
// - Subscribers receive every new entry (push, not pull). Browser
|
|
13
|
+
// SSE handlers register here; when the connection closes, they
|
|
14
|
+
// unsubscribe.
|
|
15
|
+
|
|
16
|
+
export interface LogEntry {
|
|
17
|
+
/** Monotonic id; lets clients dedupe + request "since this id." */
|
|
18
|
+
seq: number;
|
|
19
|
+
/** ISO timestamp. */
|
|
20
|
+
ts: string;
|
|
21
|
+
/** Short event name (e.g. 'dispatch', 'workflow_compile_start'). */
|
|
22
|
+
event: string;
|
|
23
|
+
/** Structured fields. Stringifies safely as JSON. */
|
|
24
|
+
fields: Record<string, unknown>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const LOG_BUFFER_CAPACITY = 500;
|
|
28
|
+
|
|
29
|
+
class LogBuffer {
|
|
30
|
+
private entries: LogEntry[] = [];
|
|
31
|
+
private nextSeq = 1;
|
|
32
|
+
private subscribers = new Set<(entry: LogEntry) => void>();
|
|
33
|
+
|
|
34
|
+
push(event: string, fields: Record<string, unknown>): LogEntry {
|
|
35
|
+
const entry: LogEntry = {
|
|
36
|
+
seq: this.nextSeq++,
|
|
37
|
+
ts: new Date().toISOString(),
|
|
38
|
+
event,
|
|
39
|
+
fields,
|
|
40
|
+
};
|
|
41
|
+
this.entries.push(entry);
|
|
42
|
+
if (this.entries.length > LOG_BUFFER_CAPACITY) {
|
|
43
|
+
this.entries.splice(0, this.entries.length - LOG_BUFFER_CAPACITY);
|
|
44
|
+
}
|
|
45
|
+
for (const sub of this.subscribers) {
|
|
46
|
+
try { sub(entry); } catch { /* subscriber threw; ignore */ }
|
|
47
|
+
}
|
|
48
|
+
return entry;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Return entries with seq > sinceSeq. Defaults to all buffered. */
|
|
52
|
+
recent(sinceSeq: number = 0): LogEntry[] {
|
|
53
|
+
if (sinceSeq <= 0) return [...this.entries];
|
|
54
|
+
return this.entries.filter((e) => e.seq > sinceSeq);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
subscribe(fn: (entry: LogEntry) => void): () => void {
|
|
58
|
+
this.subscribers.add(fn);
|
|
59
|
+
return () => { this.subscribers.delete(fn); };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
get currentSeq(): number {
|
|
63
|
+
return this.nextSeq - 1;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export const logBuffer = new LogBuffer();
|
package/src/models.ts
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
// models.ts — model registry parsing and PATH-based self-selection.
|
|
2
|
+
//
|
|
3
|
+
// Reads data/crosstalk.yaml from the transport root.
|
|
4
|
+
//
|
|
5
|
+
// providers:
|
|
6
|
+
// <provider-name>:
|
|
7
|
+
// env_file: <relpath> # optional — dotenv file under transport
|
|
8
|
+
// env: # optional — inline ${VAR} interpolation
|
|
9
|
+
// <KEY>: ${HOST_VAR}
|
|
10
|
+
// models:
|
|
11
|
+
// <model-name>: <cli command string>
|
|
12
|
+
//
|
|
13
|
+
// alpha.8 introduced env_file (per-provider auth files). alpha.10 adds
|
|
14
|
+
// the inline env: block — same per-provider scoping, but the provider→
|
|
15
|
+
// credential mapping is visible in the yaml. Inline values MUST be
|
|
16
|
+
// `${VAR}` references; raw secrets are refused at parse time so the yaml
|
|
17
|
+
// stays committable. Spawn-time precedence (last-wins) in invoke.ts:
|
|
18
|
+
// process.env → env_file → env (inline) → dispatch env
|
|
19
|
+
//
|
|
20
|
+
// ModelEntry is keyed by qualified form `provider/model` throughout the
|
|
21
|
+
// engine. resolve.ts handles bare-name auto-disambiguation at the call
|
|
22
|
+
// boundary; this module produces the canonical qualified-keyed registry
|
|
23
|
+
// plus a bareName → matches index that resolve.ts consumes.
|
|
24
|
+
|
|
25
|
+
import { existsSync, readFileSync, statSync } from 'fs';
|
|
26
|
+
import { join, delimiter, isAbsolute, resolve as pathResolve } from 'path';
|
|
27
|
+
import { parse as parseYaml } from 'yaml';
|
|
28
|
+
|
|
29
|
+
export interface ModelEntry {
|
|
30
|
+
qualified: string; // "google-personal/gemini-direct"
|
|
31
|
+
provider: string; // "google-personal"
|
|
32
|
+
name: string; // "gemini-direct"
|
|
33
|
+
envFile: string | null; // absolute path to provider's .env, or null
|
|
34
|
+
inlineEnv: Record<string, string> | null; // { KEY: "${HOST_VAR}" }, or null
|
|
35
|
+
command: string; // raw command-template string
|
|
36
|
+
cli: string; // first token — binary the dispatcher checks PATH for
|
|
37
|
+
args: string[]; // remaining tokens (the body is appended at invocation time)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ModelsRegistry {
|
|
41
|
+
all: Map<string, ModelEntry>; // keyed by qualified
|
|
42
|
+
byBareName: Map<string, ModelEntry[]>; // bare model name → all matches (for resolver)
|
|
43
|
+
claimed: Map<string, ModelEntry>; // qualified-keyed subset this dispatcher claims
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function crosstalkYamlPath(transportRoot: string): string {
|
|
47
|
+
return join(transportRoot, 'data', 'crosstalk.yaml');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface RawProvider {
|
|
51
|
+
env_file?: unknown;
|
|
52
|
+
env?: unknown;
|
|
53
|
+
models?: unknown;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function resolveEnvFilePath(transportRoot: string, envFile: string): string {
|
|
57
|
+
// Tolerate any path the operator writes (per V8-AUTH-DESIGN.md decision).
|
|
58
|
+
// Absolute path: trust as-is. Relative: resolve against transport root.
|
|
59
|
+
if (isAbsolute(envFile)) return envFile;
|
|
60
|
+
return pathResolve(transportRoot, envFile);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Inline env values must be `${VAR}` references — no raw secrets in the
|
|
64
|
+
// yaml. The regex matches a single VAR-form interpolation occupying the
|
|
65
|
+
// whole value. Mixed strings ("Bearer ${TOKEN}") are intentionally
|
|
66
|
+
// rejected for v7 simplicity; if the use case shows up, alpha.11+ can
|
|
67
|
+
// relax to full template-style interpolation.
|
|
68
|
+
const INLINE_ENV_REF_RE = /^\$\{([A-Z_][A-Z0-9_]*)\}$/;
|
|
69
|
+
|
|
70
|
+
function parseInlineEnv(
|
|
71
|
+
providerName: string,
|
|
72
|
+
raw: unknown,
|
|
73
|
+
): Record<string, string> | null {
|
|
74
|
+
if (raw == null) return null;
|
|
75
|
+
// F-LOG (alpha.14): inner throw messages no longer carry the
|
|
76
|
+
// `crosstalk:` prefix — the outer skip-provider log wrapper at the
|
|
77
|
+
// readModels() catch site is the single prefix-bearer. Mac caught
|
|
78
|
+
// the doubled prefix during the alpha.13 beta-readiness sweep.
|
|
79
|
+
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`provider '${providerName}' 'env:' must be a mapping ` +
|
|
82
|
+
`(got ${Array.isArray(raw) ? 'array' : typeof raw})`,
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
const out: Record<string, string> = {};
|
|
86
|
+
for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
|
|
87
|
+
if (typeof value !== 'string') {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`provider '${providerName}' env.${key} must be a string ` +
|
|
90
|
+
`(got ${typeof value})`,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
if (!INLINE_ENV_REF_RE.test(value)) {
|
|
94
|
+
throw new Error(
|
|
95
|
+
`provider '${providerName}' env.${key} = '${value}' is not a ` +
|
|
96
|
+
`\${VAR} reference. Inline env values must be of the form \${HOST_VAR} ` +
|
|
97
|
+
`so the yaml stays committable. Use env_file: <path> if you need raw values.`,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
out[key] = value;
|
|
101
|
+
}
|
|
102
|
+
return Object.keys(out).length > 0 ? out : null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function readModels(transportRoot: string): {
|
|
106
|
+
all: Map<string, ModelEntry>;
|
|
107
|
+
byBareName: Map<string, ModelEntry[]>;
|
|
108
|
+
} {
|
|
109
|
+
const path = crosstalkYamlPath(transportRoot);
|
|
110
|
+
if (!existsSync(path)) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
`crosstalk: data/crosstalk.yaml not found at ${path}. ` +
|
|
113
|
+
`Run from a transport root, or regenerate via 'crosstalk transport init'.`,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
const raw = readFileSync(path, 'utf8');
|
|
117
|
+
const parsed = parseYaml(raw);
|
|
118
|
+
if (parsed == null || typeof parsed !== 'object') {
|
|
119
|
+
throw new Error(`crosstalk: data/crosstalk.yaml parsed to non-object (got ${typeof parsed})`);
|
|
120
|
+
}
|
|
121
|
+
const root = parsed as Record<string, unknown>;
|
|
122
|
+
const providers = root['providers'];
|
|
123
|
+
if (providers === undefined) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
`crosstalk: data/crosstalk.yaml missing top-level 'providers:' key. ` +
|
|
126
|
+
`Schema is providers.<name>.{env_file?, models}. ` +
|
|
127
|
+
`See transport/auth/README.md for the expected shape.`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const all = new Map<string, ModelEntry>();
|
|
132
|
+
const byBareName = new Map<string, ModelEntry[]>();
|
|
133
|
+
|
|
134
|
+
// Empty providers: (operator hasn't uncommented any yet). Valid — engine
|
|
135
|
+
// boots with no claimed models; operators see this in `crosstalk status`.
|
|
136
|
+
if (providers === null) return { all, byBareName };
|
|
137
|
+
|
|
138
|
+
if (typeof providers !== 'object' || Array.isArray(providers)) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`crosstalk: data/crosstalk.yaml 'providers:' must be a mapping (got ${typeof providers})`,
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// N3 (alpha.11): per-provider try/catch so one typo doesn't tank the
|
|
145
|
+
// whole registry. Bad providers are warned to stderr and skipped; the
|
|
146
|
+
// rest of the registry loads. Operators still see the diagnostic via
|
|
147
|
+
// `crosstalk logs` (stderr → docker logs) and the dispatcher continues
|
|
148
|
+
// serving the agents that ARE valid. Before alpha.11, one
|
|
149
|
+
// raw-value-instead-of-${VAR} mistake on a single provider produced
|
|
150
|
+
// total_models:0 — every agent went dark.
|
|
151
|
+
const errors: string[] = [];
|
|
152
|
+
for (const [providerName, providerRaw] of Object.entries(providers as Record<string, unknown>)) {
|
|
153
|
+
try {
|
|
154
|
+
parseProvider(transportRoot, providerName, providerRaw, all, byBareName);
|
|
155
|
+
} catch (err) {
|
|
156
|
+
const msg = (err as Error).message;
|
|
157
|
+
errors.push(msg);
|
|
158
|
+
console.error(`crosstalk: skipping provider '${providerName}': ${msg}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return { all, byBareName };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function parseProvider(
|
|
166
|
+
transportRoot: string,
|
|
167
|
+
providerName: string,
|
|
168
|
+
providerRaw: unknown,
|
|
169
|
+
all: Map<string, ModelEntry>,
|
|
170
|
+
byBareName: Map<string, ModelEntry[]>,
|
|
171
|
+
): void {
|
|
172
|
+
if (providerRaw == null || typeof providerRaw !== 'object') {
|
|
173
|
+
throw new Error(
|
|
174
|
+
`data/crosstalk.yaml provider '${providerName}' is not a mapping`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
const p = providerRaw as RawProvider;
|
|
178
|
+
const envFile = (typeof p.env_file === 'string' && p.env_file.trim())
|
|
179
|
+
? resolveEnvFilePath(transportRoot, p.env_file.trim())
|
|
180
|
+
: null;
|
|
181
|
+
const inlineEnv = parseInlineEnv(providerName, p.env);
|
|
182
|
+
const models = p.models;
|
|
183
|
+
if (models == null || typeof models !== 'object') {
|
|
184
|
+
throw new Error(`provider '${providerName}' has no 'models:' map`);
|
|
185
|
+
}
|
|
186
|
+
for (const [modelName, cmdRaw] of Object.entries(models as Record<string, unknown>)) {
|
|
187
|
+
if (typeof cmdRaw !== 'string') {
|
|
188
|
+
throw new Error(
|
|
189
|
+
`provider '${providerName}' model '${modelName}' has non-string value ` +
|
|
190
|
+
`(expected a command-template string, got ${typeof cmdRaw})`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
const command = cmdRaw.trim();
|
|
194
|
+
const tokens = command.split(/\s+/);
|
|
195
|
+
if (tokens.length === 0 || tokens[0] === '') {
|
|
196
|
+
throw new Error(
|
|
197
|
+
`provider '${providerName}' model '${modelName}' has empty command`,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
const qualified = `${providerName}/${modelName}`;
|
|
201
|
+
if (all.has(qualified)) {
|
|
202
|
+
throw new Error(
|
|
203
|
+
`duplicate model '${qualified}' in data/crosstalk.yaml`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
const entry: ModelEntry = {
|
|
207
|
+
qualified,
|
|
208
|
+
provider: providerName,
|
|
209
|
+
name: modelName,
|
|
210
|
+
envFile,
|
|
211
|
+
inlineEnv,
|
|
212
|
+
command,
|
|
213
|
+
cli: tokens[0]!,
|
|
214
|
+
args: tokens.slice(1),
|
|
215
|
+
};
|
|
216
|
+
all.set(qualified, entry);
|
|
217
|
+
const existing = byBareName.get(modelName) ?? [];
|
|
218
|
+
existing.push(entry);
|
|
219
|
+
byBareName.set(modelName, existing);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Resolve a model's `cli` token to an absolute, executable file path — or
|
|
224
|
+
// null if it can't be resolved. Absolute paths are stat'd directly so
|
|
225
|
+
// operators can point `cli:` at a wrapper anywhere under the persistent
|
|
226
|
+
// /crosstalk-root mount; relative tokens are looked up across $PATH
|
|
227
|
+
// (which already includes /crosstalk-root/.local/bin, the documented
|
|
228
|
+
// drop-in dir for operator-installed CLIs and wrappers).
|
|
229
|
+
export function resolveCli(binary: string, env: NodeJS.ProcessEnv = process.env): string | null {
|
|
230
|
+
if (isAbsolute(binary)) {
|
|
231
|
+
try {
|
|
232
|
+
if (existsSync(binary) && statSync(binary).isFile()) return binary;
|
|
233
|
+
} catch {
|
|
234
|
+
// unreadable, fall through
|
|
235
|
+
}
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
const pathVar = env.PATH ?? '';
|
|
239
|
+
for (const dir of pathVar.split(delimiter)) {
|
|
240
|
+
if (!dir) continue;
|
|
241
|
+
const candidate = join(dir, binary);
|
|
242
|
+
try {
|
|
243
|
+
if (existsSync(candidate) && statSync(candidate).isFile()) return candidate;
|
|
244
|
+
} catch {
|
|
245
|
+
// unreadable entry, skip
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function isOnPath(binary: string, env: NodeJS.ProcessEnv = process.env): boolean {
|
|
252
|
+
return resolveCli(binary, env) !== null;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Walk every parsed model and decide which the local dispatcher can serve.
|
|
256
|
+
// Unclaimed models are logged to stderr with the specific reason —
|
|
257
|
+
// alpha.11 fixed the silent-drop for bad-provider parse errors; alpha.12
|
|
258
|
+
// extends the same diagnostic to the claim step so an operator with a
|
|
259
|
+
// wrapper at the wrong path (or a typo in the binary name) gets a clear
|
|
260
|
+
// line in `crosstalk logs` instead of a silently-missing model.
|
|
261
|
+
export function claimModels(
|
|
262
|
+
all: Map<string, ModelEntry>,
|
|
263
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
264
|
+
): Map<string, ModelEntry> {
|
|
265
|
+
const claimed = new Map<string, ModelEntry>();
|
|
266
|
+
for (const [qualified, entry] of all) {
|
|
267
|
+
if (isOnPath(entry.cli, env)) {
|
|
268
|
+
claimed.set(qualified, entry);
|
|
269
|
+
} else {
|
|
270
|
+
const reason = isAbsolute(entry.cli)
|
|
271
|
+
? `absolute path '${entry.cli}' is not an existing file`
|
|
272
|
+
: `binary '${entry.cli}' not found on PATH (${env.PATH ?? ''})`;
|
|
273
|
+
console.error(`crosstalk: skipping model '${qualified}': ${reason}`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return claimed;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
export function loadRegistry(transportRoot: string): ModelsRegistry {
|
|
280
|
+
const { all, byBareName } = readModels(transportRoot);
|
|
281
|
+
const claimed = claimModels(all);
|
|
282
|
+
return { all, byBareName, claimed };
|
|
283
|
+
}
|