@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/agents/round-trip.ts

@@ -0,0 +1,192 @@
+import { providerHttpFailure } from './provider-errors';
+import { randomUUID } from 'node:crypto';
+import { registry } from './registry';
+import { record } from '../config';
+
+/** Reasoning models spend output tokens thinking before any text or tool call,
+ * so a tight cap ends the response without the tool call or text being
+ * verified. Prompts are tiny; this is a ceiling, not an expected length.
+ */
+export const VERIFY_OUTPUT_TOKENS = 1024;
+
+/** Supported OpenAI Responses harness route. The only tool is a local echo;
+ * model output can never select a command, file, URL or other side effect.
+ */
+export async function verifyRoundTrip(
+  model: string,
+  key: string,
+  request = fetch
+): Promise<void> {
+  await verifyStream(model, key, request);
+  const nonce = randomUUID();
+  const tool = {
+    type: 'function',
+    name: 'fleet_echo',
+    description: 'Return the provided verification nonce.',
+    parameters: {
+      type: 'object',
+      properties: { nonce: { type: 'string', enum: [nonce] } },
+      required: ['nonce'],
+      additionalProperties: false,
+    },
+    strict: true,
+  };
+  const prompt = {
+    role: 'user',
+    content: `Call fleet_echo with nonce ${nonce}, then reply with its result.`,
+  };
+  const response = await call(
+    model,
+    key,
+    {
+      input: [prompt],
+      tools: [tool],
+      tool_choice: { type: 'function', name: 'fleet_echo' },
+      max_output_tokens: VERIFY_OUTPUT_TOKENS,
+    },
+    request
+  );
+  const payload: unknown = JSON.parse(await boundedText(response));
+  const invocation = validateInvocation(payload, nonce);
+  const final = await call(
+    model,
+    key,
+    {
+      input: [
+        prompt,
+        invocation,
+        {
+          type: 'function_call_output',
+          call_id: invocation.call_id,
+          output: nonce,
+        },
+      ],
+      tools: [tool],
+      tool_choice: 'none',
+      max_output_tokens: VERIFY_OUTPUT_TOKENS,
+    },
+    request
+  );
+  const completed: unknown = JSON.parse(await boundedText(final));
+  if (
+    !record(completed) ||
+    completed['status'] !== 'completed' ||
+    !Array.isArray(completed['output']) ||
+    !JSON.stringify(completed['output']).includes(nonce)
+  )
+    throw new Error('PROVIDER_TOOL_RESULT_UNVERIFIED');
+}
+async function call(
+  model: string,
+  key: string,
+  body: object,
+  request: typeof fetch
+): Promise<Response> {
+  const response = await request(registry.proxy.endpoint + '/v1/responses', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${key}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ model, ...body }),
+    redirect: 'error',
+    signal: AbortSignal.timeout(45_000),
+  });
+  if (!response.ok) throw providerHttpFailure(response.status);
+  return response;
+}
+async function boundedText(response: Response): Promise<string> {
+  if (!response.body) throw new Error('PROVIDER_EMPTY_RESPONSE');
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let size = 0;
+  let text = '';
+  try {
+    while (true) {
+      const chunk = await reader.read();
+      if (chunk.done) break;
+      size += chunk.value.byteLength;
+      if (size > 512 * 1024) throw new Error('PROVIDER_RESPONSE_LIMIT');
+      text += decoder.decode(chunk.value, { stream: true });
+    }
+    return text + decoder.decode();
+  } finally {
+    await reader.cancel();
+  }
+}
+function validateInvocation(
+  payload: unknown,
+  nonce: string
+): {
+  type: 'function_call';
+  call_id: string;
+  name: 'fleet_echo';
+  arguments: string;
+} {
+  if (!record(payload)) throw new Error('PROVIDER_TOOL_MISSING');
+  if (!Array.isArray(payload['output']))
+    throw new Error('PROVIDER_TOOL_MISSING');
+  const calls = payload['output'].filter(
+    (item) => record(item) && item['type'] === 'function_call'
+  );
+  const call = calls[0];
+  if (
+    calls.length !== 1 ||
+    !record(call) ||
+    call['name'] !== 'fleet_echo' ||
+    !stringFields(call, ['call_id', 'arguments'])
+  )
+    throw new Error('PROVIDER_TOOL_INVALID');
+  const args: unknown = JSON.parse(String(call['arguments']));
+  if (
+    !record(args) ||
+    Object.keys(args).length !== 1 ||
+    args['nonce'] !== nonce
+  )
+    throw new Error('PROVIDER_TOOL_ARGUMENTS_INVALID');
+  return {
+    type: 'function_call',
+    name: 'fleet_echo',
+    call_id: String(call['call_id']),
+    arguments: String(call['arguments']),
+  };
+}
+
+async function verifyStream(
+  model: string,
+  key: string,
+  request: typeof fetch
+): Promise<void> {
+  const response = await call(
+    model,
+    key,
+    {
+      input: 'Reply OK.',
+      stream: true,
+      max_output_tokens: VERIFY_OUTPUT_TOKENS,
+    },
+    request
+  );
+  const body = await boundedText(response);
+  const events: unknown[] = body
+    .split(/\r?\n/)
+    .filter((line) => line.startsWith('data:'))
+    .map((line) => line.slice(5).trim())
+    .filter((line) => line !== '[DONE]')
+    .map((line) => JSON.parse(line));
+  const types = events.filter(record).map((event) => event['type']);
+  if (
+    !types.includes('response.completed') ||
+    !types.includes('response.output_text.delta') ||
+    types.includes('response.failed') ||
+    types.includes('error')
+  )
+    throw new Error('PROVIDER_STREAM_INCOMPLETE');
+}
+
+function stringFields(
+  value: Record<string, unknown>,
+  fields: string[]
+): boolean {
+  return fields.every((field) => typeof value[field] === 'string');
+}

package/src/agents/secrets.ts

@@ -0,0 +1,13 @@
+import { AgentConfigurationError } from './errors';
+import { secret } from '../fleet-config';
+
+/** The key every node presents to the CLI proxy, from this fleet's secrets
+ * provider with this node's own access.
+ */
+export async function proxyKey(): Promise<string> {
+  const value = await secret('proxy-client-key');
+  if (value) return value;
+  throw new AgentConfigurationError(
+    "Required secret unavailable: proxy-client-key. Restore this node's access to the fleet's secrets provider (see `secrets` in the fleet configuration)."
+  );
+}

package/src/agents/toolchain.ts

@@ -0,0 +1,34 @@
+import { homedir } from 'node:os';
+import { delimiter, join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+/** mise shims resolve every agent to the version pinned in the shared lock.
+ * Putting them first keeps an unmanaged copy elsewhere on PATH (a vendor
+ * installer in ~/.local/bin, an nvm global) from shadowing the pinned one.
+ */
+export function pinnedPath(env: NodeJS.ProcessEnv = process.env): string {
+  const data =
+    env['MISE_DATA_DIR'] ??
+    join(env['XDG_DATA_HOME'] ?? join(homedir(), '.local', 'share'), 'mise');
+  const shims = join(data, 'shims');
+  const rest = (env['PATH'] ?? '')
+    .split(delimiter)
+    .filter((entry) => entry && entry !== shims);
+  return [shims, ...rest].join(delimiter);
+}
+
+/** The real pinned executable behind a mise shim. Needed when a launch
+ * changes HOME: a shim run under another HOME cannot find mise's trust state.
+ */
+export function pinnedBinary(
+  tool: string,
+  env: NodeJS.ProcessEnv = process.env
+): string {
+  const result = spawnSync('mise', ['which', tool], {
+    encoding: 'utf8',
+    env: { ...env, PATH: pinnedPath(env) },
+    timeout: 15_000,
+  });
+  const path = result.status === 0 ? result.stdout.trim() : '';
+  return path.startsWith('/') ? path : tool;
+}

package/src/agents/verify-cache.ts

@@ -0,0 +1,103 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { receiptDirectory } from '../platform/ansible-receipt';
+import type { Check } from '../types';
+
+/** A verified inference or agent session is re-proven at least this often. */
+export const VERIFY_MAX_AGE_MS = 6 * 60 * 60 * 1000;
+
+export interface VerifyCache {
+  directory?: string;
+  now?: number;
+  /** Ignore cached results (a live probe failed, or a full check was asked). */
+  bypass?: boolean;
+}
+
+/** Digest of everything a result depends on (never secrets). */
+export function cacheKey(...parts: (string | object)[]): string {
+  const hash = createHash('sha256');
+  for (const part of parts)
+    hash
+      .update(typeof part === 'string' ? part : JSON.stringify(part))
+      .update('\0');
+  return hash.digest('hex');
+}
+
+/** Digest of source files, so a code change re-verifies. */
+export async function sourceKey(paths: string[]): Promise<string> {
+  const hash = createHash('sha256');
+  for (const path of [...paths].sort())
+    hash
+      .update(path)
+      .update('\0')
+      .update(await readFile(path).catch(() => ''));
+  return hash.digest('hex');
+}
+
+/** Warm verification: a passing check with an unchanged key is reused until it
+ * ages out. Only clean passes are stored; a failure, a skip or a quota-limited
+ * pass (`reuse` false) clears the entry so the next sync checks it again.
+ */
+export async function cachedCheck(
+  name: string,
+  key: string,
+  compute: () => Promise<Check>,
+  options: VerifyCache & { reuse?: (check: Check) => boolean } = {}
+): Promise<Check> {
+  const directory = options.directory ?? join(receiptDirectory(), 'verify');
+  const path = join(directory, `${name.replace(/[^a-z0-9-]/gi, '_')}.json`);
+  const now = options.now ?? Date.now();
+  if (!options.bypass) {
+    const hit = await fresh(path, key, now);
+    if (hit) return hit;
+  }
+  const check = await compute();
+  const reusable =
+    check.status === 'pass' && (options.reuse ? options.reuse(check) : true);
+  try {
+    if (!reusable) await rm(path, { force: true });
+    else {
+      await mkdir(directory, { recursive: true, mode: 0o700 });
+      await writeFile(
+        path,
+        JSON.stringify({ version: 1, key, checkedAt: now, check }) + '\n',
+        { mode: 0o600 }
+      );
+    }
+  } catch {
+    /* A missing cache entry only means the next sync verifies again. */
+  }
+  return check;
+}
+
+async function fresh(
+  path: string,
+  key: string,
+  now: number
+): Promise<Check | null> {
+  try {
+    const entry = JSON.parse(await readFile(path, 'utf8')) as {
+      version?: unknown;
+      key?: unknown;
+      checkedAt?: unknown;
+      check?: Check;
+    };
+    if (
+      entry.version !== 1 ||
+      entry.key !== key ||
+      typeof entry.checkedAt !== 'number' ||
+      now - entry.checkedAt < 0 ||
+      now - entry.checkedAt >= VERIFY_MAX_AGE_MS ||
+      entry.check?.status !== 'pass'
+    )
+      return null;
+    const minutes = Math.round((now - entry.checkedAt) / 60_000);
+    return {
+      ...entry.check,
+      summary: `${entry.check.summary} (verified ${minutes}m ago; re-proven every 6h or on any change)`,
+    };
+  } catch {
+    return null;
+  }
+}

package/src/agents/verify.ts

@@ -0,0 +1,127 @@
+import {
+  ProviderVerificationError,
+  providerHttpFailure,
+} from './provider-errors';
+import { registry, type Selection } from './registry';
+import { proxyKey } from './secrets';
+import { VERIFY_OUTPUT_TOKENS } from './round-trip';
+
+export async function verifyProvider(
+  selection: Selection,
+  smoke: boolean,
+  credential?: string
+) {
+  if (selection.profile !== 'proxy')
+    return {
+      static: 'pass',
+      reachability: 'unverified',
+      authenticatedModels: 'unverified',
+      inference: 'unverified',
+      streaming: 'unverified',
+      toolCalls: 'unverified',
+      t3EndToEnd: 'unverified',
+    };
+  const key = credential ?? (await proxyKey());
+  const headers = {
+    Authorization: `Bearer ${key}`,
+    'Content-Type': 'application/json',
+  };
+  const response = await fetch(registry.proxy.endpoint + '/v1/models', {
+    headers,
+    redirect: 'error',
+    signal: AbortSignal.timeout(15_000),
+  });
+  if (!response.ok) throw providerHttpFailure(response.status);
+  const data = (await response.json()) as { data?: Array<{ id: string }> };
+  if (!data.data?.some((model) => model.id === selection.model))
+    throw new ProviderVerificationError(
+      'PROVIDER_MODEL_UNAVAILABLE',
+      'Approved model is unavailable; review registry model selection.'
+    );
+  let inference = 'unverified';
+  if (smoke) {
+    const { path, body } = inferenceRequest(selection);
+    const result = await fetch(registry.proxy.endpoint + path, {
+      method: 'POST',
+      headers: {
+        ...headers,
+        'anthropic-version': '2023-06-01',
+        'x-goog-api-key': key,
+      },
+      body: JSON.stringify(body),
+      redirect: 'error',
+      signal: AbortSignal.timeout(30_000),
+    });
+    if (!result.ok) throw providerHttpFailure(result.status);
+    await validateOutput(result, selection.tool);
+    inference = 'pass';
+  }
+  return {
+    static: 'pass',
+    reachability: 'pass',
+    authenticatedModels: 'pass',
+    inference,
+    streaming: 'unverified',
+    toolCalls: 'unverified',
+    t3EndToEnd: 'unverified',
+  };
+}
+
+async function validateOutput(
+  result: Response,
+  tool: Selection['tool']
+): Promise<void> {
+  const payload = (await result.json()) as {
+    content?: unknown[];
+    output?: unknown[];
+    error?: unknown;
+    candidates?: unknown[];
+  };
+  const output =
+    tool === 'gemini' || tool === 'antigravity'
+      ? payload.candidates
+      : tool === 'claude'
+        ? payload.content
+        : payload.output;
+  if (payload.error || !Array.isArray(output) || output.length === 0)
+    throw new ProviderVerificationError(
+      'PROVIDER_OUTPUT_UNVERIFIED',
+      'Inference returned no output; inspect the provider locally.'
+    );
+}
+
+/** Exact pinned clients use different wire protocols; model discovery is not inference.
+ * Output uses the same ceiling as the round trip: reasoning models (Muse) spend
+ * dozens of tokens before any text, so a tiny cap returns an empty output.
+ */
+export function inferenceRequest(selection: Selection): {
+  path: string;
+  body: object;
+} {
+  // Antigravity's CLI speaks the same Gemini generateContent protocol.
+  if (selection.tool === 'gemini' || selection.tool === 'antigravity')
+    return {
+      path: `/v1beta/models/${encodeURIComponent(selection.model)}:generateContent`,
+      body: {
+        contents: [{ role: 'user', parts: [{ text: 'Reply OK.' }] }],
+        generationConfig: { maxOutputTokens: VERIFY_OUTPUT_TOKENS },
+      },
+    };
+  if (selection.tool === 'claude')
+    return {
+      path: '/v1/messages',
+      body: {
+        model: selection.model,
+        max_tokens: VERIFY_OUTPUT_TOKENS,
+        messages: [{ role: 'user', content: 'Reply OK.' }],
+      },
+    };
+  return {
+    path: '/v1/responses',
+    body: {
+      model: selection.model,
+      max_output_tokens: VERIFY_OUTPUT_TOKENS,
+      input: 'Reply OK.',
+    },
+  };
+}

package/src/auth-checks.ts

@@ -0,0 +1,163 @@
+import { resolveVaultConfig, usesVault } from './vault-config';
+import { fleet as definition } from './fleet-config';
+import { workerReadiness } from './worker-readiness';
+import { record } from './config';
+import { codingAuth, serviceAuth } from './service-auth';
+import { workstationChecks } from './workstation-checks';
+import { toolchainChecks } from './toolchain-checks';
+import type { Capability, Check, Runner, CommandResult } from './types';
+
+export async function authCheck(
+  capability: Capability,
+  run: Runner,
+  root: string,
+  nodeId?: string
+): Promise<Check[]> {
+  if (capability === 'worker') return workerReadiness(run, nodeId, root);
+  if (capability === 't3') return t3Checks(run, nodeId !== undefined);
+  if (capability === 'codex' || capability === 'claude')
+    return codingAuth(capability, run);
+  if (capability === 'services') return serviceAuth(run);
+  if (capability === 'workstation') return workstationChecks(root, run);
+  if (capability === 'toolchain')
+    return toolchainChecks(
+      root,
+      run,
+      definition.workers.some((node) => node.id === nodeId)
+    );
+  return basicAuth(capability, run, root);
+}
+
+async function basicAuth(
+  capability: 'github' | 'vault' | 'vscode',
+  run: Runner,
+  root: string
+): Promise<Check[]> {
+  const spec = {
+    github: {
+      command: ['gh', 'auth', 'status', '--active'],
+      action:
+        'Authenticate on this node with gh auth login, then gh auth setup-git.',
+    },
+    vault: {
+      command: [
+        'vault',
+        'token',
+        'lookup',
+        '-format=json',
+        `-address=${usesVault() ? resolveVaultConfig().addr : ''}`,
+      ],
+      action:
+        'Authenticate to the configured Vault on this node. Do not copy a token from another computer.',
+    },
+    vscode: {
+      command: ['code', '--list-extensions'],
+      action:
+        'Install the VS Code CLI on PATH and ms-vscode-remote.remote-ssh in the local editor profile.',
+    },
+  }[capability];
+  const result = await run(spec.command, {
+    cwd: root,
+    timeoutMs: 15_000,
+  });
+  const extensions = result.stdout.toLowerCase().split(/\s+/);
+  const ok =
+    result.ok &&
+    (capability !== 'vscode' ||
+      extensions.includes('ms-vscode-remote.remote-ssh'));
+  return [
+    {
+      id: `auth:${capability}`,
+      status: ok ? 'pass' : 'fail',
+      summary: ok
+        ? `${capability} check succeeded in this process context.`
+        : `${capability} check failed (${result.reason}); no credential output retained.`,
+      ...(ok ? {} : { remedy: { action: spec.action } }),
+    },
+  ];
+}
+
+async function t3Checks(
+  run: Runner,
+  managerChecksLinks = false
+): Promise<Check[]> {
+  const t3 = await t3Executable(run);
+  const config = await run([t3, 'status', '--json']);
+  const checks: Check[] = [t3Settings(config)];
+  const live = await run([t3, 'environment', 'list', '--json']);
+  checks.push({
+    id: 't3:api',
+    status: live.ok ? 'pass' : 'fail',
+    summary: live.ok
+      ? 'Local T3 API answered a read-only environment listing; peer connectivity is not inferred.'
+      : 'Local T3 API did not answer the environment listing.',
+    ...(!live.ok
+      ? {
+          remedy: {
+            action:
+              'Check t3 service status and t3 status. Inspect linked peers in T3 Settings → Connections; relink only with operator approval.',
+          },
+        }
+      : {}),
+  });
+  if (!managerChecksLinks)
+    checks.push({
+      id: 't3:end-to-end',
+      status: 'skip',
+      summary:
+        'Relay traversal and remote peer authorization require an operator connection test; local settings are not proof.',
+    });
+  return checks;
+}
+
+/** Workers run only the T3 desktop app, so no `t3` launcher need be on PATH.
+ * Fall back to the newest pinned native CLI the shared installer verified.
+ */
+async function t3Executable(run: Runner): Promise<string> {
+  const found = await run(
+    [
+      'sh',
+      '-c',
+      'command -v t3 || ls -d "$HOME"/.local/share/personal/tools/t3-*/t3 2>/dev/null | sort -V | tail -n 1',
+    ],
+    { timeoutMs: 5000 }
+  );
+  const path = found.ok ? (found.stdout.trim().split('\n').pop() ?? '') : '';
+  return /^\/[\x21-\x7e]+$/.test(path) ? path : 't3';
+}
+
+function t3Settings(config: CommandResult): Check {
+  const fallback: Check = {
+    id: 't3:settings',
+    status: 'fail',
+    summary: 'T3 settings unavailable or unsupported CLI response.',
+    remedy: {
+      action:
+        'Run t3 status --json on this node. Complete t3 up --connect as its operator if needed.',
+    },
+  };
+  try {
+    const data: unknown = JSON.parse(config.stdout);
+    if (config.ok && record(data) && record(data['connect'])) {
+      const enabled =
+        data['connect']['enabled'] === true &&
+        data['connect']['revoked'] === false;
+      return {
+        id: 't3:settings',
+        status: enabled ? 'pass' : 'fail',
+        summary: enabled
+          ? 'T3 Connect enabled and not marked revoked (settings only).'
+          : 'T3 Connect disabled, revoked, or unknown.',
+        ...(!enabled
+          ? {
+              remedy: {
+                action:
+                  'Review t3 status, then complete t3 up --connect as the operator. Never share pairing tokens.',
+              },
+            }
+          : {}),
+      };
+    }
+  } catch {}
+  return fallback;
+}