@xingular/harness 0.0.0-stage → 0.1.1

package/src/mcp/manager.ts

@@ -0,0 +1,130 @@
+// Live MCP client sessions: one per (session, connection), opened on first use and closed when
+// the session's run ends or the connection changes.
+import { Client, SSEClientTransport, StreamableHTTPClientTransport, UnauthorizedError } from '@modelcontextprotocol/client'
+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio'
+import { randomUUID } from 'node:crypto'
+
+import { HarnessError } from '../core/errors'
+import { StoredOAuthProvider, type OAuthHost } from './oauth'
+import type { HttpConfig, LaunchConfig, StdioConfig } from './types'
+
+interface Live {
+  client: Client
+  transport: { close(): Promise<void> }
+  config: string
+}
+
+export const signInRequired = (config: Pick<LaunchConfig, 'name'>): HarnessError => new HarnessError('mcp_unauthorized', `${config.name} needs you to sign in again in Connections.`)
+export const isSignInRequired = (error: unknown): boolean => error instanceof UnauthorizedError || (error instanceof HarnessError && error.code === 'mcp_unauthorized')
+
+export class McpManager {
+  private readonly sessions = new Map<string, Promise<Live>>()
+
+  constructor(private readonly oauth: OAuthHost) {}
+
+  async close(connectionId?: string, sessionId?: string): Promise<void> {
+    const closing = [...this.sessions].filter(([key]) => (!connectionId || key.endsWith(`:${connectionId}`)) && (!sessionId || key.startsWith(`${sessionId}:`)))
+    for (const [key] of closing) this.sessions.delete(key)
+    await Promise.allSettled(
+      closing.map(async ([, session]) => {
+        const live = await session
+        await live.client.close()
+        await live.transport.close()
+      })
+    )
+  }
+
+  /** The client for a connection in one scope, reconnecting when the connection's configuration changed. */
+  async client(scope: string, config: LaunchConfig): Promise<Client> {
+    const key = `${scope}:${config.id}`
+    const fingerprint = JSON.stringify(config)
+    const existing = this.sessions.get(key)
+    if (existing) {
+      const live = await existing.catch(() => null)
+      if (live?.config === fingerprint) return live.client
+      await live?.client.close().catch(() => undefined)
+      this.sessions.delete(key)
+    }
+    const pending: Promise<Live> = (async () => {
+      const { client, transport } = config.transport === 'http' ? await this.connectHttp(config) : await this.connectStdio(config)
+      client.onclose = () => {
+        if (this.sessions.get(key) === pending) this.sessions.delete(key)
+      }
+      return { client, transport, config: fingerprint }
+    })()
+    this.sessions.set(key, pending)
+    try {
+      return (await pending).client
+    } catch (error) {
+      this.sessions.delete(key)
+      throw error
+    }
+  }
+
+  private async connectStdio(config: StdioConfig) {
+    const client = new Client({ name: 'agent-x', version: '0.1.0' })
+    const transport = new StdioClientTransport({ command: config.command, args: config.args, cwd: config.cwd ?? undefined, env: config.env, stderr: 'pipe' })
+    try {
+      await client.connect(transport, { timeout: 30000 })
+    } catch {
+      await transport.close()
+      throw new HarnessError('mcp_unavailable', `${config.name} could not start or initialize. Check its executable and arguments.`)
+    }
+    // Servers' diagnostics may contain secrets; do not persist or forward stderr.
+    transport.stderr?.on('data', () => undefined)
+    return { client, transport }
+  }
+
+  private async connectHttp(config: HttpConfig) {
+    const authProvider = config.auth === 'oauth' ? new StoredOAuthProvider(this.oauth, config, false) : undefined
+    // Without saved tokens the only way forward is the browser, which a session may not open.
+    if (authProvider && !(await authProvider.tokens())) throw signInRequired(config)
+    const options = { authProvider, requestInit: { headers: config.headers } }
+    // Servers that predate Streamable HTTP only answer on SSE.
+    for (const Transport of [StreamableHTTPClientTransport, SSEClientTransport]) {
+      const client = new Client({ name: 'agent-x', version: '0.1.0' })
+      const transport = new Transport(new URL(config.url), options)
+      try {
+        await client.connect(transport, { timeout: 30000 })
+        return { client, transport }
+      } catch (error) {
+        await transport.close().catch(() => undefined)
+        if (isSignInRequired(error)) throw signInRequired(config)
+      }
+    }
+    throw new HarnessError('mcp_unavailable', `${config.name} could not be reached. Check its address and credentials.`)
+  }
+
+  /** Runs `work` on a connection opened only for it. */
+  private async once<T>(prefix: string, config: LaunchConfig, work: (client: Client) => Promise<T>): Promise<T> {
+    const scope = `${prefix}-${randomUUID()}`
+    try {
+      return await work(await this.client(scope, config))
+    } finally {
+      const key = `${scope}:${config.id}`
+      const session = this.sessions.get(key)
+      this.sessions.delete(key)
+      if (session) await (await session).client.close().catch(() => undefined)
+    }
+  }
+
+  /** What a connection offers, read on a connection of its own. */
+  probe(config: LaunchConfig): Promise<{ serverName: string; tools: unknown[]; vaultId: string | null }> {
+    return this.once('probe', config, async (client) => {
+      const tools = (await client.listTools()).tools
+      let vaultId: string | null = null
+      if (tools.some((tool) => tool.name === 'x_workspace_status')) {
+        const status = await client.callTool({ name: 'x_workspace_status', arguments: {} })
+        const envelope = status.structuredContent as { ok?: boolean; vaultId?: string; error?: { message?: string } } | undefined
+        if (!envelope?.ok) throw new HarnessError('mcp_unavailable', envelope?.error?.message ?? 'Workspace vault is not bound.')
+        vaultId = envelope.vaultId ?? null
+      }
+      return { serverName: client.getServerVersion()?.name ?? config.name, tools, vaultId }
+    })
+  }
+
+  /** One read on a connection of its own, outside any session. */
+  readSource(config: LaunchConfig, tool: string, args: Record<string, unknown>): Promise<unknown> {
+    return this.once('source', config, (client) => client.callTool({ name: tool, arguments: args }, { timeout: 60000 }))
+  }
+}

package/src/mcp/oauth.ts

@@ -0,0 +1,195 @@
+// OAuth sign-in for a remote connection. State is kept with the connection's other secrets.
+// Only an interactive provider (a sign-in a person started) may open a browser; a session that
+// needs a new sign-in fails instead.
+import { auth, UnauthorizedError } from '@modelcontextprotocol/client'
+import type { OAuthClientInformationContext, OAuthClientMetadata, OAuthClientProvider, OAuthDiscoveryState, StoredOAuthClientInformation, StoredOAuthTokens } from '@modelcontextprotocol/client'
+import { randomUUID } from 'node:crypto'
+import { createServer } from 'node:http'
+
+import { HarnessError } from '../core/errors'
+import { MCP_OAUTH_PORT, MCP_OAUTH_REDIRECT_URL, type HttpConfig } from './types'
+
+interface Saved {
+  client?: StoredOAuthClientInformation
+  tokens?: StoredOAuthTokens
+  discovery?: OAuthDiscoveryState
+  staticClient?: StoredOAuthClientInformation
+  staticIssuer?: string
+}
+
+/** Where a connection's OAuth state is read and written, and how a sign-in page is shown. */
+export interface OAuthHost {
+  load(connectionId: string): Promise<Record<string, unknown>>
+  save(connectionId: string, state: Record<string, unknown>): Promise<void>
+  openUrl(url: string): Promise<void> | void
+  /** The name shown on the provider's consent page. */
+  clientName: string
+}
+
+export class StoredOAuthProvider implements OAuthClientProvider {
+  readonly expectedState = randomUUID()
+  private verifier: string | null = null
+
+  constructor(
+    private readonly host: OAuthHost,
+    private readonly config: Pick<HttpConfig, 'id' | 'scopes'>,
+    private readonly interactive: boolean
+  ) {}
+
+  get redirectUrl(): string {
+    return MCP_OAUTH_REDIRECT_URL
+  }
+
+  get clientMetadata(): OAuthClientMetadata {
+    return {
+      client_name: this.host.clientName,
+      redirect_uris: [MCP_OAUTH_REDIRECT_URL],
+      grant_types: ['authorization_code', 'refresh_token'],
+      response_types: ['code'],
+      token_endpoint_auth_method: 'none',
+      ...(this.config.scopes.length ? { scope: this.config.scopes.join(' ') } : {})
+    }
+  }
+
+  state(): string {
+    return this.expectedState
+  }
+
+  private load(): Promise<Saved> {
+    return this.host.load(this.config.id) as Promise<Saved>
+  }
+
+  private async change(edit: (saved: Saved) => Saved): Promise<void> {
+    await this.host.save(this.config.id, edit(await this.load()) as Record<string, unknown>)
+  }
+
+  async clientInformation(context?: OAuthClientInformationContext): Promise<StoredOAuthClientInformation | undefined> {
+    const saved = await this.load()
+    if (!saved.staticClient) return saved.client
+    // A client the user registered themselves (providers without dynamic registration) wins. It is
+    // tied to the authorization server seen at sign-in, so a session never offers it to another one.
+    if (this.interactive && context && saved.staticIssuer !== context.issuer) await this.change((current) => ({ ...current, staticIssuer: context.issuer }))
+    const issuer = this.interactive ? context?.issuer : saved.staticIssuer
+    return issuer ? { ...saved.staticClient, issuer } : saved.staticClient
+  }
+
+  async saveClientInformation(client: StoredOAuthClientInformation): Promise<void> {
+    await this.change((saved) => ({ ...saved, client }))
+  }
+
+  async tokens(): Promise<StoredOAuthTokens | undefined> {
+    return (await this.load()).tokens
+  }
+
+  async saveTokens(tokens: StoredOAuthTokens): Promise<void> {
+    await this.change((saved) => ({ ...saved, tokens }))
+  }
+
+  async discoveryState(): Promise<OAuthDiscoveryState | undefined> {
+    return (await this.load()).discovery
+  }
+
+  async saveDiscoveryState(discovery: OAuthDiscoveryState): Promise<void> {
+    await this.change((saved) => ({ ...saved, discovery }))
+  }
+
+  saveCodeVerifier(verifier: string): void {
+    this.verifier = verifier
+  }
+
+  codeVerifier(): string {
+    if (!this.verifier) throw new UnauthorizedError('Sign-in was not started from this app.')
+    return this.verifier
+  }
+
+  async invalidateCredentials(scope: 'all' | 'client' | 'tokens' | 'verifier' | 'discovery'): Promise<void> {
+    if (scope === 'verifier' || scope === 'all') this.verifier = null
+    if (scope === 'verifier') return
+    await this.change((saved) => (scope === 'all' ? {} : { ...saved, [scope]: undefined }))
+  }
+
+  async redirectToAuthorization(url: URL): Promise<void> {
+    if (!this.interactive) throw new UnauthorizedError('Sign in again in Connections.')
+    // Google only returns a refresh token when offline access is requested with a consent prompt.
+    if (url.hostname === 'accounts.google.com') {
+      url.searchParams.set('access_type', 'offline')
+      url.searchParams.set('prompt', 'consent')
+    }
+    await this.host.openUrl(url.toString())
+  }
+}
+
+const page = (title: string, message: string): string => `<!doctype html><meta charset="utf-8"><title>${title}</title><body style="font:16px system-ui;margin:20vh auto;max-width:28rem;text-align:center"><p>${message}</p></body>`
+
+// One sign-in owns the loopback port. Starting another ends the one still waiting, so a closed
+// browser tab never blocks a retry.
+let cancelActive: (() => Promise<void>) | null = null
+
+/** Waits on the fixed loopback address for the provider to send the browser back. */
+function listenForCallback(expectedState: string, clientName: string): Promise<{ result: Promise<{ code: string; iss?: string }>; close: () => Promise<void> }> {
+  return new Promise((ready, failed) => {
+    let settle: { resolve: (value: { code: string; iss?: string }) => void; reject: (error: Error) => void }
+    const result = new Promise<{ code: string; iss?: string }>((resolve, reject) => {
+      settle = { resolve, reject }
+    })
+    result.catch(() => undefined)
+    const server = createServer((request, response) => {
+      const url = new URL(request.url ?? '/', MCP_OAUTH_REDIRECT_URL)
+      const reply = (status: number, message: string): void => {
+        response.writeHead(status, { 'content-type': 'text/html; charset=utf-8' })
+        response.end(page(clientName, message))
+      }
+      if (url.pathname !== new URL(MCP_OAUTH_REDIRECT_URL).pathname) return reply(404, 'Not found.')
+      // Anything on this machine can reach the port; only the request carrying this sign-in's state counts.
+      if (url.searchParams.get('state') !== expectedState) return reply(400, `This sign-in link is not the one ${clientName} is waiting for.`)
+      const code = url.searchParams.get('code')
+      if (!code) {
+        reply(400, 'Sign-in was not completed. You can close this window.')
+        return settle.reject(new HarnessError('mcp_unauthorized', url.searchParams.get('error') === 'access_denied' ? 'Sign-in was cancelled.' : 'The provider did not complete the sign-in.'))
+      }
+      reply(200, `Signed in. You can close this window and return to ${clientName}.`)
+      settle.resolve({ code, iss: url.searchParams.get('iss') ?? undefined })
+    })
+    const timer = setTimeout(() => settle.reject(new HarnessError('mcp_unauthorized', 'Sign-in timed out. Try again.')), 300000)
+    server.once('error', (error: NodeJS.ErrnoException) => {
+      clearTimeout(timer)
+      failed(new HarnessError('invalid_request', error.code === 'EADDRINUSE' ? `Port ${MCP_OAUTH_PORT} is in use. Finish or cancel the other sign-in, or quit the program using that port, then try again.` : 'The sign-in listener could not start.'))
+    })
+    const closed = new Promise<void>((resolve) => server.once('close', resolve))
+    const close = (): Promise<void> => {
+      clearTimeout(timer)
+      if (cancelActive === cancel) cancelActive = null
+      server.close()
+      server.closeAllConnections()
+      return closed
+    }
+    const cancel = (): Promise<void> => {
+      settle.reject(new HarnessError('mcp_unauthorized', 'Sign-in was cancelled.'))
+      return close()
+    }
+    server.listen(MCP_OAUTH_PORT, '127.0.0.1', () => {
+      cancelActive = cancel
+      ready({ result, close })
+    })
+  })
+}
+
+/** Opens the provider's sign-in page and waits for the browser to return to the loopback address. */
+export async function signIn(host: OAuthHost, config: HttpConfig): Promise<void> {
+  const provider = new StoredOAuthProvider(host, config, true)
+  const scope = config.scopes.join(' ') || undefined
+  await cancelActive?.()
+  const callback = await listenForCallback(provider.expectedState, host.clientName)
+  try {
+    if ((await auth(provider, { serverUrl: config.url, scope, forceReauthorization: true })) === 'REDIRECT') {
+      const { code, iss } = await callback.result
+      if ((await auth(provider, { serverUrl: config.url, scope, authorizationCode: code, iss })) !== 'AUTHORIZED') throw new HarnessError('mcp_unauthorized', 'The provider did not complete the sign-in.')
+    }
+  } catch (error) {
+    if (error instanceof HarnessError) throw error
+    // Provider responses can quote credentials; report only that it failed.
+    throw new HarnessError('mcp_unauthorized', `${config.name} sign-in failed. Check the connection's address and OAuth client, then try again.`)
+  } finally {
+    await callback.close()
+  }
+}

package/src/mcp/scopeMigration.ts

@@ -0,0 +1,204 @@
+// Moves who can reach each connected app out of the device-local connection records and into
+// the data root's shared capability attachment store. The originals remain as owner-only backups.
+import { chmodSync, existsSync, readFileSync, unlinkSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { atomicWrite, readJson, writeJson } from '../core/fs'
+import * as attachments from '../plugins/attachments'
+import { connectionCapabilityId, connectionSchema, type Connection } from './types'
+
+interface MigrationRecord {
+  version: 1
+  status: 'attachments-written' | 'complete'
+  connectionCount: number
+  scopeAssignmentCount: number
+  connectionsBackup: string | null
+  attachmentsBackup: string | null
+  hadAttachments: boolean
+}
+
+const recordPath = (root: string): string => join(root, 'config/capability-scope-migration.json')
+const attachmentsPath = (root: string): string => join(root, 'config/plugin-attachments.json')
+const connectionPath = (stateDir: string): string => join(stateDir, 'connections.json')
+const connectionsBackupPath = (stateDir: string): string => join(stateDir, 'connections.pre-capability-scope.json')
+const attachmentsBackupPath = (root: string): string => join(root, 'config/plugin-attachments.pre-capability-scope.json')
+
+/** Scope writes are safe only after the legacy records have been copied and stripped. */
+export function connectionScopeMigrationComplete(root: string): boolean {
+  try {
+    const marker = readJson<MigrationRecord>(recordPath(root))
+    return marker?.version === 1 && marker.status === 'complete'
+  } catch {
+    return false
+  }
+}
+
+interface ScopeRows {
+  global: boolean
+  agents: Map<string, boolean>
+  projects: Map<string, boolean>
+}
+
+function unique(values: readonly string[]): string[] {
+  return [...new Set(values)]
+}
+
+function scopeRows(connection: Connection): ScopeRows {
+  const agents = new Map<string, boolean>()
+  const projects = new Map<string, boolean>()
+  for (const id of unique([...connection.agentIds, ...connection.excludedAgentIds])) {
+    agents.set(id, !connection.excludedAgentIds.includes(id))
+  }
+  for (const id of unique([...connection.projectIds, ...connection.excludedProjectIds])) {
+    projects.set(id, !connection.excludedProjectIds.includes(id))
+  }
+  return { global: connection.global, agents, projects }
+}
+
+function assignmentCount(rows: ScopeRows): number {
+  return 1 + rows.agents.size + rows.projects.size
+}
+
+function removeId(choices: attachments.Attachments, id: string): void {
+  attachments.forgetId(choices, id)
+}
+
+function put(map: { on: string[]; off: string[] }, id: string, value: boolean): void {
+  map.on = map.on.filter((item) => item !== id)
+  map.off = map.off.filter((item) => item !== id)
+  map[value ? 'on' : 'off'].push(id)
+}
+
+function writeRows(choices: attachments.Attachments, id: string, rows: ScopeRows): void {
+  removeId(choices, id)
+  put(choices.global, id, rows.global)
+  for (const [scopeId, on] of rows.agents) {
+    const target = (choices.agents[scopeId] ??= { on: [], off: [] })
+    put(target, id, on)
+  }
+  for (const [scopeId, on] of rows.projects) {
+    const target = (choices.projects[scopeId] ??= { on: [], off: [] })
+    put(target, id, on)
+  }
+}
+
+function readValue(item: { on: string[]; off: string[] } | undefined, id: string): boolean | null {
+  if (item?.off.includes(id)) return false
+  if (item?.on.includes(id)) return true
+  return null
+}
+
+function verifyRows(choices: attachments.Attachments, id: string, expected: ScopeRows): boolean {
+  if (readValue(choices.global, id) !== expected.global) return false
+  for (const [scopeId, on] of expected.agents) {
+    if (readValue(choices.agents[scopeId], id) !== on) return false
+  }
+  for (const [scopeId, on] of expected.projects) {
+    if (readValue(choices.projects[scopeId], id) !== on) return false
+  }
+  return true
+}
+
+function backup(source: string, target: string): boolean {
+  if (!existsSync(source)) return false
+  if (!existsSync(target)) atomicWrite(target, readFileSync(source))
+  chmodSync(target, 0o600)
+  return true
+}
+
+function setPrivate(path: string): void {
+  try { chmodSync(path, 0o600) } catch { /* The atomic write remains usable if chmod is unavailable. */ }
+}
+
+function stripScopes(stateDir: string): void {
+  const path = connectionPath(stateDir)
+  if (!existsSync(path)) return
+  const stored = readJson<unknown>(path)
+  if (!Array.isArray(stored)) throw new Error('Stored connector data must be a list before its scopes can be migrated.')
+  const records = stored.map((item) => {
+    if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('A stored connector record is invalid; no scope data was removed.')
+    const { global: _global, agentIds: _agentIds, projectIds: _projectIds, excludedAgentIds: _excludedAgentIds, excludedProjectIds: _excludedProjectIds, ...connection } = item as Record<string, unknown>
+    return connection
+  })
+  writeJson(path, records)
+  setPrivate(path)
+}
+
+/**
+ * Runs once when a data root opens. Both source files are backed up before either is changed;
+ * a stage marker makes an interrupted migration safe to finish on the next open.
+ */
+export function migrateConnectionScopes(root: string, stateDir: string): MigrationRecord {
+  const marker = readJson<MigrationRecord>(recordPath(root))
+  if (marker?.version === 1 && marker.status === 'complete') return marker
+
+  if (marker?.version === 1 && marker.status === 'attachments-written') {
+    stripScopes(stateDir)
+    const complete = { ...marker, status: 'complete' as const }
+    writeJson(recordPath(root), complete)
+    return complete
+  }
+
+  const connectionsFile = connectionPath(stateDir)
+  const hadConnections = existsSync(connectionsFile)
+  const raw = hadConnections ? readJson<unknown>(connectionsFile) : []
+  if (!Array.isArray(raw)) throw new Error('Stored connector data must be a list before its scopes can be migrated.')
+  const parsed = raw.map((item) => {
+    const result = connectionSchema.safeParse(item)
+    if (!result.success) throw new Error('A stored connector record is invalid; no scope data was removed.')
+    return result.data
+  })
+  const connectionBackup = hadConnections ? connectionsBackupPath(stateDir) : null
+  const hadAttachments = existsSync(attachmentsPath(root))
+  const attachmentBackup = hadAttachments ? attachmentsBackupPath(root) : null
+  const migration: MigrationRecord = {
+    version: 1,
+    status: 'attachments-written',
+    connectionCount: parsed.length,
+    scopeAssignmentCount: parsed.reduce((count, connection) => count + assignmentCount(scopeRows(connection)), 0),
+    connectionsBackup: connectionBackup,
+    attachmentsBackup: attachmentBackup,
+    hadAttachments
+  }
+  backup(connectionsFile, connectionBackup ?? '')
+  backup(attachmentsPath(root), attachmentBackup ?? '')
+
+  try {
+    const choices = attachments.load(root)
+    const expected = new Map<string, ScopeRows>()
+    for (const connection of parsed) {
+      const id = connectionCapabilityId(connection.id)
+      const rows = scopeRows(connection)
+      expected.set(id, rows)
+      writeRows(choices, id, rows)
+    }
+    const migratedAssignments = [...expected.values()].reduce((count, rows) => count + assignmentCount(rows), 0)
+    if (migratedAssignments !== migration.scopeAssignmentCount || [...expected].some(([id, rows]) => !verifyRows(choices, id, rows))) {
+      throw new Error('Connector scope count check failed; the original files were restored.')
+    }
+    attachments.save(root, choices)
+    const saved = attachments.load(root)
+    if ([...expected].some(([id, rows]) => !verifyRows(saved, id, rows))) throw new Error('Connector scope count check failed after saving; the original files were restored.')
+    writeJson(recordPath(root), migration)
+    stripScopes(stateDir)
+    const complete = { ...migration, status: 'complete' as const }
+    writeJson(recordPath(root), complete)
+    return complete
+  } catch (error) {
+    if (attachmentBackup && existsSync(attachmentBackup)) atomicWrite(attachmentsPath(root), readFileSync(attachmentBackup))
+    else if (!hadAttachments && existsSync(attachmentsPath(root))) unlinkSync(attachmentsPath(root))
+    if (connectionBackup && existsSync(connectionBackup)) atomicWrite(connectionsFile, readFileSync(connectionBackup))
+    try { unlinkSync(recordPath(root)) } catch { /* No marker is also a valid retry state. */ }
+    throw error
+  }
+}
+
+/** Restores the byte-identical pre-migration files from the backups if an operator rolls back. */
+export function rollbackConnectionScopeMigration(root: string, stateDir: string): void {
+  const marker = readJson<MigrationRecord>(recordPath(root))
+  if (!marker || marker.version !== 1) return
+  if (marker.attachmentsBackup && existsSync(marker.attachmentsBackup)) atomicWrite(attachmentsPath(root), readFileSync(marker.attachmentsBackup))
+  else if (!marker.hadAttachments && existsSync(attachmentsPath(root))) unlinkSync(attachmentsPath(root))
+  if (marker.connectionsBackup && existsSync(marker.connectionsBackup)) atomicWrite(connectionPath(stateDir), readFileSync(marker.connectionsBackup))
+  unlinkSync(recordPath(root))
+}