nebula-ai-core 0.1.0

package/src/locks.ts

@@ -0,0 +1,233 @@
+// Process-scoped advisory locks via PID-file pattern.
+//
+// Used by long-running listeners (telegram bot poller, etc.) to ensure only
+// one process per scope+identity holds the resource. Mirrors hermes's
+// acquire_scoped_lock from gateway/status.py.
+//
+// Lock file lives at ~/.nebula/locks/<scope>-<sha256(identity).slice(0,16)>.lock
+// O_CREAT|O_EXCL atomic create. Stale-detection via process.kill(pid, 0).
+// TTL eviction is a belt-and-suspenders fallback against crashed holders that
+// the kernel hasn't reaped yet (rare but real on macOS).
+
+import { createHash } from 'node:crypto'
+import { closeSync, mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+export interface AcquireScopedLockOpts {
+  scope: string
+  identity: string
+  ttl?: number
+  rootDir?: string
+}
+
+export interface ScopedLockHandle {
+  releaseFn: () => void
+  refreshFn: () => boolean
+}
+
+export interface AcquireScopedLockResult {
+  acquired: boolean
+  handle?: ScopedLockHandle
+  existing?: { pid: number; startedAt: number; updatedAt: number }
+}
+
+export const DEFAULT_LOCK_TTL_SECONDS = 300
+
+interface LockRecord {
+  pid: number
+  scope: string
+  identityHash: string
+  startedAt: number
+  updatedAt: number
+  ttl: number
+}
+
+function lockDir(rootDir?: string): string {
+  return rootDir ?? join(homedir(), '.nebula', 'locks')
+}
+
+function lockPath(scope: string, identity: string, rootDir?: string): string {
+  const hash = createHash('sha256').update(identity).digest('hex').slice(0, 16)
+  return join(lockDir(rootDir), `${scope}-${hash}.lock`)
+}
+
+function readLock(path: string): LockRecord | null {
+  try {
+    return JSON.parse(readFileSync(path, 'utf8')) as LockRecord
+  } catch {
+    return null
+  }
+}
+
+// process.kill(pid, 0) succeeds against zombie (defunct) processes on Linux
+// because the kernel keeps the PID slot until the parent reaps it. A zombie
+// can never refresh its lock or service work, so we treat it as gone. See
+// feedback-tg-token-lock-zombie-after-upgrade.md. Exported for tests; not
+// public API.
+export function isZombieLinux(pid: number): boolean {
+  try {
+    const status = readFileSync(`/proc/${pid}/status`, 'utf8')
+    const m = status.match(/^State:\s+(\S)/m)
+    return m?.[1] === 'Z'
+  } catch {
+    return false
+  }
+}
+
+type StaleReason = 'live' | 'pid-dead' | 'zombie' | 'ttl'
+
+// Single source of truth for "is this lock record dead, and if so, why?"
+// `attemptOnce` only cares whether it can reclaim; `clearStaleScopedLock`
+// reports the reason back to operators. Both go through this so the policy
+// (TTL > kill(0) ESRCH > linux zombie) stays in one place.
+function classifyStale(record: LockRecord, now: number): StaleReason {
+  if (now - record.updatedAt > record.ttl) return 'ttl'
+  if (record.pid === process.pid) return 'live'
+  try {
+    process.kill(record.pid, 0)
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException).code
+    if (code === 'EPERM') return 'live'
+    return 'pid-dead'
+  }
+  if (process.platform === 'linux' && isZombieLinux(record.pid)) return 'zombie'
+  return 'live'
+}
+
+function isStale(record: LockRecord, now: number): boolean {
+  return classifyStale(record, now) !== 'live'
+}
+
+function attemptOnce(
+  path: string,
+  scope: string,
+  identityHash: string,
+  ttl: number,
+): AcquireScopedLockResult {
+  let fd: number
+  try {
+    fd = openSync(path, 'wx')
+  } catch {
+    const existing = readLock(path)
+    const now = Math.floor(Date.now() / 1000)
+    if (!existing || isStale(existing, now)) {
+      try {
+        unlinkSync(path)
+      } catch {
+        /* race with another reclaimer */
+      }
+      return { acquired: false }
+    }
+    return {
+      acquired: false,
+      existing: {
+        pid: existing.pid,
+        startedAt: existing.startedAt,
+        updatedAt: existing.updatedAt,
+      },
+    }
+  }
+
+  const now = Math.floor(Date.now() / 1000)
+  const record: LockRecord = {
+    pid: process.pid,
+    scope,
+    identityHash,
+    startedAt: now,
+    updatedAt: now,
+    ttl,
+  }
+  try {
+    writeSync(fd, JSON.stringify(record))
+  } finally {
+    closeSync(fd)
+  }
+
+  let released = false
+  const releaseFn = (): void => {
+    if (released) return
+    released = true
+    try {
+      const current = readLock(path)
+      if (current?.pid === process.pid) unlinkSync(path)
+    } catch {
+      /* best-effort */
+    }
+  }
+  const refreshFn = (): boolean => {
+    if (released) return false
+    try {
+      const current = readLock(path)
+      if (current?.pid !== process.pid) return false
+      const next: LockRecord = { ...current, updatedAt: Math.floor(Date.now() / 1000) }
+      const fd2 = openSync(path, 'w')
+      try {
+        writeSync(fd2, JSON.stringify(next))
+      } finally {
+        closeSync(fd2)
+      }
+      return true
+    } catch {
+      return false
+    }
+  }
+  return { acquired: true, handle: { releaseFn, refreshFn } }
+}
+
+export function acquireScopedLock(opts: AcquireScopedLockOpts): AcquireScopedLockResult {
+  const ttl = opts.ttl ?? DEFAULT_LOCK_TTL_SECONDS
+  const path = lockPath(opts.scope, opts.identity, opts.rootDir)
+  mkdirSync(lockDir(opts.rootDir), { recursive: true })
+  const identityHash = createHash('sha256').update(opts.identity).digest('hex').slice(0, 16)
+
+  for (let i = 0; i < 3; i++) {
+    const result = attemptOnce(path, opts.scope, identityHash, ttl)
+    if (result.acquired) return result
+    if (result.existing) return result
+  }
+  return { acquired: false }
+}
+
+export type ClearStaleScopedLockReason =
+  | 'no-lock'
+  | 'alive-pid'
+  | 'cleared-stale'
+  | 'cleared-zombie'
+  | 'cleared-ttl'
+  | 'cleared-unreadable'
+
+export interface ClearStaleScopedLockResult {
+  cleared: boolean
+  reason: ClearStaleScopedLockReason
+}
+
+/**
+ * Inspect the lock file at `(scope, identity)` and remove it iff it's stale
+ * (PID dead, zombie on Linux, or TTL expired). Never deletes a lock held by a
+ * live foreign PID — caller must wait for it.
+ *
+ * Used at gateway boot to proactively reap zombie/crashed listener locks so
+ * the new TG listener can acquire its bot-token slot without the 30s/6min
+ * retry waltz from `recovery.ts:scheduleStartRetry`.
+ */
+export function clearStaleScopedLock(opts: AcquireScopedLockOpts): ClearStaleScopedLockResult {
+  const path = lockPath(opts.scope, opts.identity, opts.rootDir)
+  const existing = readLock(path)
+  if (!existing) return { cleared: false, reason: 'no-lock' }
+  const reason = classifyStale(existing, Math.floor(Date.now() / 1000))
+  if (reason === 'live') return { cleared: false, reason: 'alive-pid' }
+  try {
+    unlinkSync(path)
+  } catch {
+    return { cleared: false, reason: 'cleared-unreadable' }
+  }
+  switch (reason) {
+    case 'zombie':
+      return { cleared: true, reason: 'cleared-zombie' }
+    case 'pid-dead':
+      return { cleared: true, reason: 'cleared-stale' }
+    case 'ttl':
+      return { cleared: true, reason: 'cleared-ttl' }
+  }
+}

package/src/mcp/discovery.ts

@@ -0,0 +1,150 @@
+import type { Dirent } from 'node:fs'
+import { readFile, readdir, stat } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+import type { McpDiscoveryResult, McpServerConfig } from './types'
+
+interface RawMcpFile {
+  mcpServers?: Record<string, RawMcpServer>
+}
+
+interface RawMcpServer {
+  command?: string
+  args?: string[]
+  env?: Record<string, string>
+  type?: 'stdio' | 'http'
+  url?: string
+  headers?: Record<string, string>
+}
+
+export interface McpDiscoveryOptions {
+  /** Whether to scan ~/.claude/.mcp.json + ~/.claude/plugins/cache/. Default true. */
+  importsClaudeCode?: boolean
+  /** Override for ~/.claude/.mcp.json. */
+  claudeMcpPath?: string
+  /** Override for ~/.claude/plugins/cache/. */
+  claudePluginsCacheRoot?: string
+  /** Override for ~/.nebula/.mcp.json (nebula-native MCP servers). */
+  nebulaMcpPath?: string
+}
+
+export async function discoverMcpServers(
+  opts: McpDiscoveryOptions = {},
+): Promise<McpDiscoveryResult> {
+  const importsClaudeCode = opts.importsClaudeCode ?? true
+  const nebulaMcpPath = opts.nebulaMcpPath ?? join(homedir(), '.nebula', '.mcp.json')
+  const claudeMcpPath = opts.claudeMcpPath ?? join(homedir(), '.claude', '.mcp.json')
+  const claudePluginsCacheRoot =
+    opts.claudePluginsCacheRoot ?? join(homedir(), '.claude', 'plugins', 'cache')
+
+  const sources: McpDiscoveryResult['sources'] = []
+  const collected = new Map<string, McpServerConfig>()
+  await loadFromFile(nebulaMcpPath, undefined, collected, sources)
+  if (importsClaudeCode) {
+    await loadFromFile(claudeMcpPath, undefined, collected, sources)
+    await loadFromCache(claudePluginsCacheRoot, collected, sources)
+  }
+  return { servers: [...collected.values()], sources }
+}
+
+async function loadFromFile(
+  path: string,
+  pluginRoot: string | undefined,
+  out: Map<string, McpServerConfig>,
+  sources: McpDiscoveryResult['sources'],
+): Promise<void> {
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf8')
+  } catch {
+    return
+  }
+  let parsed: RawMcpFile
+  try {
+    parsed = JSON.parse(raw) as RawMcpFile
+  } catch {
+    return
+  }
+  if (!parsed.mcpServers) return
+  for (const [name, server] of Object.entries(parsed.mcpServers)) {
+    const config = normalize(name, server, pluginRoot)
+    if (!config) continue
+    if (out.has(name)) continue
+    out.set(name, config)
+    sources.push({ server: name, path })
+  }
+}
+
+async function loadFromCache(
+  cacheRoot: string,
+  out: Map<string, McpServerConfig>,
+  sources: McpDiscoveryResult['sources'],
+): Promise<void> {
+  let marketplaces: Dirent[]
+  try {
+    const s = await stat(cacheRoot)
+    if (!s.isDirectory()) return
+    marketplaces = (await readdir(cacheRoot, { withFileTypes: true })) as Dirent[]
+  } catch {
+    return
+  }
+  for (const market of marketplaces) {
+    if (!market.isDirectory()) continue
+    const marketDir = join(cacheRoot, market.name)
+    let plugins: Dirent[]
+    try {
+      plugins = (await readdir(marketDir, { withFileTypes: true })) as Dirent[]
+    } catch {
+      continue
+    }
+    for (const plugin of plugins) {
+      if (!plugin.isDirectory()) continue
+      const pluginDir = join(marketDir, plugin.name)
+      let versions: Dirent[]
+      try {
+        versions = (await readdir(pluginDir, { withFileTypes: true })) as Dirent[]
+      } catch {
+        continue
+      }
+      const versionDirs = versions.filter(v => v.isDirectory()).map(v => v.name)
+      // Pick the newest version dir (lexicographic, sufficient for semver).
+      versionDirs.sort()
+      const latest = versionDirs[versionDirs.length - 1]
+      if (!latest) continue
+      const versionDir = join(pluginDir, latest)
+      const mcpPath = join(versionDir, '.mcp.json')
+      await loadFromFile(mcpPath, versionDir, out, sources)
+    }
+  }
+}
+
+function normalize(
+  name: string,
+  raw: RawMcpServer,
+  pluginRoot: string | undefined,
+): McpServerConfig | null {
+  if (raw.type === 'http') {
+    if (!raw.url) return null
+    return { name, type: 'http', url: raw.url, headers: raw.headers }
+  }
+  if (!raw.command) return null
+  return {
+    name,
+    type: 'stdio',
+    command: raw.command,
+    args: raw.args?.map(a => substitutePluginRoot(a, pluginRoot)),
+    env: raw.env
+      ? Object.fromEntries(
+          Object.entries(raw.env).map(([k, v]) => [k, substitutePluginRoot(v, pluginRoot)]),
+        )
+      : undefined,
+    pluginRoot,
+  }
+}
+
+function substitutePluginRoot(s: string, pluginRoot: string | undefined): string {
+  if (!pluginRoot) return s
+  return s
+    .replaceAll('${CLAUDE_PLUGIN_ROOT}', pluginRoot)
+    .replaceAll('$CLAUDE_PLUGIN_ROOT', pluginRoot)
+}

package/src/mcp/index.ts

@@ -0,0 +1,10 @@
+export { discoverMcpServers, type McpDiscoveryOptions } from './discovery'
+export { McpStdioClient } from './stdio-client'
+export { McpManager } from './manager'
+export type {
+  McpServerConfig,
+  McpServerStdio,
+  McpServerHttp,
+  McpToolMeta,
+  McpDiscoveryResult,
+} from './types'

package/src/mcp/manager.ts

@@ -0,0 +1,110 @@
+import { z } from 'zod'
+import type { JSONSchema, ToolDef } from '../tools/types'
+import { McpStdioClient } from './stdio-client'
+import type { McpServerConfig, McpToolMeta } from './types'
+
+/**
+ * Lifecycle manager for one or more MCP servers. Spawns each subprocess at
+ * registration time, calls `tools/list`, and registers each remote tool as
+ * `mcp.<server>.<tool>` with `shouldDefer: true` so the brain only sees the
+ * full schema after `tool.search`.
+ *
+ * Lifecycle: caller invokes `closeAll()` on session end (chat exit).
+ */
+export class McpManager {
+  private readonly clients = new Map<string, McpStdioClient>()
+  private readonly toolMeta = new Map<string, McpToolMeta>()
+
+  constructor(public readonly servers: readonly McpServerConfig[]) {}
+
+  async registerAll(register: (def: ToolDef) => void): Promise<{
+    registered: number
+    failed: { server: string; error: string }[]
+  }> {
+    const failed: { server: string; error: string }[] = []
+    let registered = 0
+    await Promise.all(
+      this.servers.map(async server => {
+        if (server.type === 'http') {
+          failed.push({ server: server.name, error: 'http MCP not yet supported (Phase 9.4)' })
+          return
+        }
+        try {
+          const client = new McpStdioClient(server)
+          this.clients.set(server.name, client)
+          const tools = await client.listTools()
+          for (const t of tools) {
+            const id = `mcp.${server.name}.${t.name}`
+            const meta: McpToolMeta = {
+              server: server.name,
+              toolName: t.name,
+              description: t.description ?? '',
+              inputSchema: t.inputSchema ?? defaultSchema(),
+            }
+            this.toolMeta.set(id, meta)
+            register(this.makeToolDef(id, meta))
+            registered++
+          }
+        } catch (e) {
+          failed.push({ server: server.name, error: (e as Error).message })
+          const dead = this.clients.get(server.name)
+          dead?.close()
+          this.clients.delete(server.name)
+        }
+      }),
+    )
+    return { registered, failed }
+  }
+
+  closeAll(): void {
+    for (const client of this.clients.values()) client.close()
+    this.clients.clear()
+  }
+
+  private makeToolDef(id: string, meta: McpToolMeta): ToolDef {
+    const head = meta.description ? `${meta.description.trim()}\n\n` : ''
+    const description = `${head}(MCP tool from server '${meta.server}', mapped to '${meta.toolName}'.)`
+    return {
+      name: id,
+      description,
+      shouldDefer: true,
+      searchHint: `mcp ${meta.server} ${meta.toolName}`,
+      schema: z.unknown(),
+      parametersOverride: toJsonSchemaShape(meta.inputSchema),
+      handler: async args => {
+        const client = this.clients.get(meta.server)
+        if (!client) {
+          return { ok: false, error: `mcp server '${meta.server}' not running` }
+        }
+        try {
+          const result = await client.callTool(meta.toolName, args ?? {})
+          return { ok: true, data: result }
+        } catch (e) {
+          return { ok: false, error: (e as Error).message }
+        }
+      },
+    }
+  }
+}
+
+function defaultSchema(): JSONSchema {
+  return {
+    type: 'object',
+    properties: {},
+    additionalProperties: true,
+  }
+}
+
+function toJsonSchemaShape(raw: unknown): JSONSchema {
+  if (!raw || typeof raw !== 'object') return defaultSchema()
+  const r = raw as Record<string, unknown>
+  if (r.type !== 'object' && !('properties' in r)) return defaultSchema()
+  return {
+    type: 'object',
+    properties: (r.properties as Record<string, unknown>) ?? {},
+    required: Array.isArray(r.required) ? (r.required as string[]) : undefined,
+    additionalProperties:
+      typeof r.additionalProperties === 'boolean' ? (r.additionalProperties as boolean) : true,
+    description: typeof r.description === 'string' ? (r.description as string) : undefined,
+  }
+}

package/src/mcp/stdio-client.ts

@@ -0,0 +1,154 @@
+import { type ChildProcess, spawn } from 'node:child_process'
+import type { McpServerStdio } from './types'
+
+interface JsonRpcRequest {
+  jsonrpc: '2.0'
+  id: number
+  method: string
+  params?: unknown
+}
+
+interface JsonRpcResponse {
+  jsonrpc: '2.0'
+  id: number
+  result?: unknown
+  error?: { code: number; message: string; data?: unknown }
+}
+
+interface JsonRpcNotification {
+  jsonrpc: '2.0'
+  method: string
+  params?: unknown
+}
+
+type JsonRpcMessage = JsonRpcResponse | JsonRpcNotification
+
+const PROTOCOL_VERSION = '2024-11-05'
+const CLIENT_INFO = { name: 'nebula', version: '0.8.1' }
+
+export class McpStdioClient {
+  private proc: ChildProcess | null = null
+  private nextId = 1
+  private buffer = ''
+  private readonly pending = new Map<
+    number,
+    { resolve: (v: unknown) => void; reject: (e: Error) => void }
+  >()
+  private starting: Promise<void> | null = null
+
+  constructor(public readonly server: McpServerStdio) {}
+
+  async ensureStarted(timeoutMs = 10_000): Promise<void> {
+    if (this.proc) return
+    if (this.starting) return this.starting
+    this.starting = this.spawnAndInitialize(timeoutMs)
+    try {
+      await this.starting
+    } finally {
+      this.starting = null
+    }
+  }
+
+  private async spawnAndInitialize(timeoutMs: number): Promise<void> {
+    const proc = spawn(this.server.command, this.server.args ?? [], {
+      env: { ...process.env, ...(this.server.env ?? {}) },
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+    this.proc = proc
+    proc.stdout?.setEncoding('utf8')
+    proc.stdout?.on('data', chunk => this.onStdout(chunk as string))
+    proc.on('error', err => this.failAll(err))
+    proc.on('exit', () => this.failAll(new Error(`mcp server '${this.server.name}' exited`)))
+    // Initialize handshake
+    const initPromise = this.request('initialize', {
+      protocolVersion: PROTOCOL_VERSION,
+      capabilities: {},
+      clientInfo: CLIENT_INFO,
+    })
+    const timeout = new Promise<never>((_, reject) =>
+      setTimeout(() => reject(new Error(`mcp init timeout (${timeoutMs}ms)`)), timeoutMs),
+    )
+    await Promise.race([initPromise, timeout])
+    this.notify('notifications/initialized', {})
+  }
+
+  async listTools(): Promise<{ name: string; description?: string; inputSchema?: unknown }[]> {
+    await this.ensureStarted()
+    const result = (await this.request('tools/list', {})) as {
+      tools?: { name: string; description?: string; inputSchema?: unknown }[]
+    }
+    return result.tools ?? []
+  }
+
+  async callTool(name: string, args: unknown): Promise<unknown> {
+    await this.ensureStarted()
+    return await this.request('tools/call', { name, arguments: args })
+  }
+
+  request(method: string, params: unknown): Promise<unknown> {
+    if (!this.proc?.stdin) throw new Error('mcp server not started')
+    const id = this.nextId++
+    const req: JsonRpcRequest = { jsonrpc: '2.0', id, method, params }
+    return new Promise<unknown>((resolve, reject) => {
+      this.pending.set(id, { resolve, reject })
+      this.proc!.stdin!.write(`${JSON.stringify(req)}\n`, err => {
+        if (err) {
+          this.pending.delete(id)
+          reject(err)
+        }
+      })
+    })
+  }
+
+  notify(method: string, params: unknown): void {
+    if (!this.proc?.stdin) return
+    const note: JsonRpcNotification = { jsonrpc: '2.0', method, params }
+    this.proc.stdin.write(`${JSON.stringify(note)}\n`)
+  }
+
+  close(): void {
+    if (!this.proc) return
+    try {
+      this.proc.stdin?.end()
+    } catch {}
+    try {
+      this.proc.kill('SIGTERM')
+    } catch {}
+    this.proc = null
+    this.failAll(new Error('mcp client closed'))
+  }
+
+  private onStdout(chunk: string): void {
+    this.buffer += chunk
+    let nl = this.buffer.indexOf('\n')
+    while (nl !== -1) {
+      const line = this.buffer.slice(0, nl).trim()
+      this.buffer = this.buffer.slice(nl + 1)
+      nl = this.buffer.indexOf('\n')
+      if (!line) continue
+      let parsed: JsonRpcMessage
+      try {
+        parsed = JSON.parse(line) as JsonRpcMessage
+      } catch {
+        continue
+      }
+      this.dispatch(parsed)
+    }
+  }
+
+  private dispatch(msg: JsonRpcMessage): void {
+    if ('id' in msg) {
+      const pending = this.pending.get(msg.id)
+      if (!pending) return
+      this.pending.delete(msg.id)
+      if (msg.error) pending.reject(new Error(`${msg.error.code}: ${msg.error.message}`))
+      else pending.resolve(msg.result ?? null)
+    }
+    // Notifications from the server are ignored for now.
+  }
+
+  private failAll(err: Error): void {
+    for (const { reject } of this.pending.values()) reject(err)
+    this.pending.clear()
+  }
+}

package/src/mcp/types.ts

@@ -0,0 +1,44 @@
+/**
+ * MCP server config shape (compatible with Claude Code's `.mcp.json`).
+ *
+ * stdio: `{ command, args?, env? }`. nebula spawns a subprocess and speaks
+ * JSON-RPC over its stdin/stdout.
+ *
+ * http: `{ type: 'http', url, headers? }`. nebula posts JSON-RPC to the URL.
+ * Phase 9.2 ships stdio only; HTTP lands in 9.4 polish.
+ */
+
+export interface McpServerStdio {
+  /** Server name used as the prefix in tool ids (`mcp.<name>.<tool>`). */
+  name: string
+  type?: 'stdio'
+  command: string
+  args?: string[]
+  env?: Record<string, string>
+  /** Replacement value for `${CLAUDE_PLUGIN_ROOT}` in args. Set when scanning a plugin cache. */
+  pluginRoot?: string
+}
+
+export interface McpServerHttp {
+  name: string
+  type: 'http'
+  url: string
+  headers?: Record<string, string>
+}
+
+export type McpServerConfig = McpServerStdio | McpServerHttp
+
+export interface McpToolMeta {
+  /** server name */
+  server: string
+  /** original (unprefixed) tool name advertised by the MCP server */
+  toolName: string
+  description: string
+  inputSchema: unknown
+}
+
+export interface McpDiscoveryResult {
+  servers: McpServerConfig[]
+  /** Source path the server was discovered from (debug output only). */
+  sources: { server: string; path: string }[]
+}