@dfosco/storyboard-core 4.2.0-beta.2 → 4.2.0-beta.21

package/src/canvas/__tests__/helpers/canvas-api.js

@@ -0,0 +1,129 @@
+/**
+ * Canvas HTTP API helpers for integration tests.
+ *
+ * Wraps fetch calls to /_storyboard/canvas/* endpoints.
+ * Every call is automatically timed via perf.js.
+ */
+
+import * as perf from './perf.js'
+
+let baseUrl = 'http://localhost:1234'
+
+/** Set the base URL for all API calls. */
+export function setBaseUrl(url) {
+  baseUrl = url.replace(/\/$/, '')
+}
+
+/** Get the current base URL. */
+export function getBaseUrl() {
+  return baseUrl
+}
+
+async function apiFetch(path, options = {}) {
+  const url = `${baseUrl}/_storyboard/canvas${path}`
+  const res = await fetch(url, {
+    headers: { 'Content-Type': 'application/json', ...options.headers },
+    ...options,
+  })
+  const text = await res.text()
+  let json
+  try {
+    json = JSON.parse(text)
+  } catch {
+    json = { raw: text }
+  }
+  if (!res.ok && !options.allowFailure) {
+    throw new Error(`API ${options.method || 'GET'} ${path} returned ${res.status}: ${JSON.stringify(json)}`)
+  }
+  return { status: res.status, ok: res.ok, data: json }
+}
+
+/** Create a widget on a canvas. */
+export async function createWidget(canvasName, type, props = {}, position = { x: 0, y: 0 }) {
+  const timer = perf.start(`widget.create`, { type })
+  const res = await apiFetch('/widget', {
+    method: 'POST',
+    body: JSON.stringify({ name: canvasName, type, props, position }),
+  })
+  timer.end({ widgetId: res.data?.widget?.id })
+  return res.data
+}
+
+/** Read full canvas state. */
+export async function readCanvas(canvasName) {
+  const timer = perf.start('canvas.read', { canvas: canvasName })
+  const res = await apiFetch(`/read?name=${encodeURIComponent(canvasName)}`)
+  timer.end({ widgetCount: res.data?.widgets?.length })
+  return res.data
+}
+
+/** Read a specific widget from a canvas. */
+export async function readWidget(canvasName, widgetId) {
+  const timer = perf.start('canvas.readWidget', { canvas: canvasName, widgetId })
+  const res = await apiFetch(`/read?name=${encodeURIComponent(canvasName)}&widget=${encodeURIComponent(widgetId)}`)
+  timer.end()
+  return res.data
+}
+
+/** Update a widget's props and/or position. */
+export async function updateWidget(canvasName, widgetId, { props, position } = {}) {
+  const timer = perf.start('widget.update', { widgetId })
+  const body = { name: canvasName, widgetId }
+  if (props) body.props = props
+  if (position) body.position = position
+  const res = await apiFetch('/widget', {
+    method: 'PATCH',
+    body: JSON.stringify(body),
+  })
+  timer.end()
+  return res.data
+}
+
+/** Delete a widget from a canvas. */
+export async function deleteWidget(canvasName, widgetId) {
+  const timer = perf.start('widget.delete', { widgetId })
+  const res = await apiFetch('/widget', {
+    method: 'DELETE',
+    body: JSON.stringify({ name: canvasName, widgetId }),
+  })
+  timer.end()
+  return res.data
+}
+
+/** Add a connector between two widgets. */
+export async function addConnector(canvasName, startWidgetId, startAnchor, endWidgetId, endAnchor) {
+  const timer = perf.start('connector.add')
+  const res = await apiFetch('/connector', {
+    method: 'POST',
+    body: JSON.stringify({ name: canvasName, startWidgetId, startAnchor, endWidgetId, endAnchor }),
+  })
+  timer.end({ connectorId: res.data?.connector?.id })
+  return res.data
+}
+
+/** Remove a connector. */
+export async function removeConnector(canvasName, connectorId) {
+  const timer = perf.start('connector.remove')
+  const res = await apiFetch('/connector', {
+    method: 'DELETE',
+    body: JSON.stringify({ name: canvasName, connectorId }),
+  })
+  timer.end()
+  return res.data
+}
+
+/** List terminal sessions. */
+export async function listTerminalSessions() {
+  const res = await apiFetch('/../terminal/sessions')
+  return res.data
+}
+
+/** Check if the dev server is reachable. */
+export async function healthCheck() {
+  try {
+    const res = await fetch(`${baseUrl}/_storyboard/canvas/list`, { signal: AbortSignal.timeout(5000) })
+    return res.ok
+  } catch {
+    return false
+  }
+}

package/src/canvas/__tests__/helpers/perf.js

@@ -0,0 +1,118 @@
+/**
+ * Performance timing and reporting for integration tests.
+ *
+ * Every timed operation records metric name, duration, and optional metadata.
+ * After all tests, call report() for a summary table and toJSON() for file output.
+ */
+
+const metrics = []
+
+const THRESHOLDS = {
+  'widget.create': 1000,
+  'widget.update': 1000,
+  'widget.delete': 1000,
+  'canvas.read': 1000,
+  'connector.add': 1000,
+  'connector.remove': 1000,
+  'tmux.session.start': 5000,
+  'tmux.welcome.render': 10000,
+  'agent.startup': 60000,
+  'agent.response': 45000,
+  'agent.full_chain': 300000,
+}
+
+/** Start a timer. Returns { end() } that records the metric. */
+export function start(name, meta = {}) {
+  const t0 = performance.now()
+  return {
+    end(extraMeta = {}) {
+      const duration = performance.now() - t0
+      const entry = { name, duration, ...meta, ...extraMeta, timestamp: new Date().toISOString() }
+      metrics.push(entry)
+
+      // Check threshold — emit warning inline
+      const thresholdKey = Object.keys(THRESHOLDS).find((k) => name.startsWith(k) || name === k)
+      if (thresholdKey && duration > THRESHOLDS[thresholdKey]) {
+        console.warn(`[SLOW] ${name}: ${(duration / 1000).toFixed(1)}s (threshold: ${(THRESHOLDS[thresholdKey] / 1000).toFixed(1)}s)`)
+      }
+
+      return entry
+    },
+  }
+}
+
+/** Manually record a metric. */
+export function record(name, durationMs, meta = {}) {
+  metrics.push({ name, duration: durationMs, ...meta, timestamp: new Date().toISOString() })
+}
+
+/** Get all recorded metrics. */
+export function getMetrics() {
+  return [...metrics]
+}
+
+/** Get metrics exceeding thresholds. */
+export function getWarnings() {
+  return metrics.filter((m) => {
+    const key = Object.keys(THRESHOLDS).find((k) => m.name.startsWith(k) || m.name === k)
+    return key && m.duration > THRESHOLDS[key]
+  })
+}
+
+/** Print a summary table to console. */
+export function report() {
+  if (metrics.length === 0) {
+    console.log('\n[perf] No metrics recorded.\n')
+    return
+  }
+
+  // Group by metric name
+  const groups = {}
+  for (const m of metrics) {
+    if (!groups[m.name]) groups[m.name] = []
+    groups[m.name].push(m.duration)
+  }
+
+  console.log('\n┌─────────────────────────────────────────────────────────────────┐')
+  console.log('│                    Performance Summary                          │')
+  console.log('├───────────────────────────────┬───────┬───────┬───────┬─────────┤')
+  console.log('│ Metric                        │  Min  │  Avg  │  Max  │ Warning │')
+  console.log('├───────────────────────────────┼───────┼───────┼───────┼─────────┤')
+
+  for (const [name, durations] of Object.entries(groups).sort((a, b) => a[0].localeCompare(b[0]))) {
+    const min = Math.min(...durations)
+    const max = Math.max(...durations)
+    const avg = durations.reduce((a, b) => a + b, 0) / durations.length
+    const thresholdKey = Object.keys(THRESHOLDS).find((k) => name.startsWith(k) || name === k)
+    const warn = thresholdKey && max > THRESHOLDS[thresholdKey] ? '  ⚠️' : '  ✓'
+    const pad = (s, n) => s.slice(0, n).padEnd(n)
+    const fmt = (ms) => {
+      if (ms < 1000) return `${Math.round(ms)}ms`.padStart(5)
+      return `${(ms / 1000).toFixed(1)}s`.padStart(5)
+    }
+    console.log(`│ ${pad(name, 29)} │ ${fmt(min)} │ ${fmt(avg)} │ ${fmt(max)} │ ${pad(warn, 7)} │`)
+  }
+
+  console.log('└───────────────────────────────┴───────┴───────┴───────┴─────────┘')
+
+  const warnings = getWarnings()
+  if (warnings.length > 0) {
+    console.log(`\n⚠️  ${warnings.length} operation(s) exceeded performance thresholds.`)
+  }
+  console.log('')
+}
+
+/** Return JSON-serializable metrics for file output. */
+export function toJSON() {
+  return {
+    timestamp: new Date().toISOString(),
+    totalMetrics: metrics.length,
+    metrics: metrics.map((m) => ({ ...m, duration: Math.round(m.duration) })),
+    warnings: getWarnings().map((m) => ({ ...m, duration: Math.round(m.duration) })),
+  }
+}
+
+/** Reset all metrics (for test isolation). */
+export function reset() {
+  metrics.length = 0
+}

package/src/canvas/__tests__/helpers/setup.js

@@ -0,0 +1,176 @@
+/**
+ * Test setup helpers — canvas lifecycle, server URL, agent availability.
+ */
+
+import { writeFileSync, unlinkSync, existsSync, readFileSync, mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { execSync } from 'node:child_process'
+import * as canvasApi from './canvas-api.js'
+
+const ROOT = process.cwd()
+
+/** Resolve the dev server URL from ports.json or worktree registry. */
+export function resolveServerUrl() {
+  // Try .storyboard/ports.json
+  try {
+    const portsPath = join(ROOT, '.storyboard', 'ports.json')
+    if (existsSync(portsPath)) {
+      const ports = JSON.parse(readFileSync(portsPath, 'utf8'))
+      if (ports.vite) return `http://localhost:${ports.vite}`
+    }
+  } catch { /* ignore */ }
+
+  // Try worktree server registry
+  try {
+    const registryPath = join(ROOT, '.storyboard', 'servers.json')
+    if (existsSync(registryPath)) {
+      const servers = JSON.parse(readFileSync(registryPath, 'utf8'))
+      if (Array.isArray(servers) && servers.length > 0) {
+        return `http://localhost:${servers[0].port}`
+      }
+    }
+  } catch { /* ignore */ }
+
+  return 'http://localhost:1234'
+}
+
+/** Create a fresh test canvas file. Returns the canvas name. */
+export function createTestCanvas(name = '__test__-terminal') {
+  const canvasDir = join(ROOT, 'src', 'canvas')
+  if (!existsSync(canvasDir)) mkdirSync(canvasDir, { recursive: true })
+
+  const filePath = join(canvasDir, `${name}.canvas.jsonl`)
+  const event = JSON.stringify({
+    event: 'canvas_created',
+    timestamp: new Date().toISOString(),
+    meta: { name, description: 'Integration test canvas' },
+  })
+  writeFileSync(filePath, event + '\n')
+  return name
+}
+
+/** Delete the test canvas file. */
+export function deleteTestCanvas(name = '__test__-terminal') {
+  const filePath = join(ROOT, 'src', 'canvas', `${name}.canvas.jsonl`)
+  try {
+    if (existsSync(filePath)) unlinkSync(filePath)
+  } catch { /* ignore */ }
+}
+
+/**
+ * Load configured agents from storyboard.config.json.
+ * Returns array of { id, label, startupCommand }.
+ */
+export function loadConfiguredAgents() {
+  try {
+    const raw = readFileSync(join(ROOT, 'storyboard.config.json'), 'utf8')
+    const config = JSON.parse(raw)
+    const agents = config?.canvas?.agents
+    if (!agents || typeof agents !== 'object') return []
+    return Object.entries(agents)
+      .map(([id, cfg]) => ({
+        id,
+        label: cfg.label || id,
+        startupCommand: cfg.startupCommand || null,
+      }))
+      .filter((a) => a.startupCommand)
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Check if an agent binary is available on this machine.
+ * Returns true if the binary is found, false otherwise.
+ */
+export function checkAgentAvailability(agentId) {
+  // Map agent IDs to binary names
+  const binaryMap = {
+    copilot: 'copilot',
+    claude: 'claude',
+  }
+  const binary = binaryMap[agentId] || agentId
+  try {
+    execSync(`which ${binary} 2>/dev/null`, { encoding: 'utf8', timeout: 5000 })
+    return true
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Kill all tmux sessions for the test canvas.
+ * Finds sessions by the canvas name pattern in their tmux name.
+ */
+export function cleanupTerminalSessions() {
+  try {
+    const out = execSync('tmux list-sessions -F "#{session_name}" 2>/dev/null', { encoding: 'utf8', timeout: 5000 })
+    const sessions = out.trim().split('\n').filter(Boolean)
+    for (const s of sessions) {
+      // Kill sessions that are from our test canvas (sb- prefix + test widget IDs)
+      if (s.startsWith('sb-')) {
+        // We can't know for sure which are test sessions, so we track widget IDs
+        // in the test and clean those up specifically. This is a fallback.
+      }
+    }
+  } catch { /* no tmux sessions */ }
+}
+
+/**
+ * Full preflight check. Verifies:
+ * 1. Dev server is reachable
+ * 2. tmux is available
+ * 3. agent-browser is available
+ *
+ * Throws with a clear message if any check fails.
+ */
+export async function preflight() {
+  const url = resolveServerUrl()
+  canvasApi.setBaseUrl(url)
+
+  // Check dev server
+  const serverOk = await canvasApi.healthCheck()
+  if (!serverOk) {
+    throw new Error(
+      `[preflight] Dev server not reachable at ${url}. Start it with 'storyboard dev' or 'npm run dev' before running integration tests.`,
+    )
+  }
+
+  // Check tmux
+  try {
+    execSync('which tmux', { timeout: 5000 })
+  } catch {
+    throw new Error('[preflight] tmux is not installed. Install it with: brew install tmux')
+  }
+
+  // Check agent-browser
+  try {
+    execSync('which agent-browser', { timeout: 5000 })
+  } catch {
+    throw new Error('[preflight] agent-browser is not installed. Install it with: npm install -g agent-browser')
+  }
+
+  return { url }
+}
+
+/** Write the perf + summary results to test-results/. */
+export function writeResults(perfData, transcriptPaths = []) {
+  const dir = join(ROOT, 'test-results')
+  mkdirSync(dir, { recursive: true })
+
+  // Write perf JSON
+  writeFileSync(join(dir, 'integration-perf.json'), JSON.stringify(perfData, null, 2))
+
+  // Write summary
+  const summary = [
+    `Integration Test Run — ${new Date().toISOString()}`,
+    '',
+    `Metrics recorded: ${perfData.totalMetrics}`,
+    `Warnings: ${perfData.warnings.length}`,
+    '',
+    'Transcripts:',
+    ...transcriptPaths.map((p) => `  ${p}`),
+    '',
+  ].join('\n')
+  writeFileSync(join(dir, 'summary.txt'), summary)
+}

package/src/canvas/__tests__/helpers/tmux.js

@@ -0,0 +1,130 @@
+/**
+ * tmux helpers for integration tests.
+ *
+ * Every operation is timed via perf.js and recorded via transcript.js.
+ */
+
+import { execSync } from 'node:child_process'
+import * as perf from './perf.js'
+import * as transcript from './transcript.js'
+
+/** List all tmux sessions. Returns array of session name strings. */
+export function listSessions() {
+  try {
+    const out = execSync('tmux list-sessions -F "#{session_name}"', { encoding: 'utf8', timeout: 5000 })
+    return out.trim().split('\n').filter(Boolean)
+  } catch {
+    return []
+  }
+}
+
+/** Check if a tmux session exists. */
+export function hasSession(name) {
+  try {
+    execSync(`tmux has-session -t "${name}" 2>/dev/null`, { timeout: 5000 })
+    return true
+  } catch {
+    return false
+  }
+}
+
+/** Capture the full pane content of a tmux session. */
+export function capturePane(sessionName) {
+  const timer = perf.start('tmux.capture', { session: sessionName })
+  try {
+    const out = execSync(`tmux capture-pane -t "${sessionName}" -p`, { encoding: 'utf8', timeout: 10000 })
+    timer.end()
+    transcript.logStdout(sessionName, out)
+    return out
+  } catch (err) {
+    timer.end({ error: err.message })
+    return ''
+  }
+}
+
+/** Send keystrokes to a tmux session. */
+export function sendKeys(sessionName, keys) {
+  transcript.logStdin(sessionName, keys)
+  try {
+    execSync(`tmux send-keys -t "${sessionName}" ${keys}`, { timeout: 5000 })
+  } catch (err) {
+    transcript.logEvent(sessionName, `sendKeys error: ${err.message}`)
+    throw err
+  }
+}
+
+/** Send literal text (properly escaped) to a tmux session. */
+export function sendText(sessionName, text) {
+  transcript.logStdin(sessionName, text)
+  try {
+    // Use -- to prevent tmux from interpreting flags in the text
+    execSync(`tmux send-keys -t "${sessionName}" -l -- ${JSON.stringify(text)}`, { timeout: 5000 })
+  } catch (err) {
+    transcript.logEvent(sessionName, `sendText error: ${err.message}`)
+    throw err
+  }
+}
+
+/**
+ * Poll capturePane until the output matches a pattern, or timeout.
+ * Returns the matched output on success, throws on timeout.
+ */
+export async function waitForOutput(sessionName, pattern, timeoutMs = 30000, intervalMs = 1000) {
+  const timer = perf.start('tmux.waitForOutput', { session: sessionName, pattern: String(pattern) })
+  const regex = typeof pattern === 'string' ? new RegExp(pattern, 'i') : pattern
+  const deadline = Date.now() + timeoutMs
+
+  while (Date.now() < deadline) {
+    const output = capturePane(sessionName)
+    if (regex.test(output)) {
+      timer.end({ matched: true })
+      return output
+    }
+    await new Promise((r) => setTimeout(r, intervalMs))
+  }
+
+  // Final attempt
+  const output = capturePane(sessionName)
+  if (regex.test(output)) {
+    timer.end({ matched: true })
+    return output
+  }
+
+  timer.end({ matched: false, timedOut: true })
+  const err = new Error(`[tmux] waitForOutput timed out after ${timeoutMs}ms waiting for ${pattern}\nLast capture:\n${output}`)
+  err.lastCapture = output
+  throw err
+}
+
+/**
+ * Wait for a tmux session to appear. Polls tmux list-sessions.
+ * Returns the matching session name, or throws on timeout.
+ */
+export async function waitForSession(namePattern, timeoutMs = 15000, intervalMs = 500) {
+  const timer = perf.start('tmux.session.start')
+  const regex = typeof namePattern === 'string' ? new RegExp(namePattern) : namePattern
+  const deadline = Date.now() + timeoutMs
+
+  while (Date.now() < deadline) {
+    const sessions = listSessions()
+    const match = sessions.find((s) => regex.test(s))
+    if (match) {
+      timer.end({ session: match })
+      return match
+    }
+    await new Promise((r) => setTimeout(r, intervalMs))
+  }
+
+  timer.end({ timedOut: true })
+  throw new Error(`[tmux] No session matching ${namePattern} appeared within ${timeoutMs}ms`)
+}
+
+/** Kill a tmux session. */
+export function killSession(name) {
+  try {
+    execSync(`tmux kill-session -t "${name}" 2>/dev/null`, { timeout: 5000 })
+    transcript.logEvent(name, 'Session killed')
+  } catch {
+    // Session may already be dead
+  }
+}

package/src/canvas/__tests__/helpers/transcript.js

@@ -0,0 +1,129 @@
+/**
+ * Session transcript recording for integration tests.
+ *
+ * Wraps tmux interactions to produce a full stdin/stdout log per session.
+ * Transcripts are written to test-results/transcripts/ on every run.
+ */
+
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+const transcripts = new Map()
+
+const TRANSCRIPTS_DIR = join(process.cwd(), 'test-results', 'transcripts')
+
+function ensureDir() {
+  mkdirSync(TRANSCRIPTS_DIR, { recursive: true })
+}
+
+function elapsed(startTime) {
+  const ms = Date.now() - startTime
+  const secs = Math.floor(ms / 1000)
+  const mins = Math.floor(secs / 60)
+  const s = secs % 60
+  const tenths = Math.floor((ms % 1000) / 100)
+  if (mins > 0) return `${String(mins).padStart(2, '0')}:${String(s).padStart(2, '0')}.${tenths}`
+  return `${String(s).padStart(2, '0')}.${tenths}`
+}
+
+/** Initialize a transcript for a session. */
+export function createTranscript(sessionName, metadata = {}) {
+  const transcript = {
+    sessionName,
+    metadata,
+    startTime: Date.now(),
+    lines: [],
+  }
+  transcript.lines.push(`=== Session: ${sessionName} ===`)
+  transcript.lines.push(`Started: ${new Date().toISOString()}`)
+  if (metadata.widgetId) transcript.lines.push(`Widget ID: ${metadata.widgetId}`)
+  if (metadata.canvasName) transcript.lines.push(`Canvas: ${metadata.canvasName}`)
+  if (metadata.agentId) transcript.lines.push(`Agent: ${metadata.agentId}`)
+  transcript.lines.push('')
+  transcripts.set(sessionName, transcript)
+  return transcript
+}
+
+/** Log a test section header. */
+export function logSection(sessionName, testId) {
+  const t = transcripts.get(sessionName)
+  if (!t) return
+  t.lines.push(`--- [${testId}] ---`)
+}
+
+/** Log stdin (input sent via sendKeys). */
+export function logStdin(sessionName, input, testId = '') {
+  const t = transcripts.get(sessionName)
+  if (!t) return
+  const ts = elapsed(t.startTime)
+  const prefix = `[${ts}] [stdin] `
+  // Split multi-line input
+  const lines = String(input).split('\n')
+  for (const line of lines) {
+    t.lines.push(`${prefix}${line}`)
+  }
+}
+
+/** Log stdout (captured pane output). Only logs the diff from previous capture. */
+export function logStdout(sessionName, fullOutput, testId = '') {
+  const t = transcripts.get(sessionName)
+  if (!t) return
+  const ts = elapsed(t.startTime)
+  const prefix = `[${ts}] [stdout] `
+
+  // Compute diff from last stdout
+  const prevLen = t._lastStdoutLen || 0
+  const lines = String(fullOutput).split('\n')
+  const newLines = lines.slice(prevLen)
+  t._lastStdoutLen = lines.length
+
+  for (const line of newLines) {
+    // Skip empty trailing lines
+    if (line.trim() === '' && newLines.indexOf(line) === newLines.length - 1) continue
+    t.lines.push(`${prefix}${line}`)
+  }
+}
+
+/** Log a non-IO event (session start, resize, etc.). */
+export function logEvent(sessionName, event, testId = '') {
+  const t = transcripts.get(sessionName)
+  if (!t) return
+  const ts = elapsed(t.startTime)
+  t.lines.push(`[${ts}] [event] ${event}`)
+}
+
+/** Get the in-memory transcript text. */
+export function getTranscript(sessionName) {
+  const t = transcripts.get(sessionName)
+  if (!t) return ''
+  return t.lines.join('\n')
+}
+
+/** Flush a single transcript to disk. */
+export function flush(sessionName) {
+  const t = transcripts.get(sessionName)
+  if (!t) return null
+
+  const totalMs = Date.now() - t.startTime
+  t.lines.push('')
+  t.lines.push(`=== Session ended: ${new Date().toISOString()} (total: ${(totalMs / 1000).toFixed(1)}s) ===`)
+
+  ensureDir()
+  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
+  const filename = `${sessionName}-${ts}.log`
+  const filepath = join(TRANSCRIPTS_DIR, filename)
+  writeFileSync(filepath, t.lines.join('\n') + '\n')
+  console.log(`[transcript] Written: ${filepath}`)
+  return filepath
+}
+
+/** Flush all open transcripts. Call in afterAll. */
+export function flushAll() {
+  const paths = []
+  for (const sessionName of transcripts.keys()) {
+    const p = flush(sessionName)
+    if (p) paths.push(p)
+  }
+  transcripts.clear()
+  return paths
+}