circuschief 0.6.0 → 0.8.0

package/packages/server/src/scripts/dbUtils.js

@@ -0,0 +1,81 @@
+import { existsSync, mkdirSync, copyFileSync, mkdtempSync, rmSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { homedir, tmpdir } from 'node:os';
+import { getDefaultDbPath } from '../config.js';
+import { DatabaseManager } from '../db/DatabaseManager.js';
+
+export function getActiveDbPath() {
+  return process.env.DB_PATH || getDefaultDbPath();
+}
+
+export function getSqliteSidecarPaths(dbPath) {
+  return [`${dbPath}-wal`, `${dbPath}-shm`];
+}
+
+export function getBackupDir() {
+  return join(homedir(), '.circuschief', 'backups');
+}
+
+export function copyDatabaseBackups(dbPath = getActiveDbPath(), timestamp = new Date()) {
+  if (!existsSync(dbPath)) {
+    return { dbPath, backupDir: getBackupDir(), copied: [], missing: [dbPath] };
+  }
+
+  const backupDir = getBackupDir();
+  mkdirSync(backupDir, { recursive: true });
+
+  const stamp = timestamp.toISOString().replace(/[:.]/g, '-');
+  const candidates = [dbPath, ...getSqliteSidecarPaths(dbPath)];
+  const copied = [];
+  const missing = [];
+
+  for (const source of candidates) {
+    if (!existsSync(source)) {
+      missing.push(source);
+      continue;
+    }
+    const target = join(backupDir, `${basename(source)}.${stamp}.bak`);
+    copyFileSync(source, target);
+    copied.push({ source, target });
+  }
+
+  return { dbPath, backupDir, copied, missing };
+}
+
+export function createFreshBaselineDb() {
+  const tempDir = mkdtempSync(join(tmpdir(), 'circuschief-baseline-'));
+  const dbPath = join(tempDir, 'baseline.db');
+  const manager = new DatabaseManager();
+  const db = manager.init(dbPath);
+
+  return {
+    db,
+    dbPath,
+    close() {
+      manager.close();
+      rmSync(tempDir, { recursive: true, force: true });
+    },
+  };
+}
+
+export function getSchemaObjects(db) {
+  return db.prepare(`
+    SELECT type, name, tbl_name, sql
+    FROM sqlite_master
+    WHERE type IN ('table', 'index', 'trigger', 'view')
+      AND name NOT LIKE 'sqlite_%'
+    ORDER BY type, name
+  `).all();
+}
+
+export function getTableColumns(db, tableName) {
+  return db.prepare(`PRAGMA table_info(${tableName})`).all();
+}
+
+export function getIndexColumns(db, indexName) {
+  return db.prepare(`PRAGMA index_info(${indexName})`).all().map((row) => row.name);
+}
+
+export function normalizeSql(sql) {
+  return (sql || '').replace(/\s+/g, ' ').trim();
+}

package/packages/server/src/scripts/inspectDatabaseSchema.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import Database from 'better-sqlite3';
+import { existsSync } from 'node:fs';
+import {
+  createFreshBaselineDb,
+  getActiveDbPath,
+  getSchemaObjects,
+} from './dbUtils.js';
+
+const BASELINE_TABLES = [
+  'sessions',
+  'projects',
+  'project_session_defaults',
+  'session_templates',
+  'canvas_items',
+  'providers',
+  'provider_models',
+  'kanban_lanes',
+  'agent_call_logs',
+];
+
+function printDatabase(db, label) {
+  console.log(`# ${label}`);
+
+  for (const row of getSchemaObjects(db)) {
+    console.log(`\n-- ${row.type} ${row.name} (${row.tbl_name})`);
+    console.log(row.sql || '');
+  }
+
+  for (const table of BASELINE_TABLES) {
+    const exists = db.prepare(
+      "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
+    ).get(table);
+    if (!exists) continue;
+
+    console.log(`\n-- PRAGMA table_info(${table})`);
+    console.table(db.prepare(`PRAGMA table_info(${table})`).all());
+    console.log(`-- PRAGMA foreign_key_list(${table})`);
+    console.table(db.prepare(`PRAGMA foreign_key_list(${table})`).all());
+    console.log(`-- PRAGMA index_list(${table})`);
+    const indexes = db.prepare(`PRAGMA index_list(${table})`).all();
+    console.table(indexes);
+
+    for (const index of indexes) {
+      console.log(`-- PRAGMA index_info(${index.name})`);
+      console.table(db.prepare(`PRAGMA index_info(${index.name})`).all());
+      console.log(`-- PRAGMA index_xinfo(${index.name})`);
+      console.table(db.prepare(`PRAGMA index_xinfo(${index.name})`).all());
+    }
+  }
+}
+
+export function main(argv = process.argv.slice(2)) {
+  if (argv.includes('--fresh')) {
+    const fresh = createFreshBaselineDb();
+    try {
+      printDatabase(fresh.db, `fresh baseline ${fresh.dbPath}`);
+    } finally {
+      fresh.close();
+    }
+    return 0;
+  }
+
+  const dbPath = getActiveDbPath();
+  if (!existsSync(dbPath)) {
+    console.log(`No database found at ${dbPath}; use --fresh to inspect a fresh baseline.`);
+    return 0;
+  }
+
+  const db = new Database(dbPath, { readonly: true, fileMustExist: true });
+  try {
+    printDatabase(db, `active database ${dbPath}`);
+  } finally {
+    db.close();
+  }
+  return 0;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exit(main());
+}

package/packages/server/src/scripts/validateDatabaseBaseline.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+import Database from 'better-sqlite3';
+import { existsSync } from 'node:fs';
+import {
+  createFreshBaselineDb,
+  getActiveDbPath,
+  getIndexColumns,
+  getSchemaObjects,
+  getTableColumns,
+  normalizeSql,
+} from './dbUtils.js';
+import {
+  BUILT_IN_ANTHROPIC_MODELS,
+  BUILT_IN_OPENAI_MODELS,
+  BUILT_IN_ANTHROPIC_PROVIDER,
+  BUILT_IN_OPENAI_PROVIDER,
+} from '../db/seedBaselineData.js';
+
+export const DRIFT_SENSITIVE_TABLES = [
+  'sessions',
+  'project_session_defaults',
+  'session_templates',
+  'canvas_items',
+  'providers',
+  'provider_models',
+  'agent_call_logs',
+  'kanban_lanes',
+];
+
+export const REQUIRED_INDEXES = {
+  idx_sessions_project: ['project_id'],
+  idx_sessions_status: ['status'],
+  idx_sessions_archived: ['archived'],
+  idx_sessions_starred: ['archived', 'starred'],
+  idx_sessions_next_template: ['next_template_id'],
+  idx_sessions_parent: ['parent_session_id'],
+  idx_sessions_scheduled: ['scheduled_at'],
+  idx_messages_conversation: ['conversation_id'],
+  idx_canvas_deleted: ['deleted_at'],
+  idx_todos_conversation: ['conversation_id'],
+  idx_project_defaults_projectId: ['project_id'],
+  idx_conversations_parent: ['parent_conversation_id'],
+  idx_agent_call_logs_agent_type: ['agent_type'],
+  idx_agent_call_logs_call_type: ['call_type'],
+  idx_agent_call_logs_status: ['status'],
+  idx_agent_call_logs_model: ['model'],
+};
+
+function byName(rows) {
+  return new Map(rows.map((row) => [row.name, row]));
+}
+
+function compareTables(actualDb, expectedDb, errors) {
+  const actualObjects = byName(getSchemaObjects(actualDb).filter((row) => row.type === 'table'));
+  const expectedObjects = getSchemaObjects(expectedDb).filter((row) => row.type === 'table');
+
+  for (const expected of expectedObjects) {
+    const actual = actualObjects.get(expected.name);
+    if (!actual) {
+      errors.push(`Missing table: ${expected.name}`);
+      continue;
+    }
+
+    const actualColumns = new Map(
+      getTableColumns(actualDb, expected.name).map((col) => [col.name, col])
+    );
+    const expectedColumns = getTableColumns(expectedDb, expected.name);
+
+    for (const expectedColumn of expectedColumns) {
+      const actualColumn = actualColumns.get(expectedColumn.name);
+      if (!actualColumn) {
+        errors.push(`Missing column: ${expected.name}.${expectedColumn.name}`);
+        continue;
+      }
+
+      const actualSummary = {
+        type: actualColumn.type,
+        notnull: actualColumn.notnull,
+        dflt_value: actualColumn.dflt_value,
+        pk: actualColumn.pk,
+      };
+      const expectedSummary = {
+        type: expectedColumn.type,
+        notnull: expectedColumn.notnull,
+        dflt_value: expectedColumn.dflt_value,
+        pk: expectedColumn.pk,
+      };
+      if (JSON.stringify(actualSummary) !== JSON.stringify(expectedSummary)) {
+        errors.push(`Column mismatch for ${expected.name}.${expectedColumn.name}`);
+      }
+    }
+  }
+}
+
+function compareIndexes(actualDb, expectedDb, errors) {
+  const actualObjects = byName(getSchemaObjects(actualDb).filter((row) => row.type === 'index'));
+  const expectedObjects = byName(getSchemaObjects(expectedDb).filter((row) => row.type === 'index'));
+
+  for (const [name, expectedColumns] of Object.entries(REQUIRED_INDEXES)) {
+    const actual = actualObjects.get(name);
+    if (!actual) {
+      errors.push(`Missing index: ${name}`);
+      continue;
+    }
+    const actualColumns = getIndexColumns(actualDb, name);
+    if (JSON.stringify(actualColumns) !== JSON.stringify(expectedColumns)) {
+      errors.push(`Index column mismatch for ${name}: ${actualColumns.join(',')} !== ${expectedColumns.join(',')}`);
+    }
+  }
+
+  const expectedScheduledSql = normalizeSql(expectedObjects.get('idx_sessions_scheduled')?.sql);
+  const actualScheduledSql = normalizeSql(actualObjects.get('idx_sessions_scheduled')?.sql);
+  if (expectedScheduledSql !== actualScheduledSql) {
+    errors.push('Partial index SQL mismatch for idx_sessions_scheduled');
+  }
+}
+
+function validateProviders(actualDb, errors) {
+  for (const provider of [BUILT_IN_ANTHROPIC_PROVIDER, BUILT_IN_OPENAI_PROVIDER]) {
+    const row = actualDb.prepare(
+      'SELECT id, name, kind, base_url, auth_token, commit_attribution_override FROM providers WHERE id = ?'
+    ).get(provider.id);
+    if (!row) {
+      errors.push(`Missing provider seed row: ${provider.id}`);
+      continue;
+    }
+    if (row.name !== provider.name || row.kind !== provider.kind) {
+      errors.push(`Provider seed mismatch: ${provider.id}`);
+    }
+    if (row.base_url !== null || row.auth_token !== null) {
+      errors.push(`Provider nullable seed columns mismatch: ${provider.id}`);
+    }
+  }
+}
+
+function validateProviderModels(actualDb, errors) {
+  for (const model of [...BUILT_IN_ANTHROPIC_MODELS, ...BUILT_IN_OPENAI_MODELS]) {
+    const row = actualDb.prepare(
+      'SELECT provider_id, model_id, display_name, description, tier FROM provider_models WHERE id = ?'
+    ).get(model.id);
+    if (!row) {
+      errors.push(`Missing provider model seed row: ${model.id}`);
+      continue;
+    }
+    const expected = {
+      provider_id: model.providerId,
+      model_id: model.modelId,
+      display_name: model.displayName,
+      description: model.description,
+      tier: model.tier,
+    };
+    if (JSON.stringify(row) !== JSON.stringify(expected)) {
+      errors.push(`Provider model seed mismatch: ${model.id}`);
+    }
+  }
+}
+
+function compareSeedRows(actualDb, errors) {
+  const tables = new Set(actualDb.prepare(
+    "SELECT name FROM sqlite_master WHERE type = 'table'"
+  ).all().map((row) => row.name));
+  if (!tables.has('providers') || !tables.has('provider_models') || !tables.has('quick_responses') || !tables.has('session_templates')) {
+    errors.push('Required seed tables are missing');
+    return;
+  }
+  validateProviders(actualDb, errors);
+  validateProviderModels(actualDb, errors);
+}
+
+export function validateDatabaseBaseline(actualDb) {
+  const fresh = createFreshBaselineDb();
+  const errors = [];
+
+  try {
+    compareTables(actualDb, fresh.db, errors);
+    compareIndexes(actualDb, fresh.db, errors);
+    compareSeedRows(actualDb, errors);
+  } finally {
+    fresh.close();
+  }
+
+  return errors;
+}
+
+export function main() {
+  const dbPath = getActiveDbPath();
+  if (!existsSync(dbPath)) {
+    console.log(`No database found at ${dbPath}; nothing to validate.`);
+    return 0;
+  }
+
+  const db = new Database(dbPath, { readonly: true, fileMustExist: true });
+  try {
+    const errors = validateDatabaseBaseline(db);
+    if (errors.length === 0) {
+      console.log(`Database baseline validation passed: ${dbPath}`);
+      return 0;
+    }
+
+    console.error(`Database baseline validation failed: ${dbPath}`);
+    for (const error of errors) {
+      console.error(`- ${error}`);
+    }
+    return 1;
+  } finally {
+    db.close();
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exit(main());
+}

package/packages/server/src/services/agentCallLogger.js

@@ -20,7 +20,7 @@ export class AgentCallLogger {
     const callId = nanoid();
 
     // Build metadata object - only include keys with defined values
-    const metadata = {};
+    const metadata = { ...(meta.metadata || {}) };
     if (meta.effortLevel !== undefined && meta.effortLevel !== null) {
       metadata.effortLevel = meta.effortLevel;
     }

package/packages/server/src/services/codexSpawnHelper.js

@@ -1,5 +1,10 @@
 import { spawn } from 'child_process';
 import { createRobustEnv } from './nodeSpawnHelper.js';
+import {
+  captureSpawnAttempt,
+  createCapturedSpawnProcess,
+  isE2ESpawnCaptureEnabled,
+} from './e2eSpawnCapture.js';
 
 /**
  * Create a custom spawn function for the Codex CLI.
@@ -19,6 +24,10 @@ import { createRobustEnv } from './nodeSpawnHelper.js';
 export function createCodexSpawner() {
   return (options) => {
     const { command, args, cwd, env, signal } = options;
+    if (isE2ESpawnCaptureEnabled()) {
+      captureSpawnAttempt('codex', options);
+      return createCapturedSpawnProcess('codex');
+    }
 
     // Replace 'node' with the absolute path to the current Node executable
     const actualCommand = command === 'node' ? process.execPath : command;

package/packages/server/src/services/commandButtonPrompts.js

@@ -0,0 +1,48 @@
+import { commandButtons } from '../database.js';
+
+/**
+ * Build Command API instructions for system prompt if the project has commands.
+ * @param {string} apiUrl - Base API URL
+ * @param {string} sessionId - Current session ID
+ * @param {string} projectId - Current project ID
+ * @returns {string} Command instructions or empty string if no commands configured
+ */
+export function buildCommandButtonApiInstructions(apiUrl, sessionId, projectId) {
+  const buttons = commandButtons.getByProjectId(projectId);
+  if (!buttons || buttons.length === 0) {
+    return '';
+  }
+
+  return `## Commands API
+
+This project has commands configured - reusable shell commands you can execute. Use the Bash tool to run these curl commands.
+
+### List Available Commands
+\`\`\`bash
+curl ${apiUrl}/api/sessions/${sessionId}/command-buttons
+\`\`\`
+
+### Run a Command
+\`\`\`bash
+curl -X POST ${apiUrl}/api/sessions/${sessionId}/command-buttons/<button_id>/run
+\`\`\`
+
+Response: { runId, buttonId, status: "running", output: "" }
+
+### Check Run Status & Output
+\`\`\`bash
+curl ${apiUrl}/api/sessions/${sessionId}/command-buttons/runs/<run_id>
+\`\`\`
+
+Response: { runId, buttonId, status, exitCode, output, startedAt, completedAt }
+
+### List Command Runs
+\`\`\`bash
+curl ${apiUrl}/api/sessions/${sessionId}/command-buttons/runs
+\`\`\`
+
+### Kill a Running Command
+\`\`\`bash
+curl -X POST ${apiUrl}/api/sessions/${sessionId}/command-buttons/runs/<run_id>/kill
+\`\`\``;
+}

package/packages/server/src/services/commandRunner.js

@@ -7,6 +7,12 @@ import { TerminalOutputProcessor } from './terminalOutput.js';
 // Re-export for backward compatibility
 export { stripAnsiCodes, TerminalOutputProcessor } from './terminalOutput.js';
 
+export function createCommandRunnerEnv(baseEnv = process.env) {
+  const env = createRobustEnv(baseEnv);
+  delete env.CIRCUSCHIEF_COMMIT_ATTRIBUTION;
+  return env;
+}
+
 /**
  * Service for running commands and managing their execution
  */
@@ -141,7 +147,7 @@ export class CommandRunner {
           cwd: workingDirectory,
           stdio: ['pipe', 'pipe', 'pipe'],
           detached: true,
-          env: createRobustEnv(),
+          env: createCommandRunnerEnv(),
         });
 
         const entry = this.#createProcessEntry(child, sessionId, buttonId);

package/packages/server/src/services/draftSessionService.js

@@ -112,6 +112,7 @@ function getOrCreateInitialMessage(session, options) {
  * @param {object} options
  * @param {string} [options.prompt] - Optional new prompt to use/override
  * @param {string} [options.model] - Optional model override
+ * @param {string|null} [options.providerId] - Optional provider override
  * @returns {Promise<object>} The updated session
  */
 export async function startDraft(session, options = {}) {
@@ -144,6 +145,7 @@ export async function startDraft(session, options = {}) {
     status: 'starting',
     pendingModel: null,
     ...(model ? { model, agentType } : {}),
+    ...(options.providerId !== undefined ? { providerId: options.providerId } : {}),
   });
 
   // Resolve skill/command invocations so skill body goes into system prompt

package/packages/server/src/services/e2eSpawnCapture.js

@@ -0,0 +1,147 @@
+import { appendFileSync } from 'fs';
+import { EventEmitter } from 'events';
+import { PassThrough } from 'stream';
+
+export function isE2ESpawnCaptureEnabled() {
+  return Boolean(process.env.E2E_AGENT_SPAWN_CAPTURE_FILE);
+}
+
+export function captureSpawnAttempt(agentType, spawnOptions) {
+  const filePath = process.env.E2E_AGENT_SPAWN_CAPTURE_FILE;
+  if (!filePath) return;
+
+  const record = {
+    agentType,
+    command: spawnOptions.command,
+    args: spawnOptions.args || [],
+    cwd: spawnOptions.cwd || null,
+    env: summarizeSpawnEnv(spawnOptions.env),
+    options: summarizeSpawnOptions(agentType, spawnOptions),
+    capturedAt: new Date().toISOString(),
+  };
+
+  appendFileSync(filePath, `${JSON.stringify(record)}\n`, 'utf8');
+}
+
+export function createCapturedSpawnProcess(agentType) {
+  const processStub = new EventEmitter();
+  const stdin = new PassThrough();
+  const stdout = new PassThrough();
+  const stderr = new PassThrough();
+
+  processStub.stdin = stdin;
+  processStub.stdout = stdout;
+  processStub.stderr = stderr;
+  processStub.killed = false;
+  processStub.exitCode = null;
+  processStub.kill = (signal = 'SIGTERM') => {
+    if (processStub.killed || processStub.exitCode !== null) return true;
+    processStub.killed = true;
+    finishProcess({ processStub, stdout, stderr, code: null, signal });
+    return true;
+  };
+
+  const complete = () => {
+    setImmediate(() => {
+      if (processStub.killed || processStub.exitCode !== null) return;
+      writeCapturedAgentEvents(agentType, stdout);
+      finishProcess({ processStub, stdout, stderr, code: 0, signal: null });
+    });
+  };
+
+  if (agentType === 'claude-code') {
+    setTimeout(complete, 10);
+  } else {
+    stdin.once('finish', complete);
+  }
+
+  return processStub;
+}
+
+function summarizeSpawnEnv(env = {}) {
+  if (!Object.prototype.hasOwnProperty.call(env, 'CIRCUSCHIEF_COMMIT_ATTRIBUTION')) {
+    return {};
+  }
+  return {
+    CIRCUSCHIEF_COMMIT_ATTRIBUTION: env.CIRCUSCHIEF_COMMIT_ATTRIBUTION,
+  };
+}
+
+function summarizeSpawnOptions(agentType, spawnOptions) {
+  const args = spawnOptions.args || [];
+  if (agentType === 'claude-code') {
+    return {
+      model: valueAfter(args, '--model'),
+      settings: valueAfter(args, '--settings'),
+      permissionMode: valueAfter(args, '--permission-mode'),
+      settingSources: valueAfter(args, '--setting-sources'),
+    };
+  }
+
+  return {
+    model: valueAfter(args, '-m'),
+    sandbox: valueAfter(args, '--sandbox'),
+    config: valuesAfter(args, '-c'),
+  };
+}
+
+function valueAfter(args, flag) {
+  const index = args.indexOf(flag);
+  if (index === -1) return null;
+  return args[index + 1] ?? null;
+}
+
+function valuesAfter(args, flag) {
+  const values = [];
+  for (let index = 0; index < args.length; index += 1) {
+    if (args[index] === flag && args[index + 1] !== undefined) {
+      values.push(args[index + 1]);
+    }
+  }
+  return values;
+}
+
+function writeCapturedAgentEvents(agentType, stdout) {
+  if (agentType === 'codex') {
+    writeJsonLine(stdout, { type: 'thread.started', thread_id: `e2e-codex-${Date.now()}` });
+    writeJsonLine(stdout, { type: 'turn.started' });
+    writeJsonLine(stdout, {
+      type: 'item.completed',
+      item: { type: 'agent_message', text: 'E2E spawn capture response.' },
+    });
+    writeJsonLine(stdout, {
+      type: 'turn.completed',
+      usage: { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0 },
+    });
+    return;
+  }
+
+  writeJsonLine(stdout, {
+    type: 'system',
+    subtype: 'init',
+    session_id: `e2e-claude-${Date.now()}`,
+  });
+  writeJsonLine(stdout, {
+    type: 'assistant',
+    message: { content: [{ type: 'text', text: 'E2E spawn capture response.' }] },
+  });
+  writeJsonLine(stdout, {
+    type: 'result',
+    subtype: 'success',
+    usage: { input_tokens: 0, output_tokens: 0 },
+  });
+}
+
+function writeJsonLine(stream, value) {
+  stream.write(`${JSON.stringify(value)}\n`);
+}
+
+function finishProcess({ processStub, stdout, stderr, code, signal }) {
+  Object.assign(processStub, { exitCode: code });
+  stdout.end();
+  stderr.end();
+  setImmediate(() => {
+    processStub.emit('exit', code, signal);
+    processStub.emit('close', code, signal);
+  });
+}