engrm 0.1.0

package/src/capture/scrubber.ts

@@ -0,0 +1,181 @@
+/**
+ * Secret scrubbing pipeline.
+ *
+ * Runs before any observation is saved (even to local SQLite).
+ * Patterns from SPEC §6.
+ *
+ * Pattern definitions are stored as source/flags (not RegExp instances)
+ * to avoid shared mutable state from global regex lastIndex.
+ */
+
+export interface ScrubPatternDef {
+  source: string;
+  flags: string;
+  replacement: string;
+  description: string;
+  category: "api_key" | "token" | "password" | "db_url" | "custom";
+  severity: "critical" | "high" | "medium" | "low";
+}
+
+export const DEFAULT_PATTERNS: ScrubPatternDef[] = [
+  {
+    source: "sk-[a-zA-Z0-9]{20,}",
+    flags: "g",
+    replacement: "[REDACTED_API_KEY]",
+    description: "OpenAI API keys",
+    category: "api_key",
+    severity: "critical",
+  },
+  {
+    source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
+    flags: "g",
+    replacement: "[REDACTED_BEARER]",
+    description: "Bearer auth tokens",
+    category: "token",
+    severity: "medium",
+  },
+  {
+    source: "password[=:]\\s*\\S+",
+    flags: "gi",
+    replacement: "password=[REDACTED]",
+    description: "Passwords in config",
+    category: "password",
+    severity: "high",
+  },
+  {
+    source: "postgresql://[^\\s]+",
+    flags: "g",
+    replacement: "[REDACTED_DB_URL]",
+    description: "PostgreSQL connection strings",
+    category: "db_url",
+    severity: "high",
+  },
+  {
+    source: "mongodb://[^\\s]+",
+    flags: "g",
+    replacement: "[REDACTED_DB_URL]",
+    description: "MongoDB connection strings",
+    category: "db_url",
+    severity: "high",
+  },
+  {
+    source: "mysql://[^\\s]+",
+    flags: "g",
+    replacement: "[REDACTED_DB_URL]",
+    description: "MySQL connection strings",
+    category: "db_url",
+    severity: "high",
+  },
+  {
+    source: "AKIA[A-Z0-9]{16}",
+    flags: "g",
+    replacement: "[REDACTED_AWS_KEY]",
+    description: "AWS access keys",
+    category: "api_key",
+    severity: "critical",
+  },
+  {
+    source: "ghp_[a-zA-Z0-9]{36}",
+    flags: "g",
+    replacement: "[REDACTED_GH_TOKEN]",
+    description: "GitHub personal access tokens",
+    category: "token",
+    severity: "high",
+  },
+  {
+    source: "gho_[a-zA-Z0-9]{36}",
+    flags: "g",
+    replacement: "[REDACTED_GH_TOKEN]",
+    description: "GitHub OAuth tokens",
+    category: "token",
+    severity: "high",
+  },
+  {
+    source: "github_pat_[a-zA-Z0-9_]{22,}",
+    flags: "g",
+    replacement: "[REDACTED_GH_TOKEN]",
+    description: "GitHub fine-grained PATs",
+    category: "token",
+    severity: "high",
+  },
+  {
+    source: "cvk_[a-f0-9]{64}",
+    flags: "g",
+    replacement: "[REDACTED_CANDENGO_KEY]",
+    description: "Candengo API keys",
+    category: "api_key",
+    severity: "critical",
+  },
+  {
+    source: "xox[bpras]-[a-zA-Z0-9\\-]+",
+    flags: "g",
+    replacement: "[REDACTED_SLACK_TOKEN]",
+    description: "Slack tokens",
+    category: "token",
+    severity: "high",
+  },
+];
+
+/**
+ * Compile custom patterns from config strings into pattern definitions.
+ * Each string is treated as a regex pattern with global flag.
+ */
+function compileCustomPatterns(patterns: string[]): ScrubPatternDef[] {
+  const compiled: ScrubPatternDef[] = [];
+  for (const pattern of patterns) {
+    try {
+      // Validate the regex is parseable
+      new RegExp(pattern);
+      compiled.push({
+        source: pattern,
+        flags: "g",
+        replacement: "[REDACTED_CUSTOM]",
+        description: `Custom pattern: ${pattern}`,
+        category: "custom",
+        severity: "medium",
+      });
+    } catch {
+      // Skip invalid regex patterns — don't crash the scrubber
+    }
+  }
+  return compiled;
+}
+
+/**
+ * Scrub sensitive content from text.
+ * Returns the scrubbed text.
+ */
+export function scrubSecrets(
+  text: string,
+  customPatterns: string[] = []
+): string {
+  let result = text;
+  const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
+
+  for (const pattern of allPatterns) {
+    // Fresh RegExp per call — no shared mutable lastIndex state
+    result = result.replace(
+      new RegExp(pattern.source, pattern.flags),
+      pattern.replacement
+    );
+  }
+
+  return result;
+}
+
+/**
+ * Check if text contains any secrets that would be scrubbed.
+ * Useful for sensitivity classification.
+ */
+export function containsSecrets(
+  text: string,
+  customPatterns: string[] = []
+): boolean {
+  const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
+
+  for (const pattern of allPatterns) {
+    if (new RegExp(pattern.source, pattern.flags).test(text)) return true;
+  }
+
+  return false;
+}

package/src/cli.ts

@@ -0,0 +1,517 @@
+#!/usr/bin/env bun
+/**
+ * Engrm CLI.
+ *
+ * Commands:
+ *   init                — Browser OAuth setup (default)
+ *   init --token=cmt_x  — Setup from provisioning token
+ *   init --no-browser   — Device code flow (headless/SSH)
+ *   init --manual       — Interactive manual setup
+ *   init --config <f>   — Non-interactive setup from a JSON file
+ *   status              — Show current config and database stats
+ */
+
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { hostname, homedir } from "node:os";
+import { join } from "node:path";
+import { randomBytes } from "node:crypto";
+import {
+  loadConfig,
+  saveConfig,
+  configExists,
+  getConfigDir,
+  getSettingsPath,
+  getDbPath,
+  type Config,
+} from "./config.js";
+import { MemDatabase } from "./storage/sqlite.js";
+import { getOutboxStats } from "./storage/outbox.js";
+import {
+  provision,
+  ProvisionError,
+  DEFAULT_CANDENGO_URL,
+  type ProvisionResponse,
+} from "./provisioning/provision.js";
+import { runBrowserAuth } from "./provisioning/browser-auth.js";
+import { registerAll } from "./register.js";
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+switch (command) {
+  case "init":
+    await handleInit(args.slice(1));
+    break;
+  case "status":
+    handleStatus();
+    break;
+  default:
+    printUsage();
+    break;
+}
+
+// --- Init ---
+
+async function handleInit(flags: string[]): Promise<void> {
+  // --token=cmt_xxx or --token cmt_xxx
+  const tokenFlag = flags.find((f) => f.startsWith("--token"));
+  if (tokenFlag) {
+    let token: string;
+    if (tokenFlag.includes("=")) {
+      token = tokenFlag.split("=")[1]!;
+    } else {
+      const idx = flags.indexOf("--token");
+      token = flags[idx + 1] ?? "";
+    }
+    if (!token || !token.startsWith("cmt_")) {
+      console.error("Error: --token requires a cmt_ provisioning token");
+      process.exit(1);
+    }
+    const url = extractUrlFlag(flags) ?? DEFAULT_CANDENGO_URL;
+    await initWithToken(url, token);
+    return;
+  }
+
+  // --config <path>
+  if (flags.includes("--config")) {
+    const configIndex = flags.indexOf("--config");
+    const configPath = flags[configIndex + 1];
+    if (!configPath) {
+      console.error("Error: --config requires a file path");
+      process.exit(1);
+    }
+    initFromFile(configPath);
+    return;
+  }
+
+  // --manual
+  if (flags.includes("--manual")) {
+    await initManual();
+    return;
+  }
+
+  // --no-browser (device code flow — placeholder for Phase 4.1b)
+  if (flags.includes("--no-browser")) {
+    console.error("Device code flow is not yet implemented.");
+    console.error("Use: engrm init --token=cmt_xxx");
+    process.exit(1);
+  }
+
+  // Default: browser OAuth flow
+  const url = extractUrlFlag(flags) ?? DEFAULT_CANDENGO_URL;
+  await initWithBrowser(url);
+}
+
+/**
+ * Extract --url flag value from flags array.
+ */
+function extractUrlFlag(flags: string[]): string | undefined {
+  const urlFlag = flags.find((f) => f.startsWith("--url"));
+  if (!urlFlag) return undefined;
+  if (urlFlag.includes("=")) return urlFlag.split("=")[1];
+  const idx = flags.indexOf("--url");
+  return flags[idx + 1];
+}
+
+// --- Flow C: Provisioning token ---
+
+async function initWithToken(baseUrl: string, token: string): Promise<void> {
+  if (configExists()) {
+    console.log("Existing configuration found. Overwriting...\n");
+  }
+
+  console.log("Exchanging provisioning token...");
+
+  try {
+    const result = await provision(baseUrl, {
+      token,
+      device_name: hostname(),
+    });
+    writeConfigFromProvision(baseUrl, result);
+    console.log(`\nConnected as ${result.user_email}`);
+    printPostInit();
+  } catch (error) {
+    if (error instanceof ProvisionError) {
+      console.error(`\nProvisioning failed: ${error.detail}`);
+      process.exit(1);
+    }
+    throw error;
+  }
+}
+
+// --- Flow A: Browser OAuth ---
+
+async function initWithBrowser(baseUrl: string): Promise<void> {
+  if (configExists()) {
+    console.log("Existing configuration found. Overwriting...\n");
+  }
+
+  try {
+    const { code } = await runBrowserAuth(baseUrl);
+
+    console.log("Exchanging authorization code...");
+    const result = await provision(baseUrl, {
+      code,
+      device_name: hostname(),
+    });
+    writeConfigFromProvision(baseUrl, result);
+    console.log(`\nConnected as ${result.user_email}`);
+    printPostInit();
+  } catch (error) {
+    if (error instanceof ProvisionError) {
+      console.error(`\nProvisioning failed: ${error.detail}`);
+      process.exit(1);
+    }
+    console.error(
+      `\nAuthorization failed: ${error instanceof Error ? error.message : String(error)}`
+    );
+    console.error("Try: engrm init --token=cmt_xxx");
+    process.exit(1);
+  }
+}
+
+// --- Shared: write config from provision response ---
+
+function writeConfigFromProvision(
+  baseUrl: string,
+  result: ProvisionResponse
+): void {
+  ensureConfigDir();
+
+  const config: Config = {
+    candengo_url: baseUrl,
+    candengo_api_key: result.api_key,
+    site_id: result.site_id,
+    namespace: result.namespace,
+    user_id: result.user_id,
+    user_email: result.user_email,
+    device_id: generateDeviceId(),
+    teams: result.teams ?? [],
+    sync: {
+      enabled: true,
+      interval_seconds: 30,
+      batch_size: 50,
+    },
+    search: {
+      default_limit: 10,
+      local_boost: 1.2,
+      scope: "all",
+    },
+    scrubbing: {
+      enabled: true,
+      custom_patterns: [],
+      default_sensitivity: "shared",
+    },
+  };
+
+  saveConfig(config);
+
+  // Initialise database
+  const db = new MemDatabase(getDbPath());
+  db.close();
+
+  console.log(`Configuration saved to ${getSettingsPath()}`);
+  console.log(`Database initialised at ${getDbPath()}`);
+}
+
+// --- Flow D: Manual ---
+
+function initFromFile(configPath: string): void {
+  if (!existsSync(configPath)) {
+    console.error(`Config file not found: ${configPath}`);
+    process.exit(1);
+  }
+
+  let parsed: unknown;
+  try {
+    const raw = readFileSync(configPath, "utf-8");
+    parsed = JSON.parse(raw);
+  } catch {
+    console.error(`Invalid JSON in ${configPath}`);
+    process.exit(1);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    console.error("Config file must contain a JSON object");
+    process.exit(1);
+  }
+
+  const input = parsed as Record<string, unknown>;
+
+  const required = [
+    "candengo_url",
+    "candengo_api_key",
+    "site_id",
+    "namespace",
+    "user_id",
+  ];
+  for (const field of required) {
+    if (typeof input[field] !== "string" || !(input[field] as string).trim()) {
+      console.error(`Missing required field: ${field}`);
+      process.exit(1);
+    }
+  }
+
+  ensureConfigDir();
+
+  const config: Config = {
+    candengo_url: (input["candengo_url"] as string).trim(),
+    candengo_api_key: (input["candengo_api_key"] as string).trim(),
+    site_id: (input["site_id"] as string).trim(),
+    namespace: (input["namespace"] as string).trim(),
+    user_id: (input["user_id"] as string).trim(),
+    user_email:
+      typeof input["user_email"] === "string"
+        ? (input["user_email"] as string).trim()
+        : "",
+    device_id:
+      typeof input["device_id"] === "string"
+        ? input["device_id"]
+        : generateDeviceId(),
+    teams: [],
+    sync: {
+      enabled: true,
+      interval_seconds: 30,
+      batch_size: 50,
+    },
+    search: {
+      default_limit: 10,
+      local_boost: 1.2,
+      scope: "all",
+    },
+    scrubbing: {
+      enabled: true,
+      custom_patterns: [],
+      default_sensitivity: "shared",
+    },
+  };
+
+  saveConfig(config);
+
+  const db = new MemDatabase(getDbPath());
+  db.close();
+
+  console.log(`Configuration saved to ${getSettingsPath()}`);
+  console.log(`Database initialised at ${getDbPath()}`);
+  printPostInit();
+}
+
+async function initManual(): Promise<void> {
+  const prompt = createPrompter();
+
+  console.log("Engrm — Interactive Setup\n");
+
+  if (configExists()) {
+    const overwrite = await prompt(
+      "Config already exists. Overwrite? [y/N]: "
+    );
+    if (overwrite.toLowerCase() !== "y") {
+      console.log("Aborted.");
+      return;
+    }
+  }
+
+  const candengoUrl = await prompt(
+    "Candengo Vector URL (e.g. https://www.candengo.com): "
+  );
+  const apiKey = await prompt("API key (cvk_...): ");
+  const siteId = await prompt("Site ID: ");
+  const namespace = await prompt("Namespace: ");
+  const userId = await prompt("User ID: ");
+  const userEmail = await prompt("Email (optional): ");
+
+  if (!candengoUrl || !apiKey || !siteId || !namespace || !userId) {
+    console.error("All fields (except email) are required.");
+    process.exit(1);
+  }
+
+  ensureConfigDir();
+
+  const config: Config = {
+    candengo_url: candengoUrl.trim(),
+    candengo_api_key: apiKey.trim(),
+    site_id: siteId.trim(),
+    namespace: namespace.trim(),
+    user_id: userId.trim(),
+    user_email: userEmail.trim(),
+    device_id: generateDeviceId(),
+    teams: [],
+    sync: {
+      enabled: true,
+      interval_seconds: 30,
+      batch_size: 50,
+    },
+    search: {
+      default_limit: 10,
+      local_boost: 1.2,
+      scope: "all",
+    },
+    scrubbing: {
+      enabled: true,
+      custom_patterns: [],
+      default_sensitivity: "shared",
+    },
+  };
+
+  saveConfig(config);
+
+  const db = new MemDatabase(getDbPath());
+  db.close();
+
+  console.log(`\nConfiguration saved to ${getSettingsPath()}`);
+  console.log(`Database initialised at ${getDbPath()}`);
+  printPostInit();
+}
+
+// --- Status ---
+
+function handleStatus(): void {
+  if (!configExists()) {
+    console.log("Engrm is not configured.");
+    console.log("Run: engrm init");
+    return;
+  }
+
+  const config = loadConfig();
+  console.log("Engrm Status\n");
+  console.log(`  User:        ${config.user_id}`);
+  if (config.user_email) {
+    console.log(`  Email:       ${config.user_email}`);
+  }
+  console.log(`  Device:      ${config.device_id}`);
+  console.log(`  Candengo:    ${config.candengo_url || "(not set)"}`);
+  console.log(`  Site:        ${config.site_id}`);
+  console.log(`  Namespace:   ${config.namespace}`);
+  if (config.teams.length > 0) {
+    console.log(
+      `  Teams:       ${config.teams.map((t) => t.name).join(", ")}`
+    );
+  }
+  console.log(
+    `  Sync:        ${config.sync.enabled ? "enabled" : "disabled"}`
+  );
+  console.log(`  Config:      ${getSettingsPath()}`);
+  console.log(`  Database:    ${getDbPath()}`);
+
+  // Check Claude Code registration
+  const claudeJson = join(homedir(), ".claude.json");
+  const claudeSettings = join(homedir(), ".claude", "settings.json");
+  const mcpRegistered = existsSync(claudeJson) && readFileSync(claudeJson, "utf-8").includes('"engrm"');
+  const hooksRegistered = existsSync(claudeSettings) && readFileSync(claudeSettings, "utf-8").includes("engrm");
+  console.log(`  MCP server:  ${mcpRegistered ? "registered" : "not registered"}`);
+  console.log(`  Hooks:       ${hooksRegistered ? "registered" : "not registered"}`);
+
+  if (existsSync(getDbPath())) {
+    try {
+      const db = new MemDatabase(getDbPath());
+      const obsCount = db.getActiveObservationCount();
+      const outbox = getOutboxStats(db);
+
+      // Session summaries count
+      const summaryCount = db.db
+        .query<{ count: number }, []>(
+          "SELECT COUNT(*) as count FROM session_summaries"
+        )
+        .get()?.count ?? 0;
+
+      console.log(`\n  Active observations: ${obsCount}`);
+      console.log(`  Session summaries:   ${summaryCount}`);
+      console.log(
+        `  Outbox: ${outbox["pending"] ?? 0} pending, ${outbox["failed"] ?? 0} failed, ${outbox["synced"] ?? 0} synced`
+      );
+
+      // Security findings (try — table may not exist on old schemas)
+      try {
+        const findings = db.db
+          .query<{ severity: string; count: number }, []>(
+            "SELECT severity, COUNT(*) as count FROM security_findings GROUP BY severity"
+          )
+          .all();
+        if (findings.length > 0) {
+          const bySeverity = Object.fromEntries(findings.map((f) => [f.severity, f.count]));
+          const parts: string[] = [];
+          if (bySeverity["critical"]) parts.push(`${bySeverity["critical"]} critical`);
+          if (bySeverity["high"]) parts.push(`${bySeverity["high"]} high`);
+          if (bySeverity["medium"]) parts.push(`${bySeverity["medium"]} medium`);
+          if (bySeverity["low"]) parts.push(`${bySeverity["low"]} low`);
+          console.log(`  Security findings:   ${parts.join(", ")}`);
+        }
+      } catch {
+        // security_findings table may not exist yet
+      }
+
+      db.close();
+    } catch (error) {
+      console.log(
+        `\n  Database error: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+}
+
+// --- Helpers ---
+
+function ensureConfigDir(): void {
+  const dir = getConfigDir();
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+function generateDeviceId(): string {
+  const host = hostname().toLowerCase().replace(/[^a-z0-9-]/g, "");
+  const suffix = randomBytes(4).toString("hex");
+  return `${host}-${suffix}`;
+}
+
+function printPostInit(): void {
+  console.log("\nRegistering with Claude Code...");
+
+  try {
+    const result = registerAll();
+    console.log(`  MCP server registered → ${result.mcp.path}`);
+    console.log(`  Hooks registered → ${result.hooks.path}`);
+    console.log("\nEngrm is ready! Start a new Claude Code session to use memory.");
+  } catch (error) {
+    // Registration failed — fall back to manual instructions
+    console.log("\nCould not auto-register with Claude Code.");
+    console.log(`Error: ${error instanceof Error ? error.message : String(error)}`);
+    console.log("\nManual setup — add to ~/.claude.json:");
+    console.log(`
+{
+  "mcpServers": {
+    "engrm": {
+      "type": "stdio",
+      "command": "bun",
+      "args": ["run", "${process.cwd()}/src/server.ts"]
+    }
+  }
+}`);
+  }
+}
+
+function printUsage(): void {
+  console.log("Engrm — Memory layer for AI coding agents\n");
+  console.log("Usage:");
+  console.log("  engrm init                  Setup via browser (recommended)");
+  console.log("  engrm init --token=cmt_xxx  Setup from provisioning token");
+  console.log("  engrm init --no-browser     Setup via device code (SSH/headless)");
+  console.log("  engrm init --manual         Manual setup (enter all values)");
+  console.log("  engrm init --config <file>  Setup from JSON file");
+  console.log("  engrm status                Show status");
+}
+
+/**
+ * Simple line-based prompter using Bun's stdin.
+ */
+function createPrompter(): (question: string) => Promise<string> {
+  const decoder = new TextDecoder();
+  return async (question: string): Promise<string> => {
+    process.stdout.write(question);
+    for await (const chunk of Bun.stdin.stream()) {
+      const line = decoder.decode(chunk).trim();
+      return line;
+    }
+    return "";
+  };
+}