@indigoai-us/hq-cli 5.60.0 → 5.62.0

package/src/lib/db/migrate.test.ts

@@ -0,0 +1,133 @@
+/**
+ * Migration apply/idempotent/failure tests (vault-databases US-005).
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { openLocalDb } from "./local.js";
+import { migrateLocalDb, resolveMigrationsDir } from "./migrate.js";
+
+const tempRoots: string[] = [];
+
+function makeFixture(): { home: string; hqRoot: string } {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-db-migrate-"));
+  tempRoots.push(root);
+  const home = path.join(root, "home");
+  const hqRoot = path.join(root, "hq");
+  fs.mkdirSync(home, { recursive: true });
+  fs.mkdirSync(hqRoot, { recursive: true });
+  return { home, hqRoot };
+}
+
+function writeMigration(
+  hqRoot: string,
+  company: string,
+  name: string,
+  sql: string,
+): void {
+  const dir = resolveMigrationsDir(hqRoot, company);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(path.join(dir, name), sql, "utf8");
+}
+
+afterEach(() => {
+  while (tempRoots.length > 0) {
+    const dir = tempRoots.pop();
+    if (dir) fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe("migrateLocalDb", () => {
+  it("applies one migration and records ledger", () => {
+    const { home, hqRoot } = makeFixture();
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "001_create_t.sql",
+      "CREATE TABLE t (id INTEGER PRIMARY KEY);",
+    );
+
+    const result = migrateLocalDb({ company: "indigo", home, hqRoot });
+    expect(result.applied).toEqual(["001_create_t.sql"]);
+    expect(result.head).toBe("001_create_t");
+
+    const db = openLocalDb("indigo", { home });
+    const row = db
+      .prepare(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='t'",
+      )
+      .get() as { name: string };
+    expect(row.name).toBe("t");
+    db.close();
+  });
+
+  it("applies two migrations in order", () => {
+    const { home, hqRoot } = makeFixture();
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "001_a.sql",
+      "CREATE TABLE a (id INTEGER PRIMARY KEY);",
+    );
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "002_b.sql",
+      "CREATE TABLE b (id INTEGER PRIMARY KEY);",
+    );
+
+    const result = migrateLocalDb({ company: "indigo", home, hqRoot });
+    expect(result.applied).toEqual(["001_a.sql", "002_b.sql"]);
+    expect(result.head).toBe("002_b");
+  });
+
+  it("re-run is idempotent", () => {
+    const { home, hqRoot } = makeFixture();
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "001_t.sql",
+      "CREATE TABLE t (id INTEGER PRIMARY KEY);",
+    );
+
+    migrateLocalDb({ company: "indigo", home, hqRoot });
+    const second = migrateLocalDb({ company: "indigo", home, hqRoot });
+    expect(second.applied).toEqual([]);
+    expect(second.skipped).toContain("001_t.sql");
+    expect(second.head).toBe("001_t");
+  });
+
+  it("failed migration is not marked applied", () => {
+    const { home, hqRoot } = makeFixture();
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "001_ok.sql",
+      "CREATE TABLE ok (id INTEGER PRIMARY KEY);",
+    );
+    writeMigration(
+      hqRoot,
+      "indigo",
+      "002_bad.sql",
+      "CREATE TABLE ok (id INTEGER PRIMARY KEY);", // duplicate → fail
+    );
+
+    expect(() =>
+      migrateLocalDb({ company: "indigo", home, hqRoot }),
+    ).toThrow(/002_bad\.sql/);
+
+    const db = openLocalDb("indigo", { home });
+    const versions = db
+      .prepare("SELECT version FROM hq_schema_migrations ORDER BY version")
+      .all() as { version: string }[];
+    expect(versions.map((v) => v.version)).toEqual(["001_ok"]);
+    expect(
+      versions.map((v) => v.version).includes("002_bad"),
+    ).toBe(false);
+    db.close();
+  });
+});

package/src/lib/db/migrate.ts

@@ -0,0 +1,137 @@
+/**
+ * Vault text migrations for local SQLite (vault-databases US-005).
+ *
+ * Migrations live as reviewable text under:
+ *   companies/{company}/db/migrations/*.sql
+ * Applied versions are recorded in hq_schema_migrations inside the local DB.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+import type { LocalDb } from "./local.js";
+import { openLocalDb, readMigrationHead } from "./local.js";
+import type { LocalDbPathEnv } from "./paths.js";
+import { normalizeCompanySlugForLocalDb } from "./paths.js";
+
+export const MIGRATIONS_TABLE = "hq_schema_migrations";
+
+export interface MigrateOptions extends LocalDbPathEnv {
+  company: string;
+  /**
+   * Absolute path to HQ root containing companies/{co}/db/migrations.
+   * Required for vault-relative migrations.
+   */
+  hqRoot: string;
+}
+
+export interface MigrateResult {
+  company: string;
+  migrationsDir: string;
+  applied: string[];
+  skipped: string[];
+  head: string | null;
+}
+
+export function resolveMigrationsDir(hqRoot: string, companySlug: string): string {
+  const slug = normalizeCompanySlugForLocalDb(companySlug);
+  return path.join(hqRoot, "companies", slug, "db", "migrations");
+}
+
+/**
+ * Ensure migrations directory exists (creates empty dir on first migrate).
+ */
+export function ensureMigrationsDir(hqRoot: string, companySlug: string): string {
+  const dir = resolveMigrationsDir(hqRoot, companySlug);
+  fs.mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+/**
+ * List pending migration files in lexical/version order (filename sort).
+ */
+export function listMigrationFiles(migrationsDir: string): string[] {
+  if (!fs.existsSync(migrationsDir)) return [];
+  return fs
+    .readdirSync(migrationsDir)
+    .filter((f) => f.endsWith(".sql"))
+    .sort((a, b) => a.localeCompare(b, "en"));
+}
+
+export function ensureMigrationsLedger(db: LocalDb): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS ${MIGRATIONS_TABLE} (
+      version TEXT PRIMARY KEY NOT NULL,
+      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
+}
+
+export function listAppliedVersions(db: LocalDb): Set<string> {
+  ensureMigrationsLedger(db);
+  const rows = db
+    .prepare(`SELECT version FROM ${MIGRATIONS_TABLE} ORDER BY version`)
+    .all() as { version: string }[];
+  return new Set(rows.map((r) => r.version));
+}
+
+/**
+ * Apply pending migrations. On failure, does not mark the failed file applied.
+ */
+export function migrateLocalDb(opts: MigrateOptions): MigrateResult {
+  const company = normalizeCompanySlugForLocalDb(opts.company);
+  if (!opts.hqRoot?.trim()) {
+    throw new Error("hqRoot is required for vault migrations");
+  }
+
+  const migrationsDir = ensureMigrationsDir(opts.hqRoot, company);
+  const files = listMigrationFiles(migrationsDir);
+
+  const db = openLocalDb(company, opts);
+  const applied: string[] = [];
+  const skipped: string[] = [];
+
+  try {
+    ensureMigrationsLedger(db);
+    const already = listAppliedVersions(db);
+
+    for (const file of files) {
+      const version = file.replace(/\.sql$/i, "");
+      if (already.has(version) || already.has(file)) {
+        skipped.push(file);
+        continue;
+      }
+
+      const fullPath = path.join(migrationsDir, file);
+      const sql = fs.readFileSync(fullPath, "utf8");
+
+      const run = db.transaction(() => {
+        db.exec(sql);
+        db.prepare(
+          `INSERT INTO ${MIGRATIONS_TABLE} (version) VALUES (?)`,
+        ).run(version);
+      });
+
+      try {
+        run();
+        applied.push(file);
+        already.add(version);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `migration failed: ${file} — ${msg} (not marked applied)`,
+        );
+      }
+    }
+
+    return {
+      company,
+      migrationsDir,
+      applied,
+      skipped,
+      head: readMigrationHead(db),
+    };
+  } finally {
+    db.close();
+  }
+}

package/src/lib/db/paths.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Unit tests for local vault DB path helpers (vault-databases US-002).
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  LOCAL_DB_IGNORE_PATTERNS,
+  LOCAL_VAULT_DB_FILENAME,
+  describeLocalDbIgnoreRules,
+  ensureLocalDbDir,
+  normalizeCompanySlugForLocalDb,
+  resolveLocalDbDir,
+  resolveLocalDbPath,
+  resolveLocalDbRoot,
+} from "./paths.js";
+
+const tempHomes: string[] = [];
+
+function makeTempHome(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-db-paths-"));
+  tempHomes.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  while (tempHomes.length > 0) {
+    const dir = tempHomes.pop();
+    if (dir) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  }
+});
+
+describe("resolveLocalDbPath", () => {
+  it("resolves indigo under documented root ending with vault.db", () => {
+    const home = makeTempHome();
+    const p = resolveLocalDbPath("indigo", { home });
+    expect(p).toBe(path.join(home, ".hq", "db", "indigo", LOCAL_VAULT_DB_FILENAME));
+    expect(p.endsWith(path.join("indigo", "vault.db"))).toBe(true);
+    expect(p.startsWith(resolveLocalDbRoot({ home }))).toBe(true);
+  });
+
+  it("resolves a second company slug (acme) distinctly", () => {
+    const home = makeTempHome();
+    const indigo = resolveLocalDbPath("indigo", { home });
+    const acme = resolveLocalDbPath("acme", { home });
+    expect(acme).toBe(path.join(home, ".hq", "db", "acme", "vault.db"));
+    expect(indigo).not.toBe(acme);
+    expect(path.dirname(indigo)).not.toBe(path.dirname(acme));
+  });
+
+  it("normalizes slug casing to lowercase", () => {
+    const home = makeTempHome();
+    expect(resolveLocalDbPath("Indigo", { home })).toBe(
+      resolveLocalDbPath("indigo", { home }),
+    );
+  });
+});
+
+describe("normalizeCompanySlugForLocalDb", () => {
+  it("rejects empty and path-traversal slugs", () => {
+    expect(() => normalizeCompanySlugForLocalDb("")).toThrow(/required/i);
+    expect(() => normalizeCompanySlugForLocalDb("../etc")).toThrow(/invalid/i);
+    expect(() => normalizeCompanySlugForLocalDb("a/b")).toThrow(/invalid/i);
+  });
+});
+
+describe("ensureLocalDbDir", () => {
+  it("creates parent directories for a new company slug and is writable", () => {
+    const home = makeTempHome();
+    const dir = ensureLocalDbDir("newco", { home });
+    expect(fs.existsSync(dir)).toBe(true);
+    expect(fs.statSync(dir).isDirectory()).toBe(true);
+    // Writable: can create a probe file then remove it (no secrets left).
+    const probe = path.join(dir, ".write-probe");
+    fs.writeFileSync(probe, "ok", { mode: 0o600 });
+    expect(fs.readFileSync(probe, "utf8")).toBe("ok");
+    fs.unlinkSync(probe);
+    // Does not create the .db file itself.
+    expect(fs.existsSync(resolveLocalDbPath("newco", { home }))).toBe(false);
+  });
+
+  it("is idempotent when the directory already exists", () => {
+    const home = makeTempHome();
+    const first = ensureLocalDbDir("acme", { home });
+    const second = ensureLocalDbDir("acme", { home });
+    expect(first).toBe(second);
+    expect(fs.existsSync(first)).toBe(true);
+  });
+});
+
+describe("resolveLocalDbDir / ignore docs", () => {
+  it("dir is parent of vault.db path", () => {
+    const home = makeTempHome();
+    expect(resolveLocalDbDir("indigo", { home })).toBe(
+      path.dirname(resolveLocalDbPath("indigo", { home })),
+    );
+  });
+
+  it("documents *.db and .data ignore patterns", () => {
+    expect(LOCAL_DB_IGNORE_PATTERNS.some((p) => p.includes("*.db"))).toBe(true);
+    expect(LOCAL_DB_IGNORE_PATTERNS.some((p) => p.includes(".data"))).toBe(true);
+    const docs = describeLocalDbIgnoreRules();
+    expect(docs).toMatch(/\.hq\/db/);
+    expect(docs).toMatch(/\*\.db/);
+  });
+});

package/src/lib/db/paths.ts

@@ -0,0 +1,128 @@
+/**
+ * Vault database path conventions (C1 local tier).
+ *
+ * Canonical layout (locked by vault-databases PRD):
+ *   ~/.hq/db/{companySlug}/vault.db
+ *
+ * Binary DB files are machine-local. They must never live under companies/
+ * so hq-sync / git cannot treat them as vault text. Migrations remain vault
+ * text at companies/{co}/db/migrations/*.sql (handled by migrate stories).
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+/** Default filename for the per-company local vault SQLite file. */
+export const LOCAL_VAULT_DB_FILENAME = "vault.db";
+
+/**
+ * Glob-style ignore patterns for binary / local DB artifacts.
+ * Apply in gitignore and hq-sync ignore configuration for any tree that
+ * might accidentally host these files. Primary storage is under ~/.hq/db/
+ * (outside the vault), but skills must still ignore stray `*.db` / `.data/`.
+ */
+export const LOCAL_DB_IGNORE_PATTERNS: readonly string[] = [
+  "**/*.db",
+  "**/*.db-wal",
+  "**/*.db-shm",
+  "**/*.db-journal",
+  "**/.data/**",
+  // Explicit machine-local root (if a tool ever materializes a relative copy)
+  "**/.hq/db/**",
+];
+
+export interface LocalDbPathEnv {
+  /** Override home directory (tests inject temp dirs). Defaults to os.homedir(). */
+  home?: string;
+}
+
+/**
+ * Resolve the machine-local root that holds all company vault DBs:
+ * `{home}/.hq/db`.
+ */
+export function resolveLocalDbRoot(env?: LocalDbPathEnv): string {
+  const home = (env?.home ?? os.homedir()).trim();
+  if (!home) {
+    throw new Error(
+      "cannot resolve local DB root: home directory is empty (set home or os.homedir())",
+    );
+  }
+  return path.join(home, ".hq", "db");
+}
+
+/**
+ * Normalize and validate a company slug used in the local path.
+ * Tenant isolation still requires HQ identity/membership at the command layer;
+ * this only rejects empty / path-traversal slugs so we never open arbitrary paths.
+ */
+export function normalizeCompanySlugForLocalDb(companySlug: string): string {
+  const slug = companySlug.trim().toLowerCase();
+  if (!slug) {
+    throw new Error("company slug is required for local DB path resolution");
+  }
+  if (
+    slug.includes("/") ||
+    slug.includes("\\") ||
+    slug.includes("..") ||
+    slug === "." ||
+    slug.includes("\0")
+  ) {
+    throw new Error(
+      `invalid company slug for local DB path: ${JSON.stringify(companySlug)}`,
+    );
+  }
+  return slug;
+}
+
+/**
+ * Canonical path for a company's local vault SQLite file:
+ * `~/.hq/db/{companySlug}/vault.db`.
+ */
+export function resolveLocalDbPath(
+  companySlug: string,
+  env?: LocalDbPathEnv,
+): string {
+  const slug = normalizeCompanySlugForLocalDb(companySlug);
+  return path.join(resolveLocalDbRoot(env), slug, LOCAL_VAULT_DB_FILENAME);
+}
+
+/**
+ * Directory that contains the company's vault.db (and WAL side files).
+ */
+export function resolveLocalDbDir(
+  companySlug: string,
+  env?: LocalDbPathEnv,
+): string {
+  return path.dirname(resolveLocalDbPath(companySlug, env));
+}
+
+/**
+ * Ensure the company local DB directory exists and is writable.
+ * Creates parent directories recursively. Does not create the .db file
+ * and does not write secrets into the tree.
+ *
+ * @returns absolute path of the directory
+ */
+export function ensureLocalDbDir(
+  companySlug: string,
+  env?: LocalDbPathEnv,
+): string {
+  const dir = resolveLocalDbDir(companySlug, env);
+  fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+  // Verify writable without leaving secrets or DB content behind.
+  fs.accessSync(dir, fs.constants.W_OK);
+  return dir;
+}
+
+/**
+ * Human-readable documentation of ignore rules for operators and agents.
+ */
+export function describeLocalDbIgnoreRules(): string {
+  return [
+    "Local vault databases live at ~/.hq/db/{company}/vault.db (outside the vault tree).",
+    "Never place *.db under companies/ — binary DB state is not vault-synced.",
+    "Ignore patterns:",
+    ...LOCAL_DB_IGNORE_PATTERNS.map((p) => `  - ${p}`),
+  ].join("\n");
+}

package/src/lib/db/remote-engine.test.ts

@@ -0,0 +1,44 @@
+/**
+ * Remote engine registry + mock (vault-databases US-007).
+ */
+
+import { beforeEach, describe, expect, it } from "vitest";
+
+import {
+  _resetRemoteEngineRegistryForTests,
+  createMockRemoteEngine,
+  getRemoteEngine,
+  registerRemoteEngine,
+} from "./remote-engine.js";
+
+beforeEach(() => {
+  _resetRemoteEngineRegistryForTests();
+});
+
+describe("RemoteDbEngine mock", () => {
+  it("provision and healthCheck work without AWS", async () => {
+    const mock = createMockRemoteEngine("mock");
+    registerRemoteEngine(mock);
+
+    const engine = getRemoteEngine("mock");
+    const first = await engine.provision({
+      companyUid: "cmp_x",
+      companySlug: "x",
+    });
+    expect(first.status).toBe("ready");
+    expect(first.resourceArn).toContain("cmp_x");
+
+    const second = await engine.provision({
+      companyUid: "cmp_x",
+      companySlug: "x",
+    });
+    expect(second.status).toBe("existing");
+    expect(second.resourceArn).toBe(first.resourceArn);
+
+    const health = await engine.healthCheck(first.resourceArn);
+    expect(health.healthy).toBe(true);
+
+    // Secret payload must not be a printable postgres URL in mock either
+    expect(JSON.stringify(first.connectionSecret)).not.toMatch(/postgres:\/\//i);
+  });
+});

package/src/lib/db/remote-engine.ts

@@ -0,0 +1,148 @@
+/**
+ * Pluggable remote vault DB engine interface (vault-databases US-007).
+ *
+ * Default shipped adapter id: aurora-dsql.
+ * This module is interface + registry only — no production AWS provision here.
+ */
+
+/** Connection material written only to secrets stores — never CLI stdout. */
+export interface RemoteConnectionSecretPayload {
+  /** Opaque secret body (e.g. JSON with host/user/password or IAM token recipe). */
+  payload: Record<string, unknown>;
+  /** Suggested vault/Secrets Manager key suffix (namespaced by company elsewhere). */
+  secretNameHint: string;
+}
+
+export interface RemoteProvisionRequest {
+  companyUid: string;
+  companySlug: string;
+  region?: string;
+  /** Engine-specific options (cluster params, etc.). */
+  options?: Record<string, unknown>;
+}
+
+export interface RemoteProvisionResult {
+  engineId: string;
+  resourceArn: string;
+  region: string;
+  status: "ready" | "provisioning" | "failed" | "existing";
+  /** Secret payload to store — callers must not print. */
+  connectionSecret: RemoteConnectionSecretPayload;
+}
+
+export interface RemoteHealth {
+  engineId: string;
+  resourceArn: string;
+  healthy: boolean;
+  status: string;
+  checkedAt: string;
+}
+
+/**
+ * Remote Postgres-class engine adapter.
+ * Implementations live in control plane (hq-pro) for real AWS; CLI may mock.
+ */
+export interface RemoteDbEngine {
+  readonly id: string;
+  provision(req: RemoteProvisionRequest): Promise<RemoteProvisionResult>;
+  deprovision(companyUid: string, resourceArn: string): Promise<void>;
+  connectionSecretPayload(
+    companyUid: string,
+    resourceArn: string,
+  ): Promise<RemoteConnectionSecretPayload>;
+  healthCheck(resourceArn: string): Promise<RemoteHealth>;
+}
+
+/** Default engine selection (PRD decision). */
+export const DEFAULT_REMOTE_ENGINE_ID = "aurora-dsql";
+
+const registry = new Map<string, RemoteDbEngine>();
+
+export function registerRemoteEngine(engine: RemoteDbEngine): void {
+  registry.set(engine.id, engine);
+}
+
+export function getRemoteEngine(id: string = DEFAULT_REMOTE_ENGINE_ID): RemoteDbEngine {
+  const engine = registry.get(id);
+  if (!engine) {
+    throw new Error(
+      `remote DB engine not registered: ${id} (available: ${[...registry.keys()].join(", ") || "none"})`,
+    );
+  }
+  return engine;
+}
+
+export function listRemoteEngineIds(): string[] {
+  return [...registry.keys()];
+}
+
+/** Clear registry (tests only). */
+export function _resetRemoteEngineRegistryForTests(): void {
+  registry.clear();
+}
+
+/**
+ * In-memory mock engine for unit tests — no AWS.
+ */
+export function createMockRemoteEngine(
+  id = "mock",
+): RemoteDbEngine {
+  const resources = new Map<string, { companyUid: string; region: string }>();
+
+  return {
+    id,
+    async provision(req) {
+      const existing = [...resources.entries()].find(
+        ([, v]) => v.companyUid === req.companyUid,
+      );
+      if (existing) {
+        const [arn, meta] = existing;
+        return {
+          engineId: id,
+          resourceArn: arn,
+          region: meta.region,
+          status: "existing",
+          connectionSecret: {
+            payload: { mock: true, companyUid: req.companyUid },
+            secretNameHint: `db/remote/${req.companySlug}`,
+          },
+        };
+      }
+      const arn = `arn:mock:db:${req.companyUid}`;
+      const region = req.region ?? "us-east-1";
+      resources.set(arn, { companyUid: req.companyUid, region });
+      return {
+        engineId: id,
+        resourceArn: arn,
+        region,
+        status: "ready",
+        connectionSecret: {
+          payload: { mock: true, companyUid: req.companyUid },
+          secretNameHint: `db/remote/${req.companySlug}`,
+        },
+      };
+    },
+    async deprovision(companyUid, resourceArn) {
+      const meta = resources.get(resourceArn);
+      if (meta && meta.companyUid === companyUid) {
+        resources.delete(resourceArn);
+      }
+    },
+    async connectionSecretPayload(companyUid, resourceArn) {
+      return {
+        payload: { mock: true, companyUid, resourceArn },
+        secretNameHint: `db/remote/${companyUid}`,
+      };
+    },
+    async healthCheck(resourceArn) {
+      const ok = resources.has(resourceArn);
+      return {
+        engineId: id,
+        resourceArn,
+        healthy: ok,
+        status: ok ? "ready" : "missing",
+        checkedAt: new Date().toISOString(),
+      };
+    },
+  };
+}