fullstackgtm 0.45.0 → 0.46.0

package/src/market.ts

@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 import type { GtmEvidence } from "./types.ts";
 
 /**
@@ -445,12 +446,24 @@ const defaultFetchPage: FetchPage = async (url) => {
 };
 
 export type CaptureOptions = {
-  /** Directory for captures; defaults to <marketHome>/captures. */
+  /** Directory for captures; defaults to <marketHome>/captures. Ignored when `store` is given. */
   dir?: string;
   runLabel?: string;
   /** Injectable for tests; defaults to global fetch. */
   fetchPage?: FetchPage;
   now?: () => Date;
+  /**
+   * Storage seam (see MarketStore below). Defaults to the profile-home file
+   * store; the hosted app passes a Convex-backed store. Same engine, same
+   * evidence chain, different persistence.
+   */
+  store?: MarketStore;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES ("sources" → "capture" →
+   * "persist" here; "classify" belongs to classifyMarket). Presentation-only:
+   * a throwing listener never fails the capture.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type CaptureResult = {
@@ -459,57 +472,55 @@ export type CaptureResult = {
 };
 
 export async function captureMarket(config: MarketConfig, options: CaptureOptions = {}): Promise<CaptureResult> {
-  const dir = options.dir ?? join(marketHome(config.category), "captures");
+  const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
+  const progress = options.progress ?? nullProgressEmitter();
   const runLabel = options.runLabel ?? "run-1";
   const fetchPage = options.fetchPage ?? defaultFetchPage;
   const fetchedAt = (options.now ?? (() => new Date()))().toISOString();
-  mkdirSync(dir, { recursive: true });
 
-  const manifestPath = join(dir, "manifest.json");
-  const manifest: CaptureEntry[] = existsSync(manifestPath)
-    ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
-    : [];
+  progress.stage(MARKET_CAPTURE_STAGES[0], 0, MARKET_CAPTURE_STAGES.length);
+  const targets: Array<{ vendorId: string; kind: CaptureEntry["kind"]; url: string }> = [];
+  for (const vendor of config.vendors) {
+    targets.push({ vendorId: vendor.id, kind: "home", url: vendor.urls.home });
+    if (vendor.urls.pricing) targets.push({ vendorId: vendor.id, kind: "pricing", url: vendor.urls.pricing });
+    for (const url of vendor.urls.product) targets.push({ vendorId: vendor.id, kind: "product", url });
+  }
+  progress.note(`${config.vendors.length} vendor(s), ${targets.length} page(s)`);
 
+  progress.stage(MARKET_CAPTURE_STAGES[1], 1, MARKET_CAPTURE_STAGES.length);
   const entries: CaptureEntry[] = [];
-  for (const vendor of config.vendors) {
-    const targets: Array<{ kind: CaptureEntry["kind"]; url: string }> = [
-      { kind: "home", url: vendor.urls.home },
-    ];
-    if (vendor.urls.pricing) targets.push({ kind: "pricing", url: vendor.urls.pricing });
-    for (const url of vendor.urls.product) targets.push({ kind: "product", url });
-
-    for (const target of targets) {
-      let status: number | null = null;
-      let text = "";
-      try {
-        const page = await fetchPage(target.url);
-        status = page.status;
-        if (page.status === 200) text = extractReadableText(page.body);
-      } catch {
-        status = null;
-      }
-      let captureHash: string | null = null;
-      if (text) {
-        captureHash = createHash("sha256").update(text).digest("hex");
-        // Content-addressed: an unchanged page dedupes to the same file.
-        writeFileSync(join(dir, `${captureHash}.txt`), text);
-      }
-      const entry: CaptureEntry = {
-        runLabel,
-        vendorId: vendor.id,
-        kind: target.kind,
-        url: target.url,
-        fetchedAt,
-        httpStatus: status,
-        captureHash,
-        textChars: text.length,
-      };
-      manifest.push(entry);
-      entries.push(entry);
+  for (const target of targets) {
+    let status: number | null = null;
+    let text = "";
+    try {
+      const page = await fetchPage(target.url);
+      status = page.status;
+      if (page.status === 200) text = extractReadableText(page.body);
+    } catch {
+      status = null;
     }
+    let captureHash: string | null = null;
+    if (text) {
+      captureHash = createHash("sha256").update(text).digest("hex");
+      // Content-addressed: an unchanged page dedupes to the same key.
+      await store.saveCaptureText(captureHash, text);
+    }
+    entries.push({
+      runLabel,
+      vendorId: target.vendorId,
+      kind: target.kind,
+      url: target.url,
+      fetchedAt,
+      httpStatus: status,
+      captureHash,
+      textChars: text.length,
+    });
+    progress.items(entries.length, targets.length);
   }
-  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-  return { entries, manifestPath };
+  progress.stage(MARKET_CAPTURE_STAGES[3], 3, MARKET_CAPTURE_STAGES.length);
+  await store.appendCaptureEntries(entries);
+  progress.flush();
+  return { entries, manifestPath: store.captureLocation() };
 }
 
 // ---------------------------------------------------------------------------
@@ -651,6 +662,108 @@ export function loadCaptureTexts(
   return { entries, textByHash };
 }
 
+// ---------------------------------------------------------------------------
+// MarketStore — the storage seam for the whole market workspace (captures +
+// observation runs), mirroring the PlanStore precedent: the profile-home file
+// layout and the hosted app's Convex tables are two implementations of the
+// same contract. Engines never touch storage directly — captureMarket and
+// classifyMarket take a store — and the default file store preserves the
+// on-disk layout exactly, so the CLI's behavior is unchanged.
+//
+// loadConfig is deliberately NOT part of the store: the config is a reviewed
+// input (a versioned file, or a hosted map document) that callers load and
+// hand to the engines. The store owns only what the engines write.
+
+export interface MarketStore {
+  /** Operator-facing location of the capture manifest (file path or logical label). */
+  captureLocation(): string;
+  /** Persist one page's extracted text under its content hash (idempotent). */
+  saveCaptureText(captureHash: string, text: string): Promise<void>;
+  /** Append one capture pass's manifest entries (append-only; never rewrites history). */
+  appendCaptureEntries(entries: CaptureEntry[]): Promise<void>;
+  /** Full manifest + resolvable texts — the evidence-verification input. */
+  loadCaptureTexts(): Promise<{ entries: CaptureEntry[]; textByHash: Map<string, string> }>;
+  /** Append-only observation runs (the same ObservationStore contract as before). */
+  observations: ObservationStore;
+}
+
+/** The CLI's store: captures + observations under the profile market home. */
+export function createFileMarketStore(
+  category: string,
+  options: { capturesDir?: string; observationsDir?: string } = {},
+): MarketStore {
+  const dir = options.capturesDir ?? join(marketHome(category), "captures");
+  const manifestPath = join(dir, "manifest.json");
+  return {
+    captureLocation: () => manifestPath,
+    async saveCaptureText(captureHash, text) {
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, `${captureHash}.txt`), text);
+    },
+    async appendCaptureEntries(entries) {
+      mkdirSync(dir, { recursive: true });
+      const manifest: CaptureEntry[] = existsSync(manifestPath)
+        ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
+        : [];
+      manifest.push(...entries);
+      writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+    },
+    async loadCaptureTexts() {
+      return loadCaptureTexts(category, dir);
+    },
+    observations: createFileObservationStore(category, options.observationsDir),
+  };
+}
+
+/**
+ * In-memory store: seam tests, and throwaway grounding captures (e.g. the
+ * taxonomy proposer's bootstrap pass) that should never persist anywhere.
+ */
+export function createMemoryMarketStore(category: string): MarketStore {
+  const entries: CaptureEntry[] = [];
+  const textByHash = new Map<string, string>();
+  const sets = new Map<string, ObservationSet>();
+  const sorted = () => [...sets.values()].sort((a, b) => a.runAt.localeCompare(b.runAt));
+  return {
+    captureLocation: () => `memory://${category}/captures`,
+    async saveCaptureText(captureHash, text) {
+      textByHash.set(captureHash, text);
+    },
+    async appendCaptureEntries(newEntries) {
+      entries.push(...newEntries);
+    },
+    async loadCaptureTexts() {
+      return { entries: [...entries], textByHash: new Map(textByHash) };
+    },
+    observations: {
+      async append(set) {
+        if (set.category !== category) {
+          throw new Error(`Observation set category "${set.category}" does not match store "${category}"`);
+        }
+        if (sets.has(set.runLabel)) {
+          throw new Error(`Run "${set.runLabel}" already exists — observations are append-only; use a new run label`);
+        }
+        sets.set(set.runLabel, set);
+        return set;
+      },
+      async get(runLabel) {
+        return sets.get(runLabel) ?? null;
+      },
+      async list() {
+        return sorted().map((set) => ({
+          runLabel: set.runLabel,
+          runAt: set.runAt,
+          observations: set.observations.length,
+        }));
+      },
+      async latest() {
+        const all = sorted();
+        return all.length ? all[all.length - 1] : null;
+      },
+    },
+  };
+}
+
 /**
  * Whitespace-only normalization for span matching, plus one extraction
  * artifact: the HTML-to-text step can emit a line break before punctuation

package/src/marketClassify.ts

@@ -7,9 +7,11 @@ import {
   type MarketClaim,
   type MarketConfig,
   type MarketObservation,
+  type MarketStore,
   type ObservationSet,
   type SpanVerificationFailure,
 } from "./market.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 
 /**
  * LLM intensity classification for the market map — the same
@@ -112,9 +114,16 @@ export type ClassifyMarketOptions = {
   vendors?: string[];
   /** Captures directory override (tests); defaults to the profile market home. */
   capturesDir?: string;
+  /** Storage seam: when given, captures are read through it instead of the file layout. */
+  store?: MarketStore;
   now?: () => Date;
   /** Per-vendor progress (presentation only — a throwing callback never fails the run). */
   onVendor?: (done: number, total: number, vendorId: string) => void;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES (the "classify" stage, items
+   * per vendor). Presentation-only: a throwing listener never fails the run.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type ClassifyMarketResult = {
@@ -129,7 +138,10 @@ export async function classifyMarket(
   options: ClassifyMarketOptions,
 ): Promise<ClassifyMarketResult> {
   const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
-  const { entries, textByHash } = loadCaptureTexts(config.category, options.capturesDir);
+  const progress = options.progress ?? nullProgressEmitter();
+  const { entries, textByHash } = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(config.category, options.capturesDir);
   if (entries.length === 0) {
     throw new Error(`No captures for ${config.category} — run \`market capture\` first`);
   }
@@ -145,12 +157,14 @@ export async function classifyMarket(
   const observations: MarketObservation[] = [];
   const retriedVendorIds: string[] = [];
 
+  progress.stage(MARKET_CAPTURE_STAGES[2], 2, MARKET_CAPTURE_STAGES.length);
   for (const [vendorIndex, vendorId] of vendorIds.entries()) {
     try {
       options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
     } catch {
       // progress is presentation-only
     }
+    progress.note(vendorId);
     const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
     if (!vendor) throw new Error(`Unknown vendor "${vendorId}"`);
     const vendorEntries = runEntries.filter((entry) => entry.vendorId === vendorId);
@@ -174,6 +188,7 @@ export async function classifyMarket(
           evidence: [],
         });
       }
+      progress.items(vendorIndex + 1, vendorIds.length);
       continue;
     }
 
@@ -229,7 +244,9 @@ export async function classifyMarket(
       );
     }
     for (const reading of outcome.readings) observations.push(toObservation(reading, vendorId));
+    progress.items(vendorIndex + 1, vendorIds.length);
   }
+  progress.flush();
 
   return {
     set: {

package/src/marketTaxonomy.ts

@@ -9,6 +9,7 @@ import {
   loadCaptureTexts,
   type MarketClaim,
   type MarketConfig,
+  type MarketStore,
   type MarketVendor,
 } from "./market.ts";
 
@@ -44,6 +45,11 @@ export type SuggestTaxonomyOptions = {
   /** Test injectables. */
   fetchPage?: FetchPage;
   capturesDir?: string;
+  /**
+   * Storage seam for the bootstrap captures; defaults to the file layout.
+   * Pass createMemoryMarketStore for throwaway grounding (hosted flow).
+   */
+  store?: MarketStore;
   now?: () => Date;
 };
 
@@ -227,9 +233,11 @@ export async function suggestMarketConfig(options: SuggestTaxonomyOptions): Prom
   // fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
   await captureMarket(
     { category, vendors, claims: [] },
-    { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
+    { dir: options.capturesDir, store: options.store, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
   );
-  const capture = loadCaptureTexts(category, options.capturesDir);
+  const capture = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(category, options.capturesDir);
   const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
   if (!dossier.trim()) {
     throw new Error(

package/src/mcp-bin.ts

@@ -1,6 +1,38 @@
 #!/usr/bin/env node
+const args = process.argv.slice(2);
+
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`Usage: fullstackgtm-mcp [--help] [--version]
+
+Starts the fullstackgtm MCP stdio server.
+
+The MCP server requires optional peer dependencies:
+  @modelcontextprotocol/sdk
+  zod
+
+Zero-install with peers:
+  npx -y -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`);
+  process.exit(0);
+}
+
+if (args.includes("--version") || args.includes("-v")) {
+  console.log(await readPackageVersion());
+  process.exit(0);
+}
+
 export {};
 
+async function readPackageVersion(): Promise<string> {
+  try {
+    const { readFileSync } = await import("node:fs");
+    const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+    const parsed = JSON.parse(raw) as { version?: string };
+    return parsed.version ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
 // The MCP server needs the optional peer dependencies. Import dynamically so
 // a missing peer produces install guidance instead of a module-load stack
 // trace — `npx fullstackgtm-mcp` alone never installs optional peers.

package/src/mcp.ts

@@ -53,6 +53,8 @@ import {
 import { generateDemoSnapshot } from "./demo.ts";
 import type { FieldMappings } from "./mappings.ts";
 import { formatPatchPlanRun, patchPlanToMarkdown } from "./format.ts";
+import { verifyApprovalDigests } from "./integrity.ts";
+import { createFilePlanStore, type StoredPlan } from "./planStore.ts";
 import { builtinAuditRules } from "./rules.ts";
 import { sampleSnapshot } from "./sampleData.ts";
 import { normalizeTranscript, parseCall } from "./calls.ts";
@@ -322,7 +324,11 @@ const toolDefinitions: ToolDefinition[] = [
       description:
         "Apply explicitly approved operations from a patch plan through a provider " +
         "connector. Operations not listed in approvedOperationIds are never written, " +
-        "and requires_human_* placeholders need a value override.",
+        "and requires_human_* placeholders need a value override. When the plan is in " +
+        "the local plan store (saved via `audit --save`), approvals are verified against " +
+        "the store's HMAC approval digests — ids not approved with `plans approve` are " +
+        "refused — and the run is recorded onto the stored plan so `plans show` and " +
+        "`audit-log export` include it.",
       inputSchema: {
         provider: z.enum(["hubspot", "salesforce"]),
         planPath: z.string(),
@@ -332,14 +338,128 @@ const toolDefinitions: ToolDefinition[] = [
       },
     },
     handler: async ({ provider, planPath, approvedOperationIds, valueOverrides, output }) => {
-      const plan = JSON.parse(
-        readFileSync(resolve(process.cwd(), planPath), "utf8"),
-      ) as PatchPlan;
+      // The file may be a raw PatchPlan (audit --out) or a StoredPlan envelope
+      // (a file straight out of the plan-store directory).
+      const parsed = JSON.parse(readFileSync(resolve(process.cwd(), planPath), "utf8")) as unknown;
+      const filePlan = isStoredPlanFile(parsed) ? parsed.plan : (parsed as PatchPlan);
+      if (!filePlan || !Array.isArray(filePlan.operations)) {
+        throw new Error(`${planPath} is not a patch plan (expected { id, operations: [...] }).`);
+      }
+
+      // Governance parity with CLI apply: when this plan id exists in the local
+      // plan store, the STORE is the source of truth — the approved-id set must
+      // come from `plans approve` (which HMAC-signed each approved op), the
+      // signatures are re-verified here, and the run is recorded back onto the
+      // plan so audit-log export sees MCP applies exactly like CLI applies.
+      const store = createFilePlanStore();
+      const stored =
+        typeof filePlan.id === "string" && /^[\w.-]+$/.test(filePlan.id)
+          ? await store.get(filePlan.id)
+          : null;
+
+      let plan: PatchPlan;
+      let effectiveOverrides: Record<string, unknown>;
+      if (stored) {
+        if (stored.status !== "approved") {
+          throw new Error(
+            `Plan ${filePlan.id} is ${stored.status}; approve operations first with ` +
+              `\`fullstackgtm plans approve ${filePlan.id} --operations <ids|all>\`.`,
+          );
+        }
+        // Downgrade guard (same as CLI apply): an approved plan with no
+        // signatures either predates 0.26 or had approvalDigests stripped to
+        // skip the integrity check. Refuse rather than trust the file.
+        if (stored.approvedOperationIds.length > 0 && !stored.approvalDigests) {
+          throw new Error(
+            `Refusing to apply plan ${filePlan.id}: it was approved without integrity signatures ` +
+              "(approved before 0.26.0, or its signatures were removed). Re-approve it with " +
+              `\`fullstackgtm plans approve ${filePlan.id} --operations <ids|all>\`.`,
+          );
+        }
+        // Never widen: every id the tool wants applied must already be
+        // store-approved. (A subset is fine — narrowing never writes more.)
+        const storeApproved = new Set(stored.approvedOperationIds);
+        const unapproved = approvedOperationIds.filter((id: string) => !storeApproved.has(id));
+        if (unapproved.length > 0) {
+          throw new Error(
+            `Refusing to apply plan ${filePlan.id}: operation(s) ${unapproved.join(", ")} were never ` +
+              "approved in the plan store. Approve them first with " +
+              `\`fullstackgtm plans approve ${filePlan.id} --operations <ids>\`.`,
+          );
+        }
+        // Integrity gate: verify against the EFFECTIVE overrides (stored ∪ tool
+        // args), so a tool-supplied value that changes what the human approved
+        // is treated as tamper, not a live override — what gets written must
+        // equal what was signed.
+        effectiveOverrides = { ...stored.valueOverrides, ...(valueOverrides ?? {}) };
+        const verification = verifyApprovalDigests(
+          stored.plan.operations,
+          stored.approvedOperationIds,
+          effectiveOverrides,
+          stored.approvalDigests,
+        );
+        if (!verification.ok) {
+          const detail =
+            verification.reason === "no_key"
+              ? "the plan-signing key is missing (was this plan approved on another machine?). Re-approve it here with `fullstackgtm plans approve`."
+              : `these operations differ from what was approved: ${verification.tampered.join(", ")}. ` +
+                "If you want a different value, set it at approval (`plans approve --value <op>=<v>`) and re-approve; " +
+                "otherwise the plan was edited after approval — review and re-approve.";
+          throw new Error(`Refusing to apply plan ${filePlan.id}: ${detail}`);
+        }
+        plan = stored.plan; // apply what was signed, not what the file says now
+      } else {
+        // External plan file, not in the store: no digests exist to verify, but
+        // approvals must still reference real operations in the plan content.
+        plan = filePlan;
+        const known = new Set(plan.operations.map((operation) => operation.id));
+        const unknown = approvedOperationIds.filter((id: string) => !known.has(id));
+        if (unknown.length > 0) {
+          throw new Error(
+            `Plan ${planPath} has no operation(s) ${unknown.join(", ")} — approvedOperationIds ` +
+              "must reference operations in the plan.",
+          );
+        }
+        effectiveOverrides = valueOverrides ?? {};
+      }
+
       const run = await applyPatchPlan(await connectorFor(provider), plan, {
         approvedOperationIds,
-        valueOverrides,
+        valueOverrides: effectiveOverrides,
       });
-      return content(output === "markdown" ? formatPatchPlanRun(run) : run);
+
+      // Persist the run (same data the CLI records) so runs[] and plan status
+      // update and audit-log export includes MCP applies. External plan files
+      // have no store entry to update — say so instead of silently dropping it.
+      let runRecorded = false;
+      if (stored) {
+        await store.recordRun(plan.id, run);
+        runRecorded = true;
+      }
+      const governance = {
+        planSource: stored ? ("store" as const) : ("external" as const),
+        approvalDigestsVerified: stored !== null,
+        runRecorded,
+        ...(stored
+          ? {}
+          : {
+              note:
+                "Plan file is not in the local plan store: approvals came from tool arguments " +
+                "(verified against the plan content only — no approval-digest check) and the run " +
+                "was not recorded. Use `fullstackgtm audit --save` + `plans approve` for " +
+                "store-backed governance.",
+            }),
+      };
+      if (output === "markdown") {
+        return content(
+          `${formatPatchPlanRun(run)}\n\nGovernance: plan source ${governance.planSource}; ` +
+            (runRecorded
+              ? "approval digests verified; run recorded on the stored plan (audit-log export includes it)."
+              : governance.note!),
+        );
+      }
+      // Backward compatible: the run's own fields stay top-level; governance is additive.
+      return content({ ...run, governance });
     },
   },
   {
@@ -456,6 +576,20 @@ export async function startMcpServer() {
   await server.connect(transport);
 }
 
+/**
+ * A plan-store file is a StoredPlan envelope ({ plan, status, runs, ... });
+ * `audit --out` writes a bare PatchPlan. fullstackgtm_apply accepts either.
+ */
+function isStoredPlanFile(value: unknown): value is StoredPlan {
+  if (typeof value !== "object" || value === null || !("plan" in value)) return false;
+  const plan = (value as { plan?: unknown }).plan;
+  return (
+    typeof plan === "object" &&
+    plan !== null &&
+    Array.isArray((plan as { operations?: unknown }).operations)
+  );
+}
+
 function loadMarketConfigOrHint(path: string): ReturnType<typeof loadMarketConfig> {
   try {
     return loadMarketConfig(path);