fullstackgtm 0.44.0 → 0.46.0

package/src/market.ts

@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 import type { GtmEvidence } from "./types.ts";
 
 /**
@@ -445,12 +446,24 @@ const defaultFetchPage: FetchPage = async (url) => {
 };
 
 export type CaptureOptions = {
-  /** Directory for captures; defaults to <marketHome>/captures. */
+  /** Directory for captures; defaults to <marketHome>/captures. Ignored when `store` is given. */
   dir?: string;
   runLabel?: string;
   /** Injectable for tests; defaults to global fetch. */
   fetchPage?: FetchPage;
   now?: () => Date;
+  /**
+   * Storage seam (see MarketStore below). Defaults to the profile-home file
+   * store; the hosted app passes a Convex-backed store. Same engine, same
+   * evidence chain, different persistence.
+   */
+  store?: MarketStore;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES ("sources" → "capture" →
+   * "persist" here; "classify" belongs to classifyMarket). Presentation-only:
+   * a throwing listener never fails the capture.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type CaptureResult = {
@@ -459,57 +472,55 @@ export type CaptureResult = {
 };
 
 export async function captureMarket(config: MarketConfig, options: CaptureOptions = {}): Promise<CaptureResult> {
-  const dir = options.dir ?? join(marketHome(config.category), "captures");
+  const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
+  const progress = options.progress ?? nullProgressEmitter();
   const runLabel = options.runLabel ?? "run-1";
   const fetchPage = options.fetchPage ?? defaultFetchPage;
   const fetchedAt = (options.now ?? (() => new Date()))().toISOString();
-  mkdirSync(dir, { recursive: true });
 
-  const manifestPath = join(dir, "manifest.json");
-  const manifest: CaptureEntry[] = existsSync(manifestPath)
-    ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
-    : [];
+  progress.stage(MARKET_CAPTURE_STAGES[0], 0, MARKET_CAPTURE_STAGES.length);
+  const targets: Array<{ vendorId: string; kind: CaptureEntry["kind"]; url: string }> = [];
+  for (const vendor of config.vendors) {
+    targets.push({ vendorId: vendor.id, kind: "home", url: vendor.urls.home });
+    if (vendor.urls.pricing) targets.push({ vendorId: vendor.id, kind: "pricing", url: vendor.urls.pricing });
+    for (const url of vendor.urls.product) targets.push({ vendorId: vendor.id, kind: "product", url });
+  }
+  progress.note(`${config.vendors.length} vendor(s), ${targets.length} page(s)`);
 
+  progress.stage(MARKET_CAPTURE_STAGES[1], 1, MARKET_CAPTURE_STAGES.length);
   const entries: CaptureEntry[] = [];
-  for (const vendor of config.vendors) {
-    const targets: Array<{ kind: CaptureEntry["kind"]; url: string }> = [
-      { kind: "home", url: vendor.urls.home },
-    ];
-    if (vendor.urls.pricing) targets.push({ kind: "pricing", url: vendor.urls.pricing });
-    for (const url of vendor.urls.product) targets.push({ kind: "product", url });
-
-    for (const target of targets) {
-      let status: number | null = null;
-      let text = "";
-      try {
-        const page = await fetchPage(target.url);
-        status = page.status;
-        if (page.status === 200) text = extractReadableText(page.body);
-      } catch {
-        status = null;
-      }
-      let captureHash: string | null = null;
-      if (text) {
-        captureHash = createHash("sha256").update(text).digest("hex");
-        // Content-addressed: an unchanged page dedupes to the same file.
-        writeFileSync(join(dir, `${captureHash}.txt`), text);
-      }
-      const entry: CaptureEntry = {
-        runLabel,
-        vendorId: vendor.id,
-        kind: target.kind,
-        url: target.url,
-        fetchedAt,
-        httpStatus: status,
-        captureHash,
-        textChars: text.length,
-      };
-      manifest.push(entry);
-      entries.push(entry);
+  for (const target of targets) {
+    let status: number | null = null;
+    let text = "";
+    try {
+      const page = await fetchPage(target.url);
+      status = page.status;
+      if (page.status === 200) text = extractReadableText(page.body);
+    } catch {
+      status = null;
     }
+    let captureHash: string | null = null;
+    if (text) {
+      captureHash = createHash("sha256").update(text).digest("hex");
+      // Content-addressed: an unchanged page dedupes to the same key.
+      await store.saveCaptureText(captureHash, text);
+    }
+    entries.push({
+      runLabel,
+      vendorId: target.vendorId,
+      kind: target.kind,
+      url: target.url,
+      fetchedAt,
+      httpStatus: status,
+      captureHash,
+      textChars: text.length,
+    });
+    progress.items(entries.length, targets.length);
   }
-  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-  return { entries, manifestPath };
+  progress.stage(MARKET_CAPTURE_STAGES[3], 3, MARKET_CAPTURE_STAGES.length);
+  await store.appendCaptureEntries(entries);
+  progress.flush();
+  return { entries, manifestPath: store.captureLocation() };
 }
 
 // ---------------------------------------------------------------------------
@@ -651,6 +662,108 @@ export function loadCaptureTexts(
   return { entries, textByHash };
 }
 
+// ---------------------------------------------------------------------------
+// MarketStore — the storage seam for the whole market workspace (captures +
+// observation runs), mirroring the PlanStore precedent: the profile-home file
+// layout and the hosted app's Convex tables are two implementations of the
+// same contract. Engines never touch storage directly — captureMarket and
+// classifyMarket take a store — and the default file store preserves the
+// on-disk layout exactly, so the CLI's behavior is unchanged.
+//
+// loadConfig is deliberately NOT part of the store: the config is a reviewed
+// input (a versioned file, or a hosted map document) that callers load and
+// hand to the engines. The store owns only what the engines write.
+
+export interface MarketStore {
+  /** Operator-facing location of the capture manifest (file path or logical label). */
+  captureLocation(): string;
+  /** Persist one page's extracted text under its content hash (idempotent). */
+  saveCaptureText(captureHash: string, text: string): Promise<void>;
+  /** Append one capture pass's manifest entries (append-only; never rewrites history). */
+  appendCaptureEntries(entries: CaptureEntry[]): Promise<void>;
+  /** Full manifest + resolvable texts — the evidence-verification input. */
+  loadCaptureTexts(): Promise<{ entries: CaptureEntry[]; textByHash: Map<string, string> }>;
+  /** Append-only observation runs (the same ObservationStore contract as before). */
+  observations: ObservationStore;
+}
+
+/** The CLI's store: captures + observations under the profile market home. */
+export function createFileMarketStore(
+  category: string,
+  options: { capturesDir?: string; observationsDir?: string } = {},
+): MarketStore {
+  const dir = options.capturesDir ?? join(marketHome(category), "captures");
+  const manifestPath = join(dir, "manifest.json");
+  return {
+    captureLocation: () => manifestPath,
+    async saveCaptureText(captureHash, text) {
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, `${captureHash}.txt`), text);
+    },
+    async appendCaptureEntries(entries) {
+      mkdirSync(dir, { recursive: true });
+      const manifest: CaptureEntry[] = existsSync(manifestPath)
+        ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
+        : [];
+      manifest.push(...entries);
+      writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+    },
+    async loadCaptureTexts() {
+      return loadCaptureTexts(category, dir);
+    },
+    observations: createFileObservationStore(category, options.observationsDir),
+  };
+}
+
+/**
+ * In-memory store: seam tests, and throwaway grounding captures (e.g. the
+ * taxonomy proposer's bootstrap pass) that should never persist anywhere.
+ */
+export function createMemoryMarketStore(category: string): MarketStore {
+  const entries: CaptureEntry[] = [];
+  const textByHash = new Map<string, string>();
+  const sets = new Map<string, ObservationSet>();
+  const sorted = () => [...sets.values()].sort((a, b) => a.runAt.localeCompare(b.runAt));
+  return {
+    captureLocation: () => `memory://${category}/captures`,
+    async saveCaptureText(captureHash, text) {
+      textByHash.set(captureHash, text);
+    },
+    async appendCaptureEntries(newEntries) {
+      entries.push(...newEntries);
+    },
+    async loadCaptureTexts() {
+      return { entries: [...entries], textByHash: new Map(textByHash) };
+    },
+    observations: {
+      async append(set) {
+        if (set.category !== category) {
+          throw new Error(`Observation set category "${set.category}" does not match store "${category}"`);
+        }
+        if (sets.has(set.runLabel)) {
+          throw new Error(`Run "${set.runLabel}" already exists — observations are append-only; use a new run label`);
+        }
+        sets.set(set.runLabel, set);
+        return set;
+      },
+      async get(runLabel) {
+        return sets.get(runLabel) ?? null;
+      },
+      async list() {
+        return sorted().map((set) => ({
+          runLabel: set.runLabel,
+          runAt: set.runAt,
+          observations: set.observations.length,
+        }));
+      },
+      async latest() {
+        const all = sorted();
+        return all.length ? all[all.length - 1] : null;
+      },
+    },
+  };
+}
+
 /**
  * Whitespace-only normalization for span matching, plus one extraction
  * artifact: the HTML-to-text step can emit a line break before punctuation

package/src/marketClassify.ts

@@ -7,9 +7,11 @@ import {
   type MarketClaim,
   type MarketConfig,
   type MarketObservation,
+  type MarketStore,
   type ObservationSet,
   type SpanVerificationFailure,
 } from "./market.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 
 /**
  * LLM intensity classification for the market map — the same
@@ -112,7 +114,16 @@ export type ClassifyMarketOptions = {
   vendors?: string[];
   /** Captures directory override (tests); defaults to the profile market home. */
   capturesDir?: string;
+  /** Storage seam: when given, captures are read through it instead of the file layout. */
+  store?: MarketStore;
   now?: () => Date;
+  /** Per-vendor progress (presentation only — a throwing callback never fails the run). */
+  onVendor?: (done: number, total: number, vendorId: string) => void;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES (the "classify" stage, items
+   * per vendor). Presentation-only: a throwing listener never fails the run.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type ClassifyMarketResult = {
@@ -127,7 +138,10 @@ export async function classifyMarket(
   options: ClassifyMarketOptions,
 ): Promise<ClassifyMarketResult> {
   const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
-  const { entries, textByHash } = loadCaptureTexts(config.category, options.capturesDir);
+  const progress = options.progress ?? nullProgressEmitter();
+  const { entries, textByHash } = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(config.category, options.capturesDir);
   if (entries.length === 0) {
     throw new Error(`No captures for ${config.category} — run \`market capture\` first`);
   }
@@ -143,7 +157,14 @@ export async function classifyMarket(
   const observations: MarketObservation[] = [];
   const retriedVendorIds: string[] = [];
 
-  for (const vendorId of vendorIds) {
+  progress.stage(MARKET_CAPTURE_STAGES[2], 2, MARKET_CAPTURE_STAGES.length);
+  for (const [vendorIndex, vendorId] of vendorIds.entries()) {
+    try {
+      options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
+    } catch {
+      // progress is presentation-only
+    }
+    progress.note(vendorId);
     const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
     if (!vendor) throw new Error(`Unknown vendor "${vendorId}"`);
     const vendorEntries = runEntries.filter((entry) => entry.vendorId === vendorId);
@@ -167,6 +188,7 @@ export async function classifyMarket(
           evidence: [],
         });
       }
+      progress.items(vendorIndex + 1, vendorIds.length);
       continue;
     }
 
@@ -222,7 +244,9 @@ export async function classifyMarket(
       );
     }
     for (const reading of outcome.readings) observations.push(toObservation(reading, vendorId));
+    progress.items(vendorIndex + 1, vendorIds.length);
   }
+  progress.flush();
 
   return {
     set: {

package/src/marketTaxonomy.ts

@@ -9,6 +9,7 @@ import {
   loadCaptureTexts,
   type MarketClaim,
   type MarketConfig,
+  type MarketStore,
   type MarketVendor,
 } from "./market.ts";
 
@@ -44,6 +45,11 @@ export type SuggestTaxonomyOptions = {
   /** Test injectables. */
   fetchPage?: FetchPage;
   capturesDir?: string;
+  /**
+   * Storage seam for the bootstrap captures; defaults to the file layout.
+   * Pass createMemoryMarketStore for throwaway grounding (hosted flow).
+   */
+  store?: MarketStore;
   now?: () => Date;
 };
 
@@ -227,9 +233,11 @@ export async function suggestMarketConfig(options: SuggestTaxonomyOptions): Prom
   // fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
   await captureMarket(
     { category, vendors, claims: [] },
-    { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
+    { dir: options.capturesDir, store: options.store, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
   );
-  const capture = loadCaptureTexts(category, options.capturesDir);
+  const capture = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(category, options.capturesDir);
   const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
   if (!dossier.trim()) {
     throw new Error(

package/src/mcp-bin.ts

@@ -1,6 +1,38 @@
 #!/usr/bin/env node
+const args = process.argv.slice(2);
+
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`Usage: fullstackgtm-mcp [--help] [--version]
+
+Starts the fullstackgtm MCP stdio server.
+
+The MCP server requires optional peer dependencies:
+  @modelcontextprotocol/sdk
+  zod
+
+Zero-install with peers:
+  npx -y -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`);
+  process.exit(0);
+}
+
+if (args.includes("--version") || args.includes("-v")) {
+  console.log(await readPackageVersion());
+  process.exit(0);
+}
+
 export {};
 
+async function readPackageVersion(): Promise<string> {
+  try {
+    const { readFileSync } = await import("node:fs");
+    const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+    const parsed = JSON.parse(raw) as { version?: string };
+    return parsed.version ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
 // The MCP server needs the optional peer dependencies. Import dynamically so
 // a missing peer produces install guidance instead of a module-load stack
 // trace — `npx fullstackgtm-mcp` alone never installs optional peers.
@@ -9,8 +41,8 @@ const MISSING_PEER_HELP = `The MCP server needs the optional peer dependencies @
 In a project:
   npm install fullstackgtm @modelcontextprotocol/sdk zod
 
-Zero-install:
-  npx -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`;
+Zero-install (the fullstackgtm-mcp wrapper package bundles the peers):
+  npx -y fullstackgtm-mcp`;
 
 function isMissingPeerError(error: unknown): boolean {
   if (!(error instanceof Error)) return false;