fullstackgtm 0.45.0 → 0.47.0

package/src/llm.ts

@@ -511,3 +511,242 @@ export async function validateLlmKey(
     ? { ok: true, detail: `Key accepted by the ${provider} API.` }
     : { ok: false, detail: `HTTP ${response.status} ${response.statusText}`.trim() };
 }
+
+// ── Chunked extraction (the TamTone "langextract" method) ───────────────────
+//
+// The quality property this encodes, validated on real call corpora in
+// TamTone: transcripts are never sent whole. They are chopped into small
+// (~1,500-char) speaker-turn-respecting chunks and each chunk gets its own
+// focused extraction call — small chunks yielded ~95% more grounded signals
+// than 4,000-char chunks in TamTone's testing, and whole-transcript passes
+// (this module's original extractInsightsLlm, kept for tiny inputs) are the
+// worst case: truncation drops the tail and one context window extracts
+// shallowly. Per-chunk results pass the SAME verbatim-evidence and
+// next-step-grounding gates as the single-shot path (checked against the
+// chunk they came from), then a TamTone-style quality score
+// (0.25·length + 0.40·specificity + 0.35·confidence, reject < 0.15) filters
+// filler before per-call dedupe and ranking.
+
+const CHUNK_MAX_CHARS = 1500;
+const CHUNK_CONCURRENCY = 4;
+const MAX_CHUNKS_PER_CALL = 80; // cost bound ≈ 120K chars of transcript
+const MAX_INSIGHTS_PER_CALL = 40;
+
+/**
+ * Chop a "Speaker: text" transcript into ≤maxChars chunks on speaker-turn
+ * (line) boundaries; a single oversize turn splits on sentence boundaries.
+ * No overlap (per-chunk extraction + downstream dedupe make it unnecessary).
+ */
+export function chunkTranscript(transcript: string, maxChars = CHUNK_MAX_CHARS): string[] {
+  const lines = transcript.split("\n").filter((line) => line.trim().length > 0);
+  const pieces: string[] = [];
+  for (const line of lines) {
+    if (line.length <= maxChars) {
+      pieces.push(line);
+      continue;
+    }
+    // Oversize turn: split on sentence boundaries within the budget.
+    let rest = line;
+    while (rest.length > maxChars) {
+      const window = rest.slice(0, maxChars);
+      const cut = Math.max(
+        window.lastIndexOf(". "),
+        window.lastIndexOf("? "),
+        window.lastIndexOf("! "),
+      );
+      const at = cut >= maxChars * 0.4 ? cut + 1 : maxChars;
+      pieces.push(rest.slice(0, at).trim());
+      rest = rest.slice(at).trim();
+    }
+    if (rest) pieces.push(rest);
+  }
+
+  const chunks: string[] = [];
+  let current = "";
+  for (const piece of pieces) {
+    if (current && current.length + piece.length + 1 > maxChars) {
+      chunks.push(current);
+      current = piece;
+    } else {
+      current = current ? `${current}\n${piece}` : piece;
+    }
+  }
+  if (current) chunks.push(current);
+  return chunks;
+}
+
+// Few-shot examples are load-bearing (TamTone's extractor refuses to run
+// without an example set): they anchor the "verbatim span, not paraphrase"
+// contract and calibrate what NOT to extract far better than instructions.
+const CHUNK_FEW_SHOT = `Examples:
+
+Excerpt: "Prospect: honestly the big blocker is that our reps spend every Friday cleaning the pipeline by hand. Rep: how long does that take? Prospect: most of the day, easily five or six hours."
+→ [{"type":"pain_point","text":"Reps lose ~5-6 hours every Friday manually cleaning the pipeline.","evidence":"our reps spend every Friday cleaning the pipeline by hand","speaker":"Prospect","importance":4,"confidence":0.9}]
+
+Excerpt: "Prospect: we're also looking at Clari for this. Their forecasting is strong but the rollout quote scared us. Rep: what was the number? Prospect: about eighty thousand a year."
+→ [{"type":"competitor_mention","text":"Evaluating Clari; strong forecasting but ~$80K/yr rollout quote is a concern.","evidence":"we're also looking at Clari for this","speaker":"Prospect","importance":4,"confidence":0.9},{"type":"pricing","text":"Clari quoted about $80K per year.","evidence":"about eighty thousand a year","speaker":"Prospect","importance":3,"confidence":0.85}]
+
+Excerpt: "Rep: I'll send the security questionnaire answers by Thursday. Prospect: perfect, and I'll get you time with our CFO next week."
+→ [{"type":"next_step","text":"Rep to send security questionnaire answers by Thursday.","evidence":"I'll send the security questionnaire answers by Thursday","speaker":"Rep","importance":4,"confidence":0.95,"owner":"Rep","deadline":"Thursday","commitment":"firm"},{"type":"next_step","text":"Prospect to schedule time with their CFO next week.","evidence":"I'll get you time with our CFO next week","speaker":"Prospect","importance":4,"confidence":0.9,"owner":"Prospect","deadline":"next week","commitment":"firm"}]
+
+Excerpt: "Rep: how was the long weekend? Prospect: great, we took the kids camping. Rep: nice! okay, let me share my screen."
+→ []
+
+Do NOT extract: greetings, scheduling chatter, filler ("yeah", "makes sense"), screen-share logistics, or anything you cannot quote verbatim from THIS excerpt.`;
+
+function chunkInstructions(index: number, total: number, title?: string): string {
+  return `${EXTRACT_INSTRUCTIONS}
+
+You are seeing excerpt ${index + 1} of ${total} from a longer call${title ? ` ("${title}")` : ""}.
+Extract ONLY what this excerpt itself grounds — the other excerpts are handled separately.
+evidence MUST be a verbatim span of THIS excerpt.
+
+${CHUNK_FEW_SHOT}`;
+}
+
+// TamTone quality gate (analysis/quality_scorer.py):
+// 0.25·length + 0.40·specificity + 0.35·confidence, hard caps for
+// stopword-only/too-short, reject below 0.15. Specificity rewards the
+// concrete: numbers, money, dates/durations, named things.
+const STOPWORDS = new Set(
+  "a an and are as at be but by for from had has have i if in into is it its me my not of on or our so that the their them they this to was we were what when which who will with you your yeah okay right sure like just really kind sort".split(" "),
+);
+
+export function scoreInsightQuality(insight: { text: string; evidence: string; confidence: number }): number {
+  const text = `${insight.text} ${insight.evidence}`.trim();
+  const words = text.toLowerCase().split(/[^a-z0-9$%']+/).filter(Boolean);
+  if (text.length < 5) return 0.1;
+  const nonStop = words.filter((word) => !STOPWORDS.has(word));
+  if (nonStop.length === 0) return 0.1;
+
+  const lengthScore = Math.min(1, words.length / 20);
+  let specificity = 0;
+  if (/\d/.test(text)) specificity += 0.3; // numbers/amounts/dates
+  if (/[$€£]|\b(dollars?|thousand|million|percent|%)\b/i.test(text)) specificity += 0.2;
+  if (/\b(q[1-4]|january|february|march|april|may|june|july|august|september|october|november|december|week|month|quarter|year|monday|tuesday|wednesday|thursday|friday)\b/i.test(text)) specificity += 0.2;
+  if (/\b[A-Z][a-z]{2,}/.test(insight.text.slice(1))) specificity += 0.15; // named things
+  specificity += Math.min(0.3, nonStop.length / 40);
+  specificity = Math.min(1, specificity);
+
+  return 0.25 * lengthScore + 0.4 * specificity + 0.35 * Math.min(1, Math.max(0, insight.confidence));
+}
+
+const QUALITY_REJECT_BELOW = 0.15;
+
+export type ChunkedExtractResult = {
+  insights: LlmExtractedInsight[];
+  model: string;
+  chunks: number;
+  chunksUsed: number;
+  chunksFailed: number;
+};
+
+/**
+ * The chunked pipeline: chunk → per-chunk forced-tool extraction (bounded
+ * concurrency) → per-chunk verbatim + grounding gates → quality gate →
+ * per-call dedupe → rank. A failed chunk is skipped (best-effort); only
+ * all-chunks-failed throws.
+ */
+export async function extractInsightsChunked(
+  transcript: string,
+  options: LlmCallOptions & { title?: string; maxChunks?: number; concurrency?: number },
+): Promise<ChunkedExtractResult> {
+  const model = options.model ?? DEFAULT_MODELS[options.provider];
+  const allChunks = chunkTranscript(transcript);
+  const chunks = allChunks.slice(0, options.maxChunks ?? MAX_CHUNKS_PER_CALL);
+  if (chunks.length <= 1) {
+    // Tiny transcript: the single-shot path is identical work with one call.
+    const single = await extractInsightsLlm(transcript, options);
+    return { ...single, chunks: allChunks.length, chunksUsed: chunks.length, chunksFailed: 0 };
+  }
+
+  const concurrency = Math.max(1, options.concurrency ?? CHUNK_CONCURRENCY);
+  const perChunk: LlmExtractedInsight[][] = new Array(chunks.length);
+  let failed = 0;
+  let cursor = 0;
+  async function worker() {
+    while (cursor < chunks.length) {
+      const index = cursor;
+      cursor += 1;
+      const chunk = chunks[index];
+      try {
+        const result = (await forcedToolCall(
+          `${chunkInstructions(index, chunks.length, options.title)}\n\nExcerpt:\n${chunk}`,
+          "extract_call_insights",
+          EXTRACT_SCHEMA,
+          model,
+          options,
+        )) as { insights?: LlmExtractedInsight[] };
+        const normalizedChunk = normalizeSpan(chunk);
+        perChunk[index] = (result.insights ?? [])
+          .filter((insight) => INSIGHT_TYPES.includes(insight.type))
+          // The SAME gates as the single-shot path, applied per chunk: the
+          // evidence must be a verbatim span of the chunk it came from, and a
+          // next_step's written action must be grounded in that evidence.
+          .filter((insight) => {
+            const quote = normalizeSpan(insight.evidence ?? "");
+            return quote.length >= 12 && normalizedChunk.includes(quote);
+          })
+          .filter(
+            (insight) =>
+              insight.type !== "next_step" ||
+              actionGroundedInEvidence(insight.text, insight.evidence ?? ""),
+          )
+          .map((insight) => ({
+            ...insight,
+            title: insight.type.replace(/_/g, " "),
+            importance: clamp(Math.round(insight.importance ?? 3), 1, 5),
+            confidence: clamp(insight.confidence ?? 0.7, 0, 1),
+          }))
+          .filter((insight) => scoreInsightQuality(insight) >= QUALITY_REJECT_BELOW);
+      } catch {
+        failed += 1;
+        perChunk[index] = [];
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, chunks.length) }, worker));
+
+  if (failed === chunks.length) {
+    throw new Error(`LLM extraction failed for all ${chunks.length} chunks.`);
+  }
+
+  // Per-call dedupe: exact on (type, normalized evidence), then near-dup on
+  // (type, normalized text) containment — chunks don't overlap, so dupes are
+  // the model restating the same fact from adjacent context.
+  const seen = new Map<string, LlmExtractedInsight>();
+  const richness = (insight: LlmExtractedInsight) =>
+    insight.confidence +
+    (insight.owner ? 0.05 : 0) +
+    (insight.deadline ? 0.05 : 0) +
+    (insight.speaker ? 0.02 : 0);
+  for (const insights of perChunk) {
+    for (const insight of insights ?? []) {
+      const key = `${insight.type}:${normalizeSpan(insight.evidence ?? "")}`;
+      const kept = seen.get(key);
+      // Same grounded fact stated twice: keep the richer statement (owner/
+      // deadline/speaker attached, higher confidence), not the first seen.
+      if (!kept || richness(insight) > richness(kept)) seen.set(key, insight);
+    }
+  }
+  const merged: LlmExtractedInsight[] = [];
+  for (const insight of seen.values()) {
+    const text = normalizeSpan(insight.text);
+    const dupe = merged.find(
+      (kept) =>
+        kept.type === insight.type &&
+        (normalizeSpan(kept.text).includes(text) || text.includes(normalizeSpan(kept.text))),
+    );
+    if (!dupe) merged.push(insight);
+    else if (insight.importance > dupe.importance) merged[merged.indexOf(dupe)] = insight;
+  }
+
+  merged.sort((a, b) => b.importance - a.importance || b.confidence - a.confidence);
+  return {
+    insights: merged.slice(0, MAX_INSIGHTS_PER_CALL),
+    model,
+    chunks: allChunks.length,
+    chunksUsed: chunks.length,
+    chunksFailed: failed,
+  };
+}

package/src/market.ts

@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 import type { GtmEvidence } from "./types.ts";
 
 /**
@@ -445,12 +446,24 @@ const defaultFetchPage: FetchPage = async (url) => {
 };
 
 export type CaptureOptions = {
-  /** Directory for captures; defaults to <marketHome>/captures. */
+  /** Directory for captures; defaults to <marketHome>/captures. Ignored when `store` is given. */
   dir?: string;
   runLabel?: string;
   /** Injectable for tests; defaults to global fetch. */
   fetchPage?: FetchPage;
   now?: () => Date;
+  /**
+   * Storage seam (see MarketStore below). Defaults to the profile-home file
+   * store; the hosted app passes a Convex-backed store. Same engine, same
+   * evidence chain, different persistence.
+   */
+  store?: MarketStore;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES ("sources" → "capture" →
+   * "persist" here; "classify" belongs to classifyMarket). Presentation-only:
+   * a throwing listener never fails the capture.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type CaptureResult = {
@@ -459,57 +472,55 @@ export type CaptureResult = {
 };
 
 export async function captureMarket(config: MarketConfig, options: CaptureOptions = {}): Promise<CaptureResult> {
-  const dir = options.dir ?? join(marketHome(config.category), "captures");
+  const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
+  const progress = options.progress ?? nullProgressEmitter();
   const runLabel = options.runLabel ?? "run-1";
   const fetchPage = options.fetchPage ?? defaultFetchPage;
   const fetchedAt = (options.now ?? (() => new Date()))().toISOString();
-  mkdirSync(dir, { recursive: true });
 
-  const manifestPath = join(dir, "manifest.json");
-  const manifest: CaptureEntry[] = existsSync(manifestPath)
-    ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
-    : [];
+  progress.stage(MARKET_CAPTURE_STAGES[0], 0, MARKET_CAPTURE_STAGES.length);
+  const targets: Array<{ vendorId: string; kind: CaptureEntry["kind"]; url: string }> = [];
+  for (const vendor of config.vendors) {
+    targets.push({ vendorId: vendor.id, kind: "home", url: vendor.urls.home });
+    if (vendor.urls.pricing) targets.push({ vendorId: vendor.id, kind: "pricing", url: vendor.urls.pricing });
+    for (const url of vendor.urls.product) targets.push({ vendorId: vendor.id, kind: "product", url });
+  }
+  progress.note(`${config.vendors.length} vendor(s), ${targets.length} page(s)`);
 
+  progress.stage(MARKET_CAPTURE_STAGES[1], 1, MARKET_CAPTURE_STAGES.length);
   const entries: CaptureEntry[] = [];
-  for (const vendor of config.vendors) {
-    const targets: Array<{ kind: CaptureEntry["kind"]; url: string }> = [
-      { kind: "home", url: vendor.urls.home },
-    ];
-    if (vendor.urls.pricing) targets.push({ kind: "pricing", url: vendor.urls.pricing });
-    for (const url of vendor.urls.product) targets.push({ kind: "product", url });
-
-    for (const target of targets) {
-      let status: number | null = null;
-      let text = "";
-      try {
-        const page = await fetchPage(target.url);
-        status = page.status;
-        if (page.status === 200) text = extractReadableText(page.body);
-      } catch {
-        status = null;
-      }
-      let captureHash: string | null = null;
-      if (text) {
-        captureHash = createHash("sha256").update(text).digest("hex");
-        // Content-addressed: an unchanged page dedupes to the same file.
-        writeFileSync(join(dir, `${captureHash}.txt`), text);
-      }
-      const entry: CaptureEntry = {
-        runLabel,
-        vendorId: vendor.id,
-        kind: target.kind,
-        url: target.url,
-        fetchedAt,
-        httpStatus: status,
-        captureHash,
-        textChars: text.length,
-      };
-      manifest.push(entry);
-      entries.push(entry);
+  for (const target of targets) {
+    let status: number | null = null;
+    let text = "";
+    try {
+      const page = await fetchPage(target.url);
+      status = page.status;
+      if (page.status === 200) text = extractReadableText(page.body);
+    } catch {
+      status = null;
     }
+    let captureHash: string | null = null;
+    if (text) {
+      captureHash = createHash("sha256").update(text).digest("hex");
+      // Content-addressed: an unchanged page dedupes to the same key.
+      await store.saveCaptureText(captureHash, text);
+    }
+    entries.push({
+      runLabel,
+      vendorId: target.vendorId,
+      kind: target.kind,
+      url: target.url,
+      fetchedAt,
+      httpStatus: status,
+      captureHash,
+      textChars: text.length,
+    });
+    progress.items(entries.length, targets.length);
   }
-  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-  return { entries, manifestPath };
+  progress.stage(MARKET_CAPTURE_STAGES[3], 3, MARKET_CAPTURE_STAGES.length);
+  await store.appendCaptureEntries(entries);
+  progress.flush();
+  return { entries, manifestPath: store.captureLocation() };
 }
 
 // ---------------------------------------------------------------------------
@@ -651,6 +662,108 @@ export function loadCaptureTexts(
   return { entries, textByHash };
 }
 
+// ---------------------------------------------------------------------------
+// MarketStore — the storage seam for the whole market workspace (captures +
+// observation runs), mirroring the PlanStore precedent: the profile-home file
+// layout and the hosted app's Convex tables are two implementations of the
+// same contract. Engines never touch storage directly — captureMarket and
+// classifyMarket take a store — and the default file store preserves the
+// on-disk layout exactly, so the CLI's behavior is unchanged.
+//
+// loadConfig is deliberately NOT part of the store: the config is a reviewed
+// input (a versioned file, or a hosted map document) that callers load and
+// hand to the engines. The store owns only what the engines write.
+
+export interface MarketStore {
+  /** Operator-facing location of the capture manifest (file path or logical label). */
+  captureLocation(): string;
+  /** Persist one page's extracted text under its content hash (idempotent). */
+  saveCaptureText(captureHash: string, text: string): Promise<void>;
+  /** Append one capture pass's manifest entries (append-only; never rewrites history). */
+  appendCaptureEntries(entries: CaptureEntry[]): Promise<void>;
+  /** Full manifest + resolvable texts — the evidence-verification input. */
+  loadCaptureTexts(): Promise<{ entries: CaptureEntry[]; textByHash: Map<string, string> }>;
+  /** Append-only observation runs (the same ObservationStore contract as before). */
+  observations: ObservationStore;
+}
+
+/** The CLI's store: captures + observations under the profile market home. */
+export function createFileMarketStore(
+  category: string,
+  options: { capturesDir?: string; observationsDir?: string } = {},
+): MarketStore {
+  const dir = options.capturesDir ?? join(marketHome(category), "captures");
+  const manifestPath = join(dir, "manifest.json");
+  return {
+    captureLocation: () => manifestPath,
+    async saveCaptureText(captureHash, text) {
+      mkdirSync(dir, { recursive: true });
+      writeFileSync(join(dir, `${captureHash}.txt`), text);
+    },
+    async appendCaptureEntries(entries) {
+      mkdirSync(dir, { recursive: true });
+      const manifest: CaptureEntry[] = existsSync(manifestPath)
+        ? (JSON.parse(readFileSync(manifestPath, "utf8")) as CaptureEntry[])
+        : [];
+      manifest.push(...entries);
+      writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+    },
+    async loadCaptureTexts() {
+      return loadCaptureTexts(category, dir);
+    },
+    observations: createFileObservationStore(category, options.observationsDir),
+  };
+}
+
+/**
+ * In-memory store: seam tests, and throwaway grounding captures (e.g. the
+ * taxonomy proposer's bootstrap pass) that should never persist anywhere.
+ */
+export function createMemoryMarketStore(category: string): MarketStore {
+  const entries: CaptureEntry[] = [];
+  const textByHash = new Map<string, string>();
+  const sets = new Map<string, ObservationSet>();
+  const sorted = () => [...sets.values()].sort((a, b) => a.runAt.localeCompare(b.runAt));
+  return {
+    captureLocation: () => `memory://${category}/captures`,
+    async saveCaptureText(captureHash, text) {
+      textByHash.set(captureHash, text);
+    },
+    async appendCaptureEntries(newEntries) {
+      entries.push(...newEntries);
+    },
+    async loadCaptureTexts() {
+      return { entries: [...entries], textByHash: new Map(textByHash) };
+    },
+    observations: {
+      async append(set) {
+        if (set.category !== category) {
+          throw new Error(`Observation set category "${set.category}" does not match store "${category}"`);
+        }
+        if (sets.has(set.runLabel)) {
+          throw new Error(`Run "${set.runLabel}" already exists — observations are append-only; use a new run label`);
+        }
+        sets.set(set.runLabel, set);
+        return set;
+      },
+      async get(runLabel) {
+        return sets.get(runLabel) ?? null;
+      },
+      async list() {
+        return sorted().map((set) => ({
+          runLabel: set.runLabel,
+          runAt: set.runAt,
+          observations: set.observations.length,
+        }));
+      },
+      async latest() {
+        const all = sorted();
+        return all.length ? all[all.length - 1] : null;
+      },
+    },
+  };
+}
+
 /**
  * Whitespace-only normalization for span matching, plus one extraction
  * artifact: the HTML-to-text step can emit a line break before punctuation

package/src/marketClassify.ts

@@ -7,9 +7,11 @@ import {
   type MarketClaim,
   type MarketConfig,
   type MarketObservation,
+  type MarketStore,
   type ObservationSet,
   type SpanVerificationFailure,
 } from "./market.ts";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 
 /**
  * LLM intensity classification for the market map — the same
@@ -112,9 +114,16 @@ export type ClassifyMarketOptions = {
   vendors?: string[];
   /** Captures directory override (tests); defaults to the profile market home. */
   capturesDir?: string;
+  /** Storage seam: when given, captures are read through it instead of the file layout. */
+  store?: MarketStore;
   now?: () => Date;
   /** Per-vendor progress (presentation only — a throwing callback never fails the run). */
   onVendor?: (done: number, total: number, vendorId: string) => void;
+  /**
+   * Progress emission over MARKET_CAPTURE_STAGES (the "classify" stage, items
+   * per vendor). Presentation-only: a throwing listener never fails the run.
+   */
+  progress?: ProgressEmitter;
 };
 
 export type ClassifyMarketResult = {
@@ -129,7 +138,10 @@ export async function classifyMarket(
   options: ClassifyMarketOptions,
 ): Promise<ClassifyMarketResult> {
   const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
-  const { entries, textByHash } = loadCaptureTexts(config.category, options.capturesDir);
+  const progress = options.progress ?? nullProgressEmitter();
+  const { entries, textByHash } = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(config.category, options.capturesDir);
   if (entries.length === 0) {
     throw new Error(`No captures for ${config.category} — run \`market capture\` first`);
   }
@@ -145,12 +157,14 @@ export async function classifyMarket(
   const observations: MarketObservation[] = [];
   const retriedVendorIds: string[] = [];
 
+  progress.stage(MARKET_CAPTURE_STAGES[2], 2, MARKET_CAPTURE_STAGES.length);
   for (const [vendorIndex, vendorId] of vendorIds.entries()) {
     try {
       options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
     } catch {
       // progress is presentation-only
     }
+    progress.note(vendorId);
     const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
     if (!vendor) throw new Error(`Unknown vendor "${vendorId}"`);
     const vendorEntries = runEntries.filter((entry) => entry.vendorId === vendorId);
@@ -174,6 +188,7 @@ export async function classifyMarket(
           evidence: [],
         });
       }
+      progress.items(vendorIndex + 1, vendorIds.length);
       continue;
     }
 
@@ -229,7 +244,9 @@ export async function classifyMarket(
       );
     }
     for (const reading of outcome.readings) observations.push(toObservation(reading, vendorId));
+    progress.items(vendorIndex + 1, vendorIds.length);
   }
+  progress.flush();
 
   return {
     set: {

package/src/marketTaxonomy.ts

@@ -9,6 +9,7 @@ import {
   loadCaptureTexts,
   type MarketClaim,
   type MarketConfig,
+  type MarketStore,
   type MarketVendor,
 } from "./market.ts";
 
@@ -44,6 +45,11 @@ export type SuggestTaxonomyOptions = {
   /** Test injectables. */
   fetchPage?: FetchPage;
   capturesDir?: string;
+  /**
+   * Storage seam for the bootstrap captures; defaults to the file layout.
+   * Pass createMemoryMarketStore for throwaway grounding (hosted flow).
+   */
+  store?: MarketStore;
   now?: () => Date;
 };
 
@@ -227,9 +233,11 @@ export async function suggestMarketConfig(options: SuggestTaxonomyOptions): Prom
   // fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
   await captureMarket(
     { category, vendors, claims: [] },
-    { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
+    { dir: options.capturesDir, store: options.store, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
   );
-  const capture = loadCaptureTexts(category, options.capturesDir);
+  const capture = options.store
+    ? await options.store.loadCaptureTexts()
+    : loadCaptureTexts(category, options.capturesDir);
   const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
   if (!dossier.trim()) {
     throw new Error(

package/src/mcp-bin.ts

@@ -1,6 +1,38 @@
 #!/usr/bin/env node
+const args = process.argv.slice(2);
+
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`Usage: fullstackgtm-mcp [--help] [--version]
+
+Starts the fullstackgtm MCP stdio server.
+
+The MCP server requires optional peer dependencies:
+  @modelcontextprotocol/sdk
+  zod
+
+Zero-install with peers:
+  npx -y -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`);
+  process.exit(0);
+}
+
+if (args.includes("--version") || args.includes("-v")) {
+  console.log(await readPackageVersion());
+  process.exit(0);
+}
+
 export {};
 
+async function readPackageVersion(): Promise<string> {
+  try {
+    const { readFileSync } = await import("node:fs");
+    const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+    const parsed = JSON.parse(raw) as { version?: string };
+    return parsed.version ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
 // The MCP server needs the optional peer dependencies. Import dynamically so
 // a missing peer produces install guidance instead of a module-load stack
 // trace — `npx fullstackgtm-mcp` alone never installs optional peers.