fullstackgtm 0.44.0 → 0.46.0

package/dist/market.js

@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.js";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
 const INTENSITY_RANK = {
     loud: 3,
     quiet: 2,
@@ -283,58 +284,57 @@ const defaultFetchPage = async (url) => {
     throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
 };
 export async function captureMarket(config, options = {}) {
-    const dir = options.dir ?? join(marketHome(config.category), "captures");
+    const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
+    const progress = options.progress ?? nullProgressEmitter();
     const runLabel = options.runLabel ?? "run-1";
     const fetchPage = options.fetchPage ?? defaultFetchPage;
     const fetchedAt = (options.now ?? (() => new Date()))().toISOString();
-    mkdirSync(dir, { recursive: true });
-    const manifestPath = join(dir, "manifest.json");
-    const manifest = existsSync(manifestPath)
-        ? JSON.parse(readFileSync(manifestPath, "utf8"))
-        : [];
-    const entries = [];
+    progress.stage(MARKET_CAPTURE_STAGES[0], 0, MARKET_CAPTURE_STAGES.length);
+    const targets = [];
     for (const vendor of config.vendors) {
-        const targets = [
-            { kind: "home", url: vendor.urls.home },
-        ];
+        targets.push({ vendorId: vendor.id, kind: "home", url: vendor.urls.home });
         if (vendor.urls.pricing)
-            targets.push({ kind: "pricing", url: vendor.urls.pricing });
+            targets.push({ vendorId: vendor.id, kind: "pricing", url: vendor.urls.pricing });
         for (const url of vendor.urls.product)
-            targets.push({ kind: "product", url });
-        for (const target of targets) {
-            let status = null;
-            let text = "";
-            try {
-                const page = await fetchPage(target.url);
-                status = page.status;
-                if (page.status === 200)
-                    text = extractReadableText(page.body);
-            }
-            catch {
-                status = null;
-            }
-            let captureHash = null;
-            if (text) {
-                captureHash = createHash("sha256").update(text).digest("hex");
-                // Content-addressed: an unchanged page dedupes to the same file.
-                writeFileSync(join(dir, `${captureHash}.txt`), text);
-            }
-            const entry = {
-                runLabel,
-                vendorId: vendor.id,
-                kind: target.kind,
-                url: target.url,
-                fetchedAt,
-                httpStatus: status,
-                captureHash,
-                textChars: text.length,
-            };
-            manifest.push(entry);
-            entries.push(entry);
+            targets.push({ vendorId: vendor.id, kind: "product", url });
+    }
+    progress.note(`${config.vendors.length} vendor(s), ${targets.length} page(s)`);
+    progress.stage(MARKET_CAPTURE_STAGES[1], 1, MARKET_CAPTURE_STAGES.length);
+    const entries = [];
+    for (const target of targets) {
+        let status = null;
+        let text = "";
+        try {
+            const page = await fetchPage(target.url);
+            status = page.status;
+            if (page.status === 200)
+                text = extractReadableText(page.body);
+        }
+        catch {
+            status = null;
         }
+        let captureHash = null;
+        if (text) {
+            captureHash = createHash("sha256").update(text).digest("hex");
+            // Content-addressed: an unchanged page dedupes to the same key.
+            await store.saveCaptureText(captureHash, text);
+        }
+        entries.push({
+            runLabel,
+            vendorId: target.vendorId,
+            kind: target.kind,
+            url: target.url,
+            fetchedAt,
+            httpStatus: status,
+            captureHash,
+            textChars: text.length,
+        });
+        progress.items(entries.length, targets.length);
     }
-    writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-    return { entries, manifestPath };
+    progress.stage(MARKET_CAPTURE_STAGES[3], 3, MARKET_CAPTURE_STAGES.length);
+    await store.appendCaptureEntries(entries);
+    progress.flush();
+    return { entries, manifestPath: store.captureLocation() };
 }
 export function createFileObservationStore(category, directory) {
     const dir = directory ?? join(marketHome(category), "observations");
@@ -459,6 +459,78 @@ export function loadCaptureTexts(category, directory) {
     }
     return { entries, textByHash };
 }
+/** The CLI's store: captures + observations under the profile market home. */
+export function createFileMarketStore(category, options = {}) {
+    const dir = options.capturesDir ?? join(marketHome(category), "captures");
+    const manifestPath = join(dir, "manifest.json");
+    return {
+        captureLocation: () => manifestPath,
+        async saveCaptureText(captureHash, text) {
+            mkdirSync(dir, { recursive: true });
+            writeFileSync(join(dir, `${captureHash}.txt`), text);
+        },
+        async appendCaptureEntries(entries) {
+            mkdirSync(dir, { recursive: true });
+            const manifest = existsSync(manifestPath)
+                ? JSON.parse(readFileSync(manifestPath, "utf8"))
+                : [];
+            manifest.push(...entries);
+            writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+        },
+        async loadCaptureTexts() {
+            return loadCaptureTexts(category, dir);
+        },
+        observations: createFileObservationStore(category, options.observationsDir),
+    };
+}
+/**
+ * In-memory store: seam tests, and throwaway grounding captures (e.g. the
+ * taxonomy proposer's bootstrap pass) that should never persist anywhere.
+ */
+export function createMemoryMarketStore(category) {
+    const entries = [];
+    const textByHash = new Map();
+    const sets = new Map();
+    const sorted = () => [...sets.values()].sort((a, b) => a.runAt.localeCompare(b.runAt));
+    return {
+        captureLocation: () => `memory://${category}/captures`,
+        async saveCaptureText(captureHash, text) {
+            textByHash.set(captureHash, text);
+        },
+        async appendCaptureEntries(newEntries) {
+            entries.push(...newEntries);
+        },
+        async loadCaptureTexts() {
+            return { entries: [...entries], textByHash: new Map(textByHash) };
+        },
+        observations: {
+            async append(set) {
+                if (set.category !== category) {
+                    throw new Error(`Observation set category "${set.category}" does not match store "${category}"`);
+                }
+                if (sets.has(set.runLabel)) {
+                    throw new Error(`Run "${set.runLabel}" already exists — observations are append-only; use a new run label`);
+                }
+                sets.set(set.runLabel, set);
+                return set;
+            },
+            async get(runLabel) {
+                return sets.get(runLabel) ?? null;
+            },
+            async list() {
+                return sorted().map((set) => ({
+                    runLabel: set.runLabel,
+                    runAt: set.runAt,
+                    observations: set.observations.length,
+                }));
+            },
+            async latest() {
+                const all = sorted();
+                return all.length ? all[all.length - 1] : null;
+            },
+        },
+    };
+}
 /**
  * Whitespace-only normalization for span matching, plus one extraction
  * artifact: the HTML-to-text step can emit a line break before punctuation

package/dist/marketClassify.d.ts

@@ -1,5 +1,6 @@
 import { type LlmCallOptions } from "./llm.ts";
-import { type CaptureEntry, type MarketClaim, type MarketConfig, type ObservationSet } from "./market.ts";
+import { type CaptureEntry, type MarketClaim, type MarketConfig, type MarketStore, type ObservationSet } from "./market.ts";
+import { type ProgressEmitter } from "./progress.ts";
 export type ClassifyMarketOptions = {
     llm: LlmCallOptions;
     /** Observation run label to produce; must be new (the store is append-only). */
@@ -10,7 +11,16 @@ export type ClassifyMarketOptions = {
     vendors?: string[];
     /** Captures directory override (tests); defaults to the profile market home. */
     capturesDir?: string;
+    /** Storage seam: when given, captures are read through it instead of the file layout. */
+    store?: MarketStore;
     now?: () => Date;
+    /** Per-vendor progress (presentation only — a throwing callback never fails the run). */
+    onVendor?: (done: number, total: number, vendorId: string) => void;
+    /**
+     * Progress emission over MARKET_CAPTURE_STAGES (the "classify" stage, items
+     * per vendor). Presentation-only: a throwing listener never fails the run.
+     */
+    progress?: ProgressEmitter;
 };
 export type ClassifyMarketResult = {
     set: ObservationSet;

package/dist/marketClassify.js

@@ -1,5 +1,6 @@
 import { DEFAULT_MODELS, forcedToolCall } from "./llm.js";
 import { loadCaptureTexts, observationId, verifyEvidenceSpans, } from "./market.js";
+import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
 /**
  * LLM intensity classification for the market map — the same
  * semi-deterministic posture as call extraction, with one upgrade calls
@@ -78,7 +79,10 @@ function claimsBlock(claims) {
 }
 export async function classifyMarket(config, options) {
     const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
-    const { entries, textByHash } = loadCaptureTexts(config.category, options.capturesDir);
+    const progress = options.progress ?? nullProgressEmitter();
+    const { entries, textByHash } = options.store
+        ? await options.store.loadCaptureTexts()
+        : loadCaptureTexts(config.category, options.capturesDir);
     if (entries.length === 0) {
         throw new Error(`No captures for ${config.category} — run \`market capture\` first`);
     }
@@ -92,7 +96,15 @@ export async function classifyMarket(config, options) {
     const claimIds = config.claims.map((claim) => claim.id);
     const observations = [];
     const retriedVendorIds = [];
-    for (const vendorId of vendorIds) {
+    progress.stage(MARKET_CAPTURE_STAGES[2], 2, MARKET_CAPTURE_STAGES.length);
+    for (const [vendorIndex, vendorId] of vendorIds.entries()) {
+        try {
+            options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
+        }
+        catch {
+            // progress is presentation-only
+        }
+        progress.note(vendorId);
         const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
         if (!vendor)
             throw new Error(`Unknown vendor "${vendorId}"`);
@@ -114,6 +126,7 @@ export async function classifyMarket(config, options) {
                     evidence: [],
                 });
             }
+            progress.items(vendorIndex + 1, vendorIds.length);
             continue;
         }
         const prompt = (feedback) => `${CLASSIFY_INSTRUCTIONS}\n\nSurface rule for this category:\n${config.surfaceRule ?? "(default rule above)"}\n\nClaims to classify (all of them):\n${claimsBlock(config.claims)}\n${feedback}\nVendor: ${vendor.name}\nCaptured pages:\n${dossier}`;
@@ -159,7 +172,9 @@ export async function classifyMarket(config, options) {
         }
         for (const reading of outcome.readings)
             observations.push(toObservation(reading, vendorId));
+        progress.items(vendorIndex + 1, vendorIds.length);
     }
+    progress.flush();
     return {
         set: {
             id: `set_${config.category}_${options.runLabel}`,

package/dist/marketTaxonomy.d.ts

@@ -1,5 +1,5 @@
 import { type LlmCallOptions } from "./llm.ts";
-import { type FetchPage, type MarketConfig } from "./market.ts";
+import { type FetchPage, type MarketConfig, type MarketStore } from "./market.ts";
 /**
  * Cold-start taxonomy bootstrap. `market init` writes a stub for a human
  * analyst to fill in; the self-serve hosted map has no analyst in the loop, so
@@ -30,6 +30,11 @@ export type SuggestTaxonomyOptions = {
     /** Test injectables. */
     fetchPage?: FetchPage;
     capturesDir?: string;
+    /**
+     * Storage seam for the bootstrap captures; defaults to the file layout.
+     * Pass createMemoryMarketStore for throwaway grounding (hosted flow).
+     */
+    store?: MarketStore;
     now?: () => Date;
 };
 export type SuggestTaxonomyResult = {

package/dist/marketTaxonomy.js

@@ -147,8 +147,10 @@ export async function suggestMarketConfig(options) {
     const anchorId = anchorSeed ? vendors[options.vendors.indexOf(anchorSeed)]?.id : undefined;
     // Capture the seed homepages so the proposer only sees text we actually
     // fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
-    await captureMarket({ category, vendors, claims: [] }, { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now });
-    const capture = loadCaptureTexts(category, options.capturesDir);
+    await captureMarket({ category, vendors, claims: [] }, { dir: options.capturesDir, store: options.store, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now });
+    const capture = options.store
+        ? await options.store.loadCaptureTexts()
+        : loadCaptureTexts(category, options.capturesDir);
     const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
     if (!dossier.trim()) {
         throw new Error(`market init --auto: none of the ${vendors.length} seed pages returned readable text — check the URLs are public homepages.`);

package/dist/mcp-bin.js

@@ -1,4 +1,33 @@
 #!/usr/bin/env node
+const args = process.argv.slice(2);
+if (args.includes("--help") || args.includes("-h")) {
+    console.log(`Usage: fullstackgtm-mcp [--help] [--version]
+
+Starts the fullstackgtm MCP stdio server.
+
+The MCP server requires optional peer dependencies:
+  @modelcontextprotocol/sdk
+  zod
+
+Zero-install with peers:
+  npx -y -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`);
+    process.exit(0);
+}
+if (args.includes("--version") || args.includes("-v")) {
+    console.log(await readPackageVersion());
+    process.exit(0);
+}
+async function readPackageVersion() {
+    try {
+        const { readFileSync } = await import("node:fs");
+        const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+        const parsed = JSON.parse(raw);
+        return parsed.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
 // The MCP server needs the optional peer dependencies. Import dynamically so
 // a missing peer produces install guidance instead of a module-load stack
 // trace — `npx fullstackgtm-mcp` alone never installs optional peers.
@@ -7,8 +36,8 @@ const MISSING_PEER_HELP = `The MCP server needs the optional peer dependencies @
 In a project:
   npm install fullstackgtm @modelcontextprotocol/sdk zod
 
-Zero-install:
-  npx -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`;
+Zero-install (the fullstackgtm-mcp wrapper package bundles the peers):
+  npx -y fullstackgtm-mcp`;
 function isMissingPeerError(error) {
     if (!(error instanceof Error))
         return false;