fullstackgtm 0.15.0 → 0.17.0

package/src/cli.ts

@@ -39,6 +39,20 @@ import { auditReportToHtml, auditReportToMarkdown, type ReportOptions } from "./
 import { builtinAuditRules } from "./rules.ts";
 import { sampleSnapshot } from "./sampleData.ts";
 import { normalizeTranscript, parseCall, suggestCallDeal, type ExtractedCallInsight, type ParsedCall } from "./calls.ts";
+import {
+  captureMarket,
+  computeFrontStates,
+  createFileObservationStore,
+  diffFrontStates,
+  loadCaptureTexts,
+  loadMarketConfig,
+  starterMarketConfig,
+  validateObservationSet,
+  verifyEvidenceSpans,
+  type ObservationSet,
+} from "./market.ts";
+import { buildWorksheet, classifyMarket } from "./marketClassify.ts";
+import { marketMapToHtml, marketMapToMarkdown } from "./marketReport.ts";
 import {
   DEFAULT_RUBRIC,
   detectProviderFromKey,
@@ -94,6 +108,20 @@ Usage:
   fullstackgtm resolve <account|contact|deal> [--name N] [--domain D] [--email E] [--account-id A] [source options] [--json]
                                                the create gate: exit 0 = safe to create, exit 2 = match
                                                found (exists/ambiguous) — call before ANY record creation
+  fullstackgtm market init --category <name>   start a market map: vendors + claim taxonomy as reviewable config
+  fullstackgtm market capture [--config <path>] [--run <label>]
+  fullstackgtm market classify [--run <label>] [--vendor <id>] [--model m] [--out <path>]
+  fullstackgtm market worksheet --vendor <id> [--out <path>]
+  fullstackgtm market observe --from <observations.json> [--unverified]
+  fullstackgtm market fronts [--run <label>] [--diff <prior-run>] [--json]
+  fullstackgtm market report [--run <label>] [--format md|html] [--out <path>]
+  fullstackgtm market refresh [--run <label>] [--model m]
+                                               the live competitive map: capture vendor pages (content-addressed),
+                                               classify intensity per claim (LLM bring-your-own-key, or fill the
+                                               worksheet with any agent) — every quoted span is verified verbatim
+                                               against the stored capture it cites before it's accepted — then
+                                               compute deterministic front states and drift, render the field
+                                               report. refresh = capture → classify → drift → report in one step
   fullstackgtm suggest --plan-id <id> | --plan <path>  [source options] [--json] [--out <path>]
                                                derive values for requires_human_* placeholders
                                                from snapshot evidence, with confidence + reasons
@@ -676,15 +704,22 @@ the free keyword baseline; score always needs a key (scoring is LLM work).`);
  * TTY a missing key is captured once (validated, stored 0600 like provider
  * logins). Non-interactive contexts get an actionable error instead.
  */
-async function requireLlmCredential(command: "parse" | "score" = "parse"): Promise<{ provider: LlmProvider; apiKey: string }> {
+async function requireLlmCredential(
+  command: "parse" | "score" | "market classify" = "parse",
+): Promise<{ provider: LlmProvider; apiKey: string }> {
   const resolved = resolveLlmCredential();
   if (resolved) return resolved;
   // Scoring is inherently LLM work — there is no keyword fallback to suggest.
   const fallbackHint =
-    command === "parse" ? ", or pass --deterministic for the free keyword baseline" : " (call score has no non-LLM mode)";
+    command === "parse"
+      ? ", or pass --deterministic for the free keyword baseline"
+      : command === "score"
+        ? " (call score has no non-LLM mode)"
+        : ", or classify by hand: `market worksheet --vendor <id>` then `market observe --from`";
+  const work = command === "score" ? "scoring" : command === "parse" ? "extraction" : "classification";
   if (!process.stdin.isTTY) {
     throw new Error(
-      `LLM ${command === "score" ? "scoring" : "extraction"} needs an API key. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or run \`echo "$KEY" | fullstackgtm login anthropic\` (or \`login openai\`) once${fallbackHint}.`,
+      `LLM ${work} needs an API key. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or run \`echo "$KEY" | fullstackgtm login anthropic\` (or \`login openai\`) once${fallbackHint}.`,
     );
   }
   console.error("LLM parsing needs an API key (Anthropic or OpenAI) — yours, used directly with the provider.");
@@ -801,6 +836,228 @@ function buildCallPlan(
   };
 }
 
+/**
+ * The market map: claim taxonomy in a reviewable config file, page captures
+ * and append-only observations under the profile home, deterministic front
+ * states and reports computed from the store. Intensity readings enter as
+ * proposals through two channels — `classify` (LLM, bring-your-own-key, the
+ * call-intelligence pattern) and `worksheet`/`observe` (an agent or human
+ * fills the worksheet) — and BOTH pass the same mechanical gate: every quoted
+ * span is verified verbatim against the stored capture it cites.
+ */
+async function marketCommand(args: string[]) {
+  const [subcommand, ...rest] = args;
+  const configPath = () => resolve(process.cwd(), option(rest, "--config") ?? "market.config.json");
+
+  if (!subcommand || subcommand === "--help") {
+    console.log(`Usage:
+market init --category <name> [--out <path>]   write a starter market.config.json
+market capture [--config <path>] [--run <label>]
+market classify [--run <label>] [--capture-run <label>] [--vendor <id>] [--model m] [--out <path>]
+market worksheet --vendor <id> [--capture-run <label>] [--out <path>]
+market observe --from <observations.json> [--unverified]
+market fronts [--config <path>] [--run <label>] [--diff <prior-run>] [--json]
+market report [--config <path>] [--run <label>] [--format md|html] [--out <path>]
+market refresh [--run <label>] [--model m]     capture → classify → fronts drift → HTML report
+
+classify uses your Anthropic/OpenAI key (like call parse) to read the stored
+captures and propose intensity readings; worksheet is the no-key path (an
+agent or human fills it, submits via observe). Either way, every quoted span
+is verified character-for-character against the capture it cites before the
+observation is accepted — quotes that aren't on the page bounce.
+
+The taxonomy (vendors + claims) is config you review and version; captures
+and observations live under ~/.fullstackgtm/market/<category> (profile-scoped,
+one client's category intel never bleeds into another's). Front states are
+recomputed deterministically on every invocation — never stored.`);
+    return;
+  }
+
+  if (subcommand === "init") {
+    const category = option(rest, "--category");
+    if (!category) throw new Error("market init requires --category <name>");
+    const outPath = resolve(process.cwd(), option(rest, "--out") ?? "market.config.json");
+    if (existsSync(outPath)) throw new Error(`${outPath} already exists — refusing to overwrite`);
+    writeFileSync(outPath, `${JSON.stringify(starterMarketConfig(category), null, 2)}\n`);
+    console.log(`Wrote ${outPath}. Fill in vendors and claims, then: fullstackgtm market capture`);
+    return;
+  }
+
+  const config = loadMarketConfig(configPath());
+  const store = createFileObservationStore(config.category);
+
+  if (subcommand === "capture") {
+    const result = await captureMarket(config, { runLabel: option(rest, "--run") ?? "run-1" });
+    for (const entry of result.entries) {
+      const flag = entry.captureHash && entry.textChars > 500 ? "" : "  <-- thin/empty";
+      console.log(
+        `${entry.vendorId.padEnd(16)} ${entry.kind.padEnd(8)} ${String(entry.httpStatus ?? "ERR").padEnd(4)} ${String(entry.textChars).padStart(7)} chars  ${entry.url}${flag}`,
+      );
+    }
+    console.log(`\nmanifest: ${result.manifestPath}`);
+    return;
+  }
+
+  if (subcommand === "observe") {
+    const fromPath = option(rest, "--from");
+    if (!fromPath) throw new Error("market observe requires --from <observations.json>");
+    const set = JSON.parse(readFileSync(resolve(process.cwd(), fromPath), "utf8")) as ObservationSet;
+    const problems = validateObservationSet(config, set);
+    if (problems.length > 0) {
+      console.error(`Rejected: ${problems.length} problem(s)`);
+      for (const problem of problems.slice(0, 20)) console.error(`  - ${problem}`);
+      process.exitCode = 1;
+      return;
+    }
+    if (!rest.includes("--unverified")) {
+      const { textByHash } = loadCaptureTexts(config.category);
+      const failures = verifyEvidenceSpans(set.observations, textByHash);
+      if (failures.length > 0) {
+        console.error(`Rejected: ${failures.length} evidence span(s) failed verification against the stored captures`);
+        for (const failure of failures.slice(0, 20)) {
+          console.error(`  - ${failure.vendorId} × ${failure.claimId}: ${failure.problem}`);
+        }
+        console.error("Quotes must be copied verbatim from the captured pages. (--unverified skips this gate when the captures genuinely live elsewhere.)");
+        process.exitCode = 1;
+        return;
+      }
+    }
+    await store.append(set);
+    console.log(`Appended ${set.runLabel}: ${set.observations.length} observations (${set.extractor})`);
+    return;
+  }
+
+  if (subcommand === "worksheet") {
+    const vendorId = option(rest, "--vendor");
+    if (!vendorId) throw new Error("market worksheet requires --vendor <id>");
+    const worksheet = buildWorksheet(config, vendorId, { captureRun: option(rest, "--capture-run") ?? undefined });
+    const outPath = option(rest, "--out");
+    const payload = `${JSON.stringify(worksheet, null, 2)}\n`;
+    if (outPath) {
+      writeFileSync(resolve(process.cwd(), outPath), payload);
+      console.log(`Wrote ${outPath} (${worksheet.pages.length} captured pages, ${worksheet.claims.length} claims)`);
+    } else {
+      console.log(payload);
+    }
+    return;
+  }
+
+  if (subcommand === "classify") {
+    const credential = await requireLlmCredential("market classify");
+    const vendorFilter = option(rest, "--vendor");
+    const outPath = option(rest, "--out");
+    if (vendorFilter && !outPath) {
+      throw new Error(
+        "market classify --vendor produces a partial set (coverage validation would reject it) — pass --out <path> to inspect/merge it by hand",
+      );
+    }
+    const result = await classifyMarket(config, {
+      llm: { ...credential, model: option(rest, "--model") ?? undefined },
+      runLabel: option(rest, "--run") ?? option(rest, "--capture-run") ?? "run-1",
+      captureRun: option(rest, "--capture-run") ?? undefined,
+      vendors: vendorFilter ? [vendorFilter] : undefined,
+    });
+    if (result.retriedVendorIds.length > 0) {
+      console.error(`Span verification bounced ${result.retriedVendorIds.join(", ")} once; retry passed.`);
+    }
+    if (outPath) {
+      writeFileSync(resolve(process.cwd(), outPath), `${JSON.stringify(result.set, null, 2)}\n`);
+      console.log(`Wrote ${outPath}: ${result.set.observations.length} verified observations (${result.set.extractor})`);
+      return;
+    }
+    const problems = validateObservationSet(config, result.set);
+    if (problems.length > 0) {
+      throw new Error(`Classified set failed validation: ${problems.slice(0, 5).join("; ")}`);
+    }
+    await store.append(result.set);
+    console.log(
+      `Appended ${result.set.runLabel}: ${result.set.observations.length} observations, every span verified (${result.set.extractor})`,
+    );
+    return;
+  }
+
+  if (subcommand === "refresh") {
+    const credential = await requireLlmCredential("market classify");
+    const runLabel = option(rest, "--run") ?? `run-${new Date().toISOString().slice(0, 10)}`;
+    const prior = await store.latest();
+    console.log(`Capturing ${config.vendors.length} vendors as ${runLabel}…`);
+    const captured = await captureMarket(config, { runLabel });
+    const failed = captured.entries.filter((entry) => !entry.captureHash);
+    if (failed.length > 0) console.log(`${failed.length} page(s) failed/empty — affected cells will verify against remaining pages or read unobservable.`);
+    console.log(`Classifying with ${credential.provider}…`);
+    const result = await classifyMarket(config, {
+      llm: { ...credential, model: option(rest, "--model") ?? undefined },
+      runLabel,
+      captureRun: runLabel,
+    });
+    await store.append(result.set);
+    const fronts = computeFrontStates(config, result.set);
+    if (prior) {
+      const drift = diffFrontStates(computeFrontStates(config, prior), fronts);
+      if (drift.length === 0) console.log(`No front changes since ${prior.runLabel}.`);
+      for (const change of drift) console.log(`CHANGED   ${change.claimId}: ${change.before} → ${change.after}`);
+    }
+    const outPath = option(rest, "--out") ?? `${config.category}-${runLabel}.html`;
+    writeFileSync(resolve(process.cwd(), outPath), marketMapToHtml(config, result.set));
+    console.log(`Wrote ${outPath}`);
+    return;
+  }
+
+  const loadSet = async (): Promise<ObservationSet> => {
+    const runLabel = option(rest, "--run");
+    const set = runLabel ? await store.get(runLabel) : await store.latest();
+    if (!set) {
+      throw new Error(
+        runLabel
+          ? `No observation run "${runLabel}" for ${config.category}`
+          : `No observations stored for ${config.category} — run market observe --from <file> first`,
+      );
+    }
+    return set;
+  };
+
+  if (subcommand === "fronts") {
+    const set = await loadSet();
+    const fronts = computeFrontStates(config, set);
+    const priorLabel = option(rest, "--diff");
+    const prior = priorLabel ? await store.get(priorLabel) : null;
+    if (priorLabel && !prior) throw new Error(`No observation run "${priorLabel}" to diff against`);
+    const drift = prior ? diffFrontStates(computeFrontStates(config, prior), fronts) : null;
+    if (rest.includes("--json")) {
+      console.log(JSON.stringify({ runLabel: set.runLabel, fronts, drift }, null, 2));
+      return;
+    }
+    for (const front of fronts) {
+      const owner = front.state === "owned" ? ` → ${front.loudVendorIds[0]}` : "";
+      console.log(`${front.state.toUpperCase().padEnd(10)} ${front.claimId}${owner}`);
+    }
+    if (drift) {
+      console.log("");
+      if (drift.length === 0) console.log(`No front changes since ${priorLabel}.`);
+      for (const change of drift) console.log(`CHANGED   ${change.claimId}: ${change.before} → ${change.after}`);
+    }
+    return;
+  }
+
+  if (subcommand === "report") {
+    const set = await loadSet();
+    const format = option(rest, "--format") ?? "md";
+    const output = format === "html" ? marketMapToHtml(config, set) : marketMapToMarkdown(config, set);
+    const outPath = option(rest, "--out");
+    if (outPath) {
+      writeFileSync(resolve(process.cwd(), outPath), output);
+      console.log(`Wrote ${outPath}`);
+    } else {
+      console.log(output);
+    }
+    return;
+  }
+
+  throw new Error(
+    `Unknown market subcommand: ${subcommand} (try: init, capture, classify, worksheet, observe, fronts, report, refresh)`,
+  );
+}
+
 /**
  * The resolve gate: exit 0 = safe to create, exit 2 = match found (exists or
  * ambiguous — do NOT blind-create), exit 1 = error. Built for sync jobs and
@@ -1665,6 +1922,10 @@ export async function runCli(argv: string[]) {
     await resolveCommand(args);
     return;
   }
+  if (command === "market") {
+    await marketCommand(args);
+    return;
+  }
   if (command === "profiles") {
     profilesCommand(args);
     return;

package/src/connectors/hubspot.ts

@@ -431,7 +431,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
             created = await request(`/crm/v3/objects/companies`, {
               method: "POST",
               body: JSON.stringify({
-                properties: { name, hs_object_source_detail_2: "fullstackgtm create: operation" },
+                properties: { name, hs_object_source_detail_2: `fullstackgtm create: (${operation.id})` },
               }),
             });
           } catch {

package/src/index.ts

@@ -130,6 +130,44 @@ export {
   type ScoredDimension,
 } from "./llm.ts";
 export { resolveRecord, type ResolveCandidate, type ResolveMatch, type ResolveResult } from "./resolve.ts";
+export {
+  captureMarket,
+  computeFrontStates,
+  createFileObservationStore,
+  diffFrontStates,
+  extractReadableText,
+  loadCaptureTexts,
+  loadMarketConfig,
+  marketHome,
+  normalizeForMatch,
+  observationId,
+  parseMarketConfig,
+  starterMarketConfig,
+  validateObservationSet,
+  verifyEvidenceSpans,
+  type CaptureEntry,
+  type CaptureOptions,
+  type ClaimFront,
+  type ClaimIntensity,
+  type FrontDrift,
+  type FrontState,
+  type MarketClaim,
+  type MarketConfig,
+  type MarketObservation,
+  type MarketVendor,
+  type ObservationConfidence,
+  type ObservationSet,
+  type ObservationStore,
+  type SpanVerificationFailure,
+} from "./market.ts";
+export {
+  buildWorksheet,
+  classifyMarket,
+  type ClassifyMarketOptions,
+  type ClassifyMarketResult,
+  type MarketWorksheet,
+} from "./marketClassify.ts";
+export { marketMapToHtml, marketMapToMarkdown } from "./marketReport.ts";
 export { suggestValues, type SuggestionConfidence, type ValueSuggestion } from "./suggest.ts";
 export type {
   ApprovalStatus,

package/src/llm.ts

@@ -239,7 +239,13 @@ export function parseRubric(json: string): Rubric {
 
 // ── Provider plumbing (raw fetch, forced tool calls) ───────────────────────
 
-async function forcedToolCall(
+/**
+ * Shared constrained-tool-call plumbing: force the model to answer through a
+ * single tool whose input_schema is the output contract. Exported for other
+ * semi-deterministic features (market classification) — every LLM feature in
+ * the package goes through this one seam.
+ */
+export async function forcedToolCall(
   prompt: string,
   toolName: string,
   schema: object,