fullstackgtm 0.48.0 → 0.49.0

package/src/connectors/salesforce.ts

@@ -19,6 +19,7 @@ import type {
   PatchOperationResult,
   SnapshotProgress,
 } from "../types.ts";
+import { validateSalesforceOrigin } from "./salesforceAuth.ts";
 import { SNAPSHOT_PULL_STAGES, type ProgressEmitter } from "../progress.ts";
 
 const DEFAULT_API_VERSION = "v59.0";
@@ -152,13 +153,17 @@ export function createSalesforceConnector(
 
   async function request(path: string, init: RequestInit = {}): Promise<any> {
     const connection = await options.getConnection();
-    const url = path.startsWith("http")
-      ? path
-      : `${connection.instanceUrl.replace(/\/$/, "")}${path}`;
+    const instanceUrl = validateSalesforceOrigin(connection.instanceUrl, "Salesforce connection instance URL");
+    const resolved = new URL(path, `${instanceUrl}/`);
+    if (resolved.origin !== instanceUrl) {
+      throw new Error("Salesforce response attempted to send credentials to a different origin.");
+    }
+    const url = resolved.href;
     let response: Response;
     try {
       response = await fetchImpl(url, {
         ...init,
+        redirect: "manual",
         headers: {
           Authorization: `Bearer ${connection.accessToken}`,
           "Content-Type": "application/json",
@@ -168,7 +173,7 @@ export function createSalesforceConnector(
     } catch (error) {
       const cause = error instanceof Error && error.cause instanceof Error ? `: ${error.cause.message}` : "";
       throw new Error(
-        `Cannot reach Salesforce at ${connection.instanceUrl}${cause}. Check SALESFORCE_INSTANCE_URL (your My Domain URL, e.g. https://yourco.my.salesforce.com) and network access.`,
+        `Cannot reach Salesforce at ${instanceUrl}${cause}. Check SALESFORCE_INSTANCE_URL (your My Domain URL, e.g. https://yourco.my.salesforce.com) and network access.`,
       );
     }
     if (!response.ok) {
@@ -644,8 +649,9 @@ export function createSalesforceConnector(
     loserIds: string[],
   ): Promise<{ ok: boolean; detail?: string }> {
     const connection = await options.getConnection();
+    const instanceUrl = validateSalesforceOrigin(connection.instanceUrl, "Salesforce connection instance URL");
     const version = apiVersion.replace(/^v/, ""); // SOAP path uses "59.0", not "v59.0"
-    const url = `${connection.instanceUrl.replace(/\/$/, "")}/services/Soap/u/${version}`;
+    const url = `${instanceUrl}/services/Soap/u/${version}`;
     const toMerge = loserIds.map((id) => `<urn:recordToMergeIds>${escapeXml(id)}</urn:recordToMergeIds>`).join("");
     const envelope =
       `<?xml version="1.0" encoding="UTF-8"?>` +
@@ -659,6 +665,7 @@ export function createSalesforceConnector(
     try {
       response = await fetchImpl(url, {
         method: "POST",
+        redirect: "manual",
         headers: { "Content-Type": "text/xml; charset=UTF-8", SOAPAction: '""' },
         body: envelope,
       });

package/src/connectors/salesforceAuth.ts

@@ -9,6 +9,41 @@
 
 const DEFAULT_LOGIN_URL = "https://login.salesforce.com";
 
+/**
+ * Normalize and validate a Salesforce credential-bearing origin.
+ *
+ * Salesforce API hosts are either the standard login/sandbox hosts or a
+ * Salesforce-owned instance/My Domain below salesforce.com.  Deliberately
+ * return an origin (not the input URL) so callers cannot accidentally retain
+ * paths, queries, credentials, or fragments from configuration.
+ */
+export function validateSalesforceOrigin(value: string, label = "Salesforce URL"): string {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`${label} must be a valid HTTPS Salesforce URL.`);
+  }
+  if (url.protocol !== "https:") throw new Error(`${label} must use HTTPS.`);
+  if (url.username || url.password) throw new Error(`${label} must not contain user information.`);
+  if (url.hash) throw new Error(`${label} must not contain a fragment.`);
+  if (url.search) throw new Error(`${label} must not contain a query string.`);
+  if (url.pathname !== "/" && url.pathname !== "") throw new Error(`${label} must be an origin without a path.`);
+  if (url.port && url.port !== "443") throw new Error(`${label} must use the standard HTTPS port.`);
+
+  const hostname = url.hostname.toLowerCase();
+  const trusted =
+    hostname === "login.salesforce.com" ||
+    hostname === "test.salesforce.com" ||
+    (hostname.endsWith(".salesforce.com") && hostname.length > ".salesforce.com".length);
+  if (!trusted) {
+    throw new Error(
+      `${label} must use login.salesforce.com, test.salesforce.com, or a Salesforce instance/My Domain host.`,
+    );
+  }
+  return url.origin;
+}
+
 export type SalesforceTokenSet = {
   accessToken: string;
   refreshToken?: string;
@@ -27,7 +62,7 @@ export type SalesforceDeviceAuthorization = {
 const SESSION_TTL_MS = 2 * 60 * 60 * 1000;
 
 function tokenUrl(loginUrl: string) {
-  return `${loginUrl.replace(/\/$/, "")}/services/oauth2/token`;
+  return `${validateSalesforceOrigin(loginUrl, "Salesforce login URL")}/services/oauth2/token`;
 }
 
 /**
@@ -55,6 +90,7 @@ export async function startSalesforceDeviceLogin(options: {
   const fetchImpl = options.fetchImpl ?? fetch;
   const response = await fetchImpl(tokenUrl(options.loginUrl ?? DEFAULT_LOGIN_URL), {
     method: "POST",
+    redirect: "manual",
     headers: { "Content-Type": "application/x-www-form-urlencoded" },
     body: new URLSearchParams({
       response_type: "device_code",
@@ -91,6 +127,7 @@ export async function pollSalesforceDeviceLogin(options: {
     await new Promise((resolveSleep) => setTimeout(resolveSleep, intervalMs));
     const response = await fetchImpl(url, {
       method: "POST",
+      redirect: "manual",
       headers: { "Content-Type": "application/x-www-form-urlencoded" },
       body: new URLSearchParams({
         grant_type: "device",
@@ -100,10 +137,11 @@ export async function pollSalesforceDeviceLogin(options: {
     });
     const data = await response.json().catch(() => ({}));
     if (response.ok && data.access_token && data.instance_url) {
+      const instanceUrl = validateSalesforceOrigin(String(data.instance_url), "Salesforce token instance_url");
       return {
         accessToken: data.access_token,
         refreshToken: data.refresh_token,
-        instanceUrl: data.instance_url,
+        instanceUrl,
         expiresAt: Date.now() + SESSION_TTL_MS,
       };
     }
@@ -128,6 +166,7 @@ export async function refreshSalesforceToken(options: {
   const fetchImpl = options.fetchImpl ?? fetch;
   const response = await fetchImpl(tokenUrl(options.loginUrl ?? DEFAULT_LOGIN_URL), {
     method: "POST",
+    redirect: "manual",
     headers: { "Content-Type": "application/x-www-form-urlencoded" },
     body: new URLSearchParams({
       grant_type: "refresh_token",
@@ -145,7 +184,7 @@ export async function refreshSalesforceToken(options: {
   return {
     accessToken: data.access_token,
     refreshToken: data.refresh_token ?? options.refreshToken,
-    instanceUrl: data.instance_url,
+    instanceUrl: validateSalesforceOrigin(String(data.instance_url), "Salesforce token instance_url"),
     expiresAt: Date.now() + SESSION_TTL_MS,
   };
 }
@@ -155,17 +194,18 @@ export async function validateSalesforceToken(
   instanceUrl: string,
   fetchImpl: typeof fetch = fetch,
 ): Promise<{ ok: boolean; detail: string }> {
+  const trustedInstanceUrl = validateSalesforceOrigin(instanceUrl, "Salesforce instance URL");
   let response: Response;
   try {
     response = await fetchImpl(
-      `${instanceUrl.replace(/\/$/, "")}/services/oauth2/userinfo`,
-      { headers: { Authorization: `Bearer ${accessToken}` } },
+      `${trustedInstanceUrl}/services/oauth2/userinfo`,
+      { redirect: "manual", headers: { Authorization: `Bearer ${accessToken}` } },
     );
   } catch (error) {
     const cause = error instanceof Error && error.cause instanceof Error ? `: ${error.cause.message}` : "";
     return {
       ok: false,
-      detail: `Cannot reach Salesforce at ${instanceUrl}${cause}. Check the --instance-url (your My Domain URL, e.g. https://yourco.my.salesforce.com) and network access.`,
+      detail: `Cannot reach Salesforce at ${trustedInstanceUrl}${cause}. Check the --instance-url (your My Domain URL, e.g. https://yourco.my.salesforce.com) and network access.`,
     };
   }
   if (response.ok) {

package/src/connectors/theirstack.ts

@@ -17,6 +17,7 @@
  * `metadata.total_results` with include_total_results:true; limit must be ≥ 1
  * (limit:0 → 422, the count gotcha — cf. Explorium's `size`-caps-the-total).
  */
+import { ProviderHttpError } from "../providerError.ts";
 
 type FetchImpl = typeof fetch;
 
@@ -44,14 +45,6 @@ export function theirStackPullCost(companies: number, usdPerCredit?: number): Th
   };
 }
 
-async function safeText(res: { text(): Promise<string> }): Promise<string> {
-  try {
-    return (await res.text()).slice(0, 300);
-  } catch {
-    return "";
-  }
-}
-
 /** Firmographic + technographic filter for TheirStack company search. */
 export type TheirStackFilters = {
   /** technology slugs, OR-matched — e.g. ["salesforce","hubspot","pipedrive"]. */
@@ -136,7 +129,7 @@ export async function theirStackCountCompanies(opts: {
     body: buildBody(opts.filters, 1, 0, true),
   });
   if (!res.ok) {
-    throw new Error(`TheirStack count failed: HTTP ${res.status} ${await safeText(res)}`);
+    throw new ProviderHttpError("TheirStack", "count", res.status);
   }
   return readTheirStackTotal(await res.json());
 }
@@ -162,7 +155,7 @@ export async function theirStackSearchCompanies(opts: {
     body: buildBody(opts.filters, limit, opts.page ?? 0, true),
   });
   if (!res.ok) {
-    throw new Error(`TheirStack search failed: HTTP ${res.status} ${await safeText(res)}`);
+    throw new ProviderHttpError("TheirStack", "search", res.status);
   }
   const body = (await res.json()) as { data?: Array<Record<string, unknown>>; results?: Array<Record<string, unknown>> };
   const rows = body.data ?? body.results ?? [];

package/src/credentials.ts

@@ -3,17 +3,20 @@ import {
   existsSync,
   mkdirSync,
   readdirSync,
-  readFileSync,
-  statSync,
   unlinkSync,
-  writeFileSync,
 } from "node:fs";
 import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { refreshHubspotToken } from "./connectors/hubspotAuth.ts";
-import { refreshSalesforceToken } from "./connectors/salesforceAuth.ts";
+import { refreshSalesforceToken, validateSalesforceOrigin } from "./connectors/salesforceAuth.ts";
 import { detectKeychainBackend } from "./keychain.ts";
+import {
+  assertNoSymlinkComponents,
+  readSecureRegularFile,
+  UnsafeManagedPathError,
+  writeSecureFileAtomic,
+} from "./secureFile.ts";
 
 /**
  * Local CLI credential store: ~/.fullstackgtm/credentials.json (0600), or
@@ -126,7 +129,9 @@ export function ensureSecureHomeDir(): string {
   const levels =
     dir === baseHomeDir() ? [dir] : [baseHomeDir(), join(baseHomeDir(), "profiles"), dir];
   for (const level of levels) {
+    assertNoSymlinkComponents(level);
     mkdirSync(level, { recursive: true, mode: 0o700 });
+    assertNoSymlinkComponents(level);
     try {
       chmodSync(level, 0o700);
     } catch {
@@ -138,12 +143,7 @@ export function ensureSecureHomeDir(): string {
 
 /** Write a 0600 file under the home, enforcing the mode even on rewrite. */
 export function writeSecureFile(path: string, contents: string) {
-  writeFileSync(path, contents, { mode: 0o600 });
-  try {
-    chmodSync(path, 0o600);
-  } catch {
-    // Non-POSIX filesystems ignore chmod.
-  }
+  writeSecureFileAtomic(path, contents);
 }
 
 /**
@@ -153,19 +153,14 @@ export function writeSecureFile(path: string, contents: string) {
  * mode on read too — re-tighten to 0600 and warn once — so a world-readable
  * credential store can't sit there silently leaking the token to other users.
  */
-function enforceCredentialFileMode(path: string): void {
-  try {
-    const mode = statSync(path).mode & 0o777;
-    if ((mode & 0o077) !== 0) {
-      chmodSync(path, 0o600);
-      console.error(
-        `fullstackgtm: tightened ${path} from ${mode.toString(8).padStart(3, "0")} to 600 ` +
-          "(it was readable or writable by other users).",
-      );
-    }
-  } catch {
-    // Missing file or non-POSIX filesystem: nothing to enforce.
-  }
+function readCredentialFile(path: string): string {
+  return readSecureRegularFile(path, {
+    tightenMode: 0o600,
+    onModeTightened: (mode) => console.error(
+      `fullstackgtm: tightened ${path} from ${mode.toString(8).padStart(3, "0")} to 600 ` +
+        "(it was readable or writable by other users).",
+    ),
+  });
 }
 
 /**
@@ -190,7 +185,7 @@ function activeKeychain(): { account: string; backend: NonNullable<ReturnType<ty
 function migratePlaintextToKeychain(keychain: NonNullable<ReturnType<typeof activeKeychain>>): void {
   if (!existsSync(credentialsPath())) return;
   try {
-    const fileParsed = JSON.parse(readFileSync(credentialsPath(), "utf8"));
+    const fileParsed = JSON.parse(readCredentialFile(credentialsPath()));
     if (fileParsed && fileParsed.version === 1 && fileParsed.providers) {
       const current = keychain.backend.get(keychain.account);
       const existing = current ? (JSON.parse(current).providers ?? {}) : {};
@@ -209,14 +204,15 @@ function readFile(): CredentialsFile {
   const keychain = activeKeychain();
   if (keychain) migratePlaintextToKeychain(keychain);
   try {
-    const raw = keychain ? keychain.backend.get(keychain.account) : (enforceCredentialFileMode(credentialsPath()), readFileSync(credentialsPath(), "utf8"));
+    const raw = keychain ? keychain.backend.get(keychain.account) : readCredentialFile(credentialsPath());
     if (raw) {
       const parsed = JSON.parse(raw);
       if (parsed && typeof parsed === "object" && parsed.version === 1 && parsed.providers) {
         return parsed as CredentialsFile;
       }
     }
-  } catch {
+  } catch (error) {
+    if (error instanceof UnsafeManagedPathError) throw error;
     // Missing or unreadable store falls through to an empty one.
   }
   return { version: 1, providers: {} };
@@ -238,6 +234,18 @@ export function getCredential(provider: string): StoredCredential | null {
 }
 
 export function storeCredential(provider: string, credential: StoredCredential) {
+  if (provider === "salesforce") {
+    if (!credential.instanceUrl) {
+      throw new Error("Salesforce credentials require an instance URL.");
+    }
+    credential = {
+      ...credential,
+      instanceUrl: validateSalesforceOrigin(credential.instanceUrl, "Salesforce credential instance URL"),
+      ...(credential.loginUrl
+        ? { loginUrl: validateSalesforceOrigin(credential.loginUrl, "Salesforce credential login URL") }
+        : {}),
+    };
+  }
   const file = readFile();
   file.providers[provider] = credential;
   persist(file);
@@ -315,12 +323,19 @@ export async function resolveSalesforceConnection(
         "Stored Salesforce credential has no instance URL. Run `fullstackgtm login salesforce` again.",
       );
     }
+    const instanceUrl = validateSalesforceOrigin(
+      credential.instanceUrl,
+      "Stored Salesforce credential instance URL",
+    );
+    if (credential.loginUrl) {
+      validateSalesforceOrigin(credential.loginUrl, "Stored Salesforce credential login URL");
+    }
     const needsRefresh =
       credential.kind === "oauth" &&
       credential.expiresAt !== undefined &&
       Date.now() > credential.expiresAt - REFRESH_SKEW_MS;
     if (!needsRefresh) {
-      return { accessToken: credential.accessToken, instanceUrl: credential.instanceUrl };
+      return { accessToken: credential.accessToken, instanceUrl };
     }
     if (!credential.refreshToken || !credential.clientId) {
       throw new Error(
@@ -349,9 +364,13 @@ export async function resolveSalesforceConnection(
   if (!minted.instanceUrl) {
     throw new Error("The hosted deployment returned no Salesforce instance URL.");
   }
+  const instanceUrl = validateSalesforceOrigin(
+    minted.instanceUrl,
+    "Hosted Salesforce credential instance URL",
+  );
   return {
     accessToken: minted.accessToken,
-    instanceUrl: minted.instanceUrl,
+    instanceUrl,
     fieldMappings: minted.fieldMappings,
   };
 }

package/src/index.ts

@@ -38,8 +38,10 @@ export {
   loadConfig,
   mergePolicy,
   resolveConfiguredRules,
+  rulePackageTrustFromCli,
   type FullstackgtmConfig,
   type LoadedConfig,
+  type RulePackageTrust,
 } from "./config.ts";
 export { applyPatchPlan, type ApplyPatchPlanOptions } from "./connector.ts";
 export {
@@ -76,6 +78,7 @@ export {
   pollSalesforceDeviceLogin,
   refreshSalesforceToken,
   startSalesforceDeviceLogin,
+  validateSalesforceOrigin,
   validateSalesforceToken,
   type SalesforceDeviceAuthorization,
   type SalesforceTokenSet,
@@ -575,3 +578,4 @@ export type {
   RiskLevel,
   SourceFreshness,
 } from "./types.ts";
+export { ProviderHttpError } from "./providerError.ts";

package/src/market.ts

@@ -1,11 +1,11 @@
 import { createHash } from "node:crypto";
-import { lookup } from "node:dns/promises";
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
-import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.ts";
 import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 import type { GtmEvidence } from "./types.ts";
+import { assertPublicUrl, publicHttpGet } from "./publicHttp.ts";
+export { assertPublicUrl } from "./publicHttp.ts";
 
 /**
  * The Market Map: a live model of the competitive category a company sells
@@ -328,121 +328,17 @@ export type FetchPage = (url: string) => Promise<{ status: number; body: string
  * any host that is or resolves to a private/loopback/link-local/metadata
  * address, and (3) follow redirects manually, re-validating each hop.
  *
- * Residual gap (documented, not defended here): TOCTOU DNS rebinding between
- * our lookup and fetch's own resolution. Out of scope for fetching public
- * competitor pages; a hardened deployment should fetch through an egress proxy.
+ * The default transport resolves once, rejects mixed public/private answers,
+ * and pins a validated address into socket creation to prevent DNS rebinding.
  */
 const MAX_REDIRECTS = 5;
 const FETCH_TIMEOUT_MS = 15_000;
 const MAX_BODY_BYTES = 5_000_000;
 
-function ipv4IsPrivate(ip: string): boolean {
-  const parts = ip.split(".").map((n) => Number(n));
-  if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
-  const [a, b] = parts;
-  if (a === 0 || a === 127) return true; // this-host, loopback
-  if (a === 10) return true; // private
-  if (a === 172 && b >= 16 && b <= 31) return true; // private
-  if (a === 192 && b === 168) return true; // private
-  if (a === 169 && b === 254) return true; // link-local incl. 169.254.169.254 metadata
-  if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
-  if (a >= 224) return true; // multicast / reserved
-  return false;
-}
-
-function ipIsPrivate(ip: string): boolean {
-  const family = isIP(ip);
-  if (family === 4) return ipv4IsPrivate(ip);
-  if (family === 6) {
-    const lower = ip.toLowerCase();
-    if (lower === "::1" || lower === "::") return true; // loopback / unspecified
-    // IPv4-mapped (::ffff:…) — Node normalizes ::ffff:127.0.0.1 to ::ffff:7f00:1,
-    // so accept both the dotted and the hex-pair forms, unwrap, check the v4.
-    const mapped = lower.match(/^::ffff:(.+)$/);
-    if (mapped) {
-      const rest = mapped[1];
-      if (rest.includes(".")) return ipv4IsPrivate(rest);
-      const groups = rest.split(":");
-      if (groups.length === 2) {
-        const hi = parseInt(groups[0], 16);
-        const lo = parseInt(groups[1], 16);
-        if (Number.isNaN(hi) || Number.isNaN(lo)) return true;
-        return ipv4IsPrivate(`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`);
-      }
-      return true; // unrecognized mapped form → refuse
-    }
-    if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb")) return true; // link-local fe80::/10
-    if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // unique-local fc00::/7
-    return false;
-  }
-  return true; // not a recognizable IP literal → refuse
-}
-
-export async function assertPublicUrl(rawUrl: string): Promise<URL> {
-  let url: URL;
-  try {
-    url = new URL(rawUrl);
-  } catch {
-    throw new Error(`market capture: "${rawUrl}" is not a valid URL.`);
-  }
-  if (url.protocol !== "http:" && url.protocol !== "https:") {
-    throw new Error(`market capture refuses ${url.protocol} URLs (only http/https): ${rawUrl}`);
-  }
-  const host = url.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
-  if (isIP(host)) {
-    if (ipIsPrivate(host)) throw new Error(`market capture refuses private/loopback address ${host} (SSRF guard).`);
-    return url;
-  }
-  // Hostname: resolve and refuse if ANY address is private.
-  const addrs = await lookup(host, { all: true });
-  for (const { address } of addrs) {
-    if (ipIsPrivate(address)) {
-      throw new Error(`market capture refuses ${host} — it resolves to private/internal address ${address} (SSRF guard).`);
-    }
-  }
-  return url;
-}
-
 const defaultFetchPage: FetchPage = async (url) => {
-  let current = url;
-  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-    await assertPublicUrl(current);
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
-    let response: Response;
-    try {
-      response = await fetch(current, {
-        headers: {
-          "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)",
-          "Accept-Language": "en-US",
-        },
-        redirect: "manual",
-        signal: controller.signal,
-      });
-    } finally {
-      clearTimeout(timer);
-    }
-    if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
-      current = new URL(response.headers.get("location") as string, current).toString();
-      continue; // re-validate the redirect target on the next iteration
-    }
-    const reader = response.body?.getReader();
-    if (!reader) return { status: response.status, body: await response.text() };
-    const chunks: Uint8Array[] = [];
-    let total = 0;
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      total += value.length;
-      if (total > MAX_BODY_BYTES) {
-        await reader.cancel();
-        break;
-      }
-      chunks.push(value);
-    }
-    return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
-  }
-  throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
+  const response = await publicHttpGet(url, { maxBytes: MAX_BODY_BYTES, maxRedirects: MAX_REDIRECTS,
+    timeoutMs: FETCH_TIMEOUT_MS, headers: { "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)", "Accept-Language": "en-US" } });
+  return { status: response.status, body: Buffer.from(response.body).toString("utf8") };
 };
 
 export type CaptureOptions = {

package/src/marketClassify.ts

@@ -276,9 +276,68 @@ export type MarketWorksheet = {
   vendor: { id: string; name: string };
   claims: MarketClaim[];
   pages: Array<{ kind: CaptureEntry["kind"]; url: string; captureHash: string; text: string }>;
+  /** Deterministic retrieval hints: page lines that contain claim terms. These are not labels. */
+  claimHints: Array<{
+    claimId: string;
+    matches: Array<{ url: string; captureHash: string; term: string; quote: string }>;
+  }>;
   instructions: string;
 };
 
+const HINT_STOPWORDS = new Set([
+  "the", "and", "for", "with", "from", "that", "this", "into", "your", "you", "are", "but", "not", "when", "how", "use", "uses", "using",
+  "claim", "claims", "vendor", "vendors", "page", "pages", "loud", "quiet", "absent", "capability", "specific", "differentiating",
+  "unknown", "buyer", "buyers", "pricing", "structure", "category", "support", "supported", "primary", "secondary", "mentioned", "mentions",
+]);
+
+function normalizeHintText(value: string): string {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+}
+
+function claimHintTerms(claim: MarketClaim): string[] {
+  const exact = [claim.id.replace(/[-_]+/g, " "), claim.capability, ...(claim.terms ?? [])]
+    .map(normalizeHintText)
+    .filter((term) => term.length >= 3);
+  const tokenSource = normalizeHintText(`${claim.capability} ${claim.definition} ${(claim.terms ?? []).join(" ")}`)
+    .split(" ")
+    .filter((term) => term.length >= 4 && !HINT_STOPWORDS.has(term));
+  return Array.from(new Set([...exact, ...tokenSource])).slice(0, 24);
+}
+
+function lineSnippet(line: string, term: string): string {
+  const clean = line.trim().replace(/\s+/g, " ");
+  if (clean.length <= 300) return clean;
+  const lower = clean.toLowerCase();
+  const idx = lower.indexOf(term.toLowerCase().split(" ")[0] ?? "");
+  const start = Math.max(0, idx < 0 ? 0 : idx - 120);
+  return clean.slice(start, start + 300).trim();
+}
+
+function buildClaimHints(
+  claims: MarketClaim[],
+  pages: Array<{ kind: CaptureEntry["kind"]; url: string; captureHash: string; text: string }>,
+): MarketWorksheet["claimHints"] {
+  return claims.map((claim) => {
+    const terms = claimHintTerms(claim);
+    const matches: Array<{ url: string; captureHash: string; term: string; quote: string }> = [];
+    for (const page of pages) {
+      const lines = page.text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+      for (const term of terms) {
+        const normalizedTerm = normalizeHintText(term);
+        const line = lines.find((candidate) => normalizeHintText(candidate).includes(normalizedTerm));
+        if (!line) continue;
+        const quote = lineSnippet(line, term);
+        if (quote && !matches.some((m) => m.quote === quote && m.url === page.url)) {
+          matches.push({ url: page.url, captureHash: page.captureHash, term, quote });
+        }
+        if (matches.length >= 4) break;
+      }
+      if (matches.length >= 4) break;
+    }
+    return { claimId: claim.id, matches };
+  });
+}
+
 export function buildWorksheet(
   config: MarketConfig,
   vendorId: string,
@@ -304,7 +363,8 @@ export function buildWorksheet(
     vendor: { id: vendor.id, name: vendor.name },
     claims: config.claims,
     pages,
+    claimHints: buildClaimHints(config.claims, pages),
     instructions:
-      "Produce one observation per claim (intensity loud|quiet|absent from these pages only; unobservable only if a page you need failed to capture). Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
+      "Produce one observation per claim from these pages only. Intensity rubric: loud = hero copy, primary positioning, or a named/dedicated product area; quiet = supported by a verbatim page quote but secondary, qualified, or merely available; absent = the pages do not support the claim; unobservable = no usable captured page exists for this vendor, not a synonym for absent. claimHints are deterministic retrieval hints only: use them to find candidate support, but do not mark a claim loud/quiet unless the quote truly supports the claim. Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
   };
 }