fullstackgtm 0.48.0 → 0.50.0

package/src/hostedPatchPlan.ts

@@ -0,0 +1,286 @@
+/** Broker-authenticated mirror for locally governed patch plans. */
+import { createHash } from "node:crypto";
+import { getCredential } from "./credentials.ts";
+import type { PlanStore, StoredPlan } from "./planStore.ts";
+import type { CrmProvider, PatchOperation, PatchOperationResult, PatchPlan, PatchPlanRun } from "./types.ts";
+
+const ENDPOINT = "/api/cli/patch-plan";
+const TIMEOUT_MS = 5000;
+
+type Options = { fetchImpl?: typeof fetch; timeoutMs?: number };
+export type HostedPlanState = {
+  patchPlanId: string;
+  externalPlanId: string;
+  documentSha256: string;
+  status: string;
+  approvedOperationIds: string[];
+  valueOverrides: Record<string, unknown>;
+  updatedAt: number;
+  runRecord?: PatchPlanRun;
+  url: string;
+};
+export type HostedPlanResult =
+  | { status: "unpaired" }
+  | { status: "missing" }
+  | { status: "found" | "saved"; state: HostedPlanState }
+  | { status: "conflict"; reason: string }
+  | { status: "unavailable"; reason: string };
+
+function broker(): { baseUrl: string; token: string } | null {
+  const value = getCredential("broker");
+  if (!value?.baseUrl || !value.accessToken) return null;
+  try {
+    const url = new URL(value.baseUrl);
+    const local = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
+    if (url.protocol !== "https:" && !(url.protocol === "http:" && local)) return null;
+    return { baseUrl: url.href.replace(/\/+$/, ""), token: value.accessToken };
+  } catch { return null; }
+}
+
+/** Review document intentionally omits evidence/findings and all mutable store security state. */
+export function hostedReviewDocument(plan: PatchPlan) {
+  return {
+    id: plan.id,
+    title: plan.title,
+    createdAt: plan.createdAt,
+    status: "needs_approval",
+    dryRun: true,
+    summary: plan.summary,
+    operations: plan.operations,
+    ...(plan.filter ? { filter: plan.filter } : {}),
+    ...(plan.guards ? { guards: plan.guards } : {}),
+  };
+}
+
+export function hostedPlanDigest(plan: PatchPlan): string {
+  return createHash("sha256").update(JSON.stringify(hostedReviewDocument(plan))).digest("hex");
+}
+
+function mapOperation(op: PatchOperation) {
+  return {
+    packageOpId: op.id,
+    objectType: op.objectType,
+    ...(op.objectId ? { objectId: op.objectId } : {}),
+    fieldOrAction: op.field ?? op.operation,
+    beforeValue: op.beforeValue ?? null,
+    afterValue: op.afterValue ?? null,
+    ...(op.reason ? { reason: op.reason } : {}),
+    ...(op.sourceRuleOrPolicy ? { sourceRuleOrPolicy: op.sourceRuleOrPolicy } : {}),
+    riskLevel: op.riskLevel,
+    approvalRequired: op.approvalRequired,
+    ...(op.rollback ? { rollback: op.rollback } : {}),
+  };
+}
+
+function parseState(value: unknown, baseUrl: string): HostedPlanState | null {
+  if (!value || typeof value !== "object") return null;
+  const row = value as Record<string, unknown>;
+  const id = typeof row.patchPlanId === "string" ? row.patchPlanId : null;
+  if (!id) return null;
+  const approvedOperationIds = Array.isArray(row.approvedOperationIds) ? row.approvedOperationIds.filter((id): id is string => typeof id === "string") : [];
+  const runRecord = parseRunRecord(row.runRecord, row.operationReceipts, row.externalPlanId, row.updatedAt);
+  return {
+    patchPlanId: id,
+    externalPlanId: typeof row.externalPlanId === "string" ? row.externalPlanId : "",
+    documentSha256: typeof row.documentSha256 === "string" ? row.documentSha256 : "",
+    status: typeof row.status === "string" ? row.status : "pending_approval",
+    approvedOperationIds,
+    valueOverrides: row.valueOverrides && typeof row.valueOverrides === "object" && !Array.isArray(row.valueOverrides) ? row.valueOverrides as Record<string, unknown> : {},
+    updatedAt: typeof row.updatedAt === "number" ? row.updatedAt : Date.now(),
+    ...(runRecord ? { runRecord } : {}),
+    url: `${baseUrl}/dashboard/patch-plans?plan=${encodeURIComponent(id)}`,
+  };
+}
+
+function parseRunRecord(value: unknown, receiptsValue: unknown, externalPlanId: unknown, updatedAt: unknown): PatchPlanRun | undefined {
+  const row = value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+  const planId = typeof row?.planId === "string" ? row.planId : typeof externalPlanId === "string" ? externalPlanId : undefined;
+  if (!planId) return undefined;
+  const rawResults = Array.isArray(row?.results) ? row.results : Array.isArray(receiptsValue) ? receiptsValue : [];
+  const results = rawResults.flatMap((entry): PatchOperationResult[] => {
+    if (!entry || typeof entry !== "object") return [];
+    const receipt = entry as Record<string, unknown>;
+    const operationId = typeof receipt.operationId === "string" ? receipt.operationId : typeof receipt.packageOpId === "string" ? receipt.packageOpId : undefined;
+    const rawStatus = receipt.status;
+    const status = rawStatus === "applied" || rawStatus === "failed" || rawStatus === "skipped" || rawStatus === "conflict" ? rawStatus : undefined;
+    if (!operationId || !status) return [];
+    return [{ operationId, status, ...(typeof receipt.detail === "string" ? { detail: receipt.detail } : typeof receipt.error === "string" ? { detail: receipt.error } : {}), ...(receipt.providerData !== undefined ? { providerData: receipt.providerData } : {}) }];
+  });
+  if (results.length === 0) return undefined;
+  const timestamp = typeof updatedAt === "number" ? new Date(updatedAt).toISOString() : new Date().toISOString();
+  const provider = typeof row?.provider === "string" ? row.provider as CrmProvider : "mock";
+  const status = row?.status === "applied" || row?.status === "partial" || row?.status === "failed" || row?.status === "rejected"
+    ? row.status : results.every((result) => result.status === "applied" || result.status === "skipped") ? "applied" : "partial";
+  return {
+    planId, provider,
+    startedAt: typeof row?.startedAt === "string" ? row.startedAt : timestamp,
+    finishedAt: typeof row?.finishedAt === "string" ? row.finishedAt : timestamp,
+    status,
+    results,
+  };
+}
+
+function failure(status: number): HostedPlanResult {
+  if (status === 409) return { status: "conflict", reason: "hosted mirror has different immutable content" };
+  if (status === 401 || status === 403) return { status: "unavailable", reason: "hosted authentication was rejected; pair the CLI again" };
+  return { status: "unavailable", reason: `hosted plan request failed (HTTP ${status})` };
+}
+
+export async function uploadHostedPatchPlan(stored: StoredPlan, options: Options = {}): Promise<HostedPlanResult> {
+  const paired = broker();
+  if (!paired) return { status: "unpaired" };
+  const document = hostedReviewDocument(stored.plan);
+  const digest = hostedPlanDigest(stored.plan);
+  const riskOrder = { low: 0, medium: 1, high: 2 } as const;
+  const riskLevel = stored.plan.operations.reduce<"low" | "medium" | "high">(
+    (risk, op) => riskOrder[op.riskLevel] > riskOrder[risk] ? op.riskLevel : risk,
+    "low",
+  );
+  try {
+    const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
+      body: JSON.stringify({
+        externalPlanId: stored.plan.id, documentSha256: digest, externalRevision: stored.revision,
+        title: stored.plan.title, summary: stored.plan.summary, riskLevel, document,
+        operations: stored.plan.operations.map(mapOperation), createdAt: Date.parse(stored.plan.createdAt),
+      }),
+      signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+    });
+    if (!response.ok) return failure(response.status);
+    const raw = await response.json() as Record<string, unknown>;
+    const state = parseState({ ...raw, externalPlanId: stored.plan.id, documentSha256: digest, approvedOperationIds: [], valueOverrides: {}, updatedAt: Date.now() }, paired.baseUrl);
+    return state ? { status: "saved", state } : { status: "unavailable", reason: "hosted plan response was invalid" };
+  } catch { return { status: "unavailable", reason: "hosted plan request was unavailable" }; }
+}
+
+export async function readHostedPatchPlan(planId: string, options: Options = {}): Promise<HostedPlanResult> {
+  const paired = broker();
+  if (!paired) return { status: "unpaired" };
+  try {
+    const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}?externalPlanId=${encodeURIComponent(planId)}`, {
+      headers: { Authorization: `Bearer ${paired.token}`, Accept: "application/json" },
+      signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+    });
+    if (response.status === 404) return { status: "missing" };
+    if (!response.ok) return failure(response.status);
+    const state = parseState(await response.json(), paired.baseUrl);
+    return state ? { status: "found", state } : { status: "unavailable", reason: "hosted plan response was invalid" };
+  } catch { return { status: "unavailable", reason: "hosted plan request was unavailable" }; }
+}
+
+export type HostedReconcileResult =
+  | { status: "unchanged" | "unpaired" | "missing" }
+  | { status: "updated"; stored: StoredPlan; remoteStatus: string }
+  | { status: "conflict" | "unavailable"; reason: string };
+
+/** Pull a peer's newer lifecycle into the local replica without weakening local integrity gates. */
+export async function reconcileHostedPatchPlan(
+  store: PlanStore,
+  stored: StoredPlan,
+  options: Options = {},
+): Promise<HostedReconcileResult> {
+  const hosted = await readHostedPatchPlan(stored.plan.id, options);
+  if (hosted.status === "unpaired" || hosted.status === "missing") return { status: hosted.status };
+  if (hosted.status === "unavailable" || hosted.status === "conflict") return hosted;
+  if (hosted.state.documentSha256 !== hostedPlanDigest(stored.plan)) {
+    return { status: "conflict", reason: "hosted review content does not match the immutable local plan" };
+  }
+  if (!(["approved", "rejected", "applied"] as string[]).includes(hosted.state.status)) return { status: "unchanged" };
+  try {
+    const reconciled = await store.reconcileReplica(stored.plan.id, {
+      status: hosted.state.status as "approved" | "rejected" | "applied",
+      approvedOperationIds: hosted.state.approvedOperationIds,
+      valueOverrides: hosted.state.valueOverrides,
+      ...(hosted.state.runRecord ? { run: hosted.state.runRecord } : {}),
+    });
+    return reconciled.changed
+      ? { status: "updated", stored: reconciled.stored, remoteStatus: hosted.state.status }
+      : { status: "unchanged" };
+  } catch (error) {
+    return { status: "conflict", reason: error instanceof Error ? error.message : "replica reconciliation failed" };
+  }
+}
+
+export async function reportHostedPlanLifecycle(stored: StoredPlan, options: Options & { claimId?: string } = {}): Promise<HostedPlanResult> {
+  const paired = broker();
+  if (!paired) return { status: "unpaired" };
+  const status = stored.status === "applied" ? "applied" : stored.status === "rejected" ? "rejected" : "approved";
+  const runRecord = status === "applied" ? stored.runs.at(-1) : undefined;
+  const operationReceipts = runRecord?.results.map((result) => ({
+    packageOpId: result.operationId,
+    status: result.status,
+    ...(result.detail ? { error: result.detail.slice(0, 1000) } : {}),
+    ...(result.providerData !== undefined ? { providerData: result.providerData } : {}),
+    updatedAt: Date.parse(runRecord.finishedAt),
+  }));
+  try {
+    const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
+      method: "POST", headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ action: "lifecycle", externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan), status, approvedOperationIds: stored.approvedOperationIds, ...(options.claimId ? { claimId: options.claimId } : {}), ...(runRecord ? { runRecord, operationReceipts } : {}) }),
+      signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+    });
+    if (!response.ok) return failure(response.status);
+    const raw = await response.json() as Record<string, unknown>;
+    const state = parseState({ ...raw, externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan), approvedOperationIds: stored.approvedOperationIds, valueOverrides: stored.valueOverrides, updatedAt: Date.now() }, paired.baseUrl);
+    return state ? { status: "saved", state } : { status: "unavailable", reason: "hosted lifecycle response was invalid" };
+  } catch { return { status: "unavailable", reason: "hosted lifecycle request was unavailable" }; }
+}
+
+export type HostedApplyClaimResult =
+  | { status: "unpaired" }
+  | { status: "claimed"; claimId: string; expiresAt?: number }
+  | { status: "applied" }
+  | { status: "conflict" | "unavailable"; reason: string };
+
+/** Coordinate online replicas before provider I/O; provider idempotency remains the offline fallback. */
+export async function claimHostedPlanApply(stored: StoredPlan, options: Options = {}): Promise<HostedApplyClaimResult> {
+  const paired = broker();
+  if (!paired) return { status: "unpaired" };
+  try {
+    const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
+      method: "POST", headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ action: "claim", externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan) }),
+      signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+    });
+    if (!response.ok) return failure(response.status) as HostedApplyClaimResult;
+    const row = await response.json() as Record<string, unknown>;
+    if (row.alreadyApplied === true || row.status === "applied") return { status: "applied" };
+    if (row.status === "claimed" && typeof row.claimId === "string") {
+      return { status: "claimed", claimId: row.claimId, ...(typeof row.expiresAt === "number" ? { expiresAt: row.expiresAt } : {}) };
+    }
+    return { status: "unavailable", reason: "hosted apply claim response was invalid" };
+  } catch { return { status: "unavailable", reason: "hosted apply claim request was unavailable" }; }
+}
+
+/** Release a shared CLI claim only when preflight aborts before provider I/O. */
+export async function releaseHostedPlanApply(
+  stored: StoredPlan,
+  claimId: string,
+  reason: string,
+  options: Options = {},
+): Promise<{ status: "released" | "unpaired" } | { status: "conflict" | "unavailable"; reason: string }> {
+  const paired = broker();
+  if (!paired) return { status: "unpaired" };
+  try {
+    const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
+      body: JSON.stringify({
+        action: "release_claim",
+        externalPlanId: stored.plan.id,
+        documentSha256: hostedPlanDigest(stored.plan),
+        claimId,
+        reason,
+      }),
+      signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+    });
+    if (!response.ok) return failure(response.status) as { status: "conflict" | "unavailable"; reason: string };
+    const row = await response.json() as Record<string, unknown>;
+    return row.released === true
+      ? { status: "released" }
+      : { status: "unavailable", reason: "hosted apply claim release response was invalid" };
+  } catch {
+    return { status: "unavailable", reason: "hosted apply claim release was unavailable" };
+  }
+}

package/src/icp.ts

@@ -193,6 +193,15 @@ const CRUSTDATA_INDUSTRY: Record<string, string[]> = {
   "financial services": ["Financial Services"],
 };
 
+// Common NAICS groups translated into Crustdata's LinkedIn industry vocabulary.
+// This keeps an ICP authored with provider-neutral NAICS codes from silently
+// dropping its industry constraint on the Pipe0 path.
+const NAICS_CRUSTDATA_INDUSTRY: Record<string, string[]> = {
+  "5112": ["Software Development", "Computer Software"],
+  "5182": ["IT Services and IT Consulting", "Information Technology & Services"],
+  "5415": ["IT Services and IT Consulting", "Information Technology & Services"],
+};
+
 /**
  * pipe0 Crustdata people-search config.filters from the ICP. `current_job_titles`
  * matches real LinkedIn title strings (case-sensitive), so keywords are Title
@@ -219,9 +228,12 @@ export function icpToCrustdataFilters(icp: Icp): Record<string, unknown> {
   if (icp.firmographics.geos?.length) {
     f.locations = icp.firmographics.geos.map((g) => COUNTRY_NAMES[g.toLowerCase()] ?? g);
   }
-  if (icp.firmographics.industries?.length) {
+  if (icp.firmographics.industries?.length || icp.firmographics.naics?.length) {
     const inds = [
-      ...new Set(icp.firmographics.industries.flatMap((i) => CRUSTDATA_INDUSTRY[i.toLowerCase()] ?? [titleCase(i)])),
+      ...new Set([
+        ...(icp.firmographics.industries ?? []).flatMap((i) => CRUSTDATA_INDUSTRY[i.toLowerCase()] ?? [titleCase(i)]),
+        ...(icp.firmographics.naics ?? []).flatMap((code) => NAICS_CRUSTDATA_INDUSTRY[code] ?? []),
+      ]),
     ];
     if (inds.length) f.current_employers_linkedin_industries = inds;
   }

package/src/index.ts

@@ -1,4 +1,15 @@
 export { auditSnapshot, defaultPolicy } from "./audit.ts";
+export {
+  acquireCheckpointId,
+  acquireCheckpointsDir,
+  createFileAcquireCheckpointStore,
+  validateAcquireCheckpoint,
+  validateAcquireCheckpointKey,
+  type AcquireCheckpoint,
+  type AcquireCheckpointKey,
+  type AcquireCheckpointStore,
+  type AcquireContinuation,
+} from "./acquireCheckpoint.ts";
 export {
   matchesTerritory,
   parseAssignmentPolicy,
@@ -38,8 +49,10 @@ export {
   loadConfig,
   mergePolicy,
   resolveConfiguredRules,
+  rulePackageTrustFromCli,
   type FullstackgtmConfig,
   type LoadedConfig,
+  type RulePackageTrust,
 } from "./config.ts";
 export { applyPatchPlan, type ApplyPatchPlanOptions } from "./connector.ts";
 export {
@@ -76,6 +89,7 @@ export {
   pollSalesforceDeviceLogin,
   refreshSalesforceToken,
   startSalesforceDeviceLogin,
+  validateSalesforceOrigin,
   validateSalesforceToken,
   type SalesforceDeviceAuthorization,
   type SalesforceTokenSet,
@@ -129,6 +143,7 @@ export {
   stagedSourceRecords,
   staleDaysFor,
   type BuildEnrichPlanOptions,
+  type AcquireDiscoveryCheckpoint,
   type EnrichAmbiguity,
   type EnrichConfig,
   type EnrichCounts,
@@ -145,6 +160,14 @@ export {
   type EnrichWorkItem,
   type MatchOutcome,
 } from "./enrich.ts";
+export {
+  readHostedAcquireCheckpoint,
+  writeHostedAcquireCheckpoint,
+  type HostedAcquireCheckpoint,
+  type HostedCheckpointReadResult,
+  type HostedCheckpointWriteResult,
+} from "./hostedAcquireCheckpoint.ts";
+export { claimHostedPlanApply, hostedPlanDigest, hostedReviewDocument, readHostedPatchPlan, reconcileHostedPatchPlan, releaseHostedPlanApply, reportHostedPlanLifecycle, uploadHostedPatchPlan } from "./hostedPatchPlan.ts";
 export {
   scaffoldWorkspace,
   starterEnrichConfig,
@@ -575,3 +598,4 @@ export type {
   RiskLevel,
   SourceFreshness,
 } from "./types.ts";
+export { ProviderHttpError } from "./providerError.ts";

package/src/market.ts

@@ -1,11 +1,11 @@
 import { createHash } from "node:crypto";
-import { lookup } from "node:dns/promises";
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
-import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.ts";
 import { MARKET_CAPTURE_STAGES, nullProgressEmitter, type ProgressEmitter } from "./progress.ts";
 import type { GtmEvidence } from "./types.ts";
+import { assertPublicUrl, publicHttpGet } from "./publicHttp.ts";
+export { assertPublicUrl } from "./publicHttp.ts";
 
 /**
  * The Market Map: a live model of the competitive category a company sells
@@ -328,121 +328,17 @@ export type FetchPage = (url: string) => Promise<{ status: number; body: string
  * any host that is or resolves to a private/loopback/link-local/metadata
  * address, and (3) follow redirects manually, re-validating each hop.
  *
- * Residual gap (documented, not defended here): TOCTOU DNS rebinding between
- * our lookup and fetch's own resolution. Out of scope for fetching public
- * competitor pages; a hardened deployment should fetch through an egress proxy.
+ * The default transport resolves once, rejects mixed public/private answers,
+ * and pins a validated address into socket creation to prevent DNS rebinding.
  */
 const MAX_REDIRECTS = 5;
 const FETCH_TIMEOUT_MS = 15_000;
 const MAX_BODY_BYTES = 5_000_000;
 
-function ipv4IsPrivate(ip: string): boolean {
-  const parts = ip.split(".").map((n) => Number(n));
-  if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
-  const [a, b] = parts;
-  if (a === 0 || a === 127) return true; // this-host, loopback
-  if (a === 10) return true; // private
-  if (a === 172 && b >= 16 && b <= 31) return true; // private
-  if (a === 192 && b === 168) return true; // private
-  if (a === 169 && b === 254) return true; // link-local incl. 169.254.169.254 metadata
-  if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
-  if (a >= 224) return true; // multicast / reserved
-  return false;
-}
-
-function ipIsPrivate(ip: string): boolean {
-  const family = isIP(ip);
-  if (family === 4) return ipv4IsPrivate(ip);
-  if (family === 6) {
-    const lower = ip.toLowerCase();
-    if (lower === "::1" || lower === "::") return true; // loopback / unspecified
-    // IPv4-mapped (::ffff:…) — Node normalizes ::ffff:127.0.0.1 to ::ffff:7f00:1,
-    // so accept both the dotted and the hex-pair forms, unwrap, check the v4.
-    const mapped = lower.match(/^::ffff:(.+)$/);
-    if (mapped) {
-      const rest = mapped[1];
-      if (rest.includes(".")) return ipv4IsPrivate(rest);
-      const groups = rest.split(":");
-      if (groups.length === 2) {
-        const hi = parseInt(groups[0], 16);
-        const lo = parseInt(groups[1], 16);
-        if (Number.isNaN(hi) || Number.isNaN(lo)) return true;
-        return ipv4IsPrivate(`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`);
-      }
-      return true; // unrecognized mapped form → refuse
-    }
-    if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb")) return true; // link-local fe80::/10
-    if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // unique-local fc00::/7
-    return false;
-  }
-  return true; // not a recognizable IP literal → refuse
-}
-
-export async function assertPublicUrl(rawUrl: string): Promise<URL> {
-  let url: URL;
-  try {
-    url = new URL(rawUrl);
-  } catch {
-    throw new Error(`market capture: "${rawUrl}" is not a valid URL.`);
-  }
-  if (url.protocol !== "http:" && url.protocol !== "https:") {
-    throw new Error(`market capture refuses ${url.protocol} URLs (only http/https): ${rawUrl}`);
-  }
-  const host = url.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
-  if (isIP(host)) {
-    if (ipIsPrivate(host)) throw new Error(`market capture refuses private/loopback address ${host} (SSRF guard).`);
-    return url;
-  }
-  // Hostname: resolve and refuse if ANY address is private.
-  const addrs = await lookup(host, { all: true });
-  for (const { address } of addrs) {
-    if (ipIsPrivate(address)) {
-      throw new Error(`market capture refuses ${host} — it resolves to private/internal address ${address} (SSRF guard).`);
-    }
-  }
-  return url;
-}
-
 const defaultFetchPage: FetchPage = async (url) => {
-  let current = url;
-  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-    await assertPublicUrl(current);
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
-    let response: Response;
-    try {
-      response = await fetch(current, {
-        headers: {
-          "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)",
-          "Accept-Language": "en-US",
-        },
-        redirect: "manual",
-        signal: controller.signal,
-      });
-    } finally {
-      clearTimeout(timer);
-    }
-    if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
-      current = new URL(response.headers.get("location") as string, current).toString();
-      continue; // re-validate the redirect target on the next iteration
-    }
-    const reader = response.body?.getReader();
-    if (!reader) return { status: response.status, body: await response.text() };
-    const chunks: Uint8Array[] = [];
-    let total = 0;
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      total += value.length;
-      if (total > MAX_BODY_BYTES) {
-        await reader.cancel();
-        break;
-      }
-      chunks.push(value);
-    }
-    return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
-  }
-  throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
+  const response = await publicHttpGet(url, { maxBytes: MAX_BODY_BYTES, maxRedirects: MAX_REDIRECTS,
+    timeoutMs: FETCH_TIMEOUT_MS, headers: { "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)", "Accept-Language": "en-US" } });
+  return { status: response.status, body: Buffer.from(response.body).toString("utf8") };
 };
 
 export type CaptureOptions = {

package/src/marketClassify.ts

@@ -276,9 +276,68 @@ export type MarketWorksheet = {
   vendor: { id: string; name: string };
   claims: MarketClaim[];
   pages: Array<{ kind: CaptureEntry["kind"]; url: string; captureHash: string; text: string }>;
+  /** Deterministic retrieval hints: page lines that contain claim terms. These are not labels. */
+  claimHints: Array<{
+    claimId: string;
+    matches: Array<{ url: string; captureHash: string; term: string; quote: string }>;
+  }>;
   instructions: string;
 };
 
+const HINT_STOPWORDS = new Set([
+  "the", "and", "for", "with", "from", "that", "this", "into", "your", "you", "are", "but", "not", "when", "how", "use", "uses", "using",
+  "claim", "claims", "vendor", "vendors", "page", "pages", "loud", "quiet", "absent", "capability", "specific", "differentiating",
+  "unknown", "buyer", "buyers", "pricing", "structure", "category", "support", "supported", "primary", "secondary", "mentioned", "mentions",
+]);
+
+function normalizeHintText(value: string): string {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+}
+
+function claimHintTerms(claim: MarketClaim): string[] {
+  const exact = [claim.id.replace(/[-_]+/g, " "), claim.capability, ...(claim.terms ?? [])]
+    .map(normalizeHintText)
+    .filter((term) => term.length >= 3);
+  const tokenSource = normalizeHintText(`${claim.capability} ${claim.definition} ${(claim.terms ?? []).join(" ")}`)
+    .split(" ")
+    .filter((term) => term.length >= 4 && !HINT_STOPWORDS.has(term));
+  return Array.from(new Set([...exact, ...tokenSource])).slice(0, 24);
+}
+
+function lineSnippet(line: string, term: string): string {
+  const clean = line.trim().replace(/\s+/g, " ");
+  if (clean.length <= 300) return clean;
+  const lower = clean.toLowerCase();
+  const idx = lower.indexOf(term.toLowerCase().split(" ")[0] ?? "");
+  const start = Math.max(0, idx < 0 ? 0 : idx - 120);
+  return clean.slice(start, start + 300).trim();
+}
+
+function buildClaimHints(
+  claims: MarketClaim[],
+  pages: Array<{ kind: CaptureEntry["kind"]; url: string; captureHash: string; text: string }>,
+): MarketWorksheet["claimHints"] {
+  return claims.map((claim) => {
+    const terms = claimHintTerms(claim);
+    const matches: Array<{ url: string; captureHash: string; term: string; quote: string }> = [];
+    for (const page of pages) {
+      const lines = page.text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+      for (const term of terms) {
+        const normalizedTerm = normalizeHintText(term);
+        const line = lines.find((candidate) => normalizeHintText(candidate).includes(normalizedTerm));
+        if (!line) continue;
+        const quote = lineSnippet(line, term);
+        if (quote && !matches.some((m) => m.quote === quote && m.url === page.url)) {
+          matches.push({ url: page.url, captureHash: page.captureHash, term, quote });
+        }
+        if (matches.length >= 4) break;
+      }
+      if (matches.length >= 4) break;
+    }
+    return { claimId: claim.id, matches };
+  });
+}
+
 export function buildWorksheet(
   config: MarketConfig,
   vendorId: string,
@@ -304,7 +363,8 @@ export function buildWorksheet(
     vendor: { id: vendor.id, name: vendor.name },
     claims: config.claims,
     pages,
+    claimHints: buildClaimHints(config.claims, pages),
     instructions:
-      "Produce one observation per claim (intensity loud|quiet|absent from these pages only; unobservable only if a page you need failed to capture). Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
+      "Produce one observation per claim from these pages only. Intensity rubric: loud = hero copy, primary positioning, or a named/dedicated product area; quiet = supported by a verbatim page quote but secondary, qualified, or merely available; absent = the pages do not support the claim; unobservable = no usable captured page exists for this vendor, not a synonym for absent. claimHints are deterministic retrieval hints only: use them to find candidate support, but do not mark a claim loud/quiet unless the quote truly supports the claim. Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
   };
 }