fullstackgtm 0.47.0 → 0.49.0

package/dist/market.js

@@ -1,10 +1,10 @@
 import { createHash } from "node:crypto";
-import { lookup } from "node:dns/promises";
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
-import { isIP } from "node:net";
 import { join } from "node:path";
 import { credentialsDir } from "./credentials.js";
 import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
+import { publicHttpGet } from "./publicHttp.js";
+export { assertPublicUrl } from "./publicHttp.js";
 const INTENSITY_RANK = {
     loud: 3,
     quiet: 2,
@@ -152,136 +152,16 @@ export function extractReadableText(html) {
  * any host that is or resolves to a private/loopback/link-local/metadata
  * address, and (3) follow redirects manually, re-validating each hop.
  *
- * Residual gap (documented, not defended here): TOCTOU DNS rebinding between
- * our lookup and fetch's own resolution. Out of scope for fetching public
- * competitor pages; a hardened deployment should fetch through an egress proxy.
+ * The default transport resolves once, rejects mixed public/private answers,
+ * and pins a validated address into socket creation to prevent DNS rebinding.
  */
 const MAX_REDIRECTS = 5;
 const FETCH_TIMEOUT_MS = 15_000;
 const MAX_BODY_BYTES = 5_000_000;
-function ipv4IsPrivate(ip) {
-    const parts = ip.split(".").map((n) => Number(n));
-    if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
-        return true;
-    const [a, b] = parts;
-    if (a === 0 || a === 127)
-        return true; // this-host, loopback
-    if (a === 10)
-        return true; // private
-    if (a === 172 && b >= 16 && b <= 31)
-        return true; // private
-    if (a === 192 && b === 168)
-        return true; // private
-    if (a === 169 && b === 254)
-        return true; // link-local incl. 169.254.169.254 metadata
-    if (a === 100 && b >= 64 && b <= 127)
-        return true; // CGNAT
-    if (a >= 224)
-        return true; // multicast / reserved
-    return false;
-}
-function ipIsPrivate(ip) {
-    const family = isIP(ip);
-    if (family === 4)
-        return ipv4IsPrivate(ip);
-    if (family === 6) {
-        const lower = ip.toLowerCase();
-        if (lower === "::1" || lower === "::")
-            return true; // loopback / unspecified
-        // IPv4-mapped (::ffff:…) — Node normalizes ::ffff:127.0.0.1 to ::ffff:7f00:1,
-        // so accept both the dotted and the hex-pair forms, unwrap, check the v4.
-        const mapped = lower.match(/^::ffff:(.+)$/);
-        if (mapped) {
-            const rest = mapped[1];
-            if (rest.includes("."))
-                return ipv4IsPrivate(rest);
-            const groups = rest.split(":");
-            if (groups.length === 2) {
-                const hi = parseInt(groups[0], 16);
-                const lo = parseInt(groups[1], 16);
-                if (Number.isNaN(hi) || Number.isNaN(lo))
-                    return true;
-                return ipv4IsPrivate(`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`);
-            }
-            return true; // unrecognized mapped form → refuse
-        }
-        if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb"))
-            return true; // link-local fe80::/10
-        if (lower.startsWith("fc") || lower.startsWith("fd"))
-            return true; // unique-local fc00::/7
-        return false;
-    }
-    return true; // not a recognizable IP literal → refuse
-}
-export async function assertPublicUrl(rawUrl) {
-    let url;
-    try {
-        url = new URL(rawUrl);
-    }
-    catch {
-        throw new Error(`market capture: "${rawUrl}" is not a valid URL.`);
-    }
-    if (url.protocol !== "http:" && url.protocol !== "https:") {
-        throw new Error(`market capture refuses ${url.protocol} URLs (only http/https): ${rawUrl}`);
-    }
-    const host = url.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
-    if (isIP(host)) {
-        if (ipIsPrivate(host))
-            throw new Error(`market capture refuses private/loopback address ${host} (SSRF guard).`);
-        return url;
-    }
-    // Hostname: resolve and refuse if ANY address is private.
-    const addrs = await lookup(host, { all: true });
-    for (const { address } of addrs) {
-        if (ipIsPrivate(address)) {
-            throw new Error(`market capture refuses ${host} — it resolves to private/internal address ${address} (SSRF guard).`);
-        }
-    }
-    return url;
-}
 const defaultFetchPage = async (url) => {
-    let current = url;
-    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-        await assertPublicUrl(current);
-        const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
-        let response;
-        try {
-            response = await fetch(current, {
-                headers: {
-                    "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)",
-                    "Accept-Language": "en-US",
-                },
-                redirect: "manual",
-                signal: controller.signal,
-            });
-        }
-        finally {
-            clearTimeout(timer);
-        }
-        if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
-            current = new URL(response.headers.get("location"), current).toString();
-            continue; // re-validate the redirect target on the next iteration
-        }
-        const reader = response.body?.getReader();
-        if (!reader)
-            return { status: response.status, body: await response.text() };
-        const chunks = [];
-        let total = 0;
-        for (;;) {
-            const { done, value } = await reader.read();
-            if (done)
-                break;
-            total += value.length;
-            if (total > MAX_BODY_BYTES) {
-                await reader.cancel();
-                break;
-            }
-            chunks.push(value);
-        }
-        return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
-    }
-    throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
+    const response = await publicHttpGet(url, { maxBytes: MAX_BODY_BYTES, maxRedirects: MAX_REDIRECTS,
+        timeoutMs: FETCH_TIMEOUT_MS, headers: { "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)", "Accept-Language": "en-US" } });
+    return { status: response.status, body: Buffer.from(response.body).toString("utf8") };
 };
 export async function captureMarket(config, options = {}) {
     const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });

package/dist/marketClassify.d.ts

@@ -51,6 +51,16 @@ export type MarketWorksheet = {
         captureHash: string;
         text: string;
     }>;
+    /** Deterministic retrieval hints: page lines that contain claim terms. These are not labels. */
+    claimHints: Array<{
+        claimId: string;
+        matches: Array<{
+            url: string;
+            captureHash: string;
+            term: string;
+            quote: string;
+        }>;
+    }>;
     instructions: string;
 };
 export declare function buildWorksheet(config: MarketConfig, vendorId: string, options?: {

package/dist/marketClassify.js

@@ -188,6 +188,56 @@ export async function classifyMarket(config, options) {
         retriedVendorIds,
     };
 }
+const HINT_STOPWORDS = new Set([
+    "the", "and", "for", "with", "from", "that", "this", "into", "your", "you", "are", "but", "not", "when", "how", "use", "uses", "using",
+    "claim", "claims", "vendor", "vendors", "page", "pages", "loud", "quiet", "absent", "capability", "specific", "differentiating",
+    "unknown", "buyer", "buyers", "pricing", "structure", "category", "support", "supported", "primary", "secondary", "mentioned", "mentions",
+]);
+function normalizeHintText(value) {
+    return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+}
+function claimHintTerms(claim) {
+    const exact = [claim.id.replace(/[-_]+/g, " "), claim.capability, ...(claim.terms ?? [])]
+        .map(normalizeHintText)
+        .filter((term) => term.length >= 3);
+    const tokenSource = normalizeHintText(`${claim.capability} ${claim.definition} ${(claim.terms ?? []).join(" ")}`)
+        .split(" ")
+        .filter((term) => term.length >= 4 && !HINT_STOPWORDS.has(term));
+    return Array.from(new Set([...exact, ...tokenSource])).slice(0, 24);
+}
+function lineSnippet(line, term) {
+    const clean = line.trim().replace(/\s+/g, " ");
+    if (clean.length <= 300)
+        return clean;
+    const lower = clean.toLowerCase();
+    const idx = lower.indexOf(term.toLowerCase().split(" ")[0] ?? "");
+    const start = Math.max(0, idx < 0 ? 0 : idx - 120);
+    return clean.slice(start, start + 300).trim();
+}
+function buildClaimHints(claims, pages) {
+    return claims.map((claim) => {
+        const terms = claimHintTerms(claim);
+        const matches = [];
+        for (const page of pages) {
+            const lines = page.text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+            for (const term of terms) {
+                const normalizedTerm = normalizeHintText(term);
+                const line = lines.find((candidate) => normalizeHintText(candidate).includes(normalizedTerm));
+                if (!line)
+                    continue;
+                const quote = lineSnippet(line, term);
+                if (quote && !matches.some((m) => m.quote === quote && m.url === page.url)) {
+                    matches.push({ url: page.url, captureHash: page.captureHash, term, quote });
+                }
+                if (matches.length >= 4)
+                    break;
+            }
+            if (matches.length >= 4)
+                break;
+        }
+        return { claimId: claim.id, matches };
+    });
+}
 export function buildWorksheet(config, vendorId, options = {}) {
     const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
     if (!vendor)
@@ -211,6 +261,7 @@ export function buildWorksheet(config, vendorId, options = {}) {
         vendor: { id: vendor.id, name: vendor.name },
         claims: config.claims,
         pages,
-        instructions: "Produce one observation per claim (intensity loud|quiet|absent from these pages only; unobservable only if a page you need failed to capture). Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
+        claimHints: buildClaimHints(config.claims, pages),
+        instructions: "Produce one observation per claim from these pages only. Intensity rubric: loud = hero copy, primary positioning, or a named/dedicated product area; quiet = supported by a verbatim page quote but secondary, qualified, or merely available; absent = the pages do not support the claim; unobservable = no usable captured page exists for this vendor, not a synonym for absent. claimHints are deterministic retrieval hints only: use them to find candidate support, but do not mark a claim loud/quiet unless the quote truly supports the claim. Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
     };
 }

package/dist/marketSourcing.js

@@ -16,7 +16,7 @@
  *    Sitemaps / redirects / logo bytes are plain resources — they never need a
  *    headless browser, so the default fetch is sufficient even in the hosted layer.
  */
-import { assertPublicUrl } from "./market.js";
+import { publicHttpGet } from "./publicHttp.js";
 import { DEFAULT_MODELS, forcedToolCall } from "./llm.js";
 const USER_AGENT = "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)";
 const FETCH_TIMEOUT_MS = 15_000;
@@ -156,28 +156,9 @@ export function extractLogoUrl(html, baseUrl) {
 // ── Default SSRF-guarded fetchers ────────────────────────────────────────────
 /** Manual-redirect fetch that re-validates every hop against the SSRF guard. */
 async function guardedFetch(url) {
-    let current = url;
-    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-        await assertPublicUrl(current);
-        const res = await fetch(current, {
-            redirect: "manual",
-            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-            headers: { "User-Agent": USER_AGENT },
-        });
-        const location = res.headers.get("location");
-        if (res.status >= 300 && res.status < 400 && location) {
-            try {
-                await res.body?.cancel();
-            }
-            catch {
-                /* ignore */
-            }
-            current = new URL(location, current).toString();
-            continue;
-        }
-        return res;
-    }
-    return null;
+    const res = await publicHttpGet(url, { maxRedirects: MAX_REDIRECTS, timeoutMs: FETCH_TIMEOUT_MS,
+        headers: { "User-Agent": USER_AGENT } });
+    return new Response(res.body.length ? Buffer.from(res.body) : null, { status: res.status, headers: res.headers });
 }
 const defaultFetchText = async (url) => {
     try {
@@ -206,28 +187,9 @@ const defaultFetchBytes = async (url) => {
  * + status WITHOUT downloading the body. Used to detect identity drift.
  */
 export const resolveFinalUrl = async (rawUrl) => {
-    let current = rawUrl;
-    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
-        await assertPublicUrl(current);
-        const res = await fetch(current, {
-            redirect: "manual",
-            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-            headers: { "User-Agent": USER_AGENT },
-        });
-        try {
-            await res.body?.cancel();
-        }
-        catch {
-            /* ignore */
-        }
-        const location = res.headers.get("location");
-        if (res.status >= 300 && res.status < 400 && location) {
-            current = new URL(location, current).toString();
-            continue;
-        }
-        return { finalUrl: current, status: res.status };
-    }
-    return { finalUrl: current, status: 0 };
+    const res = await publicHttpGet(rawUrl, { maxRedirects: MAX_REDIRECTS, timeoutMs: FETCH_TIMEOUT_MS,
+        maxBytes: 0, headers: { "User-Agent": USER_AGENT } });
+    return { finalUrl: res.finalUrl, status: res.status };
 };
 // ── Fetching helpers ─────────────────────────────────────────────────────────
 /**