@taurgis/bonsai 4.0.0 → 4.1.1

package/dist/sites/salesforce-doc-fetch.js

@@ -1,9 +1,10 @@
 import { openCdpPage, waitForLoad, ResponseCapture, describeNavigationFailure, } from '../lib/research/browser.js';
-import { checkDnsSafety } from '../lib/research/fetcher.js';
+import { checkDnsSafety, fetchStatus } from '../lib/research/fetcher.js';
 import { htmlToMarkdown } from '../lib/research/markdown.js';
 import { sanitizePromptInjection } from '../lib/research/prompt-injection.js';
 import { contentMetrics, contentRicherThan } from '../lib/research/regression.js';
-import { SALESFORCE_SHADOW_DOM_HELPERS, buildPriorityContainerPickBlock, pollSalesforceContentReady, } from './salesforce-dom-probe.js';
+import { SALESFORCE_SHADOW_DOM_HELPERS, buildPriorityContainerPickBlock, pollSalesforceContentReady, looksLikeSalesforceError, } from './salesforce-dom-probe.js';
+export { looksLikeSalesforceError };
 // Both help.salesforce.com and developer.salesforce.com are LWR sites that render doc content
 // client-side inside web components (shadow DOM). document.documentElement.outerHTML captures
 // only the "Loading…" shell, and Readability prunes the structured result to a single block —
@@ -19,20 +20,11 @@ const BODY_LIMIT_BYTES = 6 * 1024 * 1024;
 /** Grace period after DOM settle for late /docs/get_document_content/ responses on guide pages. */
 const DOCS_API_GRACE_MS = 500;
 const ACCEPT_CONSENT_EXPRESSION = "(() => { const b = document.querySelector('#onetrust-accept-btn-handler'); if (b) b.click(); })()";
-const ERROR_PATTERNS = [
-    /page\s*(not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
-    /we\s*couldn'?t\s*find/i,
-    /we looked high and low/i,
-    /sorry to interrupt/i,
-];
-/** True when the extracted text is an error/not-found/loading shell rather than real content.
- *
- * @param text - Extracted Markdown text to check.
- * @returns True when the text matches a Salesforce error/not-found pattern.
- */
-export function looksLikeSalesforceError(text) {
-    return ERROR_PATTERNS.some((re) => re.test(text));
-}
+// Salesforce's WAF answers 403 unless the User-Agent leads with a recognized HTTP-tool token.
+// Same token the Markdown probe sends, so a missing-doc check sees the real status.
+const DOC_PROBE_HEADERS = {
+    'user-agent': 'curl/8.7.1 bonsai-research-cache (+https://bonsai.rhino-inquisitor.com)',
+};
 // Chrome present on every LWR doc page; stripped before serialization. Per-site noise (e.g.
 // Help's article-feedback widget) is appended via SalesforceDocOptions.removeSelectors.
 const BASE_REMOVE_SELECTORS = [
@@ -69,25 +61,28 @@ function confidenceFor(length) {
 // page. Help wraps it in a removable component (see fetch-page.ts), but Developer renders it
 // as bare divs, so we also strip its lines from the converted Markdown for both sites.
 const BOILERPLATE_LINE = /(did this article solve your issue|let us know so we can improve|share your feedback)/i;
-// Toolbar of the "View as Markdown" feature on Developer articles. When the .md route probe
-// succeeds the browser never runs, but rollout/network asymmetries can still land such a page
-// here, and the toolbar labels serialize as bare paragraph lines.
-const MARKDOWN_TOOLBAR_LINE = /^(copy as markdown|view as markdown|copy url to markdown)$/i;
+// Toolbar labels that serialize as their own paragraph: "View as Markdown" on articles that
+// missed the .md probe, and the llms.txt / "Open in ChatGPT" cluster on the newer guide chrome.
+const DOC_CHROME_LINE = /^(copy as markdown|view as markdown|copy url to markdown|view llms\.txt|copy url to llms\.txt|open in chatgpt|open in claude|open in perplexity|this page|all documentation|more actions)$/i;
 // A link with empty anchor text — e.g. an icon-only "home" logo link. It carries no readable
 // content and renders as a bare `[](url)`. (Images `![](url)` are intentionally not matched.)
 const EMPTY_LINK_LINE = /^\[\]\([^)]*\)$/;
+// Atlas books render a release switcher as one link per season. Turndown splits the anchor
+// text across lines, so the link is not a single line. Drop the whole link; the article follows.
+const VERSION_PICKER_LINK = /\[\s*(?:Spring|Summer|Winter|Fall)\s+'\d{2}\s+\(API version\s+[\d.]+\)\s*\]\([^)\n]*\)/gi;
 /**
- * Removes known boilerplate lines from converted Salesforce Markdown: the feedback widget text,
- * "View as Markdown" toolbar labels, and empty icon-anchor lines.
+ * Removes known boilerplate from converted Salesforce Markdown: the feedback widget, doc-chrome
+ * toolbar labels, the atlas release switcher, and empty icon-anchor lines.
  *
  * @param markdown - Markdown output from the DOM or content API conversion.
- * @returns Markdown with boilerplate lines stripped and excess blank lines collapsed.
+ * @returns Markdown with boilerplate stripped and excess blank lines collapsed.
  */
 export function stripBoilerplate(markdown) {
     return markdown
+        .replace(VERSION_PICKER_LINK, '')
         .split('\n')
         .filter((line) => !BOILERPLATE_LINE.test(line) &&
-        !MARKDOWN_TOOLBAR_LINE.test(line.trim()) &&
+        !DOC_CHROME_LINE.test(line.trim()) &&
         !EMPTY_LINK_LINE.test(line.trim()))
         .join('\n')
         .replace(/\n{3,}/g, '\n\n')
@@ -286,6 +281,65 @@ function buildCaptureScript(contentSelectors, extraRemove) {
     return { html: serialize(container), title };
   })()`;
 }
+/** Article identity for a Salesforce doc URL: Help `id`, otherwise the last path segment.
+ * Underscores and hyphens match so a REST rename (`intro_what_is_rest_api` →
+ * `intro-what-is-rest-api`) stays the same article.
+ *
+ * @param url - Requested or post-navigation URL.
+ * @returns Normalized article key, or null when the URL has no article identity.
+ */
+export function salesforceArticleKey(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    const id = parsed.searchParams.get('id');
+    const raw = (id ?? parsed.pathname.replace(/\/+$/, '').split('/').pop() ?? '').trim();
+    const stripped = raw.replace(/\.(html?|md)$/i, '');
+    if (!stripped)
+        return null;
+    return stripped.toLowerCase().replace(/_/g, '-');
+}
+/** True when both URLs name the same Salesforce article, or either key can't be read.
+ *
+ * @param requested - URL the caller asked for.
+ * @param finalUrl - URL the browser landed on.
+ * @returns False only when both keys exist and differ.
+ */
+export function sameSalesforceArticle(requested, finalUrl) {
+    const requestedKey = salesforceArticleKey(requested);
+    const finalKey = salesforceArticleKey(finalUrl);
+    if (!requestedKey || !finalKey)
+        return true;
+    return requestedKey === finalKey;
+}
+async function assertSalesforceDocumentExists(url) {
+    try {
+        await fetchStatus(url, { headers: DOC_PROBE_HEADERS, timeoutMs: 8_000 });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : '';
+        // Any other failure (WAF 403, timeout) still goes to the browser. Only a real
+        // not-found status is enough to skip Chrome.
+        if (/\bstatus (404|410)\b/.test(message)) {
+            throw new Error(`Salesforce doc not found: ${url}`);
+        }
+    }
+}
+async function assertLandedOnRequestedArticle(page, url) {
+    const result = await page.client
+        .send('Runtime.evaluate', { expression: 'location.href', returnByValue: true }, page.sessionId)
+        .catch(() => null);
+    const finalHref = result?.result?.value;
+    if (typeof finalHref !== 'string' || !finalHref)
+        return;
+    if (!sameSalesforceArticle(url, finalHref)) {
+        throw new Error(`Salesforce doc ${url} redirected to a different article (${finalHref}).`);
+    }
+}
 function parseSalesforceDocUrl(url, allowedHost) {
     let parsed;
     try {
@@ -300,7 +354,10 @@ function parseSalesforceDocUrl(url, allowedHost) {
     return parsed;
 }
 function assertReadableSalesforceContent(detailedMarkdown, url) {
-    if (detailedMarkdown.length < MIN_CONTAINER_CHARS || looksLikeSalesforceError(detailedMarkdown)) {
+    if (looksLikeSalesforceError(detailedMarkdown)) {
+        throw new Error(`Salesforce doc not found: ${url}`);
+    }
+    if (detailedMarkdown.length < MIN_CONTAINER_CHARS) {
         throw new Error(`Salesforce returned no readable content for ${url} (page may be missing, gated, or still loading).`);
     }
 }
@@ -351,6 +408,9 @@ async function captureSalesforceAttempt(page, url, captureScript, contentSelecto
     }
     let detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
     let usedDocsApi = false;
+    // Retired atlas leaves (apex_qs_hello.htm) stay HTTP 200, then the SPA swaps in the book
+    // home. Storing that home under the requested URL would cache the wrong article.
+    await assertLandedOnRequestedArticle(page, url);
     if (docsApiHtml) {
         const picked = preferRicherMarkdown(detailedMarkdown, docsApiHtml, title, docsApiTitle);
         detailedMarkdown = picked.markdown;
@@ -393,12 +453,14 @@ function buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityN
  * @param options - Allowed host, priority content selectors, and optional per-site remove selectors.
  * @returns Extracted Markdown, fetch metadata, and browser-capture provenance.
  * @throws {Error} When the URL is invalid, the host is not the allowed host, DNS is unsafe,
- *   Chrome/CDP navigation or evaluation fails, the response body exceeds the size limit,
- *   or the captured content is too short / looks like an error page.
+ *   the document itself is HTTP 404/410, Chrome/CDP navigation or evaluation fails, the page
+ *   redirects to a different article, the response body exceeds the size limit, or the captured
+ *   content is too short / looks like an error page.
  */
 export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
     const parsed = parseSalesforceDocUrl(url, allowedHost);
     await checkDnsSafety(parsed.hostname);
+    await assertSalesforceDocumentExists(url);
     const captureScript = buildCaptureScript(contentSelectors, removeSelectors);
     const page = await openCdpPage();
     try {

package/dist/sites/salesforce-dom-probe.d.ts

@@ -1,6 +1,12 @@
 import type { CdpPage } from '../lib/research/browser.js';
 /** Shared in-page helpers for traversing Salesforce Lightning Web Runtime (LWR) open shadow roots. */
 export declare const SALESFORCE_SHADOW_DOM_HELPERS = "\nconst deepElements = (root) => {\n  const out = [];\n  const visit = (node) => {\n    if (node.shadowRoot) visit(node.shadowRoot);\n    for (const child of Array.from(node.children || [])) {\n      out.push(child);\n      visit(child);\n    }\n  };\n  visit(root);\n  return out;\n};\n\nconst deepText = (root) => {\n  let txt = '';\n  const visit = (node) => {\n    if (node.nodeType === 3) {\n      txt += node.textContent || '';\n    } else if (node.nodeType === 1 || node.nodeType === 9 || node.nodeType === 11) {\n      const tag = node.tagName ? node.tagName.toLowerCase() : '';\n      if (tag !== 'style' && tag !== 'script') {\n        if (node.shadowRoot) visit(node.shadowRoot);\n        for (const child of Array.from(node.childNodes || [])) visit(child);\n      }\n    }\n  };\n  visit(root);\n  return txt;\n};\n\nconst textLenFor = (el) =>\n  Math.max((el.innerText || '').trim().length, deepText(el).trim().length);\n";
+/** True when extracted text is a Salesforce not-found or error shell rather than an article.
+ *
+ * @param text - Extracted Markdown or visible page text.
+ * @returns True when the text matches a Salesforce error/not-found pattern.
+ */
+export declare function looksLikeSalesforceError(text: string): boolean;
 /**
  * In-page block: pick the highest-priority content container that exists on the page.
  * Does not fall through to lower-priority selectors when a higher-priority host is present

package/dist/sites/salesforce-dom-probe.js

@@ -33,6 +33,23 @@ const deepText = (root) => {
 const textLenFor = (el) =>
   Math.max((el.innerText || '').trim().length, deepText(el).trim().length);
 `;
+// Shells Salesforce renders for a missing article. Shared with the readiness probe so a
+// not-found page ends the wait instead of burning the full content timeout.
+const SALESFORCE_ERROR_PATTERNS = [
+    /page\s*(not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
+    /we\s*couldn'?t\s*find/i,
+    /we looked high and low/i,
+    /sorry to interrupt/i,
+    /head back to the space station/i,
+];
+/** True when extracted text is a Salesforce not-found or error shell rather than an article.
+ *
+ * @param text - Extracted Markdown or visible page text.
+ * @returns True when the text matches a Salesforce error/not-found pattern.
+ */
+export function looksLikeSalesforceError(text) {
+    return SALESFORCE_ERROR_PATTERNS.some((re) => re.test(text));
+}
 /**
  * In-page block: pick the highest-priority content container that exists on the page.
  * Does not fall through to lower-priority selectors when a higher-priority host is present
@@ -64,6 +81,20 @@ export function buildPriorityContainerPickBlock(selectors, minChars, containerVa
       }
     }
     ${containerVar} = ${containerVar} || document.body || document.documentElement;
+    // A Redoc hash (#tag/…/operation/…) names one operation inside the reference.
+    // getElementById stops at the shadow boundary, so match the id while walking.
+    try {
+      const hashId = decodeURIComponent((location.hash || '').replace(/^#/, ''));
+      if (hashId) {
+        let hashBest = null, hashLen = 0;
+        for (const el of deepElements(document)) {
+          if (el.id !== hashId) continue;
+          const len = textLenFor(el);
+          if (len > hashLen) { hashBest = el; hashLen = len; }
+        }
+        if (hashBest && hashLen >= MIN_CHARS) ${containerVar} = hashBest;
+      }
+    } catch (e) {}
   `;
 }
 /**
@@ -75,8 +106,14 @@ export function buildPriorityContainerPickBlock(selectors, minChars, containerVa
  * @returns A self-invoking JS expression string that evaluates to `{ has: boolean, len: number }`.
  */
 export function buildContentReadyProbeExpression(selectors, minChars) {
+    const errorSource = SALESFORCE_ERROR_PATTERNS.map((re) => re.source).join('|');
     return `(() => {
     ${SALESFORCE_SHADOW_DOM_HELPERS}
+    const errorRe = new RegExp(${JSON.stringify(errorSource)}, 'i');
+    const pageText = deepText(document.body || document.documentElement);
+    if (errorRe.test(pageText)) {
+      return { has: true, len: pageText.length, error: true };
+    }
     const SELS = ${JSON.stringify(selectors)};
     const MIN = ${minChars};
     let has = false;
@@ -119,6 +156,9 @@ export async function pollSalesforceContentReady(page, selectors, minChars, time
             .send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
             .catch(() => null);
         const value = rawResult?.result?.value;
+        // A not-found shell is ready now; waiting for it to "settle" only delays the failure.
+        if (value?.error)
+            return;
         if (value?.has) {
             const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
             if (!grew && ++stablePolls >= 2)