gologin-web-access 0.3.1 → 0.3.2

package/CHANGELOG.md

@@ -5,6 +5,12 @@
 - browser automation is now embedded directly in `gologin-web-access`, so one repo and one install contains both Web Unlocker and Cloud Browser flows
 - doctor now reports the embedded browser runtime source and version
 
+## 0.3.2 - 2026-04-03
+
+- added unified page outcome classification across `read`, `scrape-json`, and `batch-scrape`
+- structured and readable paths now distinguish `ok`, `empty`, `incomplete`, `authwall`, `challenge`, `blocked`, and `cookie_wall`
+- batch and extract-oriented flows now propagate next-step hints and fallback metadata more consistently for agents
+
 ## 0.1.0 - 2026-03-10
 
 Initial public release of Gologin Web Access.

package/README.md

@@ -305,6 +305,7 @@ gologin-web-access snapshot -i
 - `batch-extract` reuses the same extraction path across many URLs and returns one structured result per URL, including request and fallback metadata. Add `--output <path>` to save the full array directly.
 - `scrape-json` now returns both a flat `headings` array and `headingsByLevel` buckets for `h1` through `h6`.
 - `scrape-json --fallback browser` is available for JS-heavy pages where stateless extraction returns weak heading data.
+- `scrape-json` now also classifies the page outcome as `ok`, `empty`, `incomplete`, `authwall`, `challenge`, `blocked`, or `cookie_wall`, and includes `nextActionHint` when the result is weak or gated.
 - `scrape`, `scrape-markdown`, `scrape-text`, `scrape-json`, and `batch-scrape` accept `--retry`, `--backoff-ms`, and `--timeout-ms`.
 - `batch-scrape --only-main-content` lets markdown, text, and html batch runs use the same readable-content isolation path as `read`.
 - `crawl --only-main-content` uses the same readable-fragment extraction strategy for html, markdown, and text crawl output, but stays on the stateless unlocker path.
@@ -312,6 +313,7 @@ gologin-web-access snapshot -i
 - `batch-scrape` now returns exit code `0` on partial success by default and only fails the command when every URL failed. Add `--strict` if any single failed URL should make the whole batch exit non-zero.
 - `batch-scrape --output <path>` writes the full JSON to disk so shells and agent consoles cannot truncate a large payload silently.
 - `batch-scrape --format json` now returns the same structured scrape envelope as `scrape-json`, including `renderSource`, `fallbackAttempted`, `fallbackUsed`, and `request.attemptCount/retryCount/attempts`.
+- `batch-scrape --only-main-content` now propagates `outcome`, `outcomeReason`, `nextActionHint`, and fallback metadata per URL so agents can tell "weak page" from "gated page" without scraping log text.
 - `scrape-json` now surfaces explicit `BLOCKED_PAGE` failures when structured output clearly matches a challenge or block page, instead of silently looking like a valid empty result.
 - `search` now returns `requestedLimit`, `returnedCount`, `warnings`, `cacheTtlMs`, and per-result `position`.
 - `search` may return fewer results than the requested `--limit` when the upstream SERP contains fewer valid results; inspect `returnedCount`, `warnings`, and `attempts`.

package/dist/cli.js

@@ -67,7 +67,7 @@ const wait_1 = require("./commands/wait");
 const doctor_1 = require("./doctor");
 const errors_1 = require("./lib/errors");
 const output_1 = require("./lib/output");
-const CLI_VERSION = "0.3.0";
+const CLI_VERSION = "0.3.2";
 async function main() {
     const program = new commander_1.Command();
     program

package/dist/commands/batchScrape.js

@@ -47,7 +47,16 @@ function buildBatchScrapeCommand() {
                     url,
                     ok: true,
                     format,
-                    output,
+                    output: output.output,
+                    outcome: output.outcome,
+                    outcomeReason: output.outcomeReason,
+                    nextActionHint: output.nextActionHint,
+                    renderSource: output.renderSource,
+                    fallbackAttempted: output.fallbackAttempted,
+                    fallbackUsed: output.fallbackUsed,
+                    fallbackReason: output.fallbackReason,
+                    warning: output.warning,
+                    request: output.request,
                 };
             }
             catch (error) {
@@ -59,6 +68,8 @@ function buildBatchScrapeCommand() {
                     error: error instanceof Error ? error.message : "Unknown error",
                     code: extractErrorCode(error),
                     status: extractStatusCode(error),
+                    outcome: extractOutcome(error),
+                    nextActionHint: extractNextActionHint(error),
                     request,
                 };
             }
@@ -97,31 +108,67 @@ async function formatOutput(url, config, apiKey, format, requestOptions, fallbac
         };
         switch (format) {
             case "html":
-                return (await (0, readSource_1.readHtmlContent)(url, config, apiKey, readOptions)).content;
+                return mapReadableBatchResult(await (0, readSource_1.readHtmlContent)(url, config, apiKey, readOptions));
             case "markdown":
-                return (await (0, readSource_1.readMarkdownContent)(url, config, apiKey, readOptions)).content;
+                return mapReadableBatchResult(await (0, readSource_1.readMarkdownContent)(url, config, apiKey, readOptions));
             case "text":
-                return (await (0, readSource_1.readTextContent)(url, config, apiKey, readOptions)).content;
+                return mapReadableBatchResult(await (0, readSource_1.readTextContent)(url, config, apiKey, readOptions));
             default:
                 break;
         }
     }
     switch (format) {
         case "html":
-            return (await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, requestOptions)).content;
+            return {
+                output: (await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, requestOptions)).content,
+            };
         case "markdown":
-            return (await (0, unlocker_1.scrapeMarkdown)(url, apiKey, requestOptions)).markdown;
+            return {
+                output: (await (0, unlocker_1.scrapeMarkdown)(url, apiKey, requestOptions)).markdown,
+            };
         case "text":
-            return (await (0, unlocker_1.scrapeText)(url, apiKey, requestOptions)).text;
+            return {
+                output: (await (0, unlocker_1.scrapeText)(url, apiKey, requestOptions)).text,
+            };
         case "json":
-            return await (0, structuredScrape_1.scrapeStructuredJson)(url, config, apiKey, {
+            return mapStructuredBatchResult(await (0, structuredScrape_1.scrapeStructuredJson)(url, config, apiKey, {
                 fallback,
                 request: requestOptions,
-            });
+            }));
         default:
-            return (await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, requestOptions)).content;
+            return {
+                output: (await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, requestOptions)).content,
+            };
     }
 }
+function mapReadableBatchResult(result) {
+    return {
+        output: result.content,
+        outcome: result.outcome,
+        outcomeReason: result.outcomeReason,
+        nextActionHint: result.nextActionHint,
+        renderSource: result.renderSource,
+        fallbackAttempted: result.fallbackAttempted,
+        fallbackUsed: result.fallbackUsed,
+        fallbackReason: result.fallbackReason,
+        warning: result.warning,
+        request: result.request,
+    };
+}
+function mapStructuredBatchResult(result) {
+    return {
+        output: result,
+        outcome: result.outcome,
+        outcomeReason: result.outcomeReason,
+        nextActionHint: result.nextActionHint,
+        renderSource: result.renderSource,
+        fallbackAttempted: result.fallbackAttempted,
+        fallbackUsed: result.fallbackUsed,
+        fallbackReason: result.fallbackReason,
+        warning: result.warning,
+        request: result.request,
+    };
+}
 async function mapWithConcurrency(items, concurrency, mapper) {
     const results = new Array(items.length);
     let nextIndex = 0;
@@ -180,3 +227,21 @@ function extractErrorCode(error) {
     }
     return undefined;
 }
+function extractOutcome(error) {
+    if (typeof error === "object" &&
+        error !== null &&
+        "outcome" in error &&
+        typeof error.outcome === "string") {
+        return error.outcome;
+    }
+    return undefined;
+}
+function extractNextActionHint(error) {
+    if (typeof error === "object" &&
+        error !== null &&
+        "nextActionHint" in error &&
+        typeof error.nextActionHint === "string") {
+        return error.nextActionHint;
+    }
+    return undefined;
+}

package/dist/commands/read.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildReadCommand = buildReadCommand;
 const commander_1 = require("commander");
 const config_1 = require("../config");
+const pageOutcome_1 = require("../lib/pageOutcome");
 const readSource_1 = require("../lib/readSource");
 const output_1 = require("../lib/output");
 const shared_1 = require("./shared");
@@ -27,7 +28,7 @@ function buildReadCommand() {
             : format === "markdown"
                 ? await (0, readSource_1.readMarkdownContent)(url, config, apiKey, readOptions)
                 : await (0, readSource_1.readTextContent)(url, config, apiKey, readOptions);
-        emitReadNotice(result.fallbackAttempted, result.fallbackUsed, result.fallbackReason);
+        emitReadNotice(result);
         (0, output_1.printText)(result.content);
     })));
 }
@@ -37,15 +38,24 @@ function normalizeReadFormat(value) {
     }
     throw new Error(`Unsupported read format: ${value}`);
 }
-function emitReadNotice(fallbackAttempted, fallbackUsed, fallbackReason) {
-    if (!fallbackAttempted) {
-        return;
+function emitReadNotice(result) {
+    if (result.fallbackAttempted) {
+        if (result.fallbackUsed) {
+            process.stderr.write(`JS-rendered page detected, retrying with browser. ${result.fallbackReason ?? ""}\n`);
+        }
+        else if (result.fallbackReason) {
+            process.stderr.write(`${result.fallbackReason}\n`);
+        }
+    }
+    if (result.outcome !== "ok") {
+        process.stderr.write(`Outcome: ${result.outcome}\n`);
     }
-    if (fallbackUsed) {
-        process.stderr.write(`JS-rendered page detected, retrying with browser. ${fallbackReason ?? ""}\n`);
+    if (result.warning) {
+        process.stderr.write(`${result.warning}\n`);
         return;
     }
-    if (fallbackReason) {
-        process.stderr.write(`${fallbackReason}\n`);
+    const hint = (0, pageOutcome_1.describeNextActionHint)(result.nextActionHint);
+    if (hint && result.outcome !== "ok") {
+        process.stderr.write(`${hint}\n`);
     }
 }

package/dist/commands/scrapeMarkdown.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildScrapeMarkdownCommand = buildScrapeMarkdownCommand;
 const commander_1 = require("commander");
 const config_1 = require("../config");
+const pageOutcome_1 = require("../lib/pageOutcome");
 const readSource_1 = require("../lib/readSource");
 const shared_1 = require("./shared");
 const output_1 = require("../lib/output");
@@ -19,19 +20,28 @@ function buildScrapeMarkdownCommand() {
             source,
             request: (0, shared_1.normalizeUnlockerRequestOptions)(options),
         });
-        emitReadNotice(result.fallbackAttempted, result.fallbackUsed, result.fallbackReason);
+        emitReadNotice(result);
         (0, output_1.printText)(result.content);
     }));
 }
-function emitReadNotice(fallbackAttempted, fallbackUsed, fallbackReason) {
-    if (!fallbackAttempted) {
-        return;
+function emitReadNotice(result) {
+    if (result.fallbackAttempted) {
+        if (result.fallbackUsed) {
+            process.stderr.write(`JS-rendered page detected, retrying with browser. ${result.fallbackReason ?? ""}\n`);
+        }
+        else if (result.fallbackReason) {
+            process.stderr.write(`${result.fallbackReason}\n`);
+        }
+    }
+    if (result.outcome !== "ok") {
+        process.stderr.write(`Outcome: ${result.outcome}\n`);
     }
-    if (fallbackUsed) {
-        process.stderr.write(`JS-rendered page detected, retrying with browser. ${fallbackReason ?? ""}\n`);
+    if (result.warning) {
+        process.stderr.write(`${result.warning}\n`);
         return;
     }
-    if (fallbackReason) {
-        process.stderr.write(`${fallbackReason}\n`);
+    const hint = (0, pageOutcome_1.describeNextActionHint)(result.nextActionHint);
+    if (hint && result.outcome !== "ok") {
+        process.stderr.write(`${hint}\n`);
     }
 }

package/dist/commands/scrapeText.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildScrapeTextCommand = buildScrapeTextCommand;
 const commander_1 = require("commander");
 const config_1 = require("../config");
+const pageOutcome_1 = require("../lib/pageOutcome");
 const readSource_1 = require("../lib/readSource");
 const shared_1 = require("./shared");
 const output_1 = require("../lib/output");
@@ -19,19 +20,28 @@ function buildScrapeTextCommand() {
             source,
             request: (0, shared_1.normalizeUnlockerRequestOptions)(options),
         });
-        emitReadNotice(result.fallbackAttempted, result.fallbackUsed, result.fallbackReason);
+        emitReadNotice(result);
         (0, output_1.printText)(result.content);
     }));
 }
-function emitReadNotice(fallbackAttempted, fallbackUsed, fallbackReason) {
-    if (!fallbackAttempted) {
-        return;
+function emitReadNotice(result) {
+    if (result.fallbackAttempted) {
+        if (result.fallbackUsed) {
+            process.stderr.write(`JS-rendered page detected, retrying with browser. ${result.fallbackReason ?? ""}\n`);
+        }
+        else if (result.fallbackReason) {
+            process.stderr.write(`${result.fallbackReason}\n`);
+        }
+    }
+    if (result.outcome !== "ok") {
+        process.stderr.write(`Outcome: ${result.outcome}\n`);
     }
-    if (fallbackUsed) {
-        process.stderr.write(`JS-rendered page detected, retrying with browser. ${fallbackReason ?? ""}\n`);
+    if (result.warning) {
+        process.stderr.write(`${result.warning}\n`);
         return;
     }
-    if (fallbackReason) {
-        process.stderr.write(`${fallbackReason}\n`);
+    const hint = (0, pageOutcome_1.describeNextActionHint)(result.nextActionHint);
+    if (hint && result.outcome !== "ok") {
+        process.stderr.write(`${hint}\n`);
     }
 }

package/dist/lib/extractRunner.js

@@ -16,6 +16,10 @@ async function extractUrlWithSchema(url, config, apiKey, schema, options = {}) {
         fallbackAttempted: rendered.fallbackAttempted,
         fallbackUsed: rendered.fallbackUsed,
         fallbackReason: rendered.fallbackReason,
+        outcome: rendered.outcome,
+        outcomeReason: rendered.outcomeReason,
+        nextActionHint: rendered.nextActionHint,
+        warning: rendered.warning,
         request: rendered.request,
         extracted: (0, extract_1.extractWithSchema)(rendered.html, schema),
     };

package/dist/lib/pageOutcome.js

@@ -0,0 +1,192 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assessStructuredPageOutcome = assessStructuredPageOutcome;
+exports.assessReadablePageOutcome = assessReadablePageOutcome;
+exports.detectStructuredChallengeReason = detectStructuredChallengeReason;
+exports.detectStructuredBlockedReason = detectStructuredBlockedReason;
+exports.detectStructuredAuthwallReason = detectStructuredAuthwallReason;
+exports.detectStructuredCookieWallReason = detectStructuredCookieWallReason;
+exports.describeNextActionHint = describeNextActionHint;
+const cheerio_1 = require("cheerio");
+function assessStructuredPageOutcome(data) {
+    const authwallReason = detectStructuredAuthwallReason(data);
+    if (authwallReason) {
+        return buildAssessment("authwall", authwallReason, "use_logged_in_session", true);
+    }
+    const challengeReason = detectStructuredChallengeReason(data);
+    if (challengeReason) {
+        return buildAssessment("challenge", challengeReason, "use_local_profile", true);
+    }
+    const blockedReason = detectStructuredBlockedReason(data);
+    if (blockedReason) {
+        return buildAssessment("blocked", blockedReason, "use_local_profile", true);
+    }
+    const cookieWallReason = detectStructuredCookieWallReason(data);
+    if (cookieWallReason) {
+        return buildAssessment("cookie_wall", cookieWallReason, "retry_with_browser", true);
+    }
+    if (isStructuredDataEmpty(data)) {
+        return buildAssessment("empty", "Structured output contained almost no readable fields", "retry_with_browser", true);
+    }
+    if (isStructuredDataLikelyIncomplete(data)) {
+        return buildAssessment("incomplete", "Structured output looks incomplete or client-rendered", "retry_with_browser", true);
+    }
+    return {
+        outcome: "ok",
+        browserRecommended: false,
+    };
+}
+function assessReadablePageOutcome(html, content, options = {}) {
+    const $ = (0, cheerio_1.load)(html);
+    const title = $("title").first().text().trim();
+    const firstHeading = $("h1").first().text().trim();
+    const canonical = ($("link[rel='canonical']").attr("href") ?? "").trim();
+    const normalizedText = normalizeText(content);
+    const inputCount = $("input, textarea, select").length;
+    const formCount = $("form").length;
+    const candidates = [
+        title,
+        firstHeading,
+        canonical,
+        normalizedText.slice(0, 1_500),
+    ].filter(Boolean);
+    if (canonical.includes("/authwall")) {
+        return buildAssessment("authwall", "Canonical URL points to an authwall path", "use_logged_in_session", true);
+    }
+    for (const candidate of candidates) {
+        const authwallReason = classifyAuthwallText(candidate);
+        if (authwallReason && (formCount > 0 || inputCount > 0 || normalizedText.length < 3_000)) {
+            return buildAssessment("authwall", authwallReason, "use_logged_in_session", true);
+        }
+    }
+    for (const candidate of candidates) {
+        const challengeReason = classifyChallengeText(candidate);
+        if (challengeReason) {
+            return buildAssessment("challenge", challengeReason, "use_local_profile", true);
+        }
+    }
+    for (const candidate of candidates) {
+        const blockedReason = classifyBlockedText(candidate);
+        if (blockedReason) {
+            return buildAssessment("blocked", blockedReason, "use_local_profile", true);
+        }
+    }
+    for (const candidate of candidates) {
+        const cookieWallReason = classifyCookieWallText(candidate);
+        if (cookieWallReason && (formCount > 0 || normalizedText.length < 2_500)) {
+            return buildAssessment("cookie_wall", cookieWallReason, "retry_with_browser", true);
+        }
+    }
+    if (normalizedText.length === 0) {
+        return buildAssessment("empty", "Readable output was empty", "retry_with_browser", true);
+    }
+    if (options.looksIncomplete) {
+        return buildAssessment("incomplete", options.incompleteReason ?? "Readable output looks incomplete", "retry_with_browser", true);
+    }
+    return {
+        outcome: "ok",
+        browserRecommended: false,
+    };
+}
+function detectStructuredChallengeReason(data) {
+    return findStructuredReason(data, classifyChallengeText);
+}
+function detectStructuredBlockedReason(data) {
+    return findStructuredReason(data, classifyBlockedText);
+}
+function detectStructuredAuthwallReason(data) {
+    if ((data.canonical ?? "").includes("/authwall")) {
+        return "Canonical URL points to an authwall path";
+    }
+    return findStructuredReason(data, classifyAuthwallText);
+}
+function detectStructuredCookieWallReason(data) {
+    return findStructuredReason(data, classifyCookieWallText);
+}
+function describeNextActionHint(hint) {
+    switch (hint) {
+        case "retry_with_browser":
+            return "Retry with --source browser or --fallback browser.";
+        case "use_logged_in_session":
+            return "Use a logged-in browser session or a GoLogin profile with saved cookies.";
+        case "use_local_profile":
+            return "Switch to gologin-local-agent-browser or another profile-backed browser path.";
+        default:
+            return undefined;
+    }
+}
+function buildAssessment(outcome, reason, nextActionHint, browserRecommended) {
+    const actionText = describeNextActionHint(nextActionHint);
+    return {
+        outcome,
+        reason,
+        nextActionHint,
+        browserRecommended,
+        warning: actionText ? `${capitalizeOutcome(outcome)} detected: ${reason}. ${actionText}` : `${capitalizeOutcome(outcome)} detected: ${reason}.`,
+    };
+}
+function capitalizeOutcome(value) {
+    return value.replace(/_/g, " ").replace(/^\w/, (char) => char.toUpperCase());
+}
+function isStructuredDataEmpty(data) {
+    return !data.title && !data.description && data.headings.length === 0 && data.links.length === 0;
+}
+function isStructuredDataLikelyIncomplete(data) {
+    const firstH1 = data.headingsByLevel.h1[0];
+    if (!firstH1) {
+        return true;
+    }
+    if (looksSuspiciousHeadingText(firstH1)) {
+        return true;
+    }
+    if (!data.title && data.headings.length < 2) {
+        return true;
+    }
+    return false;
+}
+function looksSuspiciousHeadingText(value) {
+    return /function\s*\(|window\.|document\.|const\s+|let\s+|var\s+|=>|import\s+/i.test(value) || value.length > 240;
+}
+function findStructuredReason(data, matcher) {
+    const candidates = [
+        data.title,
+        data.description,
+        data.canonical,
+        ...data.headingsByLevel.h1.slice(0, 2),
+        ...data.headingsByLevel.h2.slice(0, 2),
+    ].filter((value) => Boolean(value && value.trim()));
+    for (const candidate of candidates) {
+        const reason = matcher(candidate);
+        if (reason) {
+            return reason;
+        }
+    }
+    return undefined;
+}
+function normalizeText(value) {
+    return value.replace(/\s+/g, " ").trim();
+}
+function classifyChallengeText(value) {
+    if (/(verify you are human|verify you are a human|are you human|captcha|security check|attention required|just a moment|checking your browser|enable javascript and cookies to continue|one more step|security verification)/i.test(value)) {
+        return "Challenge or verification markers matched the page";
+    }
+    return undefined;
+}
+function classifyBlockedText(value) {
+    if (/(access denied|forbidden|blocked request|request blocked|request unsuccessful|temporarily blocked|temporarily unavailable|you have been blocked|access to this page has been denied)/i.test(value)) {
+        return "Blocked-page markers matched the page";
+    }
+    return undefined;
+}
+function classifyAuthwallText(value) {
+    if (/(sign in to view|sign in to continue|log in to continue|join to view|join now|join linkedin|join to continue|sign up to view|create a free account|view full profile|view profile|join to see more)/i.test(value)) {
+        return "Login or signup wall markers matched the page";
+    }
+    return undefined;
+}
+function classifyCookieWallText(value) {
+    if (/(cookie preferences|accept cookies|manage cookies|consent preferences|we use cookies|your privacy choices|respects your privacy)/i.test(value)) {
+        return "Cookie or consent wall markers matched the page";
+    }
+    return undefined;
+}

package/dist/lib/readSource.js

@@ -10,6 +10,7 @@ exports.extractReadableSegmentFromHtml = extractReadableSegmentFromHtml;
 const cheerio_1 = require("cheerio");
 const config_1 = require("../config");
 const browserRead_1 = require("./browserRead");
+const pageOutcome_1 = require("./pageOutcome");
 const unlocker_1 = require("./unlocker");
 function normalizeReadSourceMode(value, defaultMode = "auto") {
     if (!value) {
@@ -50,6 +51,7 @@ async function readRenderedHtmlContent(url, config, apiKey, options = {}) {
             renderSource: "browser",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: "ok",
         };
     }
     const unlocker = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, options.request);
@@ -59,17 +61,26 @@ async function readRenderedHtmlContent(url, config, apiKey, options = {}) {
             renderSource: "unlocker",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: "ok",
             request: unlocker.request,
         };
     }
     const unlockerText = (0, unlocker_1.htmlToText)(unlocker.content);
     const assessment = assessReadableContent(unlocker.content, unlockerText);
+    const outcomeAssessment = (0, pageOutcome_1.assessReadablePageOutcome)(unlocker.content, unlockerText, {
+        looksIncomplete: assessment.shouldFallback,
+        incompleteReason: assessment.reason,
+    });
     if (!assessment.shouldFallback) {
         return {
             html: unlocker.content,
             renderSource: "unlocker",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
@@ -80,6 +91,10 @@ async function readRenderedHtmlContent(url, config, apiKey, options = {}) {
             fallbackAttempted: true,
             fallbackUsed: false,
             fallbackReason: `${assessment.reason}; GOLOGIN_TOKEN is not configured`,
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
@@ -92,6 +107,7 @@ async function readRenderedHtmlContent(url, config, apiKey, options = {}) {
         fallbackAttempted: true,
         fallbackUsed: true,
         fallbackReason: assessment.reason,
+        outcome: "ok",
         request: unlocker.request,
     };
 }
@@ -150,27 +166,40 @@ async function readReadableContent(url, config, apiKey, options) {
             renderSource: "browser",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: "ok",
         };
     }
     const unlocker = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, options.request);
     const readable = extractReadableSegmentFromHtml(unlocker.content);
     const unlockerContent = formatReadableContent(options.format, readable.html, readable.text);
+    const assessment = assessReadableContent(unlocker.content, unlockerContent);
+    const outcomeAssessment = (0, pageOutcome_1.assessReadablePageOutcome)(unlocker.content, unlockerContent, {
+        looksIncomplete: assessment.shouldFallback,
+        incompleteReason: assessment.reason,
+    });
     if (source === "unlocker") {
         return {
             content: unlockerContent,
             renderSource: "unlocker",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
-    const assessment = assessReadableContent(unlocker.content, unlockerContent);
     if (!assessment.shouldFallback) {
         return {
             content: unlockerContent,
             renderSource: "unlocker",
             fallbackAttempted: false,
             fallbackUsed: false,
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
@@ -181,6 +210,10 @@ async function readReadableContent(url, config, apiKey, options) {
             fallbackAttempted: true,
             fallbackUsed: false,
             fallbackReason: `${assessment.reason}; GOLOGIN_TOKEN is not configured`,
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
@@ -195,6 +228,10 @@ async function readReadableContent(url, config, apiKey, options) {
             fallbackAttempted: true,
             fallbackUsed: false,
             fallbackReason: "Browser fallback did not improve readable output",
+            outcome: outcomeAssessment.outcome,
+            outcomeReason: outcomeAssessment.reason,
+            nextActionHint: outcomeAssessment.nextActionHint,
+            warning: outcomeAssessment.warning,
             request: unlocker.request,
         };
     }
@@ -204,6 +241,7 @@ async function readReadableContent(url, config, apiKey, options) {
         fallbackAttempted: true,
         fallbackUsed: true,
         fallbackReason: assessment.reason,
+        outcome: "ok",
         request: unlocker.request,
     };
 }

package/dist/lib/structuredScrape.js

@@ -9,21 +9,27 @@ exports.detectStructuredBlockReason = detectStructuredBlockReason;
 const config_1 = require("../config");
 const errors_1 = require("./errors");
 const browserStructured_1 = require("./browserStructured");
+const pageOutcome_1 = require("./pageOutcome");
 const unlocker_1 = require("./unlocker");
 class StructuredBlockedPageError extends errors_1.CliError {
     status;
     request;
-    constructor(url, status, request, reason, options) {
-        super(`Structured scrape returned a likely blocked or challenge page for ${url}.`, 1, [
+    outcome;
+    nextActionHint;
+    constructor(url, status, request, outcome, reason, nextActionHint, options) {
+        super(`Structured scrape returned ${outcome.replace(/_/g, " ")} content for ${url}.`, 1, [
             `Reason: ${reason}.`,
             options.fallbackAttempted
                 ? options.fallbackUsed
                     ? "Browser fallback was used but the page still looked blocked."
                     : `Browser fallback was attempted but not used. ${options.fallbackReason ?? "It did not improve the result."}`
-                : "Retry with --fallback browser, use read --source browser, or switch to gologin-local-agent-browser for full rendered DOM.",
-        ].join("\n"), "BLOCKED_PAGE");
+                : (0, pageOutcome_1.describeNextActionHint)(nextActionHint) ??
+                    "Retry with --fallback browser, use read --source browser, or switch to gologin-local-agent-browser for full rendered DOM.",
+        ].join("\n"), outcomeToErrorCode(outcome));
         this.status = status;
         this.request = request;
+        this.outcome = outcome;
+        this.nextActionHint = nextActionHint;
     }
 }
 async function scrapeStructuredJson(url, config, apiKey, options = {}) {
@@ -34,7 +40,7 @@ async function scrapeStructuredJson(url, config, apiKey, options = {}) {
     let fallbackAttempted = false;
     let fallbackUsed = false;
     let fallbackReason;
-    let { browserRecommended, warning } = buildStructuredFallbackAdvisory(data);
+    let { outcome, reason, nextActionHint, browserRecommended, warning } = (0, pageOutcome_1.assessStructuredPageOutcome)(data);
     if (fallbackMode === "browser" && shouldUseBrowserFallback(data)) {
         fallbackAttempted = true;
         (0, config_1.requireCloudToken)(config);
@@ -46,16 +52,14 @@ async function scrapeStructuredJson(url, config, apiKey, options = {}) {
             renderSource = "browser";
             fallbackUsed = true;
             fallbackReason = "unlocker structured data looked incomplete";
-            browserRecommended = false;
-            warning = undefined;
+            ({ outcome, reason, nextActionHint, browserRecommended, warning } = (0, pageOutcome_1.assessStructuredPageOutcome)(data));
         }
         else {
             fallbackReason = "browser fallback did not improve structured output";
         }
     }
-    const blockedReason = detectStructuredBlockReason(data);
-    if (blockedReason) {
-        throw new StructuredBlockedPageError(url, result.status, result.request, blockedReason, {
+    if (outcome === "authwall" || outcome === "challenge" || outcome === "blocked" || outcome === "cookie_wall") {
+        throw new StructuredBlockedPageError(url, result.status, result.request, outcome, reason ?? "Outcome matched page markers", nextActionHint, {
             fallbackAttempted,
             fallbackUsed,
             fallbackReason,
@@ -63,6 +67,9 @@ async function scrapeStructuredJson(url, config, apiKey, options = {}) {
     }
     return makeStructuredScrapeEnvelope(url, result, data, {
         renderSource,
+        outcome,
+        outcomeReason: reason,
+        nextActionHint,
         fallbackAttempted,
         fallbackUsed,
         fallbackReason,
@@ -74,6 +81,9 @@ function makeStructuredScrapeEnvelope(url, result, data, options = {}) {
     return {
         url,
         status: result.status,
+        outcome: options.outcome ?? "ok",
+        outcomeReason: options.outcomeReason,
+        nextActionHint: options.nextActionHint,
         renderSource: options.renderSource ?? "unlocker",
         fallbackAttempted: options.fallbackAttempted ?? false,
         fallbackUsed: options.fallbackUsed ?? false,
@@ -94,37 +104,18 @@ function normalizeStructuredFallbackMode(value) {
     throw new Error(`Unsupported scrape-json fallback mode: ${value}`);
 }
 function shouldUseBrowserFallback(data) {
-    if (detectStructuredBlockReason(data)) {
-        return true;
-    }
-    const firstH1 = data.headingsByLevel.h1[0];
-    if (!firstH1) {
-        return true;
-    }
-    return looksSuspiciousHeadingText(firstH1);
+    return (0, pageOutcome_1.assessStructuredPageOutcome)(data).outcome !== "ok";
 }
 function buildStructuredFallbackAdvisory(data) {
-    const blockedReason = detectStructuredBlockReason(data);
-    if (blockedReason) {
-        return {
-            browserRecommended: true,
-            warning: `Structured output looks blocked or challenge-gated (${blockedReason}). Retry with --fallback browser or use a rendered browser path.`,
-        };
-    }
-    if (!shouldUseBrowserFallback(data)) {
-        return { browserRecommended: false };
-    }
+    const assessment = (0, pageOutcome_1.assessStructuredPageOutcome)(data);
     return {
-        browserRecommended: true,
-        warning: "Structured output looks incomplete or client-rendered. Retry with --fallback browser or use read/open for rendered DOM.",
+        browserRecommended: assessment.browserRecommended,
+        warning: assessment.warning,
     };
 }
-function looksSuspiciousHeadingText(value) {
-    return /function\s*\(|window\.|document\.|const\s+|let\s+|var\s+|=>|import\s+/i.test(value) || value.length > 240;
-}
 function isBrowserDataBetter(current, candidate) {
-    const currentBlocked = Boolean(detectStructuredBlockReason(current));
-    const candidateBlocked = Boolean(detectStructuredBlockReason(candidate));
+    const currentBlocked = Boolean((0, pageOutcome_1.detectStructuredBlockedReason)(current));
+    const candidateBlocked = Boolean((0, pageOutcome_1.detectStructuredBlockedReason)(candidate));
     if (currentBlocked !== candidateBlocked) {
         return currentBlocked && !candidateBlocked;
     }
@@ -140,27 +131,25 @@ function isBrowserDataBetter(current, candidate) {
     return false;
 }
 function detectStructuredBlockReason(data) {
-    const candidates = [
-        data.title,
-        data.description,
-        ...data.headingsByLevel.h1.slice(0, 2),
-        ...data.headingsByLevel.h2.slice(0, 2),
-    ].filter((value) => Boolean(value && value.trim()));
-    for (const candidate of candidates) {
-        const reason = classifyBlockedText(candidate);
-        if (reason) {
-            return reason;
-        }
-    }
-    return undefined;
+    return ((0, pageOutcome_1.assessStructuredPageOutcome)(data).reason ??
+        (0, pageOutcome_1.detectStructuredBlockedReason)(data));
 }
-function classifyBlockedText(value) {
-    const text = value.trim();
-    if (/(verify you are human|verify you are a human|are you human|captcha|security check|attention required|just a moment|checking your browser|enable javascript and cookies to continue|one more step)/i.test(text)) {
-        return "challenge markers matched the page title or heading";
-    }
-    if (/(access denied|forbidden|blocked request|request blocked|request unsuccessful|temporarily blocked|temporarily unavailable|you have been blocked|access to this page has been denied)/i.test(text)) {
-        return "blocked-page markers matched the page title or heading";
+function outcomeToErrorCode(outcome) {
+    switch (outcome) {
+        case "authwall":
+            return "AUTHWALL_PAGE";
+        case "challenge":
+            return "CHALLENGE_PAGE";
+        case "cookie_wall":
+            return "COOKIE_WALL_PAGE";
+        case "blocked":
+            return "BLOCKED_PAGE";
+        case "empty":
+            return "EMPTY_PAGE";
+        case "incomplete":
+            return "INCOMPLETE_PAGE";
+        case "ok":
+        default:
+            return "PAGE_OUTCOME";
     }
-    return undefined;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin-web-access",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Unified web access CLI for developers and AI agents to read and interact with the web using Gologin Web Unlocker and Cloud Browser.",
   "main": "dist/cli.js",
   "bin": {