ask-web-fetch 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15d4465ae7ca7805003043af5b97e96ddc8c2ba19618daab313377f4a529bd54
-  data.tar.gz: 9f83f8c2258dc973457158e7d7e2288afc43403221453545a4d19c97440b85ee
+  metadata.gz: 5e7b287b5b62f821cb0b3085082b385583d3f1a7ef2dfc0816eb311931d1e218
+  data.tar.gz: 534bce458033fb046914c169b7bb7bad982f8fcd3ffb999d6feb2b495953d90c
 SHA512:
-  metadata.gz: b34dfd5f3d45b827cf8882d3c9c3b47e05ea8c16622849b2a41fc51b8c5f005f0b149ef83a87db57f4167470e85911ec58f72830afded9a84ab707ea08dcbc68
-  data.tar.gz: c8245c574d480179cd3b8ad67f1049b096fdf298f0886a790cb557f698c11e8dbaed9bb4336de4ccf84882e03eb8cf716830db069988f310c35cf4202cf01d60
+  metadata.gz: 48e47dd345f498f1a50b6dc3ddb09a5f55078691ee34a617b02936fb9b7d639c2ff55c309f6fc2ecec2a4d067c487edd0975c983b4ed301471d50536e73e6bd0
+  data.tar.gz: 7bf1c58c8fb17a9c5575b41f8b2d44c6deb1c01a8b82b56b3d73f5d1df4c5d72fadf09ca60de1fc3995b8dc62581611f4aa50bcb4c47ceb9894f5bb6749490fc

data/lib/ask/web_fetch/backend.rb

@@ -44,7 +44,9 @@ module Ask
     #      Markdown.generate, backends fed pre-converted markdown (Jina,
     #      Crawl4AI) must call it explicitly so the shared noise removal
     #      and whitespace normalization apply everywhere
-    #   5. register the class in Ask::Tools::WebFetch.backends
+    #   5. run the extracted page through #guard_page! — the parked-domain
+    #      and empty-content verdicts are identical in every backend
+    #   6. register the class in Ask::Tools::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend
@@ -92,6 +94,29 @@ module Ask
         raise NotImplementedError, "#{self.class} must implement #fetch(url)"
       end
 
+      # The shared page guard, run by EVERY backend at the same point in
+      # its flow — after extraction, before returning: a registrar parking
+      # page raises ParkedDomainError, content below the minimum raises
+      # EmptyContentError. Same verdicts, same messages, everywhere; a
+      # backend's only job is to pass the strings it has.
+      #
+      # raw_body: the raw HTML the backend saw, where it saw it (Local,
+      # Browser) — the HTML-only markers (ap:"parking", parking-lander,
+      # LANDER_SYSTEM="PW") live in scripts and assets that never survive
+      # conversion to markdown. content: what the backend would return
+      # (all four) — the prose markers survive conversion, so a backend
+      # that only ever sees rendered text (Jina, Crawl4AI) still rejects
+      # the ad.
+      #
+      # Parked is checked BEFORE the content minimum on purpose: a parking
+      # page can render above it (puncta.ai: 395c of Namecheap auction
+      # ads) and must still be rejected.
+      def guard_page!(url, content, raw_body: nil)
+        raise ParkedDomainError,
+              "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(raw_body) || parked_domain?(content)
+        raise EmptyContentError, "no readable content at #{url}" unless usable_content?(content)
+      end
+
       # --- outlinks (crawler discovery) ---
 
       # The page's raw outlinks from HTML: every <a href> resolved against

data/lib/ask/web_fetch/backends/browser.rb

@@ -201,13 +201,12 @@ module Ask
           end
           # Browser renders the parked page a JS redirect lands on (the
           # server shell hands /lander to JS) — Local never sees it. The
-          # shared parked-domain detector catches it here; the distinct
-          # ParkedDomainError lets the pipeline classify (never retry) it.
-          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
-
+          # shared guard catches it on the rendered HTML (raw_body) and
+          # the converted content; the distinct ParkedDomainError lets the
+          # pipeline classify (never retry) it.
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
+          guard_page!(url, result[:content], raw_body: body)
 
           result
         end

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -56,12 +56,12 @@ module Ask
 
           page = to_page(body, url)
           # A registrar parking page renders fine in headless Chrome too —
-          # the shared detector (0.5.7) catches the text markers that
-          # survive conversion, so the ad is rejected, not returned as
-          # the site's content. Crawl4AI leads the default chain, so this
-          # check is what keeps parked domains out of every result.
-          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(page[:content])
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # the shared guard's prose markers catch it (the HTML-only
+          # markers never reach a markdown-only backend), so the ad is
+          # rejected, not returned as the site's content. Crawl4AI leads
+          # the default chain, so this guard is what keeps parked domains
+          # out of every result.
+          guard_page!(url, page[:content])
 
           page
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,

data/lib/ask/web_fetch/backends/jina.rb

@@ -35,18 +35,17 @@ module Ask
             body = res.body.to_s
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
 
+            content = Markdown.clean(body)
             # Jina only sees rendered markdown — outlinks come from its
             # links, resolved against the requested URL. Content runs
             # through the same Markdown.clean as the converting backends,
             # so decorative symbol noise is stripped here too; a page
             # whose only "content" was noise falls through as empty. A
             # registrar parking page renders fine through Jina — the
-            # shared detector (0.5.7) catches the text markers that
-            # survive conversion, so the ad is rejected, not returned as
-            # the site's content.
-            content = Markdown.clean(body)
-            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(content)
-            raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
+            # shared guard's prose markers catch it (the HTML-only
+            # markers never reach a markdown-only backend), so the ad is
+            # rejected, not returned as the site's content.
+            guard_page!(url, content)
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'

data/lib/ask/web_fetch/backends/local.rb

@@ -48,16 +48,12 @@ module Ask
           # Parked-domain pages are not content: the domain owner parked it
           # with a registrar and the page is an ad for buying the domain
           # (GoDaddy/Namecheap/Sedo parking). A content company must never
-          # store these as if they were the site. Checked BEFORE the
-          # content-minimum — a parking page can render as "content" above
-          # the minimum (puncta.ai: 395c of Namecheap auction ads), and
-          # must still be rejected. Detectable from the server HTML —
-          # parked pages are fully server-rendered, so both Local and
-          # Browser see the same ad.
-          if parked_domain?(body)
-            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content"
-          end
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # store these as if they were the site. The shared guard checks
+          # the raw server HTML first (parked pages are fully
+          # server-rendered — the HTML-only markers live in scripts and
+          # assets), then the content minimum, then the JS-shell
+          # completeness signal below.
+          guard_page!(url, page[:content], raw_body: body)
           # The completeness signal: a JS-app shell whose server HTML
           # renders little is a TRUNCATED page, not a complete one — the
           # real content awaits client-side JS that Local cannot run.

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.6.1'
+    VERSION = '0.6.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Kaka Ruto