ask-web-fetch 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac8355feef3bd62bc048223f0390f79d95506f084817853a1a572be5eff4a78a
-  data.tar.gz: ccee31246284b6236732984c5f1d32fcf54d6ffa9203fa1f51150300b4b50702
+  metadata.gz: 15d4465ae7ca7805003043af5b97e96ddc8c2ba19618daab313377f4a529bd54
+  data.tar.gz: 9f83f8c2258dc973457158e7d7e2288afc43403221453545a4d19c97440b85ee
 SHA512:
-  metadata.gz: 181316376680bbb70d193b46dd898f48547b1846726e2d5f3d0596080ed4b41ca37ae818e3bc2963e0e2e700b47ff8a2957420222c0a737e45acedd00318e68c
-  data.tar.gz: 0b039ee976010c65274c956a3f5e3182164128b6a096eae1161ea42f266004fdd878dde12378899d68a29f26b56c2f454344e32abbd65d0444367f8948e330de
+  metadata.gz: b34dfd5f3d45b827cf8882d3c9c3b47e05ea8c16622849b2a41fc51b8c5f005f0b149ef83a87db57f4167470e85911ec58f72830afded9a84ab707ea08dcbc68
+  data.tar.gz: c8245c574d480179cd3b8ad67f1049b096fdf298f0886a790cb557f698c11e8dbaed9bb4336de4ccf84882e03eb8cf716830db069988f310c35cf4202cf01d60

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -55,6 +55,12 @@ module Ask
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
 
           page = to_page(body, url)
+          # A registrar parking page renders fine in headless Chrome too —
+          # the shared detector (0.5.7) catches the text markers that
+          # survive conversion, so the ad is rejected, not returned as
+          # the site's content. Crawl4AI leads the default chain, so this
+          # check is what keeps parked domains out of every result.
+          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(page[:content])
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
 
           page

data/lib/ask/web_fetch/backends/jina.rb

@@ -39,8 +39,13 @@ module Ask
             # links, resolved against the requested URL. Content runs
             # through the same Markdown.clean as the converting backends,
             # so decorative symbol noise is stripped here too; a page
-            # whose only "content" was noise falls through as empty.
+            # whose only "content" was noise falls through as empty. A
+            # registrar parking page renders fine through Jina — the
+            # shared detector (0.5.7) catches the text markers that
+            # survive conversion, so the ad is rejected, not returned as
+            # the site's content.
             content = Markdown.clean(body)
+            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(content)
             raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.6.0'
+    VERSION = '0.6.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Kaka Ruto