ask-web-fetch 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4c0b6735b967e38e8b837e7128bb501875ea1ce239f2df1549518396740ba8e
-  data.tar.gz: 0c6b95aba29f4914d062b35b3e21a43d1616a95ab01111a8d81d73326bd7f985
+  metadata.gz: '08275e5b901ca9aaba313cac4cd3f776efb2c16b93aa709645c0ad367a1ff720'
+  data.tar.gz: 0eaba632b36081a0bc3fbf7dd1ff2b463cadb613e44d8a280d0546aeeda40676
 SHA512:
-  metadata.gz: 6f7f82bc5b93d368395989049757d86675e89d7645f7b351f3d24edff50a8303ee86cdc04dec9240a9e71ef2f6f7dedd0d6641da93ddc3b9d5494b116fa5ccd0
-  data.tar.gz: c160aac9363740829eaceeb53b18e91b009dcdc278b194d7774ba692b3b6d14d3404cb5b4f38e0d50fb73a62390fad7b2e0bbc1ca60a47d8dce8de8c1e745072
+  metadata.gz: 3c82f4b9abfb154fb06b9a5feaaf2254ddb6caa1cac99892a034311b4113ab1775114ad7c5aa126db88682c3ecf4cdfbee13786b61a06a00f2f30d6b838a2cdf
+  data.tar.gz: f3d14810e581699b5d2f8717fa329052ad8b81f03611d637d99fde979f1b324d2939ee79217ee812018327596f07ae0659c987b2e42f197df60af4bff30c8a7b

data/lib/ask/web_fetch/backend.rb

@@ -59,6 +59,18 @@ module Ask
       # Wikipedia embeds an hcaptcha edit-config flag on every page).
       CHALLENGE_RE = /just a moment|checking your browser|cf-chl/i
 
+      # Registrar parking-page markers: the page is an ad for a parked
+      # (for-sale) domain, not the site's content. A content company must
+      # never store these as if they were the site. Observed live on the
+      # CC list, three shapes: (a) GoDaddy's parking-lander JS app
+      # (ap:"parking" flag, parking-lander asset, LANDER_SYSTEM="PW")
+      # served at /lander, (b) Namecheap's parking app (utm_campaign=
+      # nc_market + parkingpage links), and (c) static registrar pages
+      # ("is parked free, courtesy of GoDaddy.com", "is registered at
+      # Namecheap"). Deliberately specific — generic terms like "domain"
+      # or "for sale" appear on real pages.
+      PARKED_DOMAIN_MARKERS = /ap:"parking"|parking-lander|LANDER_SYSTEM="PW"|utm_campaign=nc_market|utm_source=parkingpage|is parked free, courtesy of GoDaddy|is available on GoDaddy Auctions|is registered at Namecheap/i
+
       def self.backend_name
         name.split('::').last
       end
@@ -118,6 +130,10 @@ module Ask
         body.to_s.match?(CHALLENGE_RE)
       end
 
+      def parked_domain?(body)
+        body.to_s.match?(PARKED_DOMAIN_MARKERS)
+      end
+
       def usable_content?(content)
         content.to_s.strip.length >= MIN_CONTENT_LENGTH
       end

data/lib/ask/web_fetch/backends/browser.rb

@@ -199,6 +199,10 @@ module Ask
 
             return fetch_attempt(page, url, warmed: true)
           end
+          # Browser renders the parked page a JS redirect lands on (the
+          # server shell hands /lander to JS) — Local never sees it. The
+          # shared parked-domain detector catches it here.
+          raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)

data/lib/ask/web_fetch/backends/local.rb

@@ -45,6 +45,18 @@ module Ask
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
+          # Parked-domain pages are not content: the domain owner parked it
+          # with a registrar and the page is an ad for buying the domain
+          # (GoDaddy/Namecheap/Sedo parking). A content company must never
+          # store these as if they were the site. Checked BEFORE the
+          # content-minimum — a parking page can render as "content" above
+          # the minimum (puncta.ai: 395c of Namecheap auction ads), and
+          # must still be rejected. Detectable from the server HTML —
+          # parked pages are fully server-rendered, so both Local and
+          # Browser see the same ad.
+          if parked_domain?(body)
+            raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content"
+          end
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
           # The completeness signal: a JS-app shell whose server HTML
           # renders little is a TRUNCATED page, not a complete one — the

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.6'
+    VERSION = '0.5.7'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - Kaka Ruto