ask-web-fetch 0.5.5 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db21a4a35495ff89b8b91b769d4a15743da85beb961a5ddbbc430bdde37988f5
-  data.tar.gz: 61fb64631bbfe1d478b3b6ed6587de8e5b9b3890dd5348924ca574929f85acb7
+  metadata.gz: '08275e5b901ca9aaba313cac4cd3f776efb2c16b93aa709645c0ad367a1ff720'
+  data.tar.gz: 0eaba632b36081a0bc3fbf7dd1ff2b463cadb613e44d8a280d0546aeeda40676
 SHA512:
-  metadata.gz: f53d538cf613c02e1411c9f09901c3b93db98c51c33d0d3dfca2d7ea58420f9e463f074c1f2e9a6c97da59023037abcfce59d757ef3ab689eada87fc6155939d
-  data.tar.gz: fde5f3000a2ec77294d1bceec3484d9d4ec68b3b3d0b91d8e29083b3bb711f31b391092a1cd544b329858ffb510e116d39c39ed4a23323c110af133db6cef928
+  metadata.gz: 3c82f4b9abfb154fb06b9a5feaaf2254ddb6caa1cac99892a034311b4113ab1775114ad7c5aa126db88682c3ecf4cdfbee13786b61a06a00f2f30d6b838a2cdf
+  data.tar.gz: f3d14810e581699b5d2f8717fa329052ad8b81f03611d637d99fde979f1b324d2939ee79217ee812018327596f07ae0659c987b2e42f197df60af4bff30c8a7b

data/lib/ask/web_fetch/backend.rb

@@ -59,6 +59,18 @@ module Ask
       # Wikipedia embeds an hcaptcha edit-config flag on every page).
       CHALLENGE_RE = /just a moment|checking your browser|cf-chl/i
 
+      # Registrar parking-page markers: the page is an ad for a parked
+      # (for-sale) domain, not the site's content. A content company must
+      # never store these as if they were the site. Observed live on the
+      # CC list, three shapes: (a) GoDaddy's parking-lander JS app
+      # (ap:"parking" flag, parking-lander asset, LANDER_SYSTEM="PW")
+      # served at /lander, (b) Namecheap's parking app (utm_campaign=
+      # nc_market + parkingpage links), and (c) static registrar pages
+      # ("is parked free, courtesy of GoDaddy.com", "is registered at
+      # Namecheap"). Deliberately specific — generic terms like "domain"
+      # or "for sale" appear on real pages.
+      PARKED_DOMAIN_MARKERS = /ap:"parking"|parking-lander|LANDER_SYSTEM="PW"|utm_campaign=nc_market|utm_source=parkingpage|is parked free, courtesy of GoDaddy|is available on GoDaddy Auctions|is registered at Namecheap/i
+
       def self.backend_name
         name.split('::').last
       end
@@ -118,6 +130,10 @@ module Ask
         body.to_s.match?(CHALLENGE_RE)
       end
 
+      def parked_domain?(body)
+        body.to_s.match?(PARKED_DOMAIN_MARKERS)
+      end
+
       def usable_content?(content)
         content.to_s.strip.length >= MIN_CONTENT_LENGTH
       end

data/lib/ask/web_fetch/backends/browser.rb

@@ -199,6 +199,10 @@ module Ask
 
             return fetch_attempt(page, url, warmed: true)
           end
+          # Browser renders the parked page a JS redirect lands on (the
+          # server shell hands /lander to JS) — Local never sees it. The
+          # shared parked-domain detector catches it here.
+          raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)

data/lib/ask/web_fetch/backends/local.rb

@@ -45,7 +45,32 @@ module Ask
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
+          # Parked-domain pages are not content: the domain owner parked it
+          # with a registrar and the page is an ad for buying the domain
+          # (GoDaddy/Namecheap/Sedo parking). A content company must never
+          # store these as if they were the site. Checked BEFORE the
+          # content-minimum — a parking page can render as "content" above
+          # the minimum (puncta.ai: 395c of Namecheap auction ads), and
+          # must still be rejected. Detectable from the server HTML —
+          # parked pages are fully server-rendered, so both Local and
+          # Browser see the same ad.
+          if parked_domain?(body)
+            raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content"
+          end
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # The completeness signal: a JS-app shell whose server HTML
+          # renders little is a TRUNCATED page, not a complete one — the
+          # real content awaits client-side JS that Local cannot run.
+          # Storing the shell as success would silently under-deliver
+          # (airbnb: 613KB server HTML -> 143 chars of markdown); failing
+          # through lets the chain prefer a rendering backend (Browser),
+          # and keeps a partial page from ever being stored as the real
+          # thing. Two detectors: known framework markers, or a large
+          # HTML page with almost no server-rendered text.
+          if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
+            raise EmptyContentError,
+              "JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
+          end
 
           page
         rescue Errno::ECONNREFUSED, Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
@@ -55,6 +80,40 @@ module Ask
           raise TimeoutError, "#{e.class}: #{e.message}"
         end
 
+        # Client-rendered framework markers: the server HTML is (at least
+        # partly) a shell awaiting JS. Deliberately narrow — generic terms
+        # like "script" or "root" appear on every page; these are the
+        # specific footprints of React/Vue/Next/Nuxt app shells.
+        JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
+
+        # Markdown below this from a JS-app shell is "server sent a shell",
+        # not "page is genuinely short" — a real page (even a short one)
+        # is usually server-rendered above this. Tunable; the chain turns
+        # the signal into "prefer Browser for this URL".
+        SHELL_CONTENT_THRESHOLD = 4_000
+
+        # A page whose server HTML is large but yields almost no text is a
+        # shell whatever framework it uses (airbnb: 613KB HTML -> 143 chars
+        # of markdown). Framework markers miss these; the size ratio
+        # catches them. nytimes (1.4MB -> 7.5k text) stays above the text
+        # floor and is correctly left to Local.
+        SHELL_HTML_BYTES = 20_000
+
+        def js_app_shell?(body)
+          body.to_s.match?(JS_APP_SHELL_MARKERS) ||
+            (body.to_s.bytesize > SHELL_HTML_BYTES && markdown_visible_chars(body) < SHELL_CONTENT_THRESHOLD)
+        end
+
+        # A cheap text estimate from the raw HTML (tags stripped) — the
+        # "how much did the server actually render" number. Deliberately
+        # rough: it only feeds a shell-vs-page heuristic.
+        def markdown_visible_chars(body)
+          body.to_s.gsub(/<script[\s\S]*?<\/script>/i, "")
+            .gsub(/<style[\s\S]*?<\/style>/i, "")
+            .gsub(/<[^>]+>/, " ")
+            .gsub(/\s+/, " ").strip.length
+        end
+
         # Parses +html+ and returns { title:, description:, content:,
         # licenses:, outlinks: } where content is clean markdown (pruned by
         # the ContentFilter), licenses are the page's declared license

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.5'
+    VERSION = '0.5.7'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.7
 platform: ruby
 authors:
 - Kaka Ruto