url_to_markdown 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f1e8199bcec143512716c5e781d84e5553b481d76c323375663f2ba80a05014
-  data.tar.gz: 887d2c94f48028e6ac1c9dc6a65a89dff12f519eed6a71ee2a6ba024c227fcc9
+  metadata.gz: 9717cbf5f44fb3b6c4c52d9e394aaeb6877dfb0908b39d1344d48c42d96a40d8
+  data.tar.gz: d66e680ab993c01eb41ac2acf8a6aa2f3b613fdaf0699867343cfa61b9888900
 SHA512:
-  metadata.gz: 250692441f4ec178814c0ae9a2a5546e85e21827f99c338092a583600e77bc1eb41aec7ec0bf385dfc7e2663d62ca043266a0aff692d3ce4da28eef2fb10e772
-  data.tar.gz: adc736650a0a7d1ae1f4ca9458f4e4651233b7b7325f20e0ca7c9e964127893ba495bb2448d5f9ce698acf2a9c74e91af09470a563175a7191f9d263e00541a5
+  metadata.gz: 9e55e675ab867a696d5efd23a0abaf3a8c614fa72634f934f298aa2f574fbe33c9195a90ad835aa7959eab37934fcaa71c7fbc90d3a996681dc1d2d566516ee4
+  data.tar.gz: b436fa839abbd87e20ec51400e5852fcd9acef09f2b6e4f692012af451133fc81c3b0b65cb0272e1f0a080afaa1131d4c986172f0ff2bc042a44abdf3646bd34

data/lib/url_to_markdown/cloudflare/client.rb

@@ -18,20 +18,21 @@ class UrlToMarkdown
       end
 
       def markdown(url: nil, html: nil, wait_for_selector: nil, wait_for_timeout_in_milliseconds: nil, cache_ttl: nil,
-                   actions: nil)
+                   scripts: nil, set_extra_http_headers: nil)
         validate_payload!(url: url, html: html)
 
         response = connection.post("accounts/#{@account_id}/browser-rendering/markdown") do |request|
           request.headers["Authorization"] = "Bearer #{@token}"
           request.headers["Content-Type"] = "application/json"
           request.options.timeout = @timeout
+          request.params["cacheTTL"] = cache_ttl if cache_ttl
           request.body = JSON.generate(build_payload(
                                          url: url,
                                          html: html,
                                          wait_for_selector: wait_for_selector,
                                          wait_for_timeout_in_milliseconds: wait_for_timeout_in_milliseconds,
-                                         cache_ttl: cache_ttl,
-                                         actions: actions
+                                         scripts: scripts,
+                                         set_extra_http_headers: set_extra_http_headers
                                        ))
         end
 
@@ -44,6 +45,16 @@ class UrlToMarkdown
         UrlToMarkdown::Result.failure(UrlToMarkdown::NetworkError.new(e))
       end
 
+      SECURITY_CHECKPOINT_PATTERNS = [
+        "vercel.link/security-checkpoint",  # Vercel Firewall
+        "Vercel Security Checkpoint",
+        "Just a moment",                    # Cloudflare challenge
+        "Checking your browser",            # Cloudflare challenge
+        "Enable JavaScript and cookies",    # Cloudflare challenge
+        "cf-browser-verification",          # Cloudflare legacy
+        "DDoS protection by"                # Generic DDoS protection
+      ].freeze
+
       private
 
       def validate_credentials!
@@ -58,20 +69,23 @@ class UrlToMarkdown
         raise UrlToMarkdown::ValidationError.new(nil, "Provide a URL or HTML")
       end
 
-      def build_payload(url:, html:, wait_for_selector:, wait_for_timeout_in_milliseconds:, cache_ttl:, actions: nil)
+      def build_payload(url:, html:, wait_for_selector:, wait_for_timeout_in_milliseconds:, scripts:, set_extra_http_headers:)
         payload = {}
         payload[:url] = url if url
         payload[:html] = html if html
-        payload[:wait_for_selector] = wait_for_selector if wait_for_selector
-        if wait_for_timeout_in_milliseconds
-          payload[:wait_for_timeout_in_milliseconds] =
-            wait_for_timeout_in_milliseconds
-        end
-        payload[:cache_ttl] = cache_ttl if cache_ttl
-        payload[:actions] = actions if actions&.any?
+        payload[:waitForSelector] = { selector: wait_for_selector } if wait_for_selector
+        payload[:waitForTimeout] = wait_for_timeout_in_milliseconds if wait_for_timeout_in_milliseconds
+        payload[:addScriptTag] = Array(scripts).map { { content: it } } if scripts&.any?
+        payload[:setExtraHTTPHeaders] = set_extra_http_headers if set_extra_http_headers
         payload
       end
 
+      def security_checkpoint?(content)
+        return false unless content.is_a?(String)
+
+        SECURITY_CHECKPOINT_PATTERNS.any? { |pattern| content.include?(pattern) }
+      end
+
       def connection
         @connection ||= Faraday.new(url: BASE_URL)
       end
@@ -84,7 +98,12 @@ class UrlToMarkdown
         when 200..299
           data = JSON.parse(body)
           if data.key?("result")
-            UrlToMarkdown::Result.success(data["result"])
+            content = data["result"]
+            if security_checkpoint?(content)
+              UrlToMarkdown::Result.failure(UrlToMarkdown::SecurityCheckpointError.new(nil, "Blocked by security checkpoint"))
+            else
+              UrlToMarkdown::Result.success(content)
+            end
           else
             UrlToMarkdown::Result.failure(UrlToMarkdown::MissingResultKeyInResponse.new(status, body))
           end

data/lib/url_to_markdown/cloudflare/processor.rb

@@ -15,9 +15,9 @@ class UrlToMarkdown
         )
       end
 
-      def convert(url, actions: nil)
+      def convert(url, scripts: nil)
         validate_url!(url)
-        @client.markdown(url: url, actions: actions)
+        @client.markdown(url: url, scripts: scripts)
       end
 
       private

data/lib/url_to_markdown/configuration.rb

@@ -5,7 +5,7 @@ require "logger"
 class UrlToMarkdown
   class Configuration
     attr_accessor :cloudflare_api_token, :cloudflare_account_id, :cloudflare_timeout_ms, :cloudflare_cache_ttl,
-                  :logger, :default_processor, :default_actions
+                  :logger, :default_processor, :default_scripts
 
     def initialize
       @cloudflare_api_token = ENV.fetch("CLOUDFLARE_API_TOKEN", nil)
@@ -14,7 +14,7 @@ class UrlToMarkdown
       @cloudflare_cache_ttl = 5
       @logger = Logger.new($stdout)
       @default_processor = UrlToMarkdown::Cloudflare::Processor
-      @default_actions = nil
+      @default_scripts = nil
     end
 
     def cloudflare_api_token!

data/lib/url_to_markdown/errors.rb

@@ -42,6 +42,8 @@ class UrlToMarkdown
   class ServerError < ApiError; end
   class MissingResultKeyInResponse < ApiError; end
 
+  class SecurityCheckpointError < Error; end
+
   class ValidationError < Error; end
   class InvalidUrlError < ValidationError; end
 

data/lib/url_to_markdown/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class UrlToMarkdown
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

data/lib/url_to_markdown.rb

@@ -8,12 +8,12 @@ loader.inflector.inflect("errors" => "Error", "pstore" => "PStore")
 loader.setup
 
 class UrlToMarkdown
-  def initialize(url:, processor: nil, logger: nil, cache_store: nil, actions: nil)
+  def initialize(url:, processor: nil, logger: nil, cache_store: nil, scripts: nil)
     @url = url
     @processor_class = processor || self.class.configuration.default_processor
     @logger = logger || self.class.configuration.logger
     @cache_store = cache_store
-    @actions = actions || self.class.configuration.default_actions
+    @scripts = scripts || self.class.configuration.default_scripts
   end
 
   def convert
@@ -28,7 +28,7 @@ class UrlToMarkdown
     end
 
     processor = @processor_class.new(logger: @logger, cache_store: @cache_store)
-    result = processor.convert(@url, actions: @actions)
+    result = processor.convert(@url, scripts: @scripts)
 
     @cache_store.store!(@url, result.payload) if @cache_store && result.respond_to?(:success?) && result.success?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: url_to_markdown
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Lucian Ghinda