curb 1.3.5 → 1.3.6

data/lib/curl.rb

@@ -1,11 +1,412 @@
 # frozen_string_literal: true
 require 'curb_core'
+require 'curl/download'
 require 'curl/easy'
 require 'curl/multi'
+require 'ipaddr'
 require 'uri'
 
 # expose shortcut methods
 module Curl
+  class SafetyConfig
+    DEFAULT_PROTOCOLS = [:http, :https].freeze
+
+    attr_reader :max_body_bytes, :network_policy
+    attr_accessor :allow_proxies, :allow_resolve, :allow_connect_to, :allow_doh, :allow_unix_socket
+
+    def initialize
+      @protocols = DEFAULT_PROTOCOLS
+      @redirect_protocols = nil
+      @max_body_bytes = nil
+      @network_policy = nil
+      @allowed_hosts = nil
+      @allowed_proxy_hosts = nil
+      @allowed_cidrs = nil
+      @allow_proxies = false
+      @allow_resolve = false
+      @allow_connect_to = false
+      @allow_doh = false
+      @allow_unix_socket = false
+    end
+
+    def protocols
+      @protocols.dup
+    end
+
+    def protocols=(protocols)
+      @protocols = normalize_protocols(protocols)
+    end
+
+    def redirect_protocols
+      @redirect_protocols&.dup
+    end
+
+    def redirect_protocols=(protocols)
+      @redirect_protocols = protocols.nil? ? nil : normalize_protocols(protocols)
+    end
+
+    def allowed_hosts
+      @allowed_hosts&.map(&:dup)
+    end
+
+    def allowed_proxy_hosts
+      @allowed_proxy_hosts&.map(&:dup)
+    end
+
+    def allowed_cidrs
+      @allowed_cidrs&.map(&:dup)
+    end
+
+    def max_body_bytes=(bytes)
+      if bytes.nil?
+        @max_body_bytes = nil
+        return
+      end
+
+      bytes = Integer(bytes)
+      raise ArgumentError, "max_body_bytes must be greater than or equal to zero" if bytes < 0
+
+      @max_body_bytes = bytes.zero? ? nil : bytes
+    end
+
+    def network_policy=(policy)
+      @network_policy = normalize_network_policy(policy)
+    end
+
+    def allowed_hosts=(hosts)
+      @allowed_hosts = normalize_allowed_hosts(hosts)
+    end
+
+    def allowed_proxy_hosts=(hosts)
+      @allowed_proxy_hosts = normalize_allowed_hosts(hosts)
+    end
+
+    def allowed_cidrs=(cidrs)
+      @allowed_cidrs = normalize_allowed_cidrs(cidrs)
+    end
+
+    private
+
+    def normalize_protocols(protocols)
+      protocol_names = Array(protocols).map { |protocol| protocol.to_s.downcase.to_sym }
+      raise ArgumentError, "at least one protocol is required" if protocol_names.empty?
+
+      protocol_names
+    end
+
+    def normalize_network_policy(policy)
+      return nil if policy.nil?
+
+      policy_name = policy.to_s.downcase.to_sym
+      return policy_name if [:none, :public].include?(policy_name)
+
+      raise ArgumentError, "network_policy must be one of :none, :public"
+    end
+
+    def normalize_allowed_hosts(hosts)
+      list = normalize_optional_list(hosts)
+      return nil unless list
+
+      list.map { |host| normalize_allowed_host(host) }.uniq
+    end
+
+    def normalize_allowed_cidrs(cidrs)
+      list = normalize_optional_list(cidrs)
+      return nil unless list
+
+      list.map do |cidr|
+        cidr = cidr.to_s.strip
+        raise ArgumentError, "allowed_cidrs cannot include blank entries" if cidr.empty?
+
+        IPAddr.new(cidr)
+        cidr
+      end.uniq
+    rescue IPAddr::InvalidAddressError => e
+      raise ArgumentError, "invalid CIDR range: #{e.message}"
+    end
+
+    def normalize_optional_list(values)
+      return nil if values.nil?
+
+      list = Array(values)
+      return nil if list.empty?
+
+      list
+    end
+
+    def normalize_allowed_host(host)
+      host = host.to_s.strip.downcase
+      raise ArgumentError, "allowed_hosts cannot include blank entries" if host.empty?
+
+      parsed_host = begin
+        URI.parse(host).host if host.include?("://")
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      host = parsed_host.downcase if parsed_host
+      host = normalize_host_authority(host) unless parsed_host
+      host = host[1...-1] if host.start_with?("[") && host.end_with?("]")
+      host = host.chomp(".")
+
+      raise ArgumentError, "allowed_hosts cannot include blank entries" if host.empty?
+
+      host
+    end
+
+    def normalize_host_authority(host)
+      authority = host.split(/[\/?#]/, 2).first
+      authority = authority[(authority.rindex("@") + 1)..-1] if authority.include?("@")
+
+      if authority.start_with?("[")
+        closing = authority.index("]")
+        raise ArgumentError, "invalid allowed host: #{host}" unless closing
+
+        return authority[1...closing]
+      end
+
+      if authority.count(":") <= 1
+        authority = authority.split(":", 2).first
+      end
+
+      authority
+    end
+  end
+
+  def self.safe!
+    config = SafetyConfig.new
+    yield config if block_given?
+    @safety_config = config
+    bump_safety_generation!
+  end
+
+  def self.apply_safety!(easy)
+    config = @safety_config
+    override = safety_override_for(easy)
+    return easy unless config || override
+
+    protocols = config&.protocols
+    redirect_protocols = config && (config.redirect_protocols || protocols)
+
+    if override
+      override_protocols = override[:protocols]
+      protocols = safety_protocol_intersection(protocols, override_protocols) if override_protocols
+
+      override_redirect_protocols = override[:redirect_protocols] || override_protocols
+      redirect_protocols = safety_protocol_intersection(redirect_protocols, override_redirect_protocols) if override_redirect_protocols
+    end
+
+    if protocols
+      easy.allowed_protocols = protocols
+      easy.allowed_redirect_protocols = redirect_protocols || protocols
+    end
+
+    apply_allowed_hosts!(easy, config.allowed_hosts) if config&.allowed_hosts
+    apply_allowed_cidrs!(easy, config) if config&.allowed_cidrs
+
+    if config&.network_policy
+      easy.network_policy = config.network_policy
+      apply_public_network_policy_controls!(easy, config) if config.network_policy == :public
+    end
+
+    apply_max_body_bytes!(easy, config.max_body_bytes) if config&.max_body_bytes
+    apply_max_body_bytes!(easy, override[:max_body_bytes]) if override && override.key?(:max_body_bytes)
+    easy
+  end
+
+  def self.clear_safe!
+    @safety_config = nil
+    bump_safety_generation!
+  end
+
+  def self.safety_active_for?(easy)
+    !!(@safety_config || safety_override_for(easy))
+  end
+
+  def self.safety_signature_for(easy)
+    return nil unless safety_active_for?(easy)
+
+    override_generation = if easy.respond_to?(:__curb_safety_override_generation, true)
+      easy.__send__(:__curb_safety_override_generation)
+    end
+
+    [safety_generation, override_generation.to_i]
+  end
+
+  def self.safety_override_for(easy)
+    if easy.respond_to?(:__curb_safety_override, true)
+      easy.__send__(:__curb_safety_override)
+    end
+  end
+
+  def self.safety_generation
+    @safety_generation ||= 0
+  end
+
+  def self.bump_safety_generation!
+    @safety_generation = safety_generation + 1
+  end
+
+  def self.safety_protocol_intersection(base_protocols, override_protocols)
+    return override_protocols unless base_protocols
+
+    protocols = override_protocols & base_protocols
+    raise ArgumentError, "safety policies allow no protocols" if protocols.empty?
+
+    protocols
+  end
+
+  def self.apply_max_body_bytes!(easy, max_body_bytes)
+    return unless max_body_bytes
+
+    current_max_body_bytes = easy.max_body_bytes
+    easy.max_body_bytes = max_body_bytes if current_max_body_bytes.nil? || current_max_body_bytes > max_body_bytes
+  end
+
+  def self.apply_public_network_policy_controls!(easy, config)
+    reject_resolve_override!(easy) unless config.allow_resolve
+    reject_connect_to_override!(easy) unless config.allow_connect_to
+    reject_doh_override!(easy) unless config.allow_doh
+    reject_dns_servers_override!(easy)
+    allow_proxy = config.allow_proxies || !!config.allowed_proxy_hosts
+    easy.__send__(:__curb_allow_proxy=, allow_proxy) if easy.respond_to?(:__curb_allow_proxy=, true)
+    easy.__send__(:__curb_allow_unix_socket=, config.allow_unix_socket) if easy.respond_to?(:__curb_allow_unix_socket=, true)
+    reject_unix_socket_override!(easy) unless config.allow_unix_socket
+    if config.allowed_proxy_hosts
+      apply_allowed_proxy!(easy, config.allowed_proxy_hosts)
+    elsif !config.allow_proxies
+      disable_proxy!(easy)
+    end
+  end
+
+  def self.apply_allowed_hosts!(easy, allowed_hosts)
+    if easy.respond_to?(:follow_location?) && easy.follow_location? &&
+       !Curl.const_defined?(:CURLOPT_PREREQFUNCTION)
+      raise NotImplementedError, "redirect-aware host allowlists require CURLOPT_PREREQFUNCTION support"
+    end
+
+    host = begin
+      URI.parse(easy.url.to_s).host
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    normalized_host = host.to_s.downcase.chomp(".")
+    normalized_host = normalized_host[1...-1] if normalized_host.start_with?("[") && normalized_host.end_with?("]")
+
+    unless allowed_hosts.include?(normalized_host)
+      raise Curl::Err::UnsafeDestinationError,
+            "URL host #{host.inspect} is not allowed by safe mode host allowlist"
+    end
+
+    easy.allowed_hosts = allowed_hosts if easy.respond_to?(:allowed_hosts=)
+  end
+
+  def self.apply_allowed_cidrs!(easy, config)
+    unless config.network_policy == :public
+      raise ArgumentError, "allowed_cidrs require network_policy = :public"
+    end
+
+    easy.allowed_cidrs = config.allowed_cidrs if easy.respond_to?(:allowed_cidrs=)
+  end
+
+  def self.apply_allowed_proxy!(easy, allowed_proxy_hosts)
+    proxy_url = easy.proxy_url if easy.respond_to?(:proxy_url)
+    if proxy_url.nil? || proxy_url.to_s.empty?
+      disable_proxy!(easy)
+      return
+    end
+
+    proxy_host = normalize_proxy_host(proxy_url)
+    unless allowed_proxy_hosts.include?(proxy_host)
+      raise Curl::Err::UnsafeDestinationError,
+            "proxy host #{proxy_host.inspect} is not allowed by safe mode proxy allowlist"
+    end
+
+    easy.set(Curl::CURLOPT_NOPROXY, "") if Curl.const_defined?(:CURLOPT_NOPROXY)
+  end
+
+  def self.normalize_proxy_host(proxy_url)
+    value = proxy_url.to_s.strip
+    raise Curl::Err::UnsafeDestinationError, "proxy URL is empty" if value.empty?
+
+    uri = begin
+      URI.parse(value.include?("://") ? value : "http://#{value}")
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    host = uri&.host
+    raise Curl::Err::UnsafeDestinationError, "proxy URL host is invalid" if host.nil? || host.empty?
+
+    host = host.downcase.chomp(".")
+    host = host[1...-1] if host.start_with?("[") && host.end_with?("]")
+    host
+  end
+
+  def self.reject_resolve_override!(easy)
+    return unless easy.respond_to?(:resolve)
+
+    resolve = easy.resolve
+    return if resolve.nil? || (resolve.respond_to?(:empty?) && resolve.empty?)
+
+    raise Curl::Err::UnsafeDestinationError, "resolve overrides are disabled by public network policy"
+  end
+
+  def self.reject_connect_to_override!(easy)
+    return unless easy.respond_to?(:connect_to)
+
+    connect_to = easy.connect_to
+    return if connect_to.nil? || (connect_to.respond_to?(:empty?) && connect_to.empty?)
+
+    raise Curl::Err::UnsafeDestinationError, "connect_to overrides are disabled by public network policy"
+  end
+
+  def self.reject_doh_override!(easy)
+    return unless easy.respond_to?(:doh_url)
+
+    doh_url = easy.doh_url
+    return if doh_url.nil? || (doh_url.respond_to?(:empty?) && doh_url.empty?)
+
+    raise Curl::Err::UnsafeDestinationError, "DoH URL overrides are disabled by public network policy"
+  end
+
+  def self.reject_dns_servers_override!(easy)
+    return unless easy.respond_to?(:dns_servers)
+
+    dns_servers = easy.dns_servers
+    return if dns_servers.nil? || (dns_servers.respond_to?(:empty?) && dns_servers.empty?)
+
+    raise Curl::Err::UnsafeDestinationError, "DNS server overrides are disabled by public network policy"
+  end
+
+  def self.reject_unix_socket_override!(easy)
+    return unless easy.respond_to?(:unix_socket_path)
+
+    unix_socket_path = easy.unix_socket_path
+    return if unix_socket_path.nil? || (unix_socket_path.respond_to?(:empty?) && unix_socket_path.empty?)
+
+    raise Curl::Err::UnsafeDestinationError, "Unix socket paths are disabled by public network policy"
+  end
+
+  def self.disable_proxy!(easy)
+    easy.proxy_url = "" if easy.respond_to?(:proxy_url=)
+    easy.proxy_tunnel = false if easy.respond_to?(:proxy_tunnel=)
+    easy.set(Curl::CURLOPT_NOPROXY, "*") if Curl.const_defined?(:CURLOPT_NOPROXY)
+  end
+
+  private_class_method :apply_safety!, :clear_safe!, :safety_active_for?,
+                       :safety_signature_for, :safety_override_for,
+                       :safety_generation, :bump_safety_generation!,
+                       :safety_protocol_intersection,
+                       :apply_max_body_bytes!, :apply_public_network_policy_controls!,
+                       :apply_allowed_hosts!, :apply_allowed_cidrs!,
+                       :apply_allowed_proxy!, :normalize_proxy_host,
+                       :reject_resolve_override!, :reject_connect_to_override!,
+                       :reject_doh_override!, :reject_dns_servers_override!,
+                       :reject_unix_socket_override!,
+                       :disable_proxy!
+
   def self.scheduler_active?
     Fiber.respond_to?(:scheduler) && !Fiber.scheduler.nil?
   end
@@ -201,52 +602,84 @@ module Curl
   end
 
   def self.http(verb, url, post_body=nil, put_data=nil, &block)
-    if Thread.current[:curb_curl_yielding]
-      handle = Curl::Easy.new # we can't reuse this
-    else
-      handle = Thread.current[:curb_curl] ||= Curl::Easy.new
-      handle.reset
-    end
+    handle = Curl::Easy.new
     handle.url = url
     handle.post_body = post_body if post_body
     handle.put_data = put_data if put_data
-    if block_given?
-      Thread.current[:curb_curl_yielding] = true
-      yield handle
-      Thread.current[:curb_curl_yielding] = false
-    end
+    yield handle if block_given?
     handle.http(verb)
     handle
   end
 
+  def self.safe_http(verb, url, post_body=nil, put_data=nil, options={}, &block)
+    options = safe_http_options(options)
+
+    http(verb, url, post_body, put_data) do |handle|
+      yield handle if block
+      handle.safe_http!
+      handle.max_body_bytes = options[:max_body_bytes] if options.key?(:max_body_bytes)
+    end
+  end
+
   def self.get(url, params={}, &block)
     http :GET, urlalize(url, params), nil, nil, &block
   end
 
+  def self.safe_get(url, params={}, options={}, &block)
+    params, options = split_safe_http_params_options(params, options)
+    safe_http :GET, urlalize(url, params), nil, nil, options, &block
+  end
+
   def self.post(url, params={}, &block)
     http :POST, url, postalize(params), nil, &block
   end
 
+  def self.safe_post(url, params={}, options={}, &block)
+    safe_http :POST, url, postalize(params), nil, options, &block
+  end
+
   def self.put(url, params={}, &block)
     http :PUT, url, nil, postalize(params), &block
   end
 
+  def self.safe_put(url, params={}, options={}, &block)
+    safe_http :PUT, url, nil, postalize(params), options, &block
+  end
+
   def self.delete(url, params={}, &block)
     http :DELETE, url, postalize(params), nil, &block
   end
 
+  def self.safe_delete(url, params={}, options={}, &block)
+    safe_http :DELETE, url, postalize(params), nil, options, &block
+  end
+
   def self.patch(url, params={}, &block)
     http :PATCH, url, postalize(params), nil, &block
   end
 
+  def self.safe_patch(url, params={}, options={}, &block)
+    safe_http :PATCH, url, postalize(params), nil, options, &block
+  end
+
   def self.head(url, params={}, &block)
     http :HEAD, urlalize(url, params), nil, nil, &block
   end
 
+  def self.safe_head(url, params={}, options={}, &block)
+    params, options = split_safe_http_params_options(params, options)
+    safe_http :HEAD, urlalize(url, params), nil, nil, options, &block
+  end
+
   def self.options(url, params={}, &block)
     http :OPTIONS, urlalize(url, params), nil, nil, &block
   end
 
+  def self.safe_options(url, params={}, options={}, &block)
+    params, options = split_safe_http_params_options(params, options)
+    safe_http :OPTIONS, urlalize(url, params), nil, nil, options, &block
+  end
+
   def self.urlalize(url, params={})
     uri = URI(url)
     # early return if we didn't specify any extra params
@@ -261,6 +694,33 @@ module Curl
     params.respond_to?(:map) ? URI.encode_www_form(params) : (params.respond_to?(:to_s) ? params.to_s : params)
   end
 
+  def self.safe_http_options(options)
+    options ||= {}
+    raise ArgumentError, "safe HTTP options must be a Hash" unless options.is_a?(Hash)
+
+    options = options.dup
+    unsupported = options.keys - safe_http_option_keys
+    raise ArgumentError, "unsupported safe HTTP option(s): #{unsupported.join(', ')}" unless unsupported.empty?
+
+    options
+  end
+
+  def self.split_safe_http_params_options(params, options)
+    if options == {} && safe_http_option_hash?(params)
+      [{}, params]
+    else
+      [params, options]
+    end
+  end
+
+  def self.safe_http_option_hash?(value)
+    value.is_a?(Hash) && !value.empty? && (value.keys - safe_http_option_keys).empty?
+  end
+
+  def self.safe_http_option_keys
+    [:max_body_bytes]
+  end
+
   def self.reset
     Thread.current[:curb_curl] = Curl::Easy.new
   end

data/tests/bug_poison.rb

@@ -0,0 +1,29 @@
+require 'digest'
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class BugTestPoisonResponse < Test::Unit::TestCase
+  include BugTestServerSetupTeardown
+
+  def setup
+    @port = unused_local_port
+    @response_proc = lambda do|res|
+      sleep 0.5
+      res.body = "hi"
+      res['Content-Type'] = "text/html"
+    end
+    super
+  end
+
+  def test_bug
+    res_a = Curl.get("https://google.com/")
+
+    first_a_body = Digest::MD5.hexdigest(res_a.body)
+
+    res_b = Curl.get("https://unclanked.com/")
+
+    b_body = Digest::MD5.hexdigest(res_b.body)
+    a_body = Digest::MD5.hexdigest(res_a.body)
+    assert_not_equal a_body, b_body, "we expected #{a_body} to be #{first_a_body}"
+  end
+
+end

data/tests/tc_curl_download.rb

@@ -1,4 +1,5 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+require 'tmpdir'
 
 class TestCurbCurlDownload < Test::Unit::TestCase
   include TestServerMethods 
@@ -18,6 +19,91 @@ class TestCurbCurlDownload < Test::Unit::TestCase
     File.unlink(dl_path) if File.exist?(dl_path)
   end
 
+  def test_download_default_path_does_not_overwrite_existing_file
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      Dir.chdir(dir) do
+        File.write('curb_easy.c', 'sentinel')
+
+        assert_raise(Curl::DownloadTargetExistsError) do
+          Curl::Easy.download(dl_url)
+        end
+
+        assert_equal 'sentinel', File.read('curb_easy.c')
+      end
+    end
+  end
+
+  def test_download_default_path_strips_query_from_url_basename
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c?cache=1"
+    source_path = File.expand_path(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c'))
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      Dir.chdir(dir) do
+        Curl::Easy.download(dl_url)
+
+        assert File.exist?('curb_easy.c')
+        assert !File.exist?('curb_easy.c?cache=1')
+        assert_equal File.read(source_path), File.read('curb_easy.c')
+      end
+    end
+  end
+
+  def test_download_safe_directory_rejects_unsafe_names
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      ['../escape.c', '.env', 'nested/file', '/tmp/escape.c', "bad\0name"].each do |name|
+        assert_raise(ArgumentError, "expected #{name.inspect} to be rejected") do
+          Curl::Easy.download(dl_url, name, :download_dir => dir)
+        end
+      end
+    end
+  end
+
+  def test_download_safe_directory_rejects_dotfile_from_url
+    dl_url = "http://127.0.0.1:9129/ext/.env"
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      assert_raise(ArgumentError) do
+        Curl::Easy.download(dl_url, :download_dir => dir)
+      end
+
+      assert !File.exist?(File.join(dir, '.env'))
+    end
+  end
+
+  def test_download_overwrite_true_replaces_existing_file
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      dl_path = File.join(dir, 'curb_easy.c')
+      File.write(dl_path, 'sentinel')
+
+      Curl::Easy.download(dl_url, dl_path, :overwrite => true)
+
+      assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path)
+    end
+  end
+
+  def test_download_callback_abort_does_not_replace_existing_file
+    dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
+
+    Dir.mktmpdir('curb-download-') do |dir|
+      dl_path = File.join(dir, 'curb_easy.c')
+      File.write(dl_path, 'sentinel')
+
+      assert_raise(Curl::Err::WriteError, Curl::Err::RecvError) do
+        Curl::Easy.download(dl_url, dl_path, :overwrite => true) do |curl|
+          curl.on_body { |_data| 0 }
+        end
+      end
+
+      assert_equal 'sentinel', File.read(dl_path)
+    end
+  end
+
   def test_download_url_to_file_via_file_io
     dl_url = "http://127.0.0.1:9129/ext/curb_easy.c"
     dl_path = File.join(Dir::tmpdir, "dl_url_test.file")