code-ruby 3.1.2 → 4.0.1

data/lib/code/object/html.rb

@@ -3,6 +3,146 @@
 class Code
   class Object
     class Html < Object
+      CLASS_DOCUMENTATION = {
+        name: "Html",
+        description: "builds, escapes, parses, and queries html fragments.",
+        examples: [
+          "Html.p { :hello }",
+          "Html.escape(\"<p>a</p>\")",
+          "Html.raw(\"<p>a</p>\").css(\"p\")"
+        ]
+      }.freeze
+      CLASS_FUNCTIONS = {
+        "escape" => {
+          name: "escape",
+          description: "escapes a value for html text.",
+          examples: [
+            "Html.escape(\"<p>\")",
+            "Html.escape(:hello)",
+            "Html.escape(() => { \"<strong>hello</strong>\" })"
+          ]
+        },
+        "unescape" => {
+          name: "unescape",
+          description: "converts html entities and tags to text.",
+          examples: [
+            "Html.unescape(\"&lt;p&gt;\")",
+            "Html.unescape(\"<p>hello</p>\")",
+            "Html.unescape(() => { \"&amp;\" })"
+          ]
+        },
+        "join" => {
+          name: "join",
+          description: "joins html or text values with an optional separator.",
+          examples: [
+            "Html.join([:a, :b], \", \")",
+            "Html.join([Html.text(\"a\"), Html.text(\"b\")])",
+            "Html.join(\"<br>\", () => { [\"a\", \"b\"] })"
+          ]
+        },
+        "text" => {
+          name: "text",
+          description: "builds an escaped html text fragment.",
+          examples: [
+            "Html.text(:hello)",
+            "Html.text(\"<strong>hello</strong>\")",
+            "Html.text(() => { :hello })"
+          ]
+        },
+        "raw" => {
+          name: "raw",
+          description:
+            "parses markup into an html fragment without escaping text.",
+          examples: [
+            "Html.raw(\"<strong>hello</strong>\")",
+            "Html.raw(Html.text(:hello))",
+            "Html.raw(() => { \"<em>hello</em>\" })"
+          ]
+        }
+      }.freeze
+      INSTANCE_FUNCTIONS = {
+        "css" => {
+          name: "css",
+          description: "returns all html nodes matching a css selector.",
+          examples: [
+            "Html.raw(\"<p>a</p><p>b</p>\").css(\"p\")",
+            "Html.raw(\"<main><p>a</p></main>\").css(\"main p\")",
+            "Html.raw(\"<p class='x'>a</p>\").css(\".x\")"
+          ]
+        },
+        "at_css" => {
+          name: "at_css",
+          description: "returns the first html node matching a css selector.",
+          examples: [
+            "Html.raw(\"<p>a</p><p>b</p>\").at_css(\"p\")",
+            "Html.raw(\"<main><p>a</p></main>\").at_css(\"main p\")",
+            "Html.raw(\"<p class='x'>a</p>\").at_css(\".x\")"
+          ]
+        },
+        "map" => {
+          name: "map",
+          description:
+            "returns a list by calling a function for each html node.",
+          examples: [
+            "Html.raw(\"<p>a</p><p>b</p>\").css(\"p\").map((node) => { node.to_string })",
+            "Html.raw(\"<p>a</p>\").css(:p).map((node, index) => { index })",
+            "Html.raw(\"<p>a</p>\").css(:p).map((node, index, nodes) => { nodes.to_string })"
+          ]
+        },
+        "to_string" => {
+          name: "to_string",
+          description: "returns the html fragment's text content.",
+          examples: [
+            "Html.raw(\"<p>hello</p>\").to_string",
+            "Html.text(:hello).to_string",
+            "Html.raw(\"<p><strong>hello</strong></p>\").to_string"
+          ]
+        },
+        "to_html" => {
+          name: "to_html",
+          description: "returns the html fragment as html markup.",
+          examples: [
+            "Html.raw(\"<p>hello</p>\").to_html",
+            "Html.text(:p).to_html",
+            "Html.raw(\"<strong>hello</strong>\").to_html"
+          ]
+        },
+        "inner_text" => {
+          name: "inner_text",
+          description: "returns the html fragment's text content.",
+          examples: [
+            "Html.raw(\"<p>hello</p>\").inner_text",
+            "Html.text(:hello).inner_text",
+            "Html.raw(\"<p><strong>hello</strong></p>\").inner_text"
+          ]
+        },
+        "attribute" => {
+          name: "attribute",
+          description: "returns an html node attribute by name.",
+          examples: [
+            "Html.raw(\"<a href='/'>home</a>\").at_css(:a).attribute(:href)",
+            "Html.raw(\"<p class='x'>a</p>\").at_css(:p).attribute(:class)",
+            "Html.raw(\"<img alt='logo'>\").at_css(:img).attribute(:alt)"
+          ]
+        },
+        "attributes" => {
+          name: "attributes",
+          description: "returns an html node's attributes as a dictionary.",
+          examples: [
+            "Html.raw(\"<a href='/'>home</a>\").at_css(:a).attributes",
+            "Html.raw(\"<p class='x'>a</p>\").at_css(:p).attributes",
+            "Html.raw(\"<img alt='logo'>\").at_css(:img).attributes"
+          ]
+        }
+      }.freeze
+
+      def self.function_documentation(scope)
+        return INSTANCE_FUNCTIONS if scope == :instance
+        return CLASS_FUNCTIONS if scope == :class
+
+        {}
+      end
+
       TAGS = %w[
         a
         abbr
@@ -124,7 +264,9 @@ class Code
                 args.first.is_a?(Nokogiri::XML::Node)
             args.first
           else
-            Nokogiri.HTML(args.first.to_s)
+            source = args.first.to_s
+            ::Code.ensure_input_size!(source, label: "html")
+            Nokogiri.HTML(source)
           end
       end
 
@@ -199,7 +341,7 @@ class Code
 
           content =
             if code_content.is_an?(Html)
-              Nokogiri::HTML::DocumentFragment.parse(code_content.to_html)
+              fragment_from_html(code_content)
             else
               Nokogiri::XML::Text.new(code_content.to_s, fragment.document)
             end
@@ -243,7 +385,10 @@ class Code
             value_or_function.to_code
           end
 
-        String.new(Nokogiri::HTML.fragment(code_value.to_s).text)
+        source = code_value.to_s
+        ::Code.ensure_input_size!(source, label: "html")
+
+        String.new(Nokogiri::HTML.fragment(source).text)
       rescue Error::Break => e
         e.code_value
       end
@@ -270,14 +415,14 @@ class Code
         code_contents.raw.each.with_index do |code_content, index|
           content =
             if code_content.is_an?(Html)
-              Nokogiri::HTML::DocumentFragment.parse(code_content.to_html)
+              fragment_from_html(code_content)
             else
               Nokogiri::XML::Text.new(code_content.to_s, fragment.document)
             end
 
           separator =
             if code_separator.is_an?(Html)
-              Nokogiri::HTML::DocumentFragment.parse(code_separator.to_html)
+              fragment_from_html(code_separator)
             else
               Nokogiri::XML::Text.new(code_separator.to_s, fragment.document)
             end
@@ -326,14 +471,22 @@ class Code
           end
 
         if code_value.is_an?(Html)
-          Html.new(Nokogiri::HTML::DocumentFragment.parse(code_value.to_html))
+          Html.new(fragment_from_html(code_value))
         else
-          Html.new(Nokogiri::HTML::DocumentFragment.parse(code_value.to_s))
+          source = code_value.to_s
+          ::Code.ensure_input_size!(source, label: "html")
+          Html.new(Nokogiri::HTML::DocumentFragment.parse(source))
         end
       rescue Error::Break => e
         e.code_value
       end
 
+      def self.fragment_from_html(html)
+        source = html.to_html
+        ::Code.ensure_input_size!(source, label: "html")
+        Nokogiri::HTML::DocumentFragment.parse(source)
+      end
+
       def call(**args)
         code_operator = args.fetch(:operator, nil).to_code
         code_arguments = args.fetch(:arguments, []).to_code
@@ -356,9 +509,15 @@ class Code
         when "to_html"
           sig(args)
           code_to_html
+        when "inner_text"
+          sig(args)
+          code_inner_text
         when "attribute"
           sig(args) { String }
           code_attribute(code_value)
+        when "attributes"
+          sig(args)
+          code_attributes
         else
           super
         end
@@ -409,10 +568,25 @@ class Code
         String.new(raw.text)
       end
 
+      def code_inner_text
+        code_to_string
+      end
+
       def code_attribute(value = nil)
         code_value = value.to_code
         String.new(raw.attr(code_value.to_s))
       end
+
+      def code_attributes
+        node = raw.is_a?(::Nokogiri::XML::NodeSet) ? raw.first : raw
+        return Dictionary.new if node.blank?
+
+        Dictionary.new(
+          node.attribute_nodes.to_h do |attribute|
+            [attribute.name.to_code, attribute.value.to_code]
+          end
+        )
+      end
     end
   end
 end

data/lib/code/object/http.rb

@@ -3,6 +3,112 @@
 class Code
   class Object
     class Http < Object
+      CLASS_DOCUMENTATION = {
+        name: "Http",
+        description:
+          "sends http requests and returns dictionaries with request and response details.",
+        examples: [
+          "Http.get(\"http://httpbin.org/status/200\").success?",
+          "Http.post(\"http://httpbin.org/status/200\", body: :hello).request.body",
+          "Http.fetch(:get, \"http://httpbin.org/status/204\").code"
+        ]
+      }.freeze
+      CLASS_FUNCTIONS = {
+        "get" => {
+          name: "get",
+          description:
+            "sends a get request and returns the response dictionary.",
+          examples: [
+            "Http.get(\"http://httpbin.org/status/200\").code",
+            "Http.get(\"http://httpbin.org/status/200\", query: { q: :ruby }).url",
+            "Http.get(\"http://httpbin.org/status/200\", headers: { accept: \"application/json\" }).request.headers"
+          ]
+        },
+        "head" => {
+          name: "head",
+          description:
+            "sends a head request and returns the response dictionary.",
+          examples: [
+            "Http.head(\"http://httpbin.org/status/200\").code",
+            "Http.head(\"http://httpbin.org/status/204\").status",
+            "Http.head(\"http://httpbin.org/status/200\", headers: { accept: \"text/html\" }).request.headers"
+          ]
+        },
+        "post" => {
+          name: "post",
+          description:
+            "sends a post request with optional body, form data, and headers.",
+          examples: [
+            "Http.post(\"http://httpbin.org/status/200\", body: :hello).request.body",
+            "Http.post(\"http://httpbin.org/status/200\", data: { a: 1 }).success?",
+            "Http.post(\"http://httpbin.org/status/200\", headers: { content_type: \"text/plain\" }).request.headers"
+          ]
+        },
+        "put" => {
+          name: "put",
+          description: "sends a put request with optional body or form data.",
+          examples: [
+            "Http.put(\"http://httpbin.org/status/200\", body: :hello).request.body",
+            "Http.put(\"http://httpbin.org/status/200\", data: { a: 1 }).method",
+            "Http.put(\"http://httpbin.org/status/204\").code"
+          ]
+        },
+        "delete" => {
+          name: "delete",
+          description:
+            "sends a delete request and returns the response dictionary.",
+          examples: [
+            "Http.delete(\"http://httpbin.org/status/200\").success?",
+            "Http.delete(\"http://httpbin.org/status/200\", body: :hello).request.body",
+            "Http.delete(\"http://httpbin.org/status/204\").status"
+          ]
+        },
+        "options" => {
+          name: "options",
+          description:
+            "sends an options request and returns the response dictionary.",
+          examples: [
+            "Http.options(\"http://httpbin.org/status/200\").method",
+            "Http.options(\"http://httpbin.org/status/204\").code",
+            "Http.options(\"http://httpbin.org/status/200\", headers: { accept: \"*/*\" }).request.headers"
+          ]
+        },
+        "trace" => {
+          name: "trace",
+          description:
+            "sends a trace request and returns the response dictionary.",
+          examples: [
+            "Http.trace(\"http://httpbin.org/status/200\").method",
+            "Http.trace(\"http://httpbin.org/status/204\").code",
+            "Http.trace(\"http://httpbin.org/status/200\", headers: { accept: \"message/http\" }).request.headers"
+          ]
+        },
+        "patch" => {
+          name: "patch",
+          description: "sends a patch request with optional body or form data.",
+          examples: [
+            "Http.patch(\"http://httpbin.org/status/200\", body: :hello).request.body",
+            "Http.patch(\"http://httpbin.org/status/200\", data: { a: 1 }).method",
+            "Http.patch(\"http://httpbin.org/status/204\").code"
+          ]
+        },
+        "fetch" => {
+          name: "fetch",
+          description: "sends an http request using the given method name.",
+          examples: [
+            "Http.fetch(:get, \"http://httpbin.org/status/200\").success?",
+            "Http.fetch(:post, \"http://httpbin.org/status/200\", body: :hello).request.body",
+            "Http.fetch(:patch, \"http://httpbin.org/status/200\", data: { a: 1 }).method"
+          ]
+        }
+      }.freeze
+
+      def self.function_documentation(scope)
+        return CLASS_FUNCTIONS if scope == :class
+
+        {}
+      end
+
       SIG = [
         String,
         {
@@ -85,6 +191,20 @@ class Code
         network_authentication_required: 511
       }.freeze
       DEFAULT_TIMEOUT = 1.hour.to_f
+      MAX_REQUEST_BYTES = ::Code::MAX_INPUT_BYTES
+      MAX_RESPONSE_BYTES = ::Code::MAX_INPUT_BYTES
+      MAX_HEADER_BYTES = 32.kilobytes
+      HEADER_NAME = /\A[A-Za-z0-9!#$%&'*+\-.^_`|~]+\z/
+      RESTRICTED_HEADERS = %w[
+        connection
+        content-length
+        host
+        proxy-authorization
+        te
+        trailer
+        transfer-encoding
+        upgrade
+      ].freeze
 
       def self.call(**args)
         code_operator = args.fetch(:operator, nil).to_code
@@ -163,7 +283,7 @@ class Code
         username = options.code_get("username").to_s
         password = options.code_get("password").to_s
         body = options.code_get("body").to_s
-        headers = options.code_get("headers").raw || {}
+        headers = sanitized_headers(options.code_get("headers").raw || {})
         data = options.code_get("data").raw || {}
         timeout = options.code_get("timeout")
         open_timeout = options.code_get("open_timeout")
@@ -171,31 +291,38 @@ class Code
         write_timeout = options.code_get("write_timeout")
         query = options.code_get("query").raw || {}
         query = query.to_a.flatten.map(&:to_s).each_slice(2).to_h.to_query
+        validate_payload_size!(username, label: "http username")
+        validate_payload_size!(password, label: "http password")
+        validate_payload_size!(query, label: "http query")
+        validate_payload_size!(body, label: "http request body")
+        if data.present?
+          validate_payload_size!(data.as_json.to_query, label: "http form data")
+        end
 
         url = original_url
         url = "#{url}?#{query}" if query.present?
+        validate_payload_size!(url, label: "http url")
 
         if username.present? || password.present?
           authorization = ::Base64.strict_encode64("#{username}:#{password}")
           headers["Authorization"] = "Basic #{authorization}"
+          validate_header_size!("Authorization", headers["Authorization"])
         end
 
-        uri = ::URI.parse(url)
-        http = ::Net::HTTP.new(uri.host, uri.port)
+        uri = parse_uri(url)
+        resolved_ip = ::Code::Network.validate_public_uri!(uri, service: "http")
+
+        http = ::Net::HTTP.new(uri.hostname, uri.port, nil)
+        http.ipaddr = resolved_ip
         http.use_ssl = true if uri.scheme == "https"
-        default_timeout = timeout.nothing? ? DEFAULT_TIMEOUT : timeout.to_f
-        open_timeout_value =
-          open_timeout.nothing? ? default_timeout : open_timeout.to_f
-        read_timeout_value =
-          read_timeout.nothing? ? default_timeout : read_timeout.to_f
-        write_timeout_value =
-          write_timeout.nothing? ? default_timeout : write_timeout.to_f
+        default_timeout = http_timeout(timeout, DEFAULT_TIMEOUT)
+        open_timeout_value = http_timeout(open_timeout, default_timeout)
+        read_timeout_value = http_timeout(read_timeout, default_timeout)
+        write_timeout_value = http_timeout(write_timeout, default_timeout)
 
         http.open_timeout = open_timeout_value if open_timeout_value
         http.read_timeout = read_timeout_value if read_timeout_value
-        if http.respond_to?(:write_timeout=) && write_timeout_value
-          http.write_timeout = write_timeout_value
-        end
+        http.write_timeout = write_timeout_value if write_timeout_value
 
         request_class =
           case verb
@@ -223,7 +350,19 @@ class Code
         request.set_form_data(**data.as_json) if data.present?
 
         begin
-          response = http.request(request)
+          response_body = +""
+          response =
+            http.request(request) do |http_response|
+              validate_response_headers!(http_response)
+
+              http_response.read_body do |chunk|
+                if response_body.bytesize + chunk.bytesize > MAX_RESPONSE_BYTES
+                  raise ::Code::Error, "http response is too large"
+                end
+
+                response_body << chunk
+              end
+            end
         rescue ::Timeout::Error, ::Errno::ETIMEDOUT
           raise ::Code::Error, "http timeout"
         rescue OpenSSL::SSL::SSLError, IOError, SystemCallError, SocketError
@@ -234,9 +373,10 @@ class Code
         location = response["location"].to_s
 
         if (300..399).cover?(code) && location.present? && redirects.positive?
-          new_uri = ::URI.join(uri, location)
+          new_uri = parse_uri(::URI.join(uri, location).to_s)
+          ::Code::Network.validate_public_uri!(new_uri, service: "http")
 
-          if new_uri.host == uri.host
+          if same_origin?(new_uri, uri)
             code_fetch(
               "get",
               new_uri.to_s,
@@ -248,8 +388,104 @@ class Code
           end
         else
           status = STATUS_CODES.key(code) || :ok
-          Dictionary.new(code: code, status: status, body: response.body.to_s)
+          response_headers = response.each_header.to_h
+          request_headers =
+            request.to_hash.transform_values { |values| List.new(values) }
+          body = response_body.to_s
+
+          Dictionary.new(
+            :code => code,
+            :status => status,
+            :body => body,
+            :headers => response_headers,
+            :method => verb,
+            :url => url,
+            "success?" => code.between?(200, 299),
+            "redirect?" => code.between?(300, 399),
+            :request => {
+              method: verb,
+              url: url,
+              headers: request_headers,
+              body: request.body.to_s
+            },
+            :response => {
+              code: code,
+              status: status,
+              headers: response_headers,
+              body: body
+            }
+          )
+        end
+      end
+
+      def self.parse_uri(url)
+        ::URI.parse(url)
+      rescue ::URI::InvalidURIError
+        raise ::Code::Error, "http: invalid url"
+      end
+
+      def self.same_origin?(new_uri, original_uri)
+        new_uri.scheme == original_uri.scheme &&
+          new_uri.hostname.to_s.downcase ==
+            original_uri.hostname.to_s.downcase &&
+          new_uri.port == original_uri.port
+      end
+
+      def self.sanitized_headers(headers)
+        headers.to_h.to_h do |key, value|
+          name = key.to_s
+          header_value = value.to_s
+
+          validate_header_name!(name)
+          validate_header_value!(header_value)
+          validate_header_size!(name, header_value)
+
+          [name, header_value]
+        end
+      end
+
+      def self.validate_header_name!(name)
+        normalized_name = name.downcase
+
+        unless HEADER_NAME.match?(name)
+          raise ::Code::Error, "http: invalid header name"
         end
+
+        return unless RESTRICTED_HEADERS.include?(normalized_name)
+
+        raise ::Code::Error, "http: restricted header #{name.inspect}"
+      end
+
+      def self.validate_header_value!(value)
+        return unless /[\r\n\0]/.match?(value)
+
+        raise ::Code::Error, "http: invalid header value"
+      end
+
+      def self.validate_header_size!(name, value)
+        return if name.bytesize + value.bytesize <= MAX_HEADER_BYTES
+
+        raise ::Code::Error, "http: request headers are too large"
+      end
+
+      def self.validate_response_headers!(response)
+        size =
+          response.each_header.sum do |name, value|
+            name.to_s.bytesize + value.to_s.bytesize
+          end
+        return if size <= MAX_HEADER_BYTES
+
+        raise ::Code::Error, "http response headers are too large"
+      end
+
+      def self.validate_payload_size!(value, label:)
+        return if value.to_s.bytesize <= MAX_REQUEST_BYTES
+
+        raise ::Code::Error, "#{label} is too large"
+      end
+
+      def self.http_timeout(value, default)
+        ::Code.normalize_timeout!(value.nothing? ? default : value)
       end
     end
   end