uri-whatwg_parser 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57eb48896a4f467a9b2f4d5a9b195a8df8a3b1f707d680711e777b1226d5471a
-  data.tar.gz: 90e1c4a439d6309ac14c7f8e46ace4128c8f8469d275e29b9afa06b60d1e8ac9
+  metadata.gz: 2061d819a10365a97f389d9cb70325c87b75c390c4145897ed640c3e4cc85f70
+  data.tar.gz: 905ec3bdd174039fc91ce433f84a079fe12ff93f9ac999f420b08760b3fcee53
 SHA512:
-  metadata.gz: dd95276e9e4df35db76eb196c46b72f1300d777c45161dcf58d3253e477db846d7662cd9f0e837fcc37a3ddd047700c024ecc5c1adeed83c24a478026b188936
-  data.tar.gz: 3cb8ee2b663bb4938bd099106874bed736aa709a38ff4b435803c8ff1752f229a4d8cd83ffb920fc1e29f5ebfc1815f5796105a6f54bb99d241f4b5adb605efa
+  metadata.gz: fc4a0d83fb46e959feb1f7990fa1f119428764447c6d4d455ec5287d5823b19eec845ef4b8912ceeffd6233266b3142859a837c7a26e47e20315d64bb5660006
+  data.tar.gz: bd36808311ae2330cd897ba824219d82cff0497d75531441b3679def9ff5ea1969297ccfcbb8cd02b83909a2f7a5c0de7f0d505bb79e1117494544dd3d4b3cfa

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+## 0.1.4
+
+* Support passing `encoding` option
+
+## 0.1.3
+
+* Improve performance of parsing
+
+## 0.1.2
+
+* Correctly compress IPv6 host
+
+## 0.1.1
+
+* Support `base` option
+
+## 0.1.0
+
+* Initial release

data/README.md

@@ -17,6 +17,8 @@ gem install uri-whatwg_parser
 This gem is compatible with [`uri`](https://github.com/ruby/uri) gem and automatically switches parser's behavior. So users don't need to set up.
 
 ```ruby
+require "uri/whatwg_parser"
+
 URI.parse("http://日本語.jp")
 # => #<URI::HTTP http://xn--wgv71a119e.jp>
 ```
@@ -31,7 +33,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 * Support state override
 * Support validations
-* Support encodings other than UTF-8
 
 ## Contributing
 

data/lib/uri/whatwg_parser/host_parser.rb

@@ -156,11 +156,11 @@ class URI::WhatwgParser
     end
 
     def include_forbidden_domain_code_point?(str)
-      str.chars.intersect?(FORBIDDEN_DOMAIN_CODE_POINT)
+      FORBIDDEN_DOMAIN_CODE_POINT.any? {|c| str.include?(c) }
     end
 
     def include_forbidden_host_code_point?(str)
-      str.chars.intersect?(FORBIDDEN_HOST_CODE_POINT)
+      FORBIDDEN_HOST_CODE_POINT.any? {|c| str.include?(c) }
     end
   end
 end

data/lib/uri/whatwg_parser/parser_helper.rb

@@ -2,8 +2,9 @@
 
 class URI::WhatwgParser
   module ParserHelper
-    C0_CONTROL = (0..0x1f).to_a
-    C0_CONTROL_PERCENT_ENCODE_SET = C0_CONTROL.map(&:chr)
+    C0_CONTROL_PERCENT_ENCODE_SET = (0..0x1f).map(&:chr)
+    ASCII_ALPHA = ("a".."z").to_a + ("A".."Z").to_a
+    ASCII_DIGIT = ("0".."9").to_a
 
     def ascii_alpha?(c)
       ASCII_ALPHA.include?(c)
@@ -17,11 +18,14 @@ class URI::WhatwgParser
       ASCII_DIGIT.include?(c)
     end
 
-    def percent_encode(c, encode_set)
-      if encode_set.include?(c) || c.ord > 0x7e
-        return c.unpack("C*").map { |b| sprintf("%%%02X", b) }.join
-      end
-      c
+    def percent_encode(c, encode_set, encoding = Encoding::UTF_8)
+      return c unless encode_set.include?(c) || c.ord > 0x7e
+
+      # For ASCII single-byte characters
+      return "%%%02X" % c.ord if c.bytesize == 1
+
+      bytes = c.encoding == encoding ? c.bytes : c.encode(encoding).bytes
+      bytes.map { |b| "%%%02X" % b }.join
     end
   end
 end

data/lib/uri/whatwg_parser/version.rb

@@ -2,6 +2,6 @@
 
 module URI
   class WhatwgParser
-    VERSION = "0.1.2"
+    VERSION = "0.1.4"
   end
 end

data/lib/uri/whatwg_parser.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "strscan"
 require "uri"
 require_relative "whatwg_parser/error"
 require_relative "whatwg_parser/version"
@@ -12,8 +11,6 @@ module URI
     include ParserHelper
 
     SPECIAL_SCHEME = { "ftp" => 21, "file" => nil, "http" => 80, "https" => 443, "ws" => 80, "wss" => 443 }
-    ASCII_ALPHA = ("a".."z").to_a + ("A".."Z").to_a
-    ASCII_DIGIT = ("0".."9").to_a
 
     FRAGMENT_PERCENT_ENCODE_SET = C0_CONTROL_PERCENT_ENCODE_SET + [" ", "\"", "<", ">", "`"]
     QUERY_PERCENT_ENCODE_SET = C0_CONTROL_PERCENT_ENCODE_SET + [" ", "\"", "#", "<", ">"]
@@ -37,45 +34,40 @@ module URI
       {}
     end
 
-    def parse(uri, base = nil) # :nodoc:
+    def parse(uri, base = nil, encoding = Encoding::UTF_8) # :nodoc:
       reset
-      URI.for(*self.split(uri, base))
+      URI.for(*self.split(uri, base, encoding))
     end
 
-    def split(uri, base = nil) # :nodoc:
+    def split(uri, base = nil, encoding = Encoding::UTF_8) # :nodoc:
       @base = nil
       if base != nil
-        ary = split(base)
+        ary = split(base, nil, encoding)
         @base = { scheme: ary[0], userinfo: ary[1], host: ary[2], port: ary[3], registry: ary[4], path: ary[5], opaque: ary[6], query: ary[7], fragment: ary[8]}
         @base_paths = @paths
         reset
       end
 
-      uri = uri.dup
-      uri.gsub!(/\A[\u0000-\u0020]*/, "")
-      uri.gsub!(/[\u0000-\u0020]*\z/, "")
-      uri.delete!("\t")
-      uri.delete!("\n")
-      uri.delete!("\r")
+      @encoding = encoding
+      @uri = uri.dup
+      @uri.sub!(/\A[\u0000-\u0020]*/, "")
+      @uri.sub!(/[\u0000-\u0020]*\z/, "")
+      @uri.delete!("\t")
+      @uri.delete!("\n")
+      @uri.delete!("\r")
 
       raise ParseError, "uri can't be empty" if uri.empty? && @base.nil?
 
-      @scanner = StringScanner.new(uri)
+      @pos = 0
 
-      loop do
-        c = @scanner.getch
-        send("on_#{@state}", c)
-
-        if @force_continue
-          @force_continue = false
-          next
-        end
-
-        break if c.nil? && @scanner.eos?
+      while @pos <= @uri.length
+        c = @uri[@pos]
+        send(@state, c)
+        @pos += 1
       end
 
       @parse_result[:userinfo] = "#{@username}:#{@password}" if !@username.nil? || !@password.nil?
-      @parse_result[:path] = "/#{@paths.join("/")}" if !@paths.empty?
+      @parse_result[:path] = "/#{@paths.join("/")}" if @paths && !@paths.empty?
 
       @parse_result.values
     end
@@ -95,12 +87,11 @@ module URI
     private
 
     def reset
-      @scanner = nil
       @buffer = +""
       @at_sign_seen = nil
       @password_token_seen = nil
       @inside_brackets = nil
-      @paths = []
+      @paths = nil
       @username = nil
       @password = nil
       @parse_result = { scheme: nil, userinfo: nil, host: nil, port: nil, registry: nil, path: nil, opaque: nil, query: nil, fragment: nil }
@@ -108,23 +99,19 @@ module URI
       @state = :scheme_start_state
     end
 
-    def on_scheme_start_state(c)
+    def scheme_start_state(c)
       if ascii_alpha?(c)
-        @buffer += c.downcase
+        @buffer << c.downcase
         @state = :scheme_state
       else
-        if c.nil?
-          @force_continue = true
-        else
-          @scanner.pos -= c.bytesize
-        end
+        @pos -= 1
         @state = :no_scheme_state
       end
     end
 
-    def on_scheme_state(c)
+    def scheme_state(c)
       if ascii_alphanumerica?(c) || ["+", "-", "."].include?(c)
-        @buffer += c.downcase
+        @buffer << c.downcase
       elsif c == ":"
         @parse_result[:scheme] = @buffer
         @buffer = +""
@@ -135,58 +122,58 @@ module URI
           @state = :special_relative_or_authority_state
         elsif special_url?
           @state = :special_authority_slashes_state
-        elsif @scanner.rest.start_with?("/")
+        elsif rest.start_with?("/")
           @state = :path_or_authority_state
-          @scanner.pos += c.bytesize
+          @pos += 1
         else
-          @parse_result[:path] = ""
+          @parse_result[:path] = nil
           @state = :opaque_path_state
         end
       else
-        @buffer = +""
-        decrease_pos(c)
+        @buffer.clear
+        @pos -= 1
         @state = :no_scheme_state
       end
     end
 
-    def on_no_scheme_state(c)
+    def no_scheme_state(c)
       raise ParseError, "scheme is missing" if @base.nil? || !@base[:opaque].nil? && c != "#"
 
       if !@base[:opaque].nil? && c == "#"
         @parse_result[:scheme] = @base[:scheme]
         @parse_result[:path] = @base[:path]
         @parse_result[:query] = @base[:query]
-        @parse_result[:fragment] = ""
+        @parse_result[:fragment] = nil
         @state = :fragment_state
       elsif @base[:scheme] != "file"
         @state = :relative_state
-        decrease_pos(c)
+        @pos -= 1
       else
         @state = :file_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_special_relative_or_authority_state(c)
-      if c == "/" && @scanner.rest.start_with?("/")
+    def special_relative_or_authority_state(c)
+      if c == "/" && rest.start_with?("/")
         @state = :special_authority_ignore_slashes_state
-        decrease_pos(c)
+        @pos -= 1
       else
         @state = :relative_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_path_or_authority_state(c)
+    def path_or_authority_state(c)
       if c == "/"
         @state = :authority_state
       else
         @state = :path_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_relative_state(c)
+    def relative_state(c)
       @parse_result[:scheme] = @base[:scheme]
       if c == "/"
         @state = :relative_slash_state
@@ -200,21 +187,21 @@ module URI
         @parse_result[:query] = @base[:query]
 
         if c == "?"
-          @parse_result[:query] = ""
+          @parse_result[:query] = nil
           @state = :query_state
         elsif c == "#"
-          @parse_result[:fragment] = ""
+          @parse_result[:fragment] = nil
           @state = :fragment_state
         elsif !c.nil?
           @parse_result[:query] = nil
           shorten_url_path
           @state = :path_state
-          @scanner.pos -= c.bytesize
+          @pos -= 1
         end
       end
     end
 
-    def on_relative_slash_state(c)
+    def relative_slash_state(c)
       if special_url? && (c == "/" || c == "\\")
         @state = :special_authority_ignore_slashes_state
       elsif c == "/"
@@ -224,28 +211,28 @@ module URI
         @parse_result[:host] = @base[:host]
         @parse_result[:port] = @base[:port]
         @state = :path_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_special_authority_slashes_state(c)
-      if c == "/" && @scanner.rest.start_with?("/")
+    def special_authority_slashes_state(c)
+      if c == "/" && rest.start_with?("/")
         @state = :special_authority_ignore_slashes_state
-        @scanner.pos += c.bytesize
+        @pos += 1
       else
         @state = :special_authority_ignore_slashes_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_special_authority_ignore_slashes_state(c)
+    def special_authority_ignore_slashes_state(c)
       if c != "/" && c != "\\"
         @state = :authority_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_authority_state(c)
+    def authority_state(c)
       if c == "@"
         @buffer.prepend("%40") if @at_sign_seen
         @at_sign_seen = true
@@ -255,7 +242,7 @@ module URI
             next
           end
 
-          encoded_char = percent_encode(char, USERINFO_PERCENT_ENCODE_SET)
+          encoded_char = percent_encode(char, USERINFO_PERCENT_ENCODE_SET, @encoding)
 
           if @password_token_seen
             @password = @password.to_s + encoded_char
@@ -264,48 +251,44 @@ module URI
           end
         end
 
-        @buffer = +""
+        @buffer.clear
       elsif c.nil? || ["/", "?", "#"].include?(c) || (special_url? && c == "\\")
         raise ParseError, "host is missing" if @at_sign_seen && @buffer.empty?
-        if c.nil?
-          @force_continue = true
-          @scanner.pos -= @buffer.bytesize
-        else
-          @scanner.pos -= (@buffer.bytesize + c.bytesize.to_i)
-        end
-        @buffer = +""
+
+        @pos -= (@buffer.size + 1)
+        @buffer.clear
         @state = :host_state
       else
         @buffer << c
       end
     end
 
-    def on_host_state(c)
+    def host_state(c)
       if c == ":" && !@inside_brackets
         raise ParseError, "host is missing" if @buffer.empty?
 
         @parse_result[:host] = @host_parser.parse(@buffer, !special_url?)
-        @buffer = +""
+        @buffer.clear
         @state = :port_state
       elsif c.nil? || ["/", "?", "#"].include?(c) || (special_url? && c == "\\")
-        decrease_pos(c)
+        @pos -= 1
         if special_url? && @buffer.empty?
           raise ParseError, "host is missing"
         else
           @parse_result[:host] = @host_parser.parse(@buffer, !special_url?)
-          @buffer = +""
+          @buffer.clear
           @state = :path_start_state
         end
       else
         @inside_brackets = true if c == "["
         @inside_brackets = false if c == "]"
-        @buffer += c
+        @buffer << c
       end
     end
 
-    def on_port_state(c)
+    def port_state(c)
       if ascii_digit?(c)
-        @buffer += c
+        @buffer << c
       elsif c.nil? || ["/", "?", "#"].include?(c) || (special_url? && c == "\\")
         unless @buffer.empty?
           begin
@@ -316,19 +299,19 @@ module URI
             raise ParseError, "port is invalid value"
           end
 
-          @buffer = +""
+          @buffer.clear
         end
 
         @state = :path_start_state
-        decrease_pos(c)
+        @pos -= 1
       else
         raise ParseError, "port is invalid value"
       end
     end
 
-    def on_file_state(c)
+    def file_state(c)
       @parse_result[:scheme] = "file"
-      @parse_result[:host] = ""
+      @parse_result[:host] = nil
 
       if c == "/" || c == "\\"
         @state = :file_slash_state
@@ -336,50 +319,50 @@ module URI
         @parse_result[:host] = @base[:host]
         @parse_result[:query] = @base[:query]
         if c == "?"
-          @parse_result[:query] = ""
+          @parse_result[:query] = nil
           @state = :query_state
         elsif c == "#"
-          @parse_result[:fragment] = ""
+          @parse_result[:fragment] = nil
           @state = :fragment_state
         elsif !c.nil?
           @parse_result[:query] = nil
-          if !starts_with_windows_drive_letter?(@scanner.rest)
+          if !starts_with_windows_drive_letter?(rest)
             shorten_url_path
           else
-            @paths = []
+            @paths = nil
           end
           @state = :path_state
-          decrease_pos(c)
+          @pos -= 1
         end
       else
         @state = :path_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_file_slash_state(c)
+    def file_slash_state(c)
       if c == "/" || c == "\\"
         @state = :file_host_state
       else
         if !@base.nil? && @base[:scheme] == "file"
           @parse_result[:host] = @base[:host]
-          if !starts_with_windows_drive_letter?(@scanner.rest) && normalized_windows_drive_letter?(@base_paths[0])
-            @paths[0] += @base_paths[0]
+          if !starts_with_windows_drive_letter?(rest) && @base_paths && normalized_windows_drive_letter?(@base_paths[0])
+            @paths[0] << @base_paths[0]
           end
         end
         @state = :path_state
-        decrease_pos(c)
+        @pos -= 1
       end
     end
 
-    def on_file_host_state(c)
+    def file_host_state(c)
       if c.nil? || c == "/" || c == "\\" || c == "?" || c == "#"
-        @scanner.pos -= c.bytesize unless c.nil?
+        @pos -= 1
 
         if windows_drive_letter?(@buffer)
           @state = :path_state
         elsif @buffer.empty?
-          @parse_result[:host] = ""
+          @parse_result[:host] = nil
           @state = :path_start_state
         else
           host = @host_parser.parse(@buffer, !special_url?)
@@ -387,31 +370,31 @@ module URI
             @parse_result[:host] = host
           end
 
-          @buffer = +""
+          @buffer.clear
           @state = :path_start_state
         end
       end
 
-      @buffer += c unless c.nil?
+      @buffer << c unless c.nil?
     end
 
-    def on_path_start_state(c)
-      return if c.nil?
-
+    def path_start_state(c)
       if special_url?
-        @scanner.pos -= c.bytesize if c != "/" && c != "\\"
+        @pos -= 1 if c != "/" && c != "\\"
         @state = :path_state
       elsif c == "?"
         @state = :query_state
       elsif c == "#"
         @state = :fragment_state
       elsif c != nil
-        @scanner.pos -= c.bytesize if c != "/"
+        @pos -= 1 if c != "/"
         @state = :path_state
       end
     end
 
-    def on_path_state(c)
+    def path_state(c)
+      @paths ||= []
+
       if (c.nil? || c == "/") || (special_url? && c == "\/") || (c == "?" || c == "#")
 
         if double_dot_path_segments?(@buffer)
@@ -431,57 +414,53 @@ module URI
         @buffer = +""
 
         if c == "?"
-          @parse_result[:query] = ""
+          @parse_result[:query] = nil
           @state = :query_state
         elsif c == "#"
-          @parse_result[:frament] = ""
+          @parse_result[:frament] = nil
           @state = :fragment_state
         end
       else
-        @buffer << percent_encode(c, PATH_PERCENT_ENCODE_SET)
+        @buffer << percent_encode(c, PATH_PERCENT_ENCODE_SET, @encoding)
       end
     end
 
-    def on_opaque_path_state(c)
+    def opaque_path_state(c)
       if c == "?"
-        @parse_result[:query] = ""
+        @parse_result[:query] = nil
         @state = :query_state
       elsif c == "#"
-        @parse_result[:fragment] = ""
+        @parse_result[:fragment] = nil
         @state = :fragment_state
       elsif c == " "
-        if @scanner.rest.start_with?("?") || @scanner.rest.start_with?("#")
+        if rest.start_with?("?") || rest.start_with?("#")
           @parse_result[:path] = @parse_result[:path].to_s + "%20"
         else
           @parse_result[:path] = @parse_result[:path].to_s + " "
         end
       elsif !c.nil?
-        @parse_result[:path] = @parse_result[:path].to_s + percent_encode(c, C0_CONTROL_PERCENT_ENCODE_SET)
+        @parse_result[:path] = @parse_result[:path].to_s + percent_encode(c, C0_CONTROL_PERCENT_ENCODE_SET, @encoding)
       end
     end
 
-    def on_query_state(c)
+    def query_state(c)
+      if @encoding != Encoding::UTF_8 && (!special_url? || %w[ws wss].include?(@parse_result[:scheme]))
+        @encoding = Encoding::UTF_8
+      end
+
       if c.nil? || c == "#"
         query_percent_encode_set = special_url? ? SPECIAL_QUERY_PERCENT_ENCODE_SET : QUERY_PERCENT_ENCODE_SET
-        @parse_result[:query] = @buffer.chars.map { |c| percent_encode(c, query_percent_encode_set) }.join
-        @buffer = +""
+        @parse_result[:query] = @buffer.chars.map { |c| percent_encode(c, query_percent_encode_set, @encoding) }.join
+        @buffer.clear
         @state = :fragment_state if c == "#"
       elsif !c.nil?
         @buffer << c
       end
     end
 
-    def on_fragment_state(c)
+    def fragment_state(c)
       return if c.nil?
-      @parse_result[:fragment] = @parse_result[:fragment].to_s + percent_encode(c, FRAGMENT_PERCENT_ENCODE_SET)
-    end
-
-    def c0_control_or_space?(c)
-      c0_control? || c == " "
-    end
-
-    def c0_control?(c)
-      C0_CONTROL.include?(c.ord)
+      @parse_result[:fragment] = @parse_result[:fragment].to_s + percent_encode(c, FRAGMENT_PERCENT_ENCODE_SET, @encoding)
     end
 
     def windows_drive_letter?(str)
@@ -497,7 +476,7 @@ module URI
     end
 
     def special_url?
-      SPECIAL_SCHEME.keys.include?(@parse_result[:scheme])
+      SPECIAL_SCHEME.key?(@parse_result[:scheme])
     end
 
     def single_dot_path_segments?(c)
@@ -515,12 +494,8 @@ module URI
       @parse_result[:path]&.chomp!
     end
 
-    def decrease_pos(c)
-      if c.nil?
-        @force_continue = true
-      else
-        @scanner.pos -= c.bytesize
-      end
+    def rest
+      @uri[@pos+1..]
     end
   end
 end

metadata

@@ -1,42 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: uri-whatwg_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Yuji Yaginuma
 bindir: exe
 cert_chain: []
-date: 2025-05-12 00:00:00.000000000 Z
+date: 2025-06-12 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: strscan
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: uri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: uri-idna
   requirement: !ruby/object:Gem::Requirement
@@ -71,6 +57,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE.txt
 - README.md
 - Rakefile