gammo 0.1.0

data/lib/gammo/tokenizer/escape.rb

@@ -0,0 +1,174 @@
+require 'gammo/tokenizer/entity'
+
+module Gammo
+  class Tokenizer
+    module Escape
+      LONGEST_ENTITY_WITHOUT_SEMICOLON = 6
+      ESCAPE_REPLACEMENT_TABLE = {
+        ?&  => '&',
+        ?'  => ''',
+        ?<  => '&lt;',
+        ?>  => '&gt;',
+        ?"  => '&#34;',
+        ?\r => '&#13;',
+      }.freeze
+
+      REPLACEMENT_TABLE = [
+        "\u{20AC}",
+        "\u{0081}",
+        "\u{201A}",
+        "\u{0192}",
+        "\u{201E}",
+        "\u{2026}",
+        "\u{2020}",
+        "\u{2021}",
+        "\u{02C6}",
+        "\u{2030}",
+        "\u{0160}",
+        "\u{2039}",
+        "\u{0152}",
+        "\u{008D}",
+        "\u{017D}",
+        "\u{008F}",
+        "\u{0090}",
+        "\u{2018}",
+        "\u{2019}",
+        "\u{201C}",
+        "\u{201D}",
+        "\u{2022}",
+        "\u{2013}",
+        "\u{2014}",
+        "\u{02DC}",
+        "\u{2122}",
+        "\u{0161}",
+        "\u{203A}",
+        "\u{0153}",
+        "\u{009D}",
+        "\u{017E}",
+        "\u{0178}",
+      ].freeze
+
+      # Escapes given string according to {ESCAPE_REPLACEMENT_TABLE}.
+      def escape(s)
+        s.gsub!(/[&'<>"\r]/) { |ch| ESCAPE_REPLACEMENT_TABLE[ch] }
+      end
+
+      # Unescapes given data.
+      # @param [String] data
+      # @return [String, nil]
+      def unescape(data, **options)
+        return unless data
+        data.each_byte.with_index do |byte, i|
+          next unless byte.chr == ?&
+          dst, src = unescape_entity(data, i, i, **options)
+          while src < data.bytes.length
+            byte = data.getbyte(src)
+            if byte.chr == ?&
+              dst, src = unescape_entity(data, dst, src, **options)
+            else
+              data.setbyte(dst, byte)
+              dst, src = dst + 1, src + 1
+            end
+          end
+          return data.byteslice(0, dst)
+        end
+      end
+
+      private
+
+      def unescape_entity(data, dst, src, in_attribute: false)
+        # No need to count "&".
+        i, s = 1, data.byteslice(src..-1)
+        swap(data, dst, src) if s.length <= 1
+        return unescape_sharp_entity(data, s, dst, src, i) if s[i] == ?#
+        i = consume_entity_chars(s, i)
+        name = s.byteslice(1, i - 1)
+        unless name == '' || (in_attribute && like_query_params?(name, s, i))
+          entities = Entity::CODEPOINT[name.to_sym]
+          entities = entities ? [entities] : Entity::TWO_CODEPOINTS[name.to_sym]
+          return replace_entity(entities, data, dst, src, i) if entities
+          unless in_attribute
+            max = name.length - 1
+            max = LONGEST_ENTITY_WITHOUT_SEMICOLON if max > LONGEST_ENTITY_WITHOUT_SEMICOLON
+            max.downto(1) do |n|
+              if entities = Entity::CODEPOINT[name.byteslice(0, n).to_sym]
+                return replace_entity([entities], data, dst, src, n + 1)
+              end
+            end
+          end
+        end
+        dst1, src1 = dst + i, src + i
+        data[dst, dst1] = data[src, src1]
+        [dst1, src1]
+      end
+
+      def unescape_sharp_entity(data, s, dst, src, i)
+        return swap(data, dst, src) if s.length <= 3
+        i += 1
+        ch = s[i]
+        hex = false
+        if ch == ?x || ch == ?X
+          hex = true
+          i += 1
+        end
+        x = ?\x0
+        while i < s.length
+          ch = s[i]
+          i += 1
+          if hex
+            if ?0 <= ch && ch <= ?9
+              x = 16 * x.ord + ch.ord - ?0.ord
+              next
+            elsif ?a <= ch && ch <= ?f
+              x = 16 * x.ord + ch.ord - ?a.ord + 10
+              next
+            elsif ?A <= ch && ch <= ?F
+              x = 16 * x.ord + ch.ord - ?A.ord + 10
+              next
+            end
+          elsif (?0 <= ch && ch <= ?9)
+            x = 10 * x.ord + ch.ord - ?0.ord
+            next
+          end
+          i -= 1 if ch != ?;
+          break
+        end
+        return swap(data, dst, src) if i <= 3
+        if 0x80 <= x && x <= 0x9F
+          x = REPLACEMENT_TABLE[x - 0x80].ord
+        elsif x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF
+          x = "\u{FFFD}".ord
+        end
+        x.chr(Encoding::UTF_8).each_byte.with_index { |byte, j| data.setbyte(dst + j, byte) }
+        [dst + x.chr(Encoding::UTF_8).bytes.length, src + i]
+      end
+
+      def swap(data, dst, src)
+        data[dst] = data[src]
+        [dst + 1, src + 1]
+      end
+
+      def consume_entity_chars(s, i)
+        while i < s.length
+          ch = s[i]
+          i += 1
+          next if ?a <= ch && ch <= ?z || ?A <= ch && ch <= ?Z || ?0 <= ch && ch <= ?9
+          i -= 1 if ch != ?;
+          break
+        end
+        i
+      end
+
+      def like_query_params?(name, s, i)
+        name[name.length - 1] != ?; && s.length > i && s[i] == ?=
+      end
+
+      def replace_entity(entities, t, dst, src, i)
+        [entities.inject(dst) { |sum, ch|
+          ch.each_byte.with_index { |byte, j| t.setbyte(sum + j, byte) }
+         sum + ch.bytes.length
+        }, src + i]
+      end
+    end
+  end
+end

data/lib/gammo/tokenizer/script_scanner.rb

@@ -0,0 +1,229 @@
+require 'gammo/tokenizer/debug'
+
+module Gammo
+  class Tokenizer
+    class ScriptScanner
+      include Debug
+
+      attr_reader :scanner, :buffer, :raw_tag
+
+      alias_method :debug?, :debug
+
+      def initialize(scanner, raw_tag:, debug: false)
+        @scanner = scanner
+        @buffer  = ''
+        @raw_tag = raw_tag
+        @debug   = debug
+      end
+
+      def scan
+        scan_script_data
+        buffer
+      end
+
+      private
+
+      def with_extendable_stack
+        begin
+          yield
+        rescue SystemStackError
+          Fiber.new { yield }.resume
+        end
+      end
+
+      def scan_script_data
+       with_extendable_stack do
+          consume do |byte|
+            byte == ?< ? scan_script_data_less_than_sign : scan_script_data
+          end
+       end
+      end
+
+      def scan_script_data_less_than_sign
+        consume do |byte|
+          case byte
+          when ?/ then return scan_script_data_end_tag_open
+          when ?! then return scan_script_data_escape_start
+          end
+          revert_byte
+          scan_script_data
+        end
+      end
+
+      def scan_script_data_end_tag_open
+        return if scan_raw_end_tag? || scanner.eos?
+        scan_script_data
+      end
+
+      def scan_script_data_escape_start
+        consume do |byte|
+          return scan_script_data_escape_start_dash if byte == ?-
+          revert_byte
+          scan_script_data
+        end
+      end
+
+      def scan_script_data_escape_start_dash
+        consume do |byte|
+          return scan_script_data_escaped_dash_dash if byte == ?-
+          revert_byte
+          scan_script_data
+        end
+      end
+
+      def scan_script_data_escaped
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_escaped_dash
+          when ?< then return scan_script_data_escaped_less_than_sign
+          else return scan_script_data_escaped
+          end
+        end
+      end
+
+      def scan_script_data_escaped_dash
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_escaped_dash_dash
+          when ?< then return scan_script_data_escaped_less_than_sign
+          else return scan_script_data_escaped
+          end
+        end
+      end
+
+      def scan_script_data_escaped_dash_dash
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_escaped_dash_dash
+          when ?< then return scan_script_data_escaped_less_than_sign
+          when ?> then return scan_script_data
+          else return scan_script_data_escaped
+          end
+        end
+      end
+
+      def scan_script_data_escaped_less_than_sign
+        consume do |byte|
+          return scan_script_data_escaped_end_tag_open if byte == ?/
+          return scan_script_data_double_escape_start if byte =~ /[a-zA-Z]/
+          revert_byte
+          scan_script_data
+        end
+      end
+
+      def scan_script_data_escaped_end_tag_open
+        return if scan_raw_end_tag? || scanner.eos?
+        scan_script_data_escaped
+      end
+
+      def scan_script_data_double_escape_start
+        revert_byte
+        'script'.each_char.with_index do |ch, index|
+          ch = scanner.get_byte
+          buffer << ch
+          return if scanner.eos?
+          unless ch.downcase == 'script'[index]
+            revert_byte
+            return scan_script_data_escaped
+          end
+        end
+        byte = scanner.get_byte
+        buffer << byte
+        return if scanner.eos?
+        case byte
+        when ?\s, ?/, ?> then return scan_script_data_double_escaped
+        else
+          revert_byte
+          scan_script_data_escaped
+        end
+      end
+
+      def scan_script_data_double_escaped
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_double_escaped_dash
+          when ?< then return scan_script_data_double_escaped_less_than_sign
+          else return scan_script_data_double_escaped
+          end
+        end
+      end
+
+      def scan_script_data_double_escaped_dash
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_double_escaped_dash_dash
+          when ?< then return scan_script_data_double_escaped_less_than_sign
+          else return scan_script_data_double_escaped
+          end
+        end
+      end
+
+      def scan_script_data_double_escaped_dash_dash
+        consume do |byte|
+          case byte
+          when ?- then return scan_script_data_double_escaped_dash_dash
+          when ?< then return scan_script_data_double_escaped_less_than_sign
+          when ?> then return scan_script_data
+          else return scan_script_data_double_escaped
+          end
+        end
+      end
+
+      def scan_script_data_double_escaped_less_than_sign
+        consume do |byte|
+          return scan_script_data_double_escape_end if byte == ?/
+          revert_byte
+          scan_script_data_double_escaped
+        end
+      end
+
+      def scan_script_data_double_escape_end
+        if scan_raw_end_tag?
+          end_tag = "</#{raw_tag}>"
+          # Last matched char needs to be concatenated.
+          buffer << scanner.string.slice(scanner.pos, end_tag.length)
+          scanner.pos += end_tag.length
+          return scan_script_data_escaped
+        end
+        return if scanner.eos?
+        scan_script_data_double_escaped
+      end
+
+      def consume
+        return unless byte = scanner.get_byte
+        buffer << byte
+        yield byte
+      end
+
+      def revert_byte
+        @buffer = buffer.slice(0, buffer.length - 1)
+        scanner.unscan
+      end
+
+      def scan_raw_end_tag?
+        raw_tag.each_char do |ch|
+          return false unless byte = scanner.get_byte
+          if byte.downcase != ch
+            scanner.unscan
+            return false
+          end
+          buffer << byte
+        end
+        case byte = scanner.get_byte
+        when ?>, ?\s, ?/
+          desired = 3 + raw_tag.length
+          scanner.pos -= desired
+          @buffer = buffer.slice(0, buffer.length - desired + 1)
+          return true
+        when nil
+          return false
+        else
+          buffer << byte
+        end
+        scanner.unscan
+        @buffer = buffer.slice(0, buffer.length - 1)
+        false
+      end
+    end
+  end
+end

data/lib/gammo/tokenizer/tokens.rb

@@ -0,0 +1,66 @@
+require 'gammo/tokenizer/escape'
+
+module Gammo
+  class Tokenizer
+    class BaseToken
+      attr_accessor :attributes, :data, :tag
+
+      def initialize(data = '', attributes: [], tag: nil)
+        @data       = data
+        @attributes = attributes
+        @tag        = tag
+      end
+
+      def concat(s)
+        data << s
+      end
+    end
+
+    class EscapedToken < BaseToken
+      include Escape
+
+      NULL        = ?\x00.freeze
+      REPLACEMENT = "\ufffd".freeze
+
+      attr_accessor :convert_null, :raw
+
+      def initialize(data = nil, raw: false, convert_null: false, **options)
+        super(data, **options)
+        @raw          = raw
+        @convert_null = convert_null
+        load_data(data)
+      end
+
+      def load_data(raw_data)
+        unless raw_data
+          @data = nil
+          return
+        end
+        raw_data = convert_newlines(raw_data).force_encoding(Encoding::UTF_8)
+        raw_data = raw_data.gsub(%r{#{NULL}}, REPLACEMENT) if should_convert_null?(raw_data)
+        @data = require_raw_data? ? raw_data : unescape(raw_data, in_attribute: false)
+      end
+
+      private
+
+      alias_method :convert_null?, :convert_null
+      alias_method :require_raw_data?, :raw
+
+      def should_convert_null?(data)
+        data && (convert_null? || self.class == CommentToken) && data.include?(NULL)
+      end
+
+      def convert_newlines(s)
+        s.gsub(/(\r\n|\r)/, ?\n)
+      end
+    end
+
+    ErrorToken          = Class.new(BaseToken)
+    TextToken           = Class.new(EscapedToken)
+    StartTagToken       = Class.new(BaseToken)
+    EndTagToken         = Class.new(BaseToken)
+    SelfClosingTagToken = Class.new(BaseToken)
+    CommentToken        = Class.new(EscapedToken)
+    DoctypeToken        = Class.new(EscapedToken)
+  end
+end