p_css 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df1cd693075fe04da6a0c9ce4c65c9c4ef5f85c5b84bb82bd0136cf62fe52552
-  data.tar.gz: 9a9fc875c3872c49396c5b6c753a099ba3cc344efa7aa97745958d277630c7cb
+  metadata.gz: 20afa2206ed855fdd796f19179d06a8dc8b76231c223560d59e664cb43ddf897
+  data.tar.gz: e951f89d04ff6db6f68151ad05a414e4f0f7d05f2dfc48580b22ef31b6f949de
 SHA512:
-  metadata.gz: 28c76784dac592aa39cfaa5100b69e4765861ec2ab1ed536fb95dfb963481de2bd64d6b6b954f71b14b17ea4a328cc65af6f7a63df662f2083c2ef5ec623e892
-  data.tar.gz: 291d12b999205c032e7cebaa5da4b69aff717cb9f7b9743c3c54ce99539b59ec49612b87e9bcf105d374cc79871d8955fc57f18200b8ba76c396894db14f349f
+  metadata.gz: dc533dd2a146654d7a622b3206568168bea1e404b163ef6b24ae1c841ef4cd5ff3a4621bc6794f64178ffd5c87a0c4e6a46f70fd6c7999a52539091849ae2941
+  data.tar.gz: 67d08837559466bc5713aa6a40d8e67030ff9389228527b4ed8b4bb5e1121965fa90a31e6cf5aae40167151e535ae83b9f6073068e72d495a5098559b331698d

data/lib/css/code_points.rb

@@ -1,36 +1,59 @@
 module CSS
   # Character class predicates from CSS Syntax §4.2 Definitions, plus the
   # U+FFFD replacement character used both during tokenization and
-  # serialization. Implemented with char comparisons rather than regex to
-  # avoid pattern-match overhead in the tokenizer's inner loop.
+  # serialization.
+  #
+  # ASCII bytes are looked up in a precomputed boolean table (one Array
+  # access + one branch); non-ASCII code points (>= 0x80) are always
+  # ident-cp / ident-start per spec, so the helpers fall back to a single
+  # `c.ord >= 0x80` check. Avoids the chain of `String#<=>` calls a
+  # range-style predicate would dispatch.
   module CodePoints
     REPLACEMENT = "�".freeze
 
+    def self.build_table(*ranges_or_ints)
+      Array.new(128, false).tap {|a|
+        ranges_or_ints.each {|r|
+          if r.is_a?(Range) then r.each { a[it] = true }
+          else                   a[r] = true
+          end
+        }
+      }.freeze
+    end
+
+    DIGIT_TABLE       = build_table(0x30..0x39)
+    HEX_DIGIT_TABLE   = build_table(0x30..0x39, 0x41..0x46, 0x61..0x66)
+    IDENT_START_TABLE = build_table(0x41..0x5A, 0x61..0x7A, 0x5F)
+    IDENT_CP_TABLE    = build_table(0x30..0x39, 0x41..0x5A, 0x61..0x7A, 0x5F, 0x2D)
+
     module_function
 
     def digit?(c)
-      !c.nil? && c >= '0' && c <= '9'
+      return false if c.nil?
+
+      o = c.ord
+      o < 128 && DIGIT_TABLE[o]
     end
 
     def hex_digit?(c)
       return false if c.nil?
 
-      (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')
+      o = c.ord
+      o < 128 && HEX_DIGIT_TABLE[o]
     end
 
     def ident_start_code_point?(c)
       return false if c.nil?
-      return true  if c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 
-      c.ord >= 0x80
+      o = c.ord
+      o >= 128 || IDENT_START_TABLE[o]
     end
 
     def ident_code_point?(c)
       return false if c.nil?
-      return true  if c == '_' || c == '-' || (c >= '0' && c <= '9')
-      return true  if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 
-      c.ord >= 0x80
+      o = c.ord
+      o >= 128 || IDENT_CP_TABLE[o]
     end
   end
 end

data/lib/css/token.rb

@@ -17,7 +17,7 @@ module CSS
       eof
     ].freeze
 
-    attr_reader :type, :value, :flag, :unit, :position
+    attr_reader :type, :value, :flag, :unit
 
     def initialize(type, value = nil, flag: nil, unit: nil, position: nil)
       raise ArgumentError, "unknown token type: #{type.inspect}" unless TYPES.include?(type)
@@ -58,21 +58,50 @@ module CSS
       type == :whitespace || type == :comment
     end
 
-    # Mutating: assigns the token's source position and returns self. Used
-    # by the tokenizer so each token requires only a single allocation.
-    def assign_position!(pos)
-      @position = pos
+    # Most tokens never have their `position` read after parsing, so the
+    # tokenizer plants raw offsets + a shared `@newlines` reference here
+    # via this method, and `Token#position` materializes the `Position`
+    # Data on first read.
+    def assign_source!(start_offset, end_offset, newlines)
+      @start_offset = start_offset
+      @end_offset   = end_offset
+      @newlines     = newlines
       self
     end
 
+    # Returns nil for tokens built without source info (i.e. tokens
+    # constructed by hand or via `Token.new(:eof)`).
+    def position
+      return @position if @position
+      return nil unless instance_variable_defined?(:@start_offset)
+
+      @position = compute_position
+    end
+
+    # Reads `@position` directly so debug-style introspection doesn't
+    # materialize a `Position` as a side effect.
     def inspect
       parts = ["type=#{type.inspect}"]
       parts << "value=#{value.inspect}" unless value.nil?
       parts << "flag=#{flag.inspect}"   unless flag.nil?
       parts << "unit=#{unit.inspect}"   unless unit.nil?
-      parts << "@#{position}"           unless position.nil?
+      parts << "@#{@position}"          if @position
 
       "#<CSS::Token #{parts.join(' ')}>"
     end
+
+    private
+
+    def compute_position
+      idx     = @newlines.bsearch_index { it >= @start_offset } || @newlines.size
+      prev_nl = idx.zero? ? -1 : @newlines[idx - 1]
+
+      Position.new(
+        line:       idx + 1,
+        column:     @start_offset - prev_nl,
+        offset:     @start_offset,
+        end_offset: @end_offset
+      )
+    end
   end
 end

data/lib/css/tokenizer.rb

@@ -2,9 +2,8 @@ module CSS
   # Tokenizer based on CSS Syntax Module Level 3/4 §4.
   # https://www.w3.org/TR/css-syntax-3/#tokenization
   #
-  # Not thread-safe: an instance carries mutable cursors (`@pos`,
-  # `@newline_cursor`) that advance over the input. Allocate one
-  # tokenizer per thread.
+  # Not thread-safe: an instance carries a mutable cursor (`@pos`) that
+  # advances over the input. Allocate one tokenizer per thread.
   class Tokenizer
     include CodePoints
 
@@ -26,9 +25,9 @@ module CSS
 
     def initialize(input, preserve_comments: false)
       @chars             = preprocess(input)
+      @length            = @chars.length
       @pos               = 0
       @newlines          = collect_newline_offsets(@chars)
-      @newline_cursor    = 0
       @preserve_comments = preserve_comments
     end
 
@@ -48,13 +47,12 @@ module CSS
     def next_token
       consume_comments unless @preserve_comments
 
-      return Token.new(:eof) if @pos >= @chars.length
+      return Token.new(:eof) if @pos >= @length
 
       start_offset = @pos
       tok          = consume_one_token
-      line, column = line_column_at(start_offset)
 
-      tok.assign_position!(Position.new(line:, column:, offset: start_offset, end_offset: @pos))
+      tok.assign_source!(start_offset, @pos, @newlines)
     end
 
     private
@@ -174,23 +172,6 @@ module CSS
       offsets
     end
 
-    # Newline characters themselves are reported as belonging to the
-    # line they terminate (col = offset + 1 on line 1, etc).
-    #
-    # Tokens are emitted in order, so the offsets passed in are
-    # monotonically non-decreasing. We keep a running cursor into
-    # `@newlines` and advance linearly — amortized O(1) per call,
-    # vs. O(log n) per call with a fresh `bsearch`.
-    def line_column_at(offset)
-      while @newline_cursor < @newlines.size && @newlines[@newline_cursor] < offset
-        @newline_cursor += 1
-      end
-
-      prev_nl = @newline_cursor.zero? ? -1 : @newlines[@newline_cursor - 1]
-
-      [@newline_cursor + 1, offset - prev_nl]
-    end
-
     def whitespace?(c)
       c == ' ' || c == "\n" || c == "\t"
     end
@@ -267,7 +248,7 @@ module CSS
     end
 
     def eof?
-      @pos >= @chars.length
+      @pos >= @length
     end
 
     def consume_whitespace

data/lib/css/version.rb

@@ -1,3 +1,3 @@
 module CSS
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/sig/css/token.rbs

@@ -22,7 +22,7 @@ module CSS
     def comment?: () -> bool
     def trivia?: () -> bool
 
-    def assign_position!: (Position pos) -> self
+    def assign_source!: (Integer start_offset, Integer end_offset, Array[Integer] newlines) -> self
 
     def ==: (untyped other) -> bool
     def eql?: (untyped other) -> bool

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: p_css
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Keita Urashima