tinygql 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 114da94b6827645fb92f0f3e85b9e4f34b64aaee8449c143217cc75fc903ab3c
-  data.tar.gz: 27031f9423d30f0453a6737be1184cb8c724a6290bc432b7e35dea255fa549a5
+  metadata.gz: 4a68b5713ff3ff9e39df6e5eb89f7eec5fc74f524df33f9840f58cd8ac5027ec
+  data.tar.gz: f823e82df55af3ca0327889627e99f6d896242327369b698d505b921bba819a9
 SHA512:
-  metadata.gz: a4362f8d3c3d638c715c4686ec9714d59f59d411be390cf115d0b7e8bc4f710fcbfe935ca058092f24935964fd29905643e6f1f4edeb84e238e1af0d5d2eff4e
-  data.tar.gz: 5ec4f82ff306221b2c8e008b3bb76feb1c55917ab26334e1531f7eb8e58f03e44a29901fe92765585cfa195c41a107fdd13337b3bb844f13bb9985f1394f8ee0
+  metadata.gz: a0b4cd65e5c017f820e7198479a54dde8f676eb52fac56d9672a173ef282b21862a028f4a685d59c49c6dbfbd2d213148eff27674b071cff561e1cd6fe685251
+  data.tar.gz: 266db1a7ad62bae36b375403286ec1a08b774f2fad5dc1808bacbac5e86e89ea165d9dc9d1a7598e9e4d059a7e55e0a88396df786f17aba2c6c011534a201e14

data/.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu, macos]
-        ruby: [ head, 3.2 ]
+        ruby: [ head, 3.2, truffleruby, truffleruby-head, jruby ]
 
     steps:
       - uses: actions/checkout@v3
@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu, macos]
-        ruby: [ head, 3.2 ]
+        ruby: [ head, 3.2, truffleruby, truffleruby-head ]
 
     steps:
       - uses: actions/checkout@v3

data/Gemfile

@@ -1,6 +1,5 @@
 source "https://rubygems.org"
 
 gem "benchmark-ips"
-gem "vernier"
 
 gemspec

data/README.md

@@ -66,6 +66,29 @@ ast = TinyGQL.parse "{ neat { cool } }"
 p ast.fold(Fold, []) # => ["neat", "cool"]
 ```
 
+Nodes store their position in the source GraphQL document.
+If you'd like to extract the line number of the node, you'll need to keep a reference to the document and pass it to the `line` method on the node:
+
+```ruby
+doc = <<-eod
+mutation {
+  likeStory(sturyID: 12345) {
+    story {
+      likeCount
+    }
+  }
+}
+
+eod
+
+parser = TinyGQL::Parser.new doc
+ast = parser.parse
+
+ast.find_all(&:field?).each { |node|
+  p node.name => node.line(doc)
+}
+```
+
 ## LICENSE:
 
 I've licensed this code as Apache 2.0, but the lexer is from [GraphQL-Ruby](https://github.com/rmosolgo/graphql-ruby/blob/772734dfcc7aa0513c867259912474ef0ba799c3/lib/graphql/language/lexer.rb) and is under the MIT license.

data/bin/make_hash.rb

@@ -0,0 +1,51 @@
+require "tinygql"
+
+# Calculate a perfect hash for GraphQL keywords
+
+def bits x
+  count = 0
+  while x > 0
+    count += 1
+    x >>= 1
+  end
+  count
+end
+
+# on is too short, and subscription is the longest.
+# The lexer can easily detect them by length, so lets calculate a perfect
+# hash for the rest.
+kws = TinyGQL::Lexer::KEYWORDS - ["on", "subscription"]
+MASK = (1 << bits(kws.length)) - 1
+
+prefixes = kws.map { |word| word[1, 2] }
+
+# make sure they're unique
+raise "Not unique" unless prefixes.uniq == prefixes
+
+keys = prefixes.map { |prefix|
+  prefix.bytes.reverse.inject(0) { |c, byte|
+    c << 8 | byte
+  }
+}
+
+shift = 32 - bits(kws.length) # use the top bits
+
+c = 13
+loop do
+  z = keys.map { |k| ((k * c) >> shift) & MASK }
+  break if z.uniq.length == z.length
+  c += 1
+end
+
+table = kws.zip(keys).each_with_object([]) { |(word, k), o|
+  hash = ((k * c) >> shift) & MASK
+  o[hash] = word.upcase.to_sym
+}
+
+print "KW_LUT ="
+pp table
+puts <<-eomethod
+def hash key
+  (key * #{c}) >> #{shift} & #{sprintf("%#0x", MASK)}
+end
+eomethod

data/lib/tinygql/lexer.rb

@@ -71,11 +71,36 @@ module TinyGQL
     ESCAPED_QUOTE = /\\"/;
     STRING_CHAR = /#{ESCAPED_QUOTE}|[^"\\]|#{UNICODE_ESCAPE}|#{STRING_ESCAPE}/
 
-    LIT_NAME_LUT = Literals.constants.each_with_object([]) { |n, o|
+    PUNCT_LUT = Literals.constants.each_with_object([]) { |n, o|
       o[Literals.const_get(n).ord] = n
     }
 
-    LIT = Regexp.union(Literals.constants.map { |n| Literals.const_get(n) })
+    LEAD_BYTES = Array.new(255) { 0 }
+
+    module LeadBytes
+      INT = 1
+      KW = 2
+      ELLIPSIS = 3
+      STRING = 4
+      PUNCT = 5
+      IDENT = 6
+    end
+
+    10.times { |i| LEAD_BYTES[i.to_s.ord] = LeadBytes::INT }
+
+    ("A".."Z").each { |chr| LEAD_BYTES[chr.ord] = LeadBytes::IDENT }
+    ("a".."z").each { |chr| LEAD_BYTES[chr.ord] = LeadBytes::IDENT }
+    LEAD_BYTES['_'.ord] = LeadBytes::IDENT
+
+    KEYWORDS.each { |kw| LEAD_BYTES[kw.getbyte(0)] = LeadBytes::KW }
+
+    LEAD_BYTES['.'.ord] = LeadBytes::ELLIPSIS
+
+    LEAD_BYTES['"'.ord] = LeadBytes::STRING
+
+    Literals.constants.each_with_object([]) { |n, o|
+      LEAD_BYTES[Literals.const_get(n).ord] = LeadBytes::PUNCT
+    }
 
     QUOTED_STRING = %r{#{QUOTE} ((?:#{STRING_CHAR})*) #{QUOTE}}x
     BLOCK_STRING = %r{
@@ -95,11 +120,10 @@ module TinyGQL
 
       @string = string
       @scan = StringScanner.new string
+      @start = nil
     end
 
-    def pos
-      @scan.pos
-    end
+    attr_reader :start
 
     def line
       @scan.string[0, @scan.pos].count("\n") + 1
@@ -109,66 +133,57 @@ module TinyGQL
       @scan.eos?
     end
 
-    KW_LUT = [:FRAGMENT,
-              :INTERFACE,
-              :MUTATION,
-              :EXTEND,
-              :FALSE,
-              :ENUM,
-              :TRUE,
-              :NULL,
-              nil,
-              nil,
-              nil,
-              nil,
-              nil,
-              nil,
-              :QUERY,
-              nil,
-              nil,
-              nil,
-              :REPEATABLE,
-              :IMPLEMENTS,
-              :INPUT,
-              :TYPE,
-              :SCHEMA,
-              nil,
-              nil,
-              nil,
-              :DIRECTIVE,
-              :UNION,
-              nil,
-              nil,
-              :SCALAR]
-
     def advance
       @scan.skip(IGNORE)
 
-      case
-      when @scan.eos?                      then false
-      when @scan.skip(ELLIPSIS)            then :ELLIPSIS
-      when tok = LIT_NAME_LUT[@string.getbyte(@scan.pos)] then
+      return false if @scan.eos?
+
+      @start = @scan.pos
+
+      lead_byte = @string.getbyte(@start)
+      lead_code = LEAD_BYTES[lead_byte]
+
+      if lead_code == LeadBytes::PUNCT
         @scan.pos += 1
-        tok
-      when @scan.skip(KW_RE) then
-        len = @scan.matched_size
-        return :ON if len == 2
-        return :SUBSCRIPTION if len == 12
-
-        pos = @scan.pos - len
-
-        # First 3 bytes are unique, so we'll hash on those
-        key = (@string.getbyte(pos + 2) << 16) |
-          (@string.getbyte(pos + 1) << 8) |
-          @string.getbyte(pos + 0)
-
-        KW_LUT[hash(key)]
-      when @scan.skip(IDENTIFIER)    then :IDENTIFIER
-      when @scan.skip(BLOCK_STRING)  then :STRING
-      when @scan.skip(QUOTED_STRING) then :STRING
-      when @scan.skip(NUMERIC) then (@scan[1] ? :FLOAT : :INT)
+        PUNCT_LUT[lead_byte]
+
+      elsif lead_code == LeadBytes::KW
+        if len = @scan.skip(KW_RE)
+          return :ON if len == 2
+          return :SUBSCRIPTION if len == 12
+
+          pos = @start
+
+          # Second 2 bytes are unique, so we'll hash on those
+          key = (@string.getbyte(pos + 2) << 8) | @string.getbyte(pos + 1)
+
+          KW_LUT[_hash(key)]
+        else
+          @scan.skip(IDENTIFIER)
+          :IDENTIFIER
+        end
+
+      elsif lead_code == LeadBytes::IDENT
+        @scan.skip(IDENTIFIER)
+        :IDENTIFIER
+
+      elsif lead_code == LeadBytes::INT
+        @scan.skip(NUMERIC)
+        @scan[1] ? :FLOAT : :INT
+
+      elsif lead_code == LeadBytes::ELLIPSIS
+        2.times do |i|
+          raise unless @string.getbyte(@start + i + 1) == 46
+        end
+        @scan.pos += 3
+        :ELLIPSIS
+
+      elsif lead_code == LeadBytes::STRING
+        raise unless @scan.skip(BLOCK_STRING) || @scan.skip(QUOTED_STRING)
+        :STRING
+
       else
-        @scan.getch
+        @scan.pos += 1
         :UNKNOWN_CHAR
       end
     end
@@ -193,6 +208,8 @@ module TinyGQL
       return unless tok = advance
       val = case tok
       when :STRING then string_value
+      when :ELLIPSIS then
+        @string.byteslice(@scan.pos - 3, 3)
       when *Literals.constants
         @string.byteslice(@scan.pos - 1, 1)
       else
@@ -315,10 +332,41 @@ module TinyGQL
       lines.size > 1 ? lines.join("\n") : (lines.first || "".dup)
     end
 
-    def hash key
-      # Constant came from perfect hash generation
-      m = key * 145291
-      (m >> 28) & 0x1f
+    KW_LUT =[:INTERFACE,
+             :MUTATION,
+             :EXTEND,
+             :FALSE,
+             :ENUM,
+             :TRUE,
+             :NULL,
+             nil,
+             nil,
+             nil,
+             nil,
+             nil,
+             nil,
+             nil,
+             :QUERY,
+             nil,
+             nil,
+             :REPEATABLE,
+             :IMPLEMENTS,
+             :INPUT,
+             :TYPE,
+             :SCHEMA,
+             nil,
+             nil,
+             nil,
+             :DIRECTIVE,
+             :UNION,
+             nil,
+             nil,
+             :SCALAR,
+             nil,
+             :FRAGMENT]
+
+    def _hash key
+      (key * 18592990) >> 27 & 0x1f
     end
   end
 end

data/lib/tinygql/nodes.rb

@@ -3,16 +3,16 @@ module TinyGQL
     class Node
       include Enumerable
 
-      # `pos` is the position of the _end_ of a token
-      attr_reader :pos
+      # `start` is the start position of this node in the source document
+      attr_reader :start
 
-      def initialize pos
-        @pos = pos
+      def initialize start
+        @start = start
       end
 
       # Return the line of this node given `doc`
       def line doc
-        doc[0, @pos].count("\n") + 1
+        doc[0, @start].count("\n") + 1
       end
 
       def document?; false; end

data/lib/tinygql/nodes.rb.erb

@@ -3,16 +3,16 @@ module TinyGQL
     class Node
       include Enumerable
 
-      # `pos` is the position of the _end_ of a token
-      attr_reader :pos
+      # `start` is the start position of this node in the source document
+      attr_reader :start
 
-      def initialize pos
-        @pos = pos
+      def initialize start
+        @start = start
       end
 
       # Return the line of this node given `doc`
       def line doc
-        doc[0, @pos].count("\n") + 1
+        doc[0, @start].count("\n") + 1
       end
 
 <%- nodes.each do |node| -%>

data/lib/tinygql/parser.rb

@@ -13,7 +13,7 @@ module TinyGQL
 
     def initialize doc
       @lexer = Lexer.new doc
-      @token_name = @lexer.advance
+      accept_token
     end
 
     def parse
@@ -25,7 +25,7 @@ module TinyGQL
     attr_reader :token_name
 
     def pos
-      @lexer.pos
+      @lexer.start
     end
 
     def document
@@ -477,7 +477,17 @@ module TinyGQL
     end
 
     def operation_type
-      expect_tokens([:QUERY, :MUTATION, :SUBSCRIPTION])
+      val = if at?(:QUERY)
+        "query"
+      elsif at?(:MUTATION)
+        "mutation"
+      elsif at?(:SUBSCRIPTION)
+        "subscription"
+      else
+        expect_token(:QUERY)
+      end
+      accept_token
+      val
     end
 
     def directives
@@ -528,7 +538,7 @@ module TinyGQL
 
     def variable_definition
       loc = pos
-      var = variable
+      var = variable if at?(:VAR_SIGN)
       expect_token(:COLON)
       type = self.type
       default_value = if at?(:EQUALS)
@@ -608,7 +618,7 @@ module TinyGQL
         accept_token
         Nodes::BooleanValue.new(pos, "false")
       else
-        expect_tokens([:TRUE, :FALSE])
+        expect_token(:TRUE)
       end
     end
 
@@ -643,15 +653,14 @@ module TinyGQL
     end
 
     def variable
-      return unless at?(:VAR_SIGN)
       loc = pos
-      accept_token
+      expect_token(:VAR_SIGN)
       Nodes::Variable.new loc, name
     end
 
     def name
       case token_name
-      when :IDENTIFIER then accept_token_value
+      when :IDENTIFIER then expect_token_value(:IDENTIFIER)
       when :TYPE then
         accept_token
         "type"
@@ -662,7 +671,7 @@ module TinyGQL
         accept_token
         "input"
       else
-        expect_token_value(:IDENTIFIER)
+        expect_token(:IDENTIFIER)
       end
     end
 
@@ -670,13 +679,6 @@ module TinyGQL
       @token_name = @lexer.advance
     end
 
-    # Only use when we care about the accepted token's value
-    def accept_token_value
-      token_value = @lexer.token_value
-      accept_token
-      token_value
-    end
-
     def expect_token tok
       unless at?(tok)
         raise UnexpectedToken, "Expected token #{tok}, actual: #{token_name} #{@lexer.token_value} line: #{@lexer.line}"
@@ -697,15 +699,6 @@ module TinyGQL
       token_value
     end
 
-    def expect_tokens toks
-      token_value = @lexer.token_value
-      unless toks.any? { |tok| at?(tok) }
-        raise UnexpectedToken, "Expected token #{tok}, actual: #{token_name}"
-      end
-      accept_token
-      token_value
-    end
-
     def at? tok
       token_name == tok
     end

data/lib/tinygql/version.rb

@@ -1,3 +1,3 @@
 module TinyGQL
-  VERSION = '0.2.0'
+  VERSION = '0.3.1'
 end

data/lib/tinygql/visitors.rb

@@ -461,5 +461,277 @@ module TinyGQL
       end
 
     end
+
+    module Null
+
+      def handle_document obj
+      end
+
+      def handle_operation_definition obj
+      end
+
+      def handle_variable obj
+      end
+
+      def handle_named_type obj
+      end
+
+      def handle_not_null_type obj
+      end
+
+      def handle_list_type obj
+      end
+
+      def handle_variable_definition obj
+      end
+
+      def handle_value obj
+      end
+
+      def handle_argument obj
+      end
+
+      def handle_field obj
+      end
+
+      def handle_object_field obj
+      end
+
+      def handle_int_value obj
+      end
+
+      def handle_float_value obj
+      end
+
+      def handle_string_value obj
+      end
+
+      def handle_boolean_value obj
+      end
+
+      def handle_null_value obj
+      end
+
+      def handle_enum_value obj
+      end
+
+      def handle_list_value obj
+      end
+
+      def handle_object_value obj
+      end
+
+      def handle_directive obj
+      end
+
+      def handle_type_condition obj
+      end
+
+      def handle_inline_fragment obj
+      end
+
+      def handle_fragment_spread obj
+      end
+
+      def handle_fragment_definition obj
+      end
+
+      def handle_root_operation_type_definition obj
+      end
+
+      def handle_schema_definition obj
+      end
+
+      def handle_field_definition obj
+      end
+
+      def handle_input_value_definition obj
+      end
+
+      def handle_object_type_definition obj
+      end
+
+      def handle_interface_type_definition obj
+      end
+
+      def handle_union_type_definition obj
+      end
+
+      def handle_scalar_type_definition obj
+      end
+
+      def handle_enum_value_definition obj
+      end
+
+      def handle_enum_type_definition obj
+      end
+
+      def handle_input_object_type_definition obj
+      end
+
+      def handle_object_type_extension obj
+      end
+
+      def handle_executable_directive_location obj
+      end
+
+      def handle_type_system_directive_location obj
+      end
+
+      def handle_directive_definition obj
+      end
+
+      def handle_scalar_type_extension obj
+      end
+
+      def handle_interface_type_extension obj
+      end
+
+      def handle_union_type_extension obj
+      end
+
+      def handle_enum_type_extension obj
+      end
+
+      def handle_input_object_type_extension obj
+      end
+
+    end
+
+    module NullFold
+
+      def handle_document obj, _
+      end
+
+      def handle_operation_definition obj, _
+      end
+
+      def handle_variable obj, _
+      end
+
+      def handle_named_type obj, _
+      end
+
+      def handle_not_null_type obj, _
+      end
+
+      def handle_list_type obj, _
+      end
+
+      def handle_variable_definition obj, _
+      end
+
+      def handle_value obj, _
+      end
+
+      def handle_argument obj, _
+      end
+
+      def handle_field obj, _
+      end
+
+      def handle_object_field obj, _
+      end
+
+      def handle_int_value obj, _
+      end
+
+      def handle_float_value obj, _
+      end
+
+      def handle_string_value obj, _
+      end
+
+      def handle_boolean_value obj, _
+      end
+
+      def handle_null_value obj, _
+      end
+
+      def handle_enum_value obj, _
+      end
+
+      def handle_list_value obj, _
+      end
+
+      def handle_object_value obj, _
+      end
+
+      def handle_directive obj, _
+      end
+
+      def handle_type_condition obj, _
+      end
+
+      def handle_inline_fragment obj, _
+      end
+
+      def handle_fragment_spread obj, _
+      end
+
+      def handle_fragment_definition obj, _
+      end
+
+      def handle_root_operation_type_definition obj, _
+      end
+
+      def handle_schema_definition obj, _
+      end
+
+      def handle_field_definition obj, _
+      end
+
+      def handle_input_value_definition obj, _
+      end
+
+      def handle_object_type_definition obj, _
+      end
+
+      def handle_interface_type_definition obj, _
+      end
+
+      def handle_union_type_definition obj, _
+      end
+
+      def handle_scalar_type_definition obj, _
+      end
+
+      def handle_enum_value_definition obj, _
+      end
+
+      def handle_enum_type_definition obj, _
+      end
+
+      def handle_input_object_type_definition obj, _
+      end
+
+      def handle_object_type_extension obj, _
+      end
+
+      def handle_executable_directive_location obj, _
+      end
+
+      def handle_type_system_directive_location obj, _
+      end
+
+      def handle_directive_definition obj, _
+      end
+
+      def handle_scalar_type_extension obj, _
+      end
+
+      def handle_interface_type_extension obj, _
+      end
+
+      def handle_union_type_extension obj, _
+      end
+
+      def handle_enum_type_extension obj, _
+      end
+
+      def handle_input_object_type_extension obj, _
+      end
+
+    end
   end
 end

data/lib/tinygql/visitors.rb.erb

@@ -28,6 +28,20 @@ module TinyGQL
       <%- end -%>
         seed
       end
+<% end %>
+    end
+
+    module Null
+<% nodes.each do |node| %>
+      def handle_<%= node.human_name %> obj
+      end
+<% end %>
+    end
+
+    module NullFold
+<% nodes.each do |node| %>
+      def handle_<%= node.human_name %> obj, _
+      end
 <% end %>
     end
   end

data/test/lexer_test.rb

@@ -21,7 +21,7 @@ module TinyGQL
         lexer = Lexer.new punc
         token = lexer.next_token
         expected = PUNC_LUT[punc]
-        assert_equal(expected || [:ON, "on"], token)
+        assert_equal(expected, token)
       end
     end
 
@@ -179,5 +179,28 @@ eod
 
       assert_equal words.map { |x| [x.upcase.to_sym, x] }, toks
     end
+
+    def test_looks_like_kw
+      words = ["fragment", "fragments"]
+      doc = words.join(" ")
+
+      lexer = Lexer.new doc
+      toks = []
+      while tok = lexer.advance
+        toks << tok
+      end
+
+      assert_equal [:FRAGMENT, :IDENTIFIER], toks
+    end
+
+    def test_num_with_dots
+      lexer = Lexer.new "1...2"
+      toks = []
+      while tok = lexer.advance
+        toks << tok
+      end
+
+      assert_equal [:INT, :ELLIPSIS, :INT], toks
+    end
   end
 end

data/test/parser_test.rb

@@ -58,8 +58,8 @@ mutation {
 eod
       parser = Parser.new doc
       ast = parser.parse
-      expected = ["likeStory", "story", "likeCount"].map { |str| doc.index(str) + str.bytesize }
-      assert_equal expected, ast.find_all(&:field?).map(&:pos)
+      expected = ["likeStory", "story", "likeCount"].map { |str| doc.index(str) }
+      assert_equal expected, ast.find_all(&:field?).map(&:start)
       assert_equal [2, 3, 4], ast.find_all(&:field?).map { |n| n.line(doc) }
     end
 
@@ -165,6 +165,54 @@ eod
       assert_equal 1, node.arguments.length
     end
 
+    def test_null
+      doc = <<-eod
+mutation {
+  a: likeStory(storyID: 12345) {
+    b: story {
+      c: likeCount
+    }
+  }
+}
+eod
+      viz = Module.new do
+        extend TinyGQL::Visitors::Null
+
+        def self.handle_field obj
+          true
+        end
+      end
+      parser = Parser.new doc
+      ast = parser.parse
+      ast.each { |node|
+        assert_equal(node.field?, !!node.accept(viz))
+      }
+    end
+
+    def test_null_fold
+      doc = <<-eod
+mutation {
+  a: likeStory(storyID: 12345) {
+    b: story {
+      c: likeCount
+    }
+  }
+}
+eod
+      viz = Module.new do
+        extend TinyGQL::Visitors::NullFold
+
+        def self.handle_field obj, x
+          x
+        end
+      end
+      parser = Parser.new doc
+      ast = parser.parse
+      ast.each { |node|
+        assert_equal(node.field?, !!node.fold(viz, true))
+      }
+    end
+
     def test_multiple_implements
       doc = <<-eod
 type SomeType implements a, b, c {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tinygql
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Aaron Patterson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-28 00:00:00.000000000 Z
+date: 2023-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -53,6 +53,7 @@ files:
 - Rakefile
 - benchmark/fixtures/negotiate.gql
 - bin/bench.rb
+- bin/make_hash.rb
 - bin/profile.rb
 - lib/tinygql.rb
 - lib/tinygql/lexer.rb