ruby_tree_sitter 0.20.8.1 → 0.20.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94cb2ce80adc91e98ee79fb48b89a8f35b750b57b38272d0f1256a2641b36434
-  data.tar.gz: a84e39a7e69d457f187ae20627a03a19476a6956ba05fe53d3088cd4ae899e22
+  metadata.gz: f897961a83176c564dd214e3b8c893880302aa9552cac65b9f0c2e06e396ba7c
+  data.tar.gz: 0ba1af8010641341f95bab2bc7182faa94181da5f862a4224ef5d87c44d1eff9
 SHA512:
-  metadata.gz: c9d6f4150314194818f1d61030324b32ff763d2d0c799a76373ec6e4d3d1fc85b660c8848df867c73eafb3903ac426f684010465cedcbd1e1a11d2cfbc11efdc
-  data.tar.gz: aeed7a952d3b640d8f9bc2c35abf73e05f3c0d921f13776ca938b6dbdaf21eaec97f617ee6984f399ab320c2066b383cc081aee287a3ac2123c6e0185ff5df74
+  metadata.gz: a3ff40e54f3d0b87984d14217f66a16f1c61a8c59c6a41d996e1792c513b2a8a3937f44839253d994d876bcbd6c1551b6c657b1cb6b6f3a0c84d752cf226da88
+  data.tar.gz: fac9fc60614e9ebd14e3181a4d2519cb4da9d170ea51326585204d3b65265349b3cdf964967c57d381bf795023c7b5e839ab406ba536ee5f971228efac9d3a5e

data/README.md

@@ -85,24 +85,47 @@ brew install tree-sitter
 
 ## Install
 
-We haven't released the gem on `Rubygems` as of yet, but we'e planning on doing so.
+From [rubygems](https://rubygems.org/gems/ruby_tree_sitter), in your `Gemfile`:
 
-Meanwhile, please install from `git` source, which will compile on installation.
+```ruby
+gem 'ruby_tree_sitter', '~> 0.20.8.1'
+```
 
-If you don't want to install from `git`, or if you don't want to compile on
-install, download a native gem from this repository's
-[releases](https://github.com/Faveod/ruby-tree-sitter/releases), or you can
-compile it yourself (see [Build from
-source](docs/Development.md#build-from-source) .)
+Or manually:
+
+```sh
+gem install ruby_tree_sitter
+```
 
-### Gemfile
+Or from `git` sources, which will compile on installation:
 
 ```ruby
-gem 'tree_sitter', git: 'https://github.com/Faveod/ruby-tree-sitter'
+gem 'ruby_tree_sitter', git: 'https://github.com/Faveod/ruby-tree-sitter'
+```
+
+### Disable system libraries
+
+To install with `--disable-sys-lib`, you can either:
+
+```sh
+gem install ruby_tree_sitter -- --disable-sys-libs
 ```
 
-If you chose to install a native gem, then you'd have to download it somewhere
-and then specify `path` as such:
+Or via bundle:
+
+```sh
+bundle config set build.ruby_tree_sitter --disable-sys-libs
+```
+
+### No compilation
+
+If you don't want to install from `rubygems`, `git`, or if you don't want to
+compile on install, then download a native gem from this repository's
+[releases](https://github.com/Faveod/ruby-tree-sitter/releases), or you can
+compile it yourself (see [Build from
+source](docs/Development.md#build-from-source) .)
+
+In that case, you'd have to point your `Gemfile` to the `gem` as such:
 
 ``` ruby
 gem 'tree_sitter', path: 'path/to/native/tree_sitter.gem'
@@ -118,6 +141,16 @@ You will have to install parsers yourself, either by:
    [Faveod/tree-sitter-parsers](https://github.com/Faveod/tree-sitter-parsers)
    which supports numerous architectures.
 
+### A note on static vs dynamic linking
+
+This extension will statically link against a downloaded version of
+`tree-sitter` when you use the `--disable-sys-lib`.  So any installed version of
+`tree-sitter` will not be loaded.
+
+The native gems are also statically linked.
+
+All other methods will dynamically link against the installed `tree-sitter`.
+
 ## Examples
 
 See `examples` directory.

data/ext/tree_sitter/extconf.rb

@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 require 'mkmf'
 require 'pathname'
 
+require_relative 'repo'
+
 # ################################## #
 #             Some helpers           #
 # ################################## #
@@ -14,63 +18,39 @@ def system_tree_sitter?
   enable_config('sys-libs', true)
 end
 
-def sh cmd
-  if !system(cmd)
-    abort <<~MSG
-
-      Failed to run: #{cmd}
-
-      exiting…
-
-    MSG
-  end
-end
-
 def env_var_on?(var)
   %w[1 on true t yes y].include?(ENV.fetch(var, '').downcase)
 end
 
 # ################################## #
-#            System lib              #
-#                                    #
-#                OR                  #
-#                                    #
-#          Downloaded libs           #
+#    System lib vs Downloaded lib    #
 # ################################## #
 
 dir_include, dir_lib =
   if system_tree_sitter?
-    [['/opt/include', '/opt/local/include', '/usr/include', '/usr/local/include'],
-     ['/opt/lib', '/opt/local/lib', '/usr/lib', '/usr/local/lib']]
+    [
+      %w[/opt/include /opt/local/include /usr/include /usr/local/include],
+      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib]
+    ]
   else
-    src = Pathname.pwd / "tree-sitter-#{TreeSitter::VERSION}"
-    if !Dir.exist? src
-      if find_executable('git')
-        sh "git clone https://github.com/tree-sitter/tree-sitter #{src}"
-        sh "cd #{src} && git checkout tags/v#{TreeSitter::VERSION}"
-      elsif find_executable('curl')
-        if find_executable('tar')
-          sh "curl -L https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v#{TreeSitter::VERSION}.tar.gz -o tree-sitter-v#{TreeSitter::VERSION}.tar.gz"
-          sh "tar -xf tree-sitter-v#{TreeSitter::VERSION}.tar.gz"
-        elsif find_executable('zip')
-          sh "curl -L https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v#{TreeSitter::VERSION}.zip -o tree-sitter-v#{TreeSitter::VERSION}.zip"
-          sh "unzip -q tree-sitter-v#{TreeSitter::VERSION}.zip"
-        else
-          abort('Could not find `tar` or `zip` (and `git` was not found!)')
-        end
-      else
-        abort('Could not find `git` or `curl` to download tree-sitter and build from sources.')
-      end
+    repo = TreeSitter::Repo.new
+    if !repo.download
+      msg = <<~MSG
+
+        Could not fetch tree-sitter sources:
+
+        #{repo.exe.map { |k, v| "#{k}: #{v}" }.join("\n")}
+
+      MSG
+      abort(msg)
     end
 
     # We need to make sure we're selecting the proper toolchain.
     # Especially needed for corss-compilation.
     ENV.store('CC', RbConfig::CONFIG['CC'])
-    # We need to clean because the same folder is used over and over
-    # by rake-compiler-dock
-    sh "cd #{src} && make clean && make"
-
-    [[src / 'lib' / 'include'], [src.to_s]]
+    repo.compile
+    repo.keep_static_lib
+    repo.include_and_lib_dirs
   end
 
 # ################################## #
@@ -78,7 +58,6 @@ dir_include, dir_lib =
 # ################################## #
 
 header = find_header('tree_sitter/api.h', *dir_include)
-
 library = find_library('tree-sitter',   # libtree-sitter
                        'ts_parser_new', # a symbol
                        *dir_lib)
@@ -100,9 +79,7 @@ if !header || !library
   MSG
 end
 
-if env_var_on?('TREE_SITTER_PEDANTIC')
-  cflags << '-Werror'
-end
+cflags << '-Werror' if env_var_on?('TREE_SITTER_PEDANTIC')
 
 if env_var_on?('DEBUG')
   cflags << '-fbounds-check'

data/ext/tree_sitter/logger.c

@@ -61,12 +61,12 @@ static void logger_payload_set(logger_t *logger, VALUE value) {
     if (rb_respond_to(logger->payload, rb_intern("printf"))) {
       logger->data.log = logger_log_printf;
     } else if (rb_respond_to(logger->payload, rb_intern("puts"))) {
-      logger->data.log = &logger_log_puts;
+      logger->data.log = logger_log_puts;
     } else {
-      logger->data.log = &logger_log_write;
+      logger->data.log = logger_log_write;
     }
   } else if (!NIL_P(logger->payload)) {
-    logger->data.log = &logger_log_write;
+    logger->data.log = logger_log_write;
   }
 }
 

data/ext/tree_sitter/repo.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative '../../lib/tree_sitter/version'
+
+module TreeSitter
+  # Fetches tree-sitter sources.
+  class Repo
+    attr_reader :exe, :src, :url, :version
+
+    def initialize
+      @version = TREESITTER_VERSION
+
+      # `tree-sitter-@version` is the name produced by tagged releases of sources
+      # by git, so we use it everywhere, including when cloning from git.
+      @src = Pathname.pwd / "tree-sitter-#{@version}"
+
+      @url = {
+        git: 'https://github.com/tree-sitter/tree-sitter',
+        tar: "https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v#{@version}.tar.gz",
+        zip: "https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v#{@version}.zip"
+      }
+
+      @exe = {}
+      %i[curl git tar wget zip].each do |cmd|
+        @exe[cmd] = find_executable(cmd.to_s)
+      end
+    end
+
+    def compile
+      # We need to clean because the same folder is used over and over
+      # by rake-compiler-dock
+      sh "cd #{src} && make clean && make"
+    end
+
+    def exe?(name)
+      @exe[name]
+    end
+
+    def extract?
+      !exe.filter { |k, v| %i[tar zip].include?(k) && v }.empty?
+    end
+
+    def download
+      # TODO: should we force re-download? Maybe with a flag?
+      return true if Dir.exist? src
+
+      res = false
+      %w[git curl wget].each do |cmd|
+        res =
+          if find_executable(cmd)
+            send("sources_from_#{cmd}")
+          else
+            false
+          end
+        break if res
+      end
+
+      res
+    end
+
+    def include_and_lib_dirs
+      [[src / 'lib' / 'include'], [src.to_s]]
+    end
+
+    def keep_static_lib
+      src
+        .children
+        .filter { |f| /\.(dylib|so)/ =~ f.basename.to_s }
+        .each(&:unlink)
+    end
+
+    def sh(cmd)
+      return if system(cmd)
+
+      abort <<~MSG
+
+        Failed to run: #{cmd}
+
+        exiting …
+
+      MSG
+    end
+
+    def sources_from_curl
+      return false if !exe?(:curl) || !extract?
+
+      if exe?(:tar)
+        sh "curl -L #{url[:tar]} -o tree-sitter-v#{version}.tar.gz"
+        sh "tar -xf tree-sitter-v#{version}.tar.gz"
+      elsif exe?(:zip)
+        sh "curl -L #{url[:zip]} -o tree-sitter-v#{version}.zip"
+        sh "unzip -q tree-sitter-v#{version}.zip"
+      end
+
+      true
+    end
+
+    def sources_from_git
+      return false if !exe?(:git)
+
+      sh "git clone #{url[:git]} #{src}"
+      sh "cd #{src} && git checkout tags/v#{version}"
+
+      true
+    end
+
+    def sources_from_wget
+      return false if !exe?(:wget) || !extract?
+
+      if exe?(:tar)
+        sh "wget #{url[:tar]} -O tree-sitter-v#{version}.tar.gz"
+        sh "tar -xf tree-sitter-v#{version}.tar.gz"
+      elsif exe?(:zip)
+        sh "wget #{url[:zip]} -O tree-sitter-v#{version}.zip"
+        sh "unzip -q tree-sitter-v#{version}.zip"
+      end
+
+      true
+    end
+  end
+end

data/lib/tree_sitter/node.rb

@@ -76,7 +76,6 @@ module TreeSitter
     end
 
     def respond_to_missing?(*args)
-      init_fields
       args.length == 1 && fields.include?(args[0])
     end
 
@@ -143,7 +142,12 @@ module TreeSitter
     # uses         named_child                   | field_name_for_child
     #              child_by_field_name           |   via each_node
     #              ------------------------------+----------------------
-    def fetch(*keys)
+    # @param all [Boolean] If `true`, return an array of nodes for all the
+    # demanded keys, putting `nil` for missing ones.  If `false`, return the
+    # same array after calling `compact`. Defaults to `false`.
+    #
+    # See {#fetch_all}.
+    def fetch(*keys, all: false, **_kwargs)
       dict = {}
       keys.each.with_index do |k, i|
         dict[k.to_s] = i
@@ -152,13 +156,25 @@ module TreeSitter
       res = {}
       each_field do |f, c|
         if dict.key?(f)
-          res[dict[f]] = c
+          res[f] = c
           dict.delete(f)
         end
         break if dict.empty?
       end
 
-      res.sort.map { |_, v| v }
+      res = keys.uniq.map { |k| res[k.to_s] }
+      res = res.compact if !all
+      res
+    end
+
+    # Access all named children of a node, returning `nil` for missing ones.
+    #
+    # Equivalent to `fetch(…, all: true)`.
+    #
+    # See {#fetch}.
+    def fetch_all(*keys, **kwargs)
+      kwargs[:all] = true
+      fetch(*keys, **kwargs)
     end
   end
 end

data/lib/tree_sitter/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module TreeSitter
-  VERSION = '0.20.8.1'
+  TREESITTER_VERSION = '0.20.8'
+  VERSION = "#{TREESITTER_VERSION}.3".freeze
 end

data/test/tree_sitter/js_test.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative '../test_helper'
+
+js = TreeSitter.lang('javascript')
+parser = TreeSitter::Parser.new
+parser.language = js
+
+program = <<~JS
+  let a = 42;
+JS
+
+describe 'loading a language' do
+  before do
+    parser.reset
+  end
+
+  it 'must set/get the same language' do
+    parser.language = js
+    assert_equal js, parser.language
+  end
+end
+
+describe 'parse_string' do
+  before do
+    parser.reset
+  end
+
+  it 'must parse nil' do
+    res = parser.parse_string(nil, nil)
+    assert_nil res
+  end
+
+  [
+    ['empty', '', 0],
+    ['valid', program, 1],
+    # ['invalid', margorp, 3]
+  ].each do |q, p, c|
+    it "must parse #{q} programs" do
+      res = parser.parse_string(nil, p)
+      assert_instance_of TreeSitter::Tree, res
+
+      root = res.root_node
+      assert_instance_of TreeSitter::Node, root
+      assert_equal c, root.child_count
+    end
+  end
+end

data/test/tree_sitter/language_test.rb

@@ -27,7 +27,12 @@ describe 'language' do
       if p = ENV.fetch('TREE_SITTER_PARSERS', nil)
         Pathname(p) / "libtree-sitter-ruby.#{TreeSitter.ext}"
       else
-        Pathname('tree-sitter-parsers') / 'ruby' / "libtree-sitter-ruby.#{TreeSitter.ext}"
+        downloaded = Pathname('tree-sitter-parsers') / "libtree-sitter-ruby.#{TreeSitter.ext}"
+        if !downloaded.exist?
+          Pathname('tree-sitter-parsers') / "ruby" / "libtree-sitter-ruby.#{TreeSitter.ext}"
+        else
+          downloaded
+        end
       end
     ll = TreeSitter::Language.load('ruby', path)
     assert ll.field_count.positive?

data/test/tree_sitter/logger_test.rb

@@ -62,6 +62,7 @@ describe 'logging' do
     backend = StringIO.new
     parser.logger = TreeSitter::Logger.new(backend, "%s#{delim}%s")
     parser.parse_string(nil, program)
+    refute_empty backend.string, 'the backend should be filled with logs'
     backend.each_line do |l|
       assert (/#{delim}/ =~ l), 'delimiter must be in every single line'
     end

data/test/tree_sitter/node_test.rb

@@ -352,4 +352,60 @@ describe 'fetch' do
     assert_equal 1, m.length
     assert_equal method, m.first
   end
+
+  it 'should return an empty array when asked for non-existent fields' do
+    b = @child.fetch(:a)
+    assert_empty b
+
+    b = @child.fetch(:b, :focus)
+    assert_empty b
+  end
+
+  it 'should return an array of `nil` values when asked for non-existent fields with `all: true`' do
+    b = @child.fetch(:d, all: true)
+    refute_empty b
+    assert b.all?(&:nil?)
+
+    b = @child.fetch(:e, :f, all: true)
+    refute_empty b
+    assert b.all?(&:nil?)
+  end
+
+  it 'should return values, even if `nil`, for all keys when `all: true`' do
+    method = @child.child(0)
+    arguments = @child.child(1)
+
+    m, a, f = @child.fetch(:method, :arguments, :fake, all: true)
+
+    assert_equal method, m
+    assert_equal arguments, a
+    assert_nil f
+  end
+end
+
+describe 'fetch_all' do
+  before do
+    @child = root.child(0).child(4)
+  end
+
+  it 'should return an array of `nil` values when asked for non-existent fields' do
+    b = @child.fetch_all(:d)
+    refute_empty b
+    assert b.all?(&:nil?)
+
+    b = @child.fetch_all(:e, :f)
+    refute_empty b
+    assert b.all?(&:nil?)
+  end
+
+  it 'should return values, even if `nil`, for all keys' do
+    method = @child.child(0)
+    arguments = @child.child(1)
+
+    m, a, f = @child.fetch_all(:method, :arguments, :fake)
+
+    assert_equal method, m
+    assert_equal arguments, a
+    assert_nil f
+  end
 end

data/tree_sitter.gemspec

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 lib = File.expand_path('lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+$LOAD_PATH.unshift(lib) if !$LOAD_PATH.include?(lib)
 
 require 'tree_sitter/version'
 
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.version       = TreeSitter::VERSION
 
   spec.extensions    = %(ext/tree_sitter/extconf.rb)
-  spec.files         = %w(LICENSE README.md tree_sitter.gemspec)
-  spec.files        += Dir.glob('ext/**/*.[ch]')
+  spec.files         = %w[LICENSE README.md tree_sitter.gemspec]
+  spec.files        += Dir.glob('ext/**/*.{c,h,rb}')
   spec.files        += Dir.glob('lib/**/*.rb')
   spec.test_files    = Dir.glob('test/**/*')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_tree_sitter
 version: !ruby/object:Gem::Version
-  version: 0.20.8.1
+  version: 0.20.8.3
 platform: ruby
 authors:
 - Firas al-Khalil
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-20 00:00:00.000000000 Z
+date: 2023-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -136,6 +136,7 @@ files:
 - ext/tree_sitter/query_match.c
 - ext/tree_sitter/query_predicate_step.c
 - ext/tree_sitter/range.c
+- ext/tree_sitter/repo.rb
 - ext/tree_sitter/symbol_type.c
 - ext/tree_sitter/tree.c
 - ext/tree_sitter/tree_cursor.c
@@ -146,6 +147,7 @@ files:
 - lib/tree_sitter/version.rb
 - test/README.md
 - test/test_helper.rb
+- test/tree_sitter/js_test.rb
 - test/tree_sitter/language_test.rb
 - test/tree_sitter/logger_test.rb
 - test/tree_sitter/node_test.rb
@@ -180,6 +182,7 @@ summary: Ruby bindings for Tree-Sitter
 test_files:
 - test/README.md
 - test/test_helper.rb
+- test/tree_sitter/js_test.rb
 - test/tree_sitter/language_test.rb
 - test/tree_sitter/logger_test.rb
 - test/tree_sitter/node_test.rb