ruby_tree_sitter 1.6.0-x86_64-darwin

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e12ffe66cb647b428b52cfd62e7a039b66b352e5e991aed2e3521df13180a04e
+  data.tar.gz: 397a39cfbfe8e4347421dac6b0f3046cd385acf48c98c2c7c851571cd86a010f
+SHA512:
+  metadata.gz: 6f366afd18a2b039c9645f7ef9ef40be6d6a62ccec33ea3aaf80dc1407fda07d284d076040de3c3be1cd0197c576d3cb7b834b2068883d6ca99580099dd93784
+  data.tar.gz: 46cc1414c1e8f571eb0fd9449d56d1ac0e33f6fb25f6b55f4a07675f7c471fe2d87eaa7a84e9df89752ba4f641936b684294fa3f9bb82fb660ea60b78c6eebb3

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2022-present, Faveod SAS.
+Copyright (c) 2022-present, Shopify Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,213 @@
+# Ruby tree-sitter bindings
+
+[![docs](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/publish-docs.yml/badge.svg)](https://faveod.github.io/ruby-tree-sitter)
+[![rubygems.org](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/publish.yml/badge.svg)](https://rubygems.org/gems/ruby_tree_sitter)
+[![ci](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/ci.yml/badge.svg)](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/ci.yml)
+<!--
+[![valgrind](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/valgrind.yml/badge.svg)](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/valgrind.yml)
+[![asan/ubsan](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/asan.yml/badge.svg)](https://github.com/Faveod/ruby-tree-sitter/actions/workflows/asan.yml)
+-->
+
+Ruby bindings for [tree-sitter](https://github.com/tree-sitter/tree-sitter).
+
+The [official bindings](https://github.com/tree-sitter/ruby-tree-sitter) are
+very old, unmaintained, and don't work with modern `tree-sitter` APIs.
+
+
+## Usage
+
+### TreeSitter API
+
+The TreeSitter API is a low-level Ruby binding for tree-sitter.
+
+``` ruby
+require 'tree_sitter'
+
+parser = TreeSitter::Parser.new
+language = TreeSitter::Language.load('javascript', 'path/to/libtree-sitter-javascript.{so,dylib}')
+# Or simply
+language = TreeSitter.lang('javascript')
+# Which will try to look in your local directory and the system for installed parsers.
+# See TreeSitter::Mixin::Language#lib_dirs
+
+src = "[1, null]"
+
+parser.language = language
+
+tree = parser.parse_string(nil, src)
+root = tree.root_node
+
+root.each do |child|
+  # ...
+end
+```
+
+The main philosophy behind the TreeSitter bindings is to do a 1:1 mapping between
+tree-sitter's `C` API and `Ruby`, which makes it easier to experiment and port
+ideas from different languages/bindings.
+
+But it feels like writing some managed `C` with `Ruby`, and that's why we provide
+a high-level API ([TreeStand](#treestand-api)) as well.
+
+### TreeStand API
+
+The TreeStand API is a high-level Ruby wrapper for the [TreeSitter](#treesitter-api) bindings. It
+makes it easier to configure the parsers, and work with the underlying syntax tree.
+```ruby
+require 'tree_stand'
+
+TreeStand.configure do
+  config.parser_path = "path/to/parser/folder/"
+end
+
+sql_parser = TreeStand::Parser.new("sql")
+ruby_parser = TreeStand::Parser.new("ruby")
+```
+
+TreeStand provides an idiomatic Ruby interface to work with tree-sitter parsers.
+
+## Dependencies
+
+This gem is a binding for `tree-sitter`, and comes with a pre-built version
+of `tree-sitter` bundled with it, so you can start using it without any
+special configuration.
+
+We support the following platforms:
+
+- `aarch64-linux-gnu`
+- `aarch64-linux-musl`
+- `arm-linux-gnu`
+- `arm-linux-musl`
+- `x86_64-linux-gnu`
+- `x86_64-linux-musl`
+- `x86-linux-musl`
+- `arm64-darwin`
+- `x86_64-darwin`
+
+(see [Build from source](docs/Contributing.md#build-from-source)).
+
+You can either install `tree-sitter` from source or through your go-to package manager.
+
+### Linux
+
+`ubuntu >= 22.04`
+
+```console
+sudo apt install libtree-sitter-dev
+```
+
+`arch`
+
+```console
+sudo pacman -S tree-sitter
+```
+
+### MacOS
+
+```console
+sudo port install tree-sitter
+```
+
+or
+
+```console
+brew install tree-sitter
+```
+
+## Install
+
+From [rubygems](https://rubygems.org/gems/ruby_tree_sitter), in your `Gemfile`:
+
+```ruby
+gem 'ruby_tree_sitter', '~> 1.6'
+```
+
+Or manually:
+
+```sh
+gem install ruby_tree_sitter
+```
+
+Or from `git` sources, which will compile on installation:
+
+```ruby
+gem 'ruby_tree_sitter', git: 'https://github.com/Faveod/ruby-tree-sitter'
+```
+
+### Enable system libraries
+
+To install with `--enable-sys-libs`, you can either:
+
+```sh
+gem install ruby_tree_sitter --platform=ruby -- --enable-sys-libs
+```
+
+Or via bundle:
+
+```sh
+bundle config force_ruby_platform true
+bundle config set build.ruby_tree_sitter --enable-sys-libs
+```
+
+### No compilation
+
+If you don't want to install from `rubygems`, `git`, or if you don't want to
+compile on install, then download a native gem from this repository's
+[releases](https://github.com/Faveod/ruby-tree-sitter/releases), or you can
+compile it yourself (see [Build from source](docs/Contributing.md#build-from-source) .)
+
+In that case, you'd have to point your `Gemfile` to the `gem` as such:
+
+``` ruby
+gem 'tree_sitter', path: 'path/to/native/tree_sitter.gem'
+```
+
+⚠️ We're currently missing a lot of platforms and architectures. Cross-build
+will come back in the near future.
+
+### Parsers
+
+You will have to install parsers yourself, either by:
+
+1. Downloading and building from source.
+1. Downloading from your package manager, if available.
+1. Downloading a pre-built binary from
+   [Faveod/tree-sitter-parsers](https://github.com/Faveod/tree-sitter-parsers)
+   which supports numerous architectures.
+
+## Examples
+
+See `examples` directory.
+
+## Development
+
+See [`docs/Contributing.md`](docs/Contributing.md).
+
+## 🚧 👷‍♀️ Notes 👷 🚧
+
+Since we're doing a 1:1 mapping between the `tree-sitter` API and the bindings,
+you need to be extra-careful when playing with the provided objects.  Some of
+them have their underlying `C` data-structure automatically freed, so you might
+get yourself in undesirable situations if you don't pay attention to what you're
+doing.
+
+We're only talking about `Tree`, `TreeCursor`, `Query`, and `QueryCursor`.  Just
+don't copy them left and right, and then expect them to work without
+`SEGFAULT`ing and creating a black-hole in your living-room.  Assume that you
+have to work locally with them. If you get a `SEGFAULT`, you can debug the
+native `C` code using `gdb`.  You can read more on `SEGFAULT`s
+[here](docs/SIGSEGV.md), and debugging [here](docs/Contributing#Debugging.md).
+
+That said, we do aim at providing an idiomatic `Ruby` interface.  It should also
+provide a _safer_ interface, where you don't have to worry about when and how
+resources are freed.
+
+To See a full list of the ruby-specific APIs, see [here](lib/README.md).
+
+## Sponsors
+
+- <a href="https://faveod.com">https://faveod.com</a>
+
+<a href="https://faveod.com">
+  <img src="img/faveod.jpg" width="66%" />
+</a>

data/ext/tree_sitter/encoding.c

@@ -0,0 +1,29 @@
+#include "tree_sitter.h"
+
+extern VALUE mTreeSitter;
+
+VALUE mEncoding;
+
+const char *utf8 = "utf8";
+const char *utf16 = "utf16";
+
+TSInputEncoding value_to_encoding(VALUE encoding) {
+  VALUE enc = SYM2ID(encoding);
+  /* VALUE u8 = rb_const_get_at(mEncoding, rb_intern(utf8)); */
+  VALUE u16 = SYM2ID(rb_const_get_at(mEncoding, rb_intern("UTF16")));
+
+  // NOTE: should we emit a warning instead of defaulting to UTF8?
+  if (enc == u16) {
+    return TSInputEncodingUTF16;
+  } else {
+    return TSInputEncodingUTF8;
+  }
+}
+
+void init_encoding(void) {
+  mEncoding = rb_define_module_under(mTreeSitter, "Encoding");
+
+  /* Constants */
+  rb_define_const(mEncoding, "UTF8", ID2SYM(rb_intern(utf8)));
+  rb_define_const(mEncoding, "UTF16", ID2SYM(rb_intern(utf16)));
+}

data/ext/tree_sitter/extconf.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'pathname'
+
+require_relative 'repo'
+
+# ################################## #
+#             Some helpers           #
+# ################################## #
+
+RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
+
+cflags = []
+ldflags = []
+
+def system_tree_sitter?
+  enable_config('sys-libs', false)
+end
+
+def env_var_on?(var)
+  %w[1 on true t yes y].include?(ENV.fetch(var, '').downcase)
+end
+
+# ################################## #
+#    System lib vs Downloaded lib    #
+# ################################## #
+
+repo = TreeSitter::Repo.new
+
+dir_include, dir_lib =
+  if system_tree_sitter?
+    [
+      %w[/opt/include /opt/local/include /usr/include /usr/local/include],
+      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib],
+    ]
+  else
+    if !repo.download
+      msg = <<~MSG
+
+        Could not fetch tree-sitter sources:
+
+        #{repo.exe.map { |k, v| "#{k}: #{v}" }.join("\n")}
+
+      MSG
+      abort(msg)
+    end
+
+    # We need to make sure we're selecting the proper toolchain.
+    # Especially needed for corss-compilation.
+    ENV.store('CC', RbConfig::CONFIG['CC'])
+    repo.patch
+    repo.compile
+    repo.include_and_lib_dirs
+  end
+
+dir_config('tree-sitter', dir_include&.first, dir_lib&.first)
+
+# ################################## #
+#          Generate Makefile         #
+# ################################## #
+
+header = find_header('tree_sitter/api.h')
+library = find_library('tree-sitter', 'ts_parser_new')
+
+if !header || !library
+  abort <<~MSG
+
+    * Missing header         : #{header ? 'no' : 'yes'}
+    * Missing lib            : #{library ? 'no' : 'yes'}
+    * Use system tree-sitter : #{system_tree_sitter?}
+
+    Try to install tree-sitter from source, or through your package manager,
+    or even try one of the following options to extconf.rb/rake compile:
+
+      --with-tree-sitter-dir=/path/to/tree-sitter
+      --with-tree-sitter-lib=/path/to/tree-sitter/lib
+      --with-tree-sitter-include=/path/to/tree-sitter/include
+
+  MSG
+end
+
+cflags << '-Werror' if env_var_on?('TREE_SITTER_PEDANTIC')
+
+if env_var_on?('DEBUG')
+  cflags << '-fbounds-check'
+  CONFIG['optflags'].gsub!(/-O\d/, '-O0')
+else
+  cflags << '-DNDEBUG'
+  CONFIG['optflags'].gsub!(/-O\d/, '-O3')
+end
+
+sanitizers = ENV.fetch('SANITIZE', nil)
+
+if sanitizers =~ /memory/
+  puts <<~MSG
+
+    We do not support memory sanitizers as of yet.
+    It requires building ruby with the same sanitizer, and maybe its dependencies.
+
+    exiting…
+
+  MSG
+  exit 1
+end
+
+if sanitizers
+  # NOTE: when sanitizing, the default generated warning flags emit a lot of …
+  # warnings.
+  #
+  # I couldn't make mkmf understand it's running with clang and not gcc, so
+  # I'm omitting the warning generating warnings.
+  #
+  # It should be harmless, since sanitization is meant for CI and dev builds.
+  %w[
+    -Wduplicated-cond
+    -Wimplicit-fallthrough=\d+
+    -Wno-cast-function-type
+    -Wno-packed-bitfield-compat
+    -Wsuggest-attribute=\w+
+  ].each do |r|
+    $warnflags.gsub!(/#{r}/, '')
+  end
+
+  cflags.concat %W[
+    -fms-extensions
+    -fdeclspec
+    -fsanitize=#{sanitizers}
+    -fsanitize-blacklist=../../../../.asanignore
+    -fsanitize-recover=#{sanitizers}
+    -fno-sanitize-recover=all
+    -fno-sanitize=null
+    -fno-sanitize=alignment
+    -fno-omit-frame-pointer
+  ]
+
+  ldflags.concat %W[
+    -fsanitize=#{sanitizers}
+    -dynamic-libasan
+  ]
+end
+
+cflags.concat %w[-std=c99 -fPIC -Wall]
+
+append_cflags(cflags)
+append_ldflags(ldflags)
+
+create_header
+create_makefile('tree_sitter/tree_sitter')

data/ext/tree_sitter/input.c

@@ -0,0 +1,127 @@
+#include "tree_sitter.h"
+
+extern VALUE mTreeSitter;
+
+VALUE cInput;
+
+typedef struct {
+  TSInput data;
+  VALUE payload;
+  VALUE last_result;
+} input_t;
+
+const char *input_read(void *payload, uint32_t byte_index, TSPoint position,
+                       uint32_t *bytes_read) {
+  input_t *input = (input_t *)payload;
+  VALUE read = rb_funcall(input->payload, rb_intern("read"), 2,
+                          UINT2NUM(byte_index), new_point(&position));
+  if (NIL_P(read)) {
+    *bytes_read = 0;
+    input->last_result = Qnil;
+    return NULL;
+  }
+
+  VALUE size = rb_funcall(read, rb_intern("bytesize"), 0);
+  *bytes_read = NUM2UINT(size);
+  input->last_result = rb_funcall(read, rb_intern("to_str"), 0);
+
+  return StringValueCStr(input->last_result);
+}
+
+static void input_payload_set(input_t *input, VALUE value) {
+  input->payload = value;
+  input->last_result = Qnil;
+  input->data.payload = (void *)input;
+  input->data.read = input_read;
+}
+
+static void input_free(void *ptr) { xfree(ptr); }
+
+static size_t input_memsize(const void *ptr) {
+  input_t *type = (input_t *)ptr;
+  return sizeof(type);
+}
+
+static void input_mark(void *ptr) {
+  input_t *input = (input_t *)ptr;
+  rb_gc_mark_movable(input->payload);
+  // we don't want last_result to move because its const char* content will be
+  // consumed by the parser.
+  //
+  // No funny things please.
+  rb_gc_mark(input->last_result);
+}
+
+static void input_compact(void *ptr) {
+  input_t *input = (input_t *)ptr;
+  input->payload = rb_gc_location(input->payload);
+}
+
+const rb_data_type_t input_data_type = {
+    .wrap_struct_name = "input",
+    .function =
+        {
+            .dmark = input_mark,
+            .dfree = input_free,
+            .dsize = input_memsize,
+            .dcompact = input_compact,
+        },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+DATA_UNWRAP(input)
+
+static VALUE input_allocate(VALUE klass) {
+  input_t *input;
+  return TypedData_Make_Struct(klass, input_t, &input_data_type, input);
+}
+
+TSInput value_to_input(VALUE self) { return SELF; }
+
+VALUE new_input(const TSInput *ptr) {
+  if (ptr != NULL) {
+    VALUE res = input_allocate(cInput);
+    input_t *input = unwrap(res);
+    VALUE payload = Qnil;
+    if (ptr->payload != NULL) {
+      input_t *old_input = (input_t *)ptr->payload;
+      payload = old_input->payload;
+    }
+    input_payload_set(input, payload);
+    return res;
+  } else {
+    return Qnil;
+  }
+}
+
+static VALUE input_initialize(int argc, VALUE *argv, VALUE self) {
+  input_t *input = unwrap(self);
+  VALUE payload;
+  rb_scan_args(argc, argv, "01", &payload);
+  input_payload_set(input, payload);
+  return self;
+}
+
+static VALUE input_inspect(VALUE self) {
+  return rb_sprintf("{payload=%+" PRIsVALUE "}", unwrap(self)->payload);
+}
+
+DEFINE_GETTER(input, payload)
+
+static VALUE input_set_payload(VALUE self, VALUE payload) {
+  input_payload_set(unwrap(self), payload);
+  return Qnil;
+}
+
+// FIXME: Missing encoding and read!
+void init_input(void) {
+  cInput = rb_define_class_under(mTreeSitter, "Input", rb_cObject);
+
+  rb_define_alloc_func(cInput, input_allocate);
+
+  /* Class methods */
+  DECLARE_ACCESSOR(cInput, input, payload)
+  rb_define_method(cInput, "initialize", input_initialize, -1);
+  rb_define_method(cInput, "inspect", input_inspect, 0);
+  rb_define_method(cInput, "to_s", input_inspect, 0);
+}

data/ext/tree_sitter/input_edit.c

@@ -0,0 +1,42 @@
+#include "tree_sitter.h"
+
+extern VALUE mTreeSitter;
+
+VALUE cInputEdit;
+
+DATA_WRAP(InputEdit, input_edit)
+DATA_DEFINE_ACCESSOR(input_edit, start_byte, UINT2NUM, NUM2UINT)
+DATA_DEFINE_ACCESSOR(input_edit, old_end_byte, UINT2NUM, NUM2UINT)
+DATA_DEFINE_ACCESSOR(input_edit, new_end_byte, UINT2NUM, NUM2UINT)
+DATA_DEFINE_ACCESSOR(input_edit, start_point, new_point_by_val, value_to_point)
+DATA_DEFINE_ACCESSOR(input_edit, old_end_point, new_point_by_val, value_to_point)
+DATA_DEFINE_ACCESSOR(input_edit, new_end_point, new_point_by_val, value_to_point)
+
+static VALUE input_edit_inspect(VALUE self) {
+  input_edit_t *input_edit = unwrap(self);
+  return rb_sprintf("{start_byte=%i, old_end_byte=%i , new_end_byte=%i, "
+                    "start_point=%+" PRIsVALUE ", old_end_point=%+" PRIsVALUE
+                    ", new_end_point=%+" PRIsVALUE "}",
+                    input_edit->data.start_byte, input_edit->data.old_end_byte,
+                    input_edit->data.new_end_byte,
+                    new_point_by_val(input_edit->data.start_point),
+                    new_point_by_val(input_edit->data.old_end_point),
+                    new_point_by_val(input_edit->data.new_end_point));
+}
+
+void init_input_edit(void) {
+  cInputEdit = rb_define_class_under(mTreeSitter, "InputEdit", rb_cObject);
+
+  rb_define_alloc_func(cInputEdit, input_edit_allocate);
+
+  /* Class methods */
+  DECLARE_ACCESSOR(cInputEdit, input_edit, start_byte)
+  DECLARE_ACCESSOR(cInputEdit, input_edit, old_end_byte)
+  DECLARE_ACCESSOR(cInputEdit, input_edit, new_end_byte)
+  DECLARE_ACCESSOR(cInputEdit, input_edit, start_point)
+  DECLARE_ACCESSOR(cInputEdit, input_edit, old_end_point)
+  DECLARE_ACCESSOR(cInputEdit, input_edit, new_end_point)
+
+  rb_define_method(cInputEdit, "inspect", input_edit_inspect, 0);
+  rb_define_method(cInputEdit, "to_s", input_edit_inspect, 0);
+}