ruby_tree_sitter 0.20.8.2-x86_64-linux → 1.0.0-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bef2d7e3652becc17fcb08340d73614e56bfd5b77147b980c979a7960714d25f
-  data.tar.gz: bb8c3834127f3b439bf4e6dfd531621e7802af4ac7dca7762161a9e56be02b71
+  metadata.gz: 5237780e4fc6628d21e5671a2259320bbb307520a690edba10e776e57f3c886b
+  data.tar.gz: 730d07e399f4e5aee47e8c71a8dd346d83f77d0de4d2d00311a8cca7e2f554ca
 SHA512:
-  metadata.gz: 70ce5e93ff22bc3631bde3d891f85d45665673852332dbc2f5bd3c448234682e26222435d7ec041d08c83fb206818fc974cae66f6bac3d8d7a216cf7bccec8aa
-  data.tar.gz: 1004f4ec5b4ed3819d0a32bb557f8abfb71020833f1e181070aaa48b661d14f1ae820dba79ba3ba9530dade622839c87fad17a6ec0e02cc53c70e0d5caeefb27
+  metadata.gz: cc2878922995a67d5e95029b135f3b9cefb5769e4bf82d5bc94f5e0a282f79429f4aedc7f2ea18301979aefceed576be52f940c357d3844e82e6f7fe872b5c36
+  data.tar.gz: 04cb391ecddc7323a53764aafda6d60789584d6b3dad0272ea9632108d2a7d55f491811e20bb7ce4241fbbd0f193a7a5f0a92899a5a2ca7b2a1e51264f9e3e3c

data/LICENSE

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2021 Max Brunsfeld
+Copyright (c) 2022-present, Faveod SAS.
+Copyright (c) 2022-present, Shopify Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -10,6 +10,10 @@ very old, unmaintained, and don't work with modern `tree-sitter` APIs.
 
 ## Usage
 
+### TreeSitter API
+
+The TreeSitter API is a low-level Ruby binding for tree-sitter.
+
 ``` ruby
 require 'tree_sitter'
 
@@ -28,32 +32,39 @@ root.each do |child|
 end
 ```
 
-## About
+The main philosophy behind the TreeSitter bindings is to do a 1:1 mapping between
+tree-sitter's `C` API and `Ruby`, which makes it easier to experiment and port
+ideas from different languages/bindings.
 
-The main philosophy behind these bindings is to do a 1:1 mapping between
-tree-sitter's `C` API and `Ruby`.
+But it feels like writing some managed `C` with `Ruby`, and that's why we provide
+a high-level API ([TreeStand](#treestand-api)) as well.
 
-It doesn't mean that we're going to yield the best perormance, but this design
-allows us to better experiment, and easily port ideas from other projects.
+### TreeStand API
 
-### Versioning
+The TreeStand API is a high-level Ruby wrapper for the [TreeSitter](#treesitter-api) bindings. It
+makes it easier to configure the parsers, and work with the underlying syntax tree.
+```ruby
+require 'tree_stand'
 
-This gem follows the `tree-sitter` versioning scheme, and appends its own
-version at the end.
+TreeStand.configure do
+  config.parser_path = "path/to/parser/folder/"
+end
 
-For instance, `tree-sitter` is now at version `0.20.8`, so this gem's version
-will be `0.20.8.x` where x is incremented with every notable batch of
-bugfixes or some ruby-only additions.
+sql_parser = TreeStand::Parser.new("sql")
+ruby_parser = TreeStand::Parser.new("ruby")
+```
+
+TreeStand provides an idiomatic Ruby interface to work with tree-sitter parsers.
 
 ## Dependencies
 
-This gem is a binding for `tree-sitter`.  It doesn't have a version of
-`tree-sitter` baked in it.
+This gem is a binding for `tree-sitter`. It doesn't have a version of
+`tree-sitter` baked in it by default.
 
 You must install `tree-sitter` and make sure that their dynamic library is
 accessible from `$PATH`, or build the gem with `--disable-sys-libs`, which will
 download the latest tagged `tree-sitter` and build against it (see [Build from
-source](docs/Development.md#build-from-source) .)
+source](docs/Contributing.md#build-from-source) .)
 
 You can either install `tree-sitter` from source or through your go-to package manager.
 
@@ -88,7 +99,7 @@ brew install tree-sitter
 From [rubygems](https://rubygems.org/gems/ruby_tree_sitter), in your `Gemfile`:
 
 ```ruby
-gem 'ruby_tree_sitter', '~> 0.20.8.1'
+gem 'ruby_tree_sitter', '~> 1.0'
 ```
 
 Or manually:
@@ -123,7 +134,7 @@ If you don't want to install from `rubygems`, `git`, or if you don't want to
 compile on install, then download a native gem from this repository's
 [releases](https://github.com/Faveod/ruby-tree-sitter/releases), or you can
 compile it yourself (see [Build from
-source](docs/Development.md#build-from-source) .)
+source](docs/Contributing.md#build-from-source) .)
 
 In that case, you'd have to point your `Gemfile` to the `gem` as such:
 
@@ -131,6 +142,9 @@ In that case, you'd have to point your `Gemfile` to the `gem` as such:
 gem 'tree_sitter', path: 'path/to/native/tree_sitter.gem'
 ```
 
+⚠️ We're currently missing a lot of platforms and architectures. Cross-build
+will come back in the near future.
+
 ### Parsers
 
 You will have to install parsers yourself, either by:
@@ -157,7 +171,7 @@ See `examples` directory.
 
 ## Development
 
-See [`docs/README.md`](docs/Development.md).
+See [`docs/README.md`](docs/Contributing.md).
 
 ## 🚧 👷‍♀️ Notes 👷 🚧
 
@@ -172,7 +186,7 @@ don't copy them left and right, and then expect them to work without
 `SEGFAULT`ing and creating a black-hole in your living-room.  Assume that you
 have to work locally with them. If you get a `SEGFAULT`, you can debug the
 native `C` code using `gdb`.  You can read more on `SEGFAULT`s
-[here](docs/SIGSEGV.md), and debugging [here](docs/Development.md#Debugging).
+[here](docs/SIGSEGV.md), and debugging [here](docs/Contributing.md#Debugging).
 
 That said, we do aim at providing an idiomatic `Ruby` interface.  It should also
 provide a _safer_ interface, where you don't have to worry about when and how

data/ext/tree_sitter/extconf.rb

@@ -30,7 +30,7 @@ dir_include, dir_lib =
   if system_tree_sitter?
     [
       %w[/opt/include /opt/local/include /usr/include /usr/local/include],
-      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib]
+      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib],
     ]
   else
     repo = TreeSitter::Repo.new
@@ -53,48 +53,6 @@ dir_include, dir_lib =
     repo.include_and_lib_dirs
   end
 
-# TREESITTER_SPEC = Bundler.load_gemspec('../../../../tree_sitter.gemspec')
-
-# # def version = TREESITTER_SPEC.version.gsub(/\A(\d+\.\d+\.\d+)(\.\d+)?\z/, '\1')
-
-# def version = '0.20.8'
-
-# LINUX_PLATFORM_REGEX = /linux/
-# DARWIN_PLATFORM_REGEX = /darwin/
-
-# def platform = RUBY_PLATFORM
-
-# def darwin?
-#   !!(platform =~ DARWIN_PLATFORM_REGEX)
-# end
-
-# def dll_ext
-#   darwin? ? 'dylib' : 'so'
-# end
-
-# def staging_path
-#   '../../stage/lib/tree_sitter'
-# end
-
-# def downloaded_dll_path
-#   "tree-sitter-#{version}"
-# end
-
-# def add_tree_sitter_dll_to_gem
-#   puts ">>>>>>>>>>>>> #{`pwd`}"
-#   path = "#{downloaded_dll_path}/libtree-sitter*.#{dll_ext}"
-#   files =
-#     Dir.glob(path)
-#        .map { |f| Pathname(f) }
-#        .filter { |f| !f.symlink? && f.file? }
-#   dll = files.first
-#   dst = Pathname(staging_path) / "libtree-sitter.#{dll_ext}"
-#   FileUtils.cp(dll, dst)
-#   TREESITTER_SPEC.files << dst
-# end
-
-# add_tree_sitter_dll_to_gem
-
 # ################################## #
 #          Generate Makefile         #
 # ################################## #

data/ext/tree_sitter/input.c

@@ -113,6 +113,7 @@ static VALUE input_set_payload(VALUE self, VALUE payload) {
   return Qnil;
 }
 
+// FIXME: Missing encoding and read!
 void init_input(void) {
   cInput = rb_define_class_under(mTreeSitter, "Input", rb_cObject);
 

data/ext/tree_sitter/language.c

@@ -25,45 +25,17 @@ VALUE new_language(const TSLanguage *language) {
   return res;
 }
 
-static VALUE language_symbol_count(VALUE self) {
-  return UINT2NUM(ts_language_symbol_count(SELF));
-}
-
-static VALUE language_symbol_name(VALUE self, VALUE symbol) {
-  return safe_str(ts_language_symbol_name(SELF, NUM2UINT(symbol)));
-}
-
-static VALUE language_symbol_for_name(VALUE self, VALUE string,
-                                      VALUE is_named) {
-  const char *str = rb_id2name(SYM2ID(string));
-  uint32_t length = (uint32_t)strlen(str);
-  bool named = RTEST(is_named);
-  return UINT2NUM(ts_language_symbol_for_name(SELF, str, length, named));
-}
-
-static VALUE language_field_count(VALUE self) {
-  return UINT2NUM(ts_language_field_count(SELF));
-}
-
-static VALUE language_field_name_for_id(VALUE self, VALUE field_id) {
-  return safe_str(ts_language_field_name_for_id(SELF, NUM2UINT(field_id)));
-}
-
-static VALUE language_field_id_for_name(VALUE self, VALUE name) {
-  TSLanguage *language = SELF;
-  const char *str = StringValuePtr(name);
-  uint32_t length = (uint32_t)RSTRING_LEN(name);
-  return UINT2NUM(ts_language_field_id_for_name(language, str, length));
-}
-
-static VALUE language_symbol_type(VALUE self, VALUE symbol) {
-  return new_symbol_type(ts_language_symbol_type(SELF, NUM2UINT(symbol)));
-}
-
-static VALUE language_version(VALUE self) {
-  return UINT2NUM(ts_language_version(SELF));
-}
-
+/**
+ * Load a language parser from disk.
+ *
+ * @raise [RuntimeError] if the parser was not found, or if it's incompatible
+ * with this gem.
+ *
+ * @param name [String] the parser's name.
+ * @param path [String] the parser's shared library (so, dylib) path on disk.
+ *
+ * @return [Language]
+ */
 static VALUE language_load(VALUE self, VALUE name, VALUE path) {
   VALUE path_s = rb_funcall(path, rb_intern("to_s"), 0);
   char *path_cstr = StringValueCStr(path_s);
@@ -113,22 +85,135 @@ static VALUE language_equal(VALUE self, VALUE other) {
   return this == that ? Qtrue : Qfalse;
 }
 
+/**
+ * Get the number of distinct field names in the language.
+ *
+ * @return [Integer]
+ */
+static VALUE language_field_count(VALUE self) {
+  return UINT2NUM(ts_language_field_count(SELF));
+}
+
+/**
+ * Get the numerical id for the given field name string.
+ *
+ * @param name [String]
+ *
+ * @return [Integer]
+ */
+static VALUE language_field_id_for_name(VALUE self, VALUE name) {
+  TSLanguage *language = SELF;
+  const char *str = StringValuePtr(name);
+  uint32_t length = (uint32_t)RSTRING_LEN(name);
+  return UINT2NUM(ts_language_field_id_for_name(language, str, length));
+}
+
+/**
+ * Get the field name string for the given numerical id.
+ *
+ * @param field_id [Integer]
+ *
+ * @return [String]
+ */
+static VALUE language_field_name_for_id(VALUE self, VALUE field_id) {
+  return safe_str(ts_language_field_name_for_id(SELF, NUM2UINT(field_id)));
+}
+
+/**
+ * Get the next parse state. Combine this with lookahead iterators to generate
+ * completion suggestions or valid symbols in error nodes. Use
+ * {Node#grammar_symbol} for valid symbols.
+ */
+static VALUE language_next_state(VALUE self, VALUE state, VALUE symbol) {
+  uint16_t sta = (uint16_t)NUM2UINT(state);
+  uint16_t sym = (uint16_t)NUM2UINT(symbol);
+  return UINT2NUM(ts_language_next_state(SELF, sta, sym));
+}
+
+/**
+ * Get the number of distinct node types in the language.
+ *
+ * @return [Integer]
+ */
+static VALUE language_symbol_count(VALUE self) {
+  return UINT2NUM(ts_language_symbol_count(SELF));
+}
+
+/**
+ * Get a node type string for the given numerical id.
+ *
+ * @param symbol [Integer]
+ *
+ * @return [String]
+ */
+static VALUE language_symbol_name(VALUE self, VALUE symbol) {
+  return safe_str(ts_language_symbol_name(SELF, NUM2UINT(symbol)));
+}
+
+/**
+ * Get the numerical id for the given node type string.
+ *
+ * @param string   [Symbol]
+ * @param is_named [Boolean]
+ *
+ * @return [Integer]
+ */
+static VALUE language_symbol_for_name(VALUE self, VALUE string,
+                                      VALUE is_named) {
+  const char *str = rb_id2name(SYM2ID(string));
+  uint32_t length = (uint32_t)strlen(str);
+  bool named = RTEST(is_named);
+  return UINT2NUM(ts_language_symbol_for_name(SELF, str, length, named));
+}
+
+/**
+ * Check whether the given node type id belongs to named nodes, anonymous nodes,
+ * or a hidden nodes.
+ *
+ * Hidden nodes are never returned from the API.
+ *
+ * @see Node#named?
+ *
+ * @param symbol [Integer]
+ *
+ * @return [SymbolType]
+ */
+static VALUE language_symbol_type(VALUE self, VALUE symbol) {
+  return new_symbol_type(ts_language_symbol_type(SELF, NUM2UINT(symbol)));
+}
+
+/**
+ * Get the ABI version number for this language. This version number is used
+ * to ensure that languages were generated by a compatible version of
+ * Tree-sitter.
+ *
+ * @see Parser#language=
+ */
+static VALUE language_version(VALUE self) {
+  return UINT2NUM(ts_language_version(SELF));
+}
+
 void init_language(void) {
   cLanguage = rb_define_class_under(mTreeSitter, "Language", rb_cObject);
 
   rb_define_alloc_func(cLanguage, language_allocate);
 
+  /* Module methods */
+  rb_define_module_function(cLanguage, "load", language_load, 2);
+
+  /* Operators */
+  rb_define_method(cLanguage, "==", language_equal, 1);
+
   /* Class methods */
-  rb_define_method(cLanguage, "symbol_count", language_symbol_count, 0);
-  rb_define_method(cLanguage, "symbol_name", language_symbol_name, 1);
-  rb_define_method(cLanguage, "symbol_for_name", language_symbol_for_name, 2);
   rb_define_method(cLanguage, "field_count", language_field_count, 0);
-  rb_define_method(cLanguage, "field_name_for_id", language_field_name_for_id,
-                   1);
   rb_define_method(cLanguage, "field_id_for_name", language_field_id_for_name,
                    1);
+  rb_define_method(cLanguage, "field_name_for_id", language_field_name_for_id,
+                   1);
+  rb_define_method(cLanguage, "next_state", language_next_state, 2);
+  rb_define_method(cLanguage, "symbol_count", language_symbol_count, 0);
+  rb_define_method(cLanguage, "symbol_for_name", language_symbol_for_name, 2);
+  rb_define_method(cLanguage, "symbol_name", language_symbol_name, 1);
   rb_define_method(cLanguage, "symbol_type", language_symbol_type, 1);
   rb_define_method(cLanguage, "version", language_version, 0);
-  rb_define_module_function(cLanguage, "load", language_load, 2);
-  rb_define_method(cLanguage, "==", language_equal, 1);
 }

data/ext/tree_sitter/logger.c

@@ -147,19 +147,35 @@ static void logger_initialize_stderr(logger_t *logger) {
   }
 }
 
-// For now, we only take:
-// argv[0] = stream
-// argv[1] = format : String
-//
-// We need to add support for argv[1] : lambda/block
-//
-// case argv[1]
-// in lambda => lambda
-// in String => puts || printf || write
-// else      => write
-// end
-//
+/**
+ * Create a new logger.
+ *
+ * By default, it logs to stderr.
+ *
+ * You can provide your proper backend. You have to make sure that it
+ * exposes a +printf+, +puts+, or +write+ (lookup is done in that specific
+ * order). {StringIO} is a perfect candidate.
+ *
+ * You can also provide a format ({String}) if your backend supports a +printf+.
+ *
+ * @example
+ *   backend = StringIO.new
+ *   parser.logger = TreeSitter::Logger.new(backend)
+ *
+ * @param args [Array] The first argument is always a backend. The second
+ *                     argument is the format.
+ */
 static VALUE logger_initialize(int argc, VALUE *argv, VALUE self) {
+  // TODO:
+  // For now, we only take:
+  //   argv[0] = stream
+  //   argv[1] = format : String
+  // We need to add support for argv[1] : lambda/block
+  //   case argv[1]
+  //   in lambda => lambda
+  //   in String => puts || printf || write
+  //   else      => write
+  //   end
   logger_t *logger = unwrap(self);
 
   VALUE payload;