ruby_tree_sitter 0.20.8.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f897961a83176c564dd214e3b8c893880302aa9552cac65b9f0c2e06e396ba7c
-  data.tar.gz: 0ba1af8010641341f95bab2bc7182faa94181da5f862a4224ef5d87c44d1eff9
+  metadata.gz: '08ade679a8ed94e7099d2cd14a06b3c889c4a3961431c2421be67b362083e0ad'
+  data.tar.gz: 01abc16071189e237e4b20587a42a8d021e34d8be1df52361a77753b8e02dc00
 SHA512:
-  metadata.gz: a3ff40e54f3d0b87984d14217f66a16f1c61a8c59c6a41d996e1792c513b2a8a3937f44839253d994d876bcbd6c1551b6c657b1cb6b6f3a0c84d752cf226da88
-  data.tar.gz: fac9fc60614e9ebd14e3181a4d2519cb4da9d170ea51326585204d3b65265349b3cdf964967c57d381bf795023c7b5e839ab406ba536ee5f971228efac9d3a5e
+  metadata.gz: a94c4a808efefb8855c7b6385ec74f48ae1ae35a2038ef4f0736675a4181ee72a405859880508c31ebef7514f706709cdd1589c386cb0d21bce1b3f82ab4f70a
+  data.tar.gz: a4af07d96b5f1356a5f71e532335b44c7d5a0eb4c59a2b09cca3b49148b6437874b76ba55266161f1567b459a8b755d9ecdba625fd6d59c598ae89dfa4a1803f

data/LICENSE

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2021 Max Brunsfeld
+Copyright (c) 2022-present, Faveod SAS.
+Copyright (c) 2022-present, Shopify Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -10,6 +10,10 @@ very old, unmaintained, and don't work with modern `tree-sitter` APIs.
 
 ## Usage
 
+### TreeSitter API
+
+The TreeSitter API is a low-level Ruby binding for tree-sitter.
+
 ``` ruby
 require 'tree_sitter'
 
@@ -28,32 +32,39 @@ root.each do |child|
 end
 ```
 
-## About
+The main philosophy behind the TreeSitter bindings is to do a 1:1 mapping between
+tree-sitter's `C` API and `Ruby`, which makes it easier to experiment and port
+ideas from different languages/bindings.
 
-The main philosophy behind these bindings is to do a 1:1 mapping between
-tree-sitter's `C` API and `Ruby`.
+But it feels like writing some managed `C` with `Ruby`, and that's why we provide
+a high-level API ([TreeStand](#treestand-api)) as well.
 
-It doesn't mean that we're going to yield the best perormance, but this design
-allows us to better experiment, and easily port ideas from other projects.
+### TreeStand API
 
-### Versioning
+The TreeStand API is a high-level Ruby wrapper for the [TreeSitter](#treesitter-api) bindings. It
+makes it easier to configure the parsers, and work with the underlying syntax tree.
+```ruby
+require 'tree_stand'
 
-This gem follows the `tree-sitter` versioning scheme, and appends its own
-version at the end.
+TreeStand.configure do
+  config.parser_path = "path/to/parser/folder/"
+end
 
-For instance, `tree-sitter` is now at version `0.20.8`, so this gem's version
-will be `0.20.8.x` where x is incremented with every notable batch of
-bugfixes or some ruby-only additions.
+sql_parser = TreeStand::Parser.new("sql")
+ruby_parser = TreeStand::Parser.new("ruby")
+```
+
+TreeStand provides an idiomatic Ruby interface to work with tree-sitter parsers.
 
 ## Dependencies
 
-This gem is a binding for `tree-sitter`.  It doesn't have a version of
-`tree-sitter` baked in it.
+This gem is a binding for `tree-sitter`. It doesn't have a version of
+`tree-sitter` baked in it by default.
 
 You must install `tree-sitter` and make sure that their dynamic library is
 accessible from `$PATH`, or build the gem with `--disable-sys-libs`, which will
 download the latest tagged `tree-sitter` and build against it (see [Build from
-source](docs/Development.md#build-from-source) .)
+source](docs/Contributing.md#build-from-source) .)
 
 You can either install `tree-sitter` from source or through your go-to package manager.
 
@@ -88,7 +99,7 @@ brew install tree-sitter
 From [rubygems](https://rubygems.org/gems/ruby_tree_sitter), in your `Gemfile`:
 
 ```ruby
-gem 'ruby_tree_sitter', '~> 0.20.8.1'
+gem 'ruby_tree_sitter', '~> 1.0'
 ```
 
 Or manually:
@@ -123,7 +134,7 @@ If you don't want to install from `rubygems`, `git`, or if you don't want to
 compile on install, then download a native gem from this repository's
 [releases](https://github.com/Faveod/ruby-tree-sitter/releases), or you can
 compile it yourself (see [Build from
-source](docs/Development.md#build-from-source) .)
+source](docs/Contributing.md#build-from-source) .)
 
 In that case, you'd have to point your `Gemfile` to the `gem` as such:
 
@@ -131,6 +142,9 @@ In that case, you'd have to point your `Gemfile` to the `gem` as such:
 gem 'tree_sitter', path: 'path/to/native/tree_sitter.gem'
 ```
 
+⚠️ We're currently missing a lot of platforms and architectures. Cross-build
+will come back in the near future.
+
 ### Parsers
 
 You will have to install parsers yourself, either by:
@@ -157,7 +171,7 @@ See `examples` directory.
 
 ## Development
 
-See [`docs/README.md`](docs/Development.md).
+See [`docs/README.md`](docs/Contributing.md).
 
 ## 🚧 👷‍♀️ Notes 👷 🚧
 
@@ -172,7 +186,7 @@ don't copy them left and right, and then expect them to work without
 `SEGFAULT`ing and creating a black-hole in your living-room.  Assume that you
 have to work locally with them. If you get a `SEGFAULT`, you can debug the
 native `C` code using `gdb`.  You can read more on `SEGFAULT`s
-[here](docs/SIGSEGV.md), and debugging [here](docs/Development.md#Debugging).
+[here](docs/SIGSEGV.md), and debugging [here](docs/Contributing.md#Debugging).
 
 That said, we do aim at providing an idiomatic `Ruby` interface.  It should also
 provide a _safer_ interface, where you don't have to worry about when and how

data/ext/tree_sitter/extconf.rb

@@ -30,7 +30,7 @@ dir_include, dir_lib =
   if system_tree_sitter?
     [
       %w[/opt/include /opt/local/include /usr/include /usr/local/include],
-      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib]
+      %w[/opt/lib /opt/local/lib /usr/lib /usr/local/lib],
     ]
   else
     repo = TreeSitter::Repo.new

data/ext/tree_sitter/input.c

@@ -113,6 +113,7 @@ static VALUE input_set_payload(VALUE self, VALUE payload) {
   return Qnil;
 }
 
+// FIXME: Missing encoding and read!
 void init_input(void) {
   cInput = rb_define_class_under(mTreeSitter, "Input", rb_cObject);
 

data/ext/tree_sitter/language.c

@@ -25,45 +25,17 @@ VALUE new_language(const TSLanguage *language) {
   return res;
 }
 
-static VALUE language_symbol_count(VALUE self) {
-  return UINT2NUM(ts_language_symbol_count(SELF));
-}
-
-static VALUE language_symbol_name(VALUE self, VALUE symbol) {
-  return safe_str(ts_language_symbol_name(SELF, NUM2UINT(symbol)));
-}
-
-static VALUE language_symbol_for_name(VALUE self, VALUE string,
-                                      VALUE is_named) {
-  const char *str = rb_id2name(SYM2ID(string));
-  uint32_t length = (uint32_t)strlen(str);
-  bool named = RTEST(is_named);
-  return UINT2NUM(ts_language_symbol_for_name(SELF, str, length, named));
-}
-
-static VALUE language_field_count(VALUE self) {
-  return UINT2NUM(ts_language_field_count(SELF));
-}
-
-static VALUE language_field_name_for_id(VALUE self, VALUE field_id) {
-  return safe_str(ts_language_field_name_for_id(SELF, NUM2UINT(field_id)));
-}
-
-static VALUE language_field_id_for_name(VALUE self, VALUE name) {
-  TSLanguage *language = SELF;
-  const char *str = StringValuePtr(name);
-  uint32_t length = (uint32_t)RSTRING_LEN(name);
-  return UINT2NUM(ts_language_field_id_for_name(language, str, length));
-}
-
-static VALUE language_symbol_type(VALUE self, VALUE symbol) {
-  return new_symbol_type(ts_language_symbol_type(SELF, NUM2UINT(symbol)));
-}
-
-static VALUE language_version(VALUE self) {
-  return UINT2NUM(ts_language_version(SELF));
-}
-
+/**
+ * Load a language parser from disk.
+ *
+ * @raise [RuntimeError] if the parser was not found, or if it's incompatible
+ * with this gem.
+ *
+ * @param name [String] the parser's name.
+ * @param path [String] the parser's shared library (so, dylib) path on disk.
+ *
+ * @return [Language]
+ */
 static VALUE language_load(VALUE self, VALUE name, VALUE path) {
   VALUE path_s = rb_funcall(path, rb_intern("to_s"), 0);
   char *path_cstr = StringValueCStr(path_s);
@@ -113,22 +85,135 @@ static VALUE language_equal(VALUE self, VALUE other) {
   return this == that ? Qtrue : Qfalse;
 }
 
+/**
+ * Get the number of distinct field names in the language.
+ *
+ * @return [Integer]
+ */
+static VALUE language_field_count(VALUE self) {
+  return UINT2NUM(ts_language_field_count(SELF));
+}
+
+/**
+ * Get the numerical id for the given field name string.
+ *
+ * @param name [String]
+ *
+ * @return [Integer]
+ */
+static VALUE language_field_id_for_name(VALUE self, VALUE name) {
+  TSLanguage *language = SELF;
+  const char *str = StringValuePtr(name);
+  uint32_t length = (uint32_t)RSTRING_LEN(name);
+  return UINT2NUM(ts_language_field_id_for_name(language, str, length));
+}
+
+/**
+ * Get the field name string for the given numerical id.
+ *
+ * @param field_id [Integer]
+ *
+ * @return [String]
+ */
+static VALUE language_field_name_for_id(VALUE self, VALUE field_id) {
+  return safe_str(ts_language_field_name_for_id(SELF, NUM2UINT(field_id)));
+}
+
+/**
+ * Get the next parse state. Combine this with lookahead iterators to generate
+ * completion suggestions or valid symbols in error nodes. Use
+ * {Node#grammar_symbol} for valid symbols.
+ */
+static VALUE language_next_state(VALUE self, VALUE state, VALUE symbol) {
+  uint16_t sta = (uint16_t)NUM2UINT(state);
+  uint16_t sym = (uint16_t)NUM2UINT(symbol);
+  return UINT2NUM(ts_language_next_state(SELF, sta, sym));
+}
+
+/**
+ * Get the number of distinct node types in the language.
+ *
+ * @return [Integer]
+ */
+static VALUE language_symbol_count(VALUE self) {
+  return UINT2NUM(ts_language_symbol_count(SELF));
+}
+
+/**
+ * Get a node type string for the given numerical id.
+ *
+ * @param symbol [Integer]
+ *
+ * @return [String]
+ */
+static VALUE language_symbol_name(VALUE self, VALUE symbol) {
+  return safe_str(ts_language_symbol_name(SELF, NUM2UINT(symbol)));
+}
+
+/**
+ * Get the numerical id for the given node type string.
+ *
+ * @param string   [Symbol]
+ * @param is_named [Boolean]
+ *
+ * @return [Integer]
+ */
+static VALUE language_symbol_for_name(VALUE self, VALUE string,
+                                      VALUE is_named) {
+  const char *str = rb_id2name(SYM2ID(string));
+  uint32_t length = (uint32_t)strlen(str);
+  bool named = RTEST(is_named);
+  return UINT2NUM(ts_language_symbol_for_name(SELF, str, length, named));
+}
+
+/**
+ * Check whether the given node type id belongs to named nodes, anonymous nodes,
+ * or a hidden nodes.
+ *
+ * Hidden nodes are never returned from the API.
+ *
+ * @see Node#named?
+ *
+ * @param symbol [Integer]
+ *
+ * @return [SymbolType]
+ */
+static VALUE language_symbol_type(VALUE self, VALUE symbol) {
+  return new_symbol_type(ts_language_symbol_type(SELF, NUM2UINT(symbol)));
+}
+
+/**
+ * Get the ABI version number for this language. This version number is used
+ * to ensure that languages were generated by a compatible version of
+ * Tree-sitter.
+ *
+ * @see Parser#language=
+ */
+static VALUE language_version(VALUE self) {
+  return UINT2NUM(ts_language_version(SELF));
+}
+
 void init_language(void) {
   cLanguage = rb_define_class_under(mTreeSitter, "Language", rb_cObject);
 
   rb_define_alloc_func(cLanguage, language_allocate);
 
+  /* Module methods */
+  rb_define_module_function(cLanguage, "load", language_load, 2);
+
+  /* Operators */
+  rb_define_method(cLanguage, "==", language_equal, 1);
+
   /* Class methods */
-  rb_define_method(cLanguage, "symbol_count", language_symbol_count, 0);
-  rb_define_method(cLanguage, "symbol_name", language_symbol_name, 1);
-  rb_define_method(cLanguage, "symbol_for_name", language_symbol_for_name, 2);
   rb_define_method(cLanguage, "field_count", language_field_count, 0);
-  rb_define_method(cLanguage, "field_name_for_id", language_field_name_for_id,
-                   1);
   rb_define_method(cLanguage, "field_id_for_name", language_field_id_for_name,
                    1);
+  rb_define_method(cLanguage, "field_name_for_id", language_field_name_for_id,
+                   1);
+  rb_define_method(cLanguage, "next_state", language_next_state, 2);
+  rb_define_method(cLanguage, "symbol_count", language_symbol_count, 0);
+  rb_define_method(cLanguage, "symbol_for_name", language_symbol_for_name, 2);
+  rb_define_method(cLanguage, "symbol_name", language_symbol_name, 1);
   rb_define_method(cLanguage, "symbol_type", language_symbol_type, 1);
   rb_define_method(cLanguage, "version", language_version, 0);
-  rb_define_module_function(cLanguage, "load", language_load, 2);
-  rb_define_method(cLanguage, "==", language_equal, 1);
 }

data/ext/tree_sitter/logger.c

@@ -147,19 +147,35 @@ static void logger_initialize_stderr(logger_t *logger) {
   }
 }
 
-// For now, we only take:
-// argv[0] = stream
-// argv[1] = format : String
-//
-// We need to add support for argv[1] : lambda/block
-//
-// case argv[1]
-// in lambda => lambda
-// in String => puts || printf || write
-// else      => write
-// end
-//
+/**
+ * Create a new logger.
+ *
+ * By default, it logs to stderr.
+ *
+ * You can provide your proper backend. You have to make sure that it
+ * exposes a +printf+, +puts+, or +write+ (lookup is done in that specific
+ * order). {StringIO} is a perfect candidate.
+ *
+ * You can also provide a format ({String}) if your backend supports a +printf+.
+ *
+ * @example
+ *   backend = StringIO.new
+ *   parser.logger = TreeSitter::Logger.new(backend)
+ *
+ * @param args [Array] The first argument is always a backend. The second
+ *                     argument is the format.
+ */
 static VALUE logger_initialize(int argc, VALUE *argv, VALUE self) {
+  // TODO:
+  // For now, we only take:
+  //   argv[0] = stream
+  //   argv[1] = format : String
+  // We need to add support for argv[1] : lambda/block
+  //   case argv[1]
+  //   in lambda => lambda
+  //   in String => puts || printf || write
+  //   else      => write
+  //   end
   logger_t *logger = unwrap(self);
 
   VALUE payload;