rroonga 1.3.1-x86-mingw32 → 2.0.0-x86-mingw32

data/lib/groonga/schema.rb

@@ -151,11 +151,13 @@ module Groonga
       # スキーマを定義する。ブロックにはGroonga::Schemaオブ
       # ジェクトがわたるので、そのオブジェクトを利用してスキー
       # マを定義する。以下の省略形。
+      #
       # <pre>
-      #  schema = Groonga::Scheme.new(options)
-      #  ==...==
-      #  schema.define
+      # schema = Groonga::Scheme.new(options)
+      # # ...
+      # schema.define
       # </pre>
+      #
       # @param [::Hash] options The name and value
       #   pairs. Omitted names are initialized as the default value.
       # @option options [Groonga::Context] :content (Groonga::Context.default) The context
@@ -170,7 +172,7 @@ module Groonga
       # @example 名前が _name_ のテーブルを作成する。以下の省略形。
       #  Groonga::Schema.define do |schema|
       #    schema.create_table(name, options) do |table|
-      #      ...
+      #      # ...
       #    end
       #  end
       # ブロックにはGroonga::Schema::TableDefinitionオブジェ
@@ -328,12 +330,13 @@ module Groonga
       end
 
       # 名前が _name_ のテーブルを変更する。以下の省略形。
+      #
       # <pre>
-      #   Groonga::Schema.define do |schema|
-      #     schema.change_table(name, options) do |table|
-      #      ==...==
-      #     end
+      # Groonga::Schema.define do |schema|
+      #   schema.change_table(name, options) do |table|
+      #     # ...
       #   end
+      # end
       # </pre>
       #
       # ブロックにはGroonga::Schema::TableDefinitionオブジェ
@@ -353,12 +356,12 @@ module Groonga
 
       # (See Groonga::Schema#rename_table)
       #
-      # This is a syntax sugar the following code:
+      # This is a syntax sugar of the following code:
       #
       # <pre>
-      #   Groonga::Schema.define do |schema|
-      #     schema.rename_table(current_name, new_name, options)
-      #   end
+      # Groonga::Schema.define do |schema|
+      #   schema.rename_table(current_name, new_name, options)
+      # end
       # </pre>
       def rename_table(current_name, new_name, options={})
         define do |schema|
@@ -367,12 +370,13 @@ module Groonga
       end
 
       # 名前が_name_のビューを作成する。以下の省略形。
+      #
       # <pre>
-      #   Groonga::Schema.define do |schema|
-      #     schema.create_view(name, options) do |view|
-      #       ==...==
-      #     end
+      # Groonga::Schema.define do |schema|
+      #   schema.create_view(name, options) do |view|
+      #     # ...
       #   end
+      # end
       # </pre>
       # ブロックにはGroonga::Schema::ViewDefinitionオブジェ
       # クトがわたるので、そのオブジェクトを利用してビュー
@@ -415,12 +419,13 @@ module Groonga
       end
 
       # 名前が_name_のビューを変更する。以下の省略形。
+      #
       # <pre>
-      #   Groonga::Schema.define do |schema|
-      #     schema.change_view(name, options) do |view|
-      #      ==...==
-      #     end
+      # Groonga::Schema.define do |schema|
+      #   schema.change_view(name, options) do |view|
+      #     # ...
       #   end
+      # end
       # </pre>
       #
       # ブロックにはGroonga::Schema::ViewDefinitionオブジェ
@@ -440,9 +445,11 @@ module Groonga
 
       # 以下と同様:
       #
-      #   Groonga::Schema.change_table(table_name) do |table|
-      #     table.remove_column(column_name)
-      #   end
+      # <pre>
+      # Groonga::Schema.change_table(table_name) do |table|
+      #   table.remove_column(column_name)
+      # end
+      # </pre>
       def remove_column(table_name, column_name)
         change_table(table_name) do |table|
           table.remove_column(column_name)
@@ -451,10 +458,12 @@ module Groonga
 
       # This is a syntax sugar of the following:
       #
-      #   Groonga::Schema.define do |schema|
-      #     schema.rename_column(table_name,
-      #                          current_column_name, new_column_name)
-      #   end
+      # <pre>
+      # Groonga::Schema.define do |schema|
+      #   schema.rename_column(table_name,
+      #                        current_column_name, new_column_name)
+      # end
+      # </pre>
       def rename_column(table_name, current_column_name, new_column_name)
         define do |schema|
           schema.rename_column(table_name, current_column_name, new_column_name)
@@ -466,25 +475,39 @@ module Groonga
       # Rubyスクリプト形式で返された値は
       # Groonga::Schema.restoreすることによりスキーマ内に組み
       # 込むことができる。
+      #
+      # dump.rb:
+      #
       # <pre>
-      #   ==dump.rb:
-      #     File.open("/tmp/groonga-schema.rb", "w") do |schema|
-      #       dumped_text = Groonga::Schema.dump
-      #     end
-      #   restore.rb:
-      #     dumped_text = Groonga::Schema.dump
-      #     Groonga::Database.create(:path => "/tmp/new-db.grn")
-      #     Groonga::Schema.restore(dumped_text)==
+      # File.open("/tmp/groonga-schema.rb", "w") do |schema|
+      #   dumped_text = Groonga::Schema.dump
+      # end
       # </pre>
+      #
+      # restore.rb:
+      #
+      # <pre>
+      # dumped_text = Groonga::Schema.dump
+      # Groonga::Database.create(:path => "/tmp/new-db.grn")
+      # Groonga::Schema.restore(dumped_text)
+      # </pre>
+      #
       # grn式形式で返された値はgroongaコマンドで読み込むこと
       # ができる。
+      #
+      # dump.rb:
+      #
       # <pre>
-      #   ==dump.rb:
-      #     File.open("/tmp/groonga-schema.grn", "w") do |schema|
-      #       dumped_text = Groonga::Schema.dump(:syntax => :command)
-      #     end
-      #   % groonga db/path < /tmp/groonga-schema.grn==
+      # File.open("/tmp/groonga-schema.grn", "w") do |schema|
+      #   dumped_text = Groonga::Schema.dump(:syntax => :command)
+      # end
       # </pre>
+      #
+      # <pre>
+      # !!!text
+      # % groonga db/path < /tmp/groonga-schema.grn
+      # </pre>
+      #
       # @param options [::Hash] The name and value
       #   pairs. Omitted names are initialized as the default value.
       # @option options :context (Groonga::Context.default) The context
@@ -914,9 +937,12 @@ module Groonga
     end
 
     # 以下と同様:
-    #   schema.change_table(table_name) do |table|
-    #     table.remove_column(column_name)
-    #   end
+    #
+    # <pre>
+    # schema.change_table(table_name) do |table|
+    #   table.remove_column(column_name)
+    # end
+    # </pre>
     def remove_column(table_name, column_name)
       change_table(table_name) do |table|
         table.remove_column(column_name)
@@ -924,9 +950,12 @@ module Groonga
     end
 
     # It is a syntax sugar of the following:
-    #   schema.change_table(table_name) do |table|
-    #     table.rename_column(current_column_name, new_column_name)
-    #   end
+    #
+    # <pre>
+    # schema.change_table(table_name) do |table|
+    #   table.rename_column(current_column_name, new_column_name)
+    # end
+    # </pre>
     def rename_column(table_name, current_column_name, new_column_name)
       change_table(table_name) do |table|
         table.rename_column(current_column_name, new_column_name)

data/lib/groonga.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2009-2011  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2009-2012  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -58,12 +58,15 @@ module Groonga
       BUILD_VERSION.join(".")
     end
 
-    ##
-    # VERSIONを"."で結合して<tt>"メジャーバージョン.マイナー
-    # バージョン.マイクロバージョン"</tt>という形式の文字列に
-    # したもの。
+    # Format version.
+    #
+    # @return [String] If Groonga::VERSION has tag,
+    #   @MAJOR.MINOR.MICRO-TAG@. Otherwise, @MAJOR.MINOR.MICRO@.
     def version
-      VERSION.join(".")
+      major, minor, micro, tag = VERSION
+      version_string = [major, minor, micro].join(".")
+      version_string << "-#{tag}" if tag
+      version_string
     end
 
     ##

data/rroonga-build.rb

@@ -17,8 +17,8 @@
 
 module RroongaBuild
   module RequiredGroongaVersion
-    MAJOR = 1
-    MINOR = 3
+    MAJOR = 2
+    MINOR = 0
     MICRO = 0
     VERSION = [MAJOR, MINOR, MICRO]
   end

data/test/run-test.rb

@@ -36,8 +36,7 @@ if make
   system("cd #{base_dir.dump} && #{make} > /dev/null") or exit(false)
 end
 
-require "rubygems"
-gem "test-unit-notify"
+require "test-unit"
 require "test/unit/notify"
 
 Test::Unit::Priority.enable

data/test/test-table-select.rb

@@ -58,11 +58,6 @@ class TableSelectTest < Test::Unit::TestCase
     @result = nil
   end
 
-  teardown
-  def teardown_expression_close
-    @result.expression.close if @result # TODO: We want to remove it.
-  end
-
   def test_sub_expression
     @result = @comments.select do |record|
       record.match("Hello", "content") &

data/test/test-type.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2009-2012  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -28,7 +28,7 @@ class TypeTest < Test::Unit::TestCase
       Groonga::Type.new("user id", :type => :integer)
     end
     message =
-      "name can't start with '_' and contains only 0-9, A-Z, a-z, #, - or _"
+      "name can't start with '_' and contains only 0-9, A-Z, a-z, #, @, - or _"
     assert_match(/#{Regexp.escape(message)}/,
                  exception.message)
   end

data/test/test-version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2009-2012  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,18 +14,48 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 class VersionTest < Test::Unit::TestCase
-  def test_build_version
-    assert_match(/\A\d+\.\d+\.\d+\z/, Groonga.build_version)
-    assert_match(Groonga.build_version, Groonga::BUILD_VERSION.join("."))
+  def test_build_version_format
+    assert_equal("X.X.X", normalize(Groonga.build_version))
   end
 
-  def test_version
-    assert_match(/\A\d+\.\d+\.\d+(?:\.\d+-[a-zA-Z\d]+)?\z/, Groonga.version)
-    assert_match(Groonga.version, Groonga::VERSION.join("."))
+  def test_build_version_value
+    assert_equal(Groonga.build_version, Groonga::BUILD_VERSION.join("."))
   end
 
-  def test_bindings_version
-    assert_match(/\A\d+\.\d+\.\d+\z/, Groonga.bindings_version)
-    assert_match(Groonga.bindings_version, Groonga::BINDINGS_VERSION.join("."))
+  def test_version_format
+    if /-/ =~ Groonga.version
+      assert_equal("X.X.X-XXX", normalize(Groonga.version))
+    else
+      assert_equal("X.X.X", normalize(Groonga.version))
+    end
+  end
+
+  def test_version_value
+    major, minor, micro, tag = Groonga::VERSION
+    versions = [major, minor, micro]
+    assert_equal(Groonga.version, [versions.join("."), tag].compact.join("-"))
+  end
+
+  def test_bindings_version_format
+    assert_equal("X.X.X", normalize(Groonga.bindings_version))
+  end
+
+  def test_bindings_version_value
+    assert_equal(Groonga.bindings_version, Groonga::BINDINGS_VERSION.join("."))
+  end
+
+  private
+  def normalize(version)
+    versions_component, tag_component = version.split(/-/, 2)
+    versions = versions_component.split(/\./)
+    normalized_versions = versions.collect do |component|
+      component.gsub(/\d+/, "X")
+    end
+    if tag_component
+      normalized_tag = "XXX"
+    else
+      normalized_tag = nil
+    end
+    [normalized_versions.join("."), normalized_tag].compact.join("-")
   end
 end

data/vendor/local/include/groonga/groonga/plugin.h

@@ -18,6 +18,8 @@
 #ifndef GRN_PLUGIN_H
 #define GRN_PLUGIN_H
 
+#include <stddef.h>
+
 #include <groonga.h>
 
 #ifdef __cplusplus
@@ -38,6 +40,117 @@ GRN_PLUGIN_EXPORT grn_rc GRN_PLUGIN_INIT(grn_ctx *ctx);
 GRN_PLUGIN_EXPORT grn_rc GRN_PLUGIN_REGISTER(grn_ctx *ctx);
 GRN_PLUGIN_EXPORT grn_rc GRN_PLUGIN_FIN(grn_ctx *ctx);
 
+/*
+  Don't call these functions directly. Use GRN_PLUGIN_MALLOC(),
+  GRN_PLUGIN_REALLOC() and GRN_PLUGIN_FREE() instead.
+ */
+void *grn_plugin_malloc(grn_ctx *ctx, size_t size, const char *file,
+                        int line, const char *func);
+void *grn_plugin_realloc(grn_ctx *ctx, void *ptr, size_t size,
+                         const char *file, int line, const char *func);
+void grn_plugin_free(grn_ctx *ctx, void *ptr, const char *file,
+                     int line, const char *func);
+
+/*
+  GRN_PLUGIN_MALLOC() allocates `size' bytes and returns a pointer to the
+  allocated memory space. Note that the memory space is associated with `ctx'.
+ */
+#define GRN_PLUGIN_MALLOC(ctx, size) \
+  grn_plugin_malloc((ctx), (size), __FILE__, __LINE__, __FUNCTION__)
+/*
+  GRN_PLUGIN_REALLOC() resizes the memory space pointed to by `ptr' or
+  allocates a new memory space of `size' bytes. GRN_PLUGIN_REALLOC() returns
+  a pointer to the memory space. The contents is unchanged or copied from the
+  old memory space to the new memory space.
+ */
+#define GRN_PLUGIN_REALLOC(ctx, ptr, size) \
+  grn_plugin_realloc((ctx), (ptr), (size), __FILE__, __LINE__, __FUNCTION__)
+/*
+  GRN_PLUGIN_FREE() frees a memory space allocated by GRN_PLUGIN_MALLOC() or
+  GRN_PLUGIN_REALLOC(). This means that `ptr' must be a pointer returned by
+  GRN_PLUGIN_MALLOC() or GRN_PLUGIN_REALLOC().
+ */
+#define GRN_PLUGIN_FREE(ctx, ptr) \
+  grn_plugin_free((ctx), (ptr), __FILE__, __LINE__, __FUNCTION__)
+
+/*
+  GRN_PLUGIN_LOG() reports a log of `level'. Its error message is generated
+  from the varying number of arguments, in which the first one is the format
+  string and the rest are its arguments. See grn_log_level in "groonga.h" for
+  more details of `level'.
+ */
+#define GRN_PLUGIN_LOG(ctx, level, ...) \
+  GRN_LOG((ctx), (level), __VA_ARGS__)
+
+/*
+  Don't call grn_plugin_set_error() directly. This function is used in
+  GRN_PLUGIN_SET_ERROR().
+ */
+void grn_plugin_set_error(grn_ctx *ctx, grn_log_level level,
+                          grn_rc error_code,
+                          const char *file, int line, const char *func,
+                          const char *format, ...);
+
+/*
+  Don't call these functions directly. grn_plugin_backtrace() and
+  grn_plugin_logtrace() are used in GRN_PLUGIN_SET_ERROR().
+ */
+void grn_plugin_backtrace(grn_ctx *ctx);
+void grn_plugin_logtrace(grn_ctx *ctx, grn_log_level level);
+
+/*
+  Don't use GRN_PLUGIN_SET_ERROR() directly. This macro is used in
+  GRN_PLUGIN_ERROR().
+ */
+#define GRN_PLUGIN_SET_ERROR(ctx, level, error_code, ...) do { \
+  grn_plugin_set_error(ctx, level, error_code, \
+                       __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__); \
+  GRN_LOG(ctx, level, __VA_ARGS__); \
+  grn_plugin_backtrace(ctx); \
+  grn_plugin_logtrace(ctx, level); \
+} while (0)
+
+/*
+  GRN_PLUGIN_ERROR() reports an error of `error_code'. Its error message is
+  generated from the varying number of arguments, in which the first one is the
+  format string and the rest are its arguments. See grn_rc in "groonga.h" for
+  more details of `error_code'.
+ */
+#define GRN_PLUGIN_ERROR(ctx, error_code, ...) \
+  GRN_PLUGIN_SET_ERROR(ctx, GRN_LOG_ERROR, error_code, __VA_ARGS__)
+
+/*
+  grn_plugin_mutex is available to make a critical section. See the
+  following functions.
+ */
+typedef struct _grn_plugin_mutex grn_plugin_mutex;
+
+/*
+  grn_plugin_mutex_create() returns a pointer to a new object of
+  grn_plugin_mutex. Memory for the new object is obtained with
+  GRN_PLUGIN_MALLOC(). grn_plugin_mutex_create() returns NULL if sufficient
+  memory is not available.
+ */
+grn_plugin_mutex *grn_plugin_mutex_create(grn_ctx *ctx);
+
+/*
+  grn_plugin_mutex_destroy() finalizes an object of grn_plugin_mutex and then
+  frees memory allocated for that object.
+ */
+void grn_plugin_mutex_destroy(grn_ctx *ctx, grn_plugin_mutex *mutex);
+
+/*
+  grn_plugin_mutex_lock() locks a mutex object. If the object is already
+  locked, the calling thread waits until the object will be unlocked.
+ */
+void grn_plugin_mutex_lock(grn_ctx *ctx, grn_plugin_mutex *mutex);
+
+/*
+  grn_plugin_mutex_unlock() unlocks a mutex object. grn_plugin_mutex_unlock()
+  should not be called for an unlocked object.
+ */
+void grn_plugin_mutex_unlock(grn_ctx *ctx, grn_plugin_mutex *mutex);
+
 #ifdef __cplusplus
 }
 #endif

data/vendor/local/include/groonga/groonga/tokenizer.h

@@ -0,0 +1,146 @@
+/* -*- c-basic-offset: 2 -*- */
+/*
+  Copyright(C) 2012 Brazil
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License version 2.1 as published by the Free Software Foundation.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+#ifndef GRN_PLUGIN_TOKENIZER_H
+#define GRN_PLUGIN_TOKENIZER_H
+
+#include <stddef.h>
+
+#include <groonga/plugin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+/*
+  grn_tokenizer_charlen() returns the length (#bytes) of the first character
+  in the string specified by `str_ptr' and `str_length'. If the starting bytes
+  are invalid as a character, grn_tokenizer_charlen() returns 0. See
+  grn_encoding in "groonga.h" for more details of `encoding'
+ */
+int grn_tokenizer_charlen(grn_ctx *ctx, const char *str_ptr,
+                          unsigned int str_length, grn_encoding encoding);
+
+/*
+  grn_tokenizer_isspace() returns the length (#bytes) of the first character
+  in the string specified by `str_ptr' and `str_length' if it is a space
+  character. Otherwise, grn_tokenizer_isspace() returns 0.
+ */
+int grn_tokenizer_isspace(grn_ctx *ctx, const char *str_ptr,
+                          unsigned int str_length, grn_encoding encoding);
+
+/*
+  grn_tokenizer_query is a structure for storing a query. See the following
+  functions.
+ */
+typedef struct _grn_tokenizer_query grn_tokenizer_query;
+
+struct _grn_tokenizer_query {
+  grn_obj *normalized_query;
+  char *query_buf;
+  const char *ptr;
+  unsigned int length;
+  grn_encoding encoding;
+};
+
+/*
+  grn_tokenizer_query_create() parses `args' and returns a new object of
+  grn_tokenizer_query. The new object stores information of the query.
+  grn_tokenizer_query_create() normalizes the query if the target table
+  requires normalization. grn_tokenizer_query_create() returns NULL if
+  something goes wrong. Note that grn_tokenizer_query_create() must be called
+  just once in the function that initializes a tokenizer.
+ */
+grn_tokenizer_query *grn_tokenizer_query_create(grn_ctx *ctx,
+                                                int num_args, grn_obj **args);
+
+/*
+  grn_tokenizer_mutex_destroy() finalizes an object of grn_tokenizer_mutex
+  and then frees memory allocated for that object.
+ */
+void grn_tokenizer_query_destroy(grn_ctx *ctx, grn_tokenizer_query *query);
+
+/*
+  grn_tokenizer_token is needed to return tokens. A grn_tokenizer_token object
+  stores a token to be returned and it must be maintained until a request for
+  next token or finalization comes.
+ */
+typedef struct _grn_tokenizer_token grn_tokenizer_token;
+
+struct _grn_tokenizer_token {
+  grn_obj str;
+  grn_obj status;
+};
+
+/*
+  grn_tokenizer_token_init() initializes `token'. Note that an initialized
+  object must be finalized by grn_tokenizer_token_fin().
+ */
+void grn_tokenizer_token_init(grn_ctx *ctx, grn_tokenizer_token *token);
+
+/*
+  grn_tokenizer_token_fin() finalizes `token' that has been initialized by
+  grn_tokenizer_token_init().
+ */
+void grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token *token);
+
+/*
+  grn_tokenizer_status provides a list of tokenizer status codes.
+  GRN_TOKENIZER_CONTINUE means that the next token is not the last one and
+  GRN_TOKENIZER_LAST means that the next token is the last one. If a document
+  or query contains no tokens, please push an empty string with
+  GRN_TOKENIZER_LAST as a token.
+ */
+typedef enum {
+  GRN_TOKENIZER_CONTINUE = 0,
+  GRN_TOKENIZER_LAST     = 1
+} grn_tokenizer_status;
+
+/*
+  grn_tokenizer_token_push() pushes the next token in `*token'. Note that
+  grn_tokenizer_token_push() does not make a copy of the given string. This
+  means that you have to maintain a memory space allocated to the string.
+  Also note that the grn_tokenizer_token object must be maintained until the
+  request for the next token or finalization comes. See grn_tokenizer_status in
+  this header for more details of `status'.
+ */
+void grn_tokenizer_token_push(grn_ctx *ctx, grn_tokenizer_token *token,
+                              const char *str_ptr, unsigned int str_length,
+                              grn_tokenizer_status status);
+
+/*
+  grn_tokenizer_register() registers a plugin to the database which is
+  associated with `ctx'. `plugin_name_ptr' and `plugin_name_length' specify the
+  plugin name. Alphabetic letters ('A'-'Z' and 'a'-'z'), digits ('0'-'9') and
+  an underscore ('_') are capable characters. `init', `next' and `fin' specify
+  the plugin functions. `init' is called for initializing a tokenizer for a
+  document or query. `next' is called for extracting tokens one by one. `fin'
+  is called for finalizing a tokenizer. grn_tokenizer_register() returns
+  GRN_SUCCESS on success, an error code on failure. See "groonga.h" for more
+  details of grn_proc_func and grn_user_data, that is used as an argument of
+  grn_proc_func.
+ */
+grn_rc grn_tokenizer_register(grn_ctx *ctx, const char *plugin_name_ptr,
+                              unsigned int plugin_name_length,
+                              grn_proc_func *init, grn_proc_func *next,
+                              grn_proc_func *fin);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */
+
+#endif  /* GRN_PLUGIN_TOKENIZER_H */