langscan 1.2-x86-mswin32-60

data/ext/langscan/_make_h.rb

@@ -0,0 +1,30 @@
+require 'fileutils'
+raise unless ARGV.length == 5
+template_file_name = ARGV.shift
+output_directory = ARGV.shift
+token_list_file_name = ARGV.shift
+lang_abbrev = ARGV.shift
+module_name = ARGV.shift
+
+token_list = File.open(token_list_file_name).readlines.map {|line| line.chomp }
+
+lines = ["#define LANGSCAN_LANGNAME_TOKEN_LIST"]
+token_list.each {|token|
+  lines.push("     LANGSCAN_LANGNAME_TOKEN(#{token})")
+}
+macro = lines.join(" \\\n")
+
+template = File.read(template_file_name)
+template.gsub!(/@LANGSCAN_LANGNAME_TOKEN_LIST@/, macro)
+template.gsub!(/LANGNAME/, lang_abbrev.upcase)
+template.gsub!(/langname/, lang_abbrev.downcase)
+template.gsub!(/LangName/, module_name)
+
+h_file_name = File.join( output_directory, lang_abbrev + ".h" )
+FileUtils.rm_rf(h_file_name)
+File.open(h_file_name, "w") {|f|
+#  f.printf("/* %s generated automatically from %s */\n",
+#           h_file_name, template_file_name)
+  f.print template
+}
+#File.chmod(0444, h_file_name)

data/ext/langscan/_template.c

@@ -0,0 +1,134 @@
+/* -*- mode: C; indent-tabs-mode: nil; c-basic-offset: 2 c-style: "BSD" -*- */
+/*
+ * _template.c - a template file for LangScan modules
+ * 
+ * Copyright (C) 2004-2005 Akira Tanaka <akr@m17n.org> 
+ *     All rights reserved.
+ *     This is free software with ABSOLUTELY NO WARRANTY.
+ * 
+ * You can redistribute it and/or modify it under the terms of 
+ * the GNU General Public License version 2.
+ */
+
+
+#include <ruby.h>
+#include "langname.h"
+
+static VALUE token_symbol_list[
+#define LANGSCAN_LANGNAME_TOKEN(token) 1 +
+  1 + LANGSCAN_LANGNAME_TOKEN_LIST 0
+#undef LANGSCAN_LANGNAME_TOKEN
+];
+
+#ifndef RSTRING_PTR
+# define RSTRING_PTR(str) (RSTRING(str)->ptr)
+#endif
+#ifndef RSTRING_LEN
+# define RSTRING_LEN(str) (RSTRING(str)->len)
+#endif
+
+#ifndef RARRAY_PTR
+# define RARRAY_PTR(str) (RARRAY(str)->ptr)
+#endif
+#ifndef RARRAY_LEN
+# define RARRAY_LEN(str) (RARRAY(str)->len)
+#endif
+
+static size_t user_read_str(void **user_data_p, char *buf, size_t maxlen)
+{
+  VALUE user_data = (VALUE)*user_data_p;
+  VALUE user_str = RARRAY_PTR(user_data)[0];
+  VALUE user_off = RARRAY_PTR(user_data)[1];
+  long off = NUM2LONG(user_off);
+  if (RSTRING_LEN(user_str)-off < maxlen) {
+    maxlen = RSTRING_LEN(user_str)-off;
+  }
+  memcpy(buf, RSTRING_PTR(user_str)+off, maxlen);
+  RARRAY_PTR(user_data)[1] = LONG2NUM(off+maxlen);
+  return maxlen;
+}
+
+static void tokenizer_mark(langscan_langname_tokenizer_t *tokenizer)
+{
+  if (tokenizer == NULL)
+    return;
+  rb_gc_mark((VALUE)langscan_langname_tokenizer_get_user_data(tokenizer));
+}
+
+static void tokenizer_free(langscan_langname_tokenizer_t *tokenizer)
+{
+  if (tokenizer == NULL)
+    return;
+  langscan_langname_free_tokenizer(tokenizer);
+}
+
+static VALUE tokenizer_s_allocate(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, tokenizer_mark, tokenizer_free, NULL);
+}
+
+static VALUE tokenizer_initialize(VALUE self, VALUE user_data)
+{
+  VALUE tmp;
+  user_read_t user_read;
+  langscan_langname_tokenizer_t *tokenizer;
+  Data_Get_Struct(self, langscan_langname_tokenizer_t, tokenizer);
+  StringValue(user_data);
+  user_read = user_read_str;
+  user_data = rb_ary_new3(2, rb_str_new4(user_data), INT2FIX(0));
+  RBASIC(user_data)->klass = 0;
+  DATA_PTR(self) = langscan_langname_make_tokenizer(user_read, (void *)user_data);
+  return self;
+}
+
+static VALUE tokenizer_get_token(VALUE self)
+{
+  langscan_langname_tokenizer_t *tokenizer;
+  langscan_langname_token_t token;
+  Data_Get_Struct(self, langscan_langname_tokenizer_t, tokenizer);
+  if (tokenizer == NULL) { return Qnil; }
+  token = langscan_langname_get_token(tokenizer);
+  if (token == langscan_langname_eof) {
+    DATA_PTR(self) = NULL;
+    langscan_langname_free_tokenizer(tokenizer);
+    return Qnil;
+  }
+  return rb_ary_new3(8,
+    token_symbol_list[token],
+    rb_str_new(langscan_langname_curtoken_text(tokenizer), langscan_langname_curtoken_leng(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_beg_lineno(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_beg_columnno(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_beg_byteno(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_end_lineno(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_end_columnno(tokenizer)),
+    INT2NUM(langscan_langname_curtoken_end_byteno(tokenizer)));
+}
+
+static VALUE tokenizer_close(VALUE self)
+{
+  langscan_langname_tokenizer_t *tokenizer;
+  Data_Get_Struct(self, langscan_langname_tokenizer_t, tokenizer);
+  if (tokenizer == NULL) { return Qnil; }
+  DATA_PTR(self) = NULL;
+  langscan_langname_free_tokenizer(tokenizer);
+  return Qnil;
+}
+
+void Init_langname()
+{
+  VALUE LangScan = rb_define_module("LangScan");
+  VALUE LangScan_LANGNAME = rb_define_module_under(LangScan, "LangName");
+  VALUE Tokenizer = rb_define_class_under(LangScan_LANGNAME, "Tokenizer", rb_cData);
+  langscan_langname_token_t token_id;
+
+  token_id = 0;
+  token_symbol_list[token_id++] = Qnil;
+#define LANGSCAN_LANGNAME_TOKEN(token) token_symbol_list[token_id++] = ID2SYM(rb_intern(#token));
+  LANGSCAN_LANGNAME_TOKEN_LIST
+#undef LANGSCAN_LANGNAME_TOKEN
+
+  rb_define_alloc_func(Tokenizer, tokenizer_s_allocate);
+  rb_define_method(Tokenizer, "initialize", tokenizer_initialize, 1);
+  rb_define_method(Tokenizer, "get_token", tokenizer_get_token, 0);
+  rb_define_method(Tokenizer, "close", tokenizer_close, 0);
+}

data/ext/langscan/_template.h

@@ -0,0 +1,53 @@
+#ifndef LANGSCAN_LANGNAME_H
+#define LANGSCAN_LANGNAME_H
+
+@LANGSCAN_LANGNAME_TOKEN_LIST@
+
+typedef enum {
+  langscan_langname_eof = 0,
+#define LANGSCAN_LANGNAME_TOKEN(name) langscan_langname_##name,
+  LANGSCAN_LANGNAME_TOKEN_LIST
+#undef LANGSCAN_LANGNAME_TOKEN
+} langscan_langname_token_t;
+
+typedef struct {
+  int beg_lineno;
+  int beg_columnno;
+  int beg_byteno;
+  int end_lineno;
+  int end_columnno;
+  int end_byteno;
+  int eof;
+  char *text;
+  int leng;
+  size_t (*user_read)(void **user_data_p, char *buf, size_t maxlen);
+  void *user_data;
+} langscan_langname_lex_extra_t;
+
+typedef struct langscan_langname_tokenizer_tag {
+  langscan_langname_lex_extra_t *extra;
+  void *scanner;
+} langscan_langname_tokenizer_t;
+
+typedef size_t (*user_read_t)(void **user_data_p, char *buf, size_t maxlen);
+
+langscan_langname_tokenizer_t *langscan_langname_make_tokenizer(user_read_t user_read, void *user_data);
+langscan_langname_token_t langscan_langname_get_token(langscan_langname_tokenizer_t *tokenizer);
+void langscan_langname_free_tokenizer(langscan_langname_tokenizer_t *tokenizer);
+
+user_read_t langscan_langname_tokenizer_get_user_read(langscan_langname_tokenizer_t *tokenizer);
+void *langscan_langname_tokenizer_get_user_data(langscan_langname_tokenizer_t *tokenizer);
+
+const char *langscan_langname_token_name(langscan_langname_token_t token);
+#define langscan_langname_curtoken_beg_lineno(tokenizer) ((tokenizer)->extra->beg_lineno)
+#define langscan_langname_curtoken_beg_columnno(tokenizer) ((tokenizer)->extra->beg_columnno)
+#define langscan_langname_curtoken_beg_byteno(tokenizer) ((tokenizer)->extra->beg_byteno)
+#define langscan_langname_curtoken_end_lineno(tokenizer) ((tokenizer)->extra->end_lineno)
+#define langscan_langname_curtoken_end_columnno(tokenizer) ((tokenizer)->extra->end_columnno)
+#define langscan_langname_curtoken_end_byteno(tokenizer) ((tokenizer)->extra->end_byteno)
+#define langscan_langname_curtoken_text(tokenizer) ((tokenizer)->extra->text)
+#define langscan_langname_curtoken_leng(tokenizer) ((tokenizer)->extra->leng)
+
+void langscan_langname_extract_functions(langscan_langname_tokenizer_t *);
+
+#endif

data/ext/langscan/c/c/Makefile

@@ -0,0 +1,188 @@
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = C:/usr/ruby/lib/ruby/1.8/i386-mswin32
+hdrdir = $(topdir)
+VPATH = $(srcdir);$(topdir);$(hdrdir)
+
+DESTDIR = C:
+exec_prefix = $(prefix)
+prefix = $(DESTDIR)/usr/ruby
+sharedstatedir = $(DESTDIR)/etc
+mandir = $(prefix)/man
+oldincludedir = $(DESTDIR)/usr/include
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+sitedir = $(prefix)/lib/ruby/site_ruby
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+includedir = $(prefix)/include
+infodir = $(prefix)/info
+vendorlibdir = $(vendordir)/$(ruby_version)
+sysconfdir = $(prefix)/etc
+libdir = $(exec_prefix)/lib
+sbindir = $(exec_prefix)/sbin
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+vendordir = $(prefix)/lib/ruby/vendor_ruby
+archdir = $(rubylibdir)/$(arch)
+sitearchdir = $(sitelibdir)/$(sitearch)
+datadir = $(prefix)/share
+localstatedir = $(DESTDIR)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+
+CC = cl -nologo
+LIBRUBY = $(RUBY_SO_NAME).lib
+LIBRUBY_A = $(RUBY_SO_NAME)-static.lib
+LIBRUBYARG_SHARED = $(LIBRUBY)
+LIBRUBYARG_STATIC = $(LIBRUBY_A)
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -MT -Zi  -O2b2xg- -G6 
+INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =     
+CXXFLAGS = $(CFLAGS) 
+ldflags  = 
+dldflags = -link -incremental:no -debug -opt:ref -opt:icf -dll $(LIBPATH)
+archflag = 
+DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
+LDSHARED = cl -nologo -LD
+AR = lib -nologo
+EXEEXT = .exe
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = msvcrt-ruby18
+arch = i386-mswin32
+sitearch = i386-msvcrt
+ruby_version = 1.8
+ruby = C:/usr/ruby/bin/ruby
+RUBY = $(ruby:/=\)
+RM = $(RUBY) -run -e rm -- -f
+MAKEDIRS = @$(RUBY) -run -e mkdir -- -p
+INSTALL = @$(RUBY) -run -e install -- -vp
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 0644
+COPY = copy > nul
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -libpath:"." -libpath:"$(libdir)"
+DEFFILE = $(TARGET)-$(arch).def
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = vc*.pdb $(DEFFILE)
+
+extout = 
+extout_prefix = 
+target_prefix = /langscan/c
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  oldnames.lib user32.lib advapi32.lib shell32.lib ws2_32.lib  
+SRCS = c.c ctok.c
+OBJS = c.obj ctok.obj
+TARGET = c
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR = C:/devel/gem/git/langscan/pkg/langscan-1.2.gem.build/lib$(target_prefix)
+RUBYARCHDIR = C:/devel/gem/git/langscan/pkg/langscan-1.2.gem.build/lib$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).so $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.obj *.lib *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS:/=\) $(CLEANOBJS:/=\) $(CLEANFILES:/=\)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES:/=\)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB:/=\) $(RUBYARCHDIR:/=\)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .obj
+
+{$(hdrdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cc.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cxx.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cpp.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+{$(topdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+{$(srcdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+.c.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+$(DLLIB): $(DEFFILE) $(OBJS) Makefile
+	@-$(RM) $@
+	$(LDSHARED) -Fe$(@) $(OBJS) $(LIBS) $(LOCAL_LIBS) $(DLDFLAGS) -implib:$(*F:.so=)-$(arch).lib -pdb:$(*F:.so=)-$(arch).pdb -def:$(DEFFILE)
+
+
+
+$(DEFFILE): 
+	$(RUBY) -e "puts 'EXPORTS', 'Init_$(TARGET)'"  > $@
+
+$(OBJS): {.;$(VPATH)}ruby.h {.;$(VPATH)}defines.h

data/ext/langscan/c/c/c.c

@@ -0,0 +1,134 @@
+/* -*- mode: C; indent-tabs-mode: nil; c-basic-offset: 2 c-style: "BSD" -*- */
+/*
+ * _template.c - a template file for LangScan modules
+ * 
+ * Copyright (C) 2004-2005 Akira Tanaka <akr@m17n.org> 
+ *     All rights reserved.
+ *     This is free software with ABSOLUTELY NO WARRANTY.
+ * 
+ * You can redistribute it and/or modify it under the terms of 
+ * the GNU General Public License version 2.
+ */
+
+
+#include <ruby.h>
+#include "c.h"
+
+static VALUE token_symbol_list[
+#define LANGSCAN_C_TOKEN(token) 1 +
+  1 + LANGSCAN_C_TOKEN_LIST 0
+#undef LANGSCAN_C_TOKEN
+];
+
+#ifndef RSTRING_PTR
+# define RSTRING_PTR(str) (RSTRING(str)->ptr)
+#endif
+#ifndef RSTRING_LEN
+# define RSTRING_LEN(str) (RSTRING(str)->len)
+#endif
+
+#ifndef RARRAY_PTR
+# define RARRAY_PTR(str) (RARRAY(str)->ptr)
+#endif
+#ifndef RARRAY_LEN
+# define RARRAY_LEN(str) (RARRAY(str)->len)
+#endif
+
+static size_t user_read_str(void **user_data_p, char *buf, size_t maxlen)
+{
+  VALUE user_data = (VALUE)*user_data_p;
+  VALUE user_str = RARRAY_PTR(user_data)[0];
+  VALUE user_off = RARRAY_PTR(user_data)[1];
+  long off = NUM2LONG(user_off);
+  if (RSTRING_LEN(user_str)-off < maxlen) {
+    maxlen = RSTRING_LEN(user_str)-off;
+  }
+  memcpy(buf, RSTRING_PTR(user_str)+off, maxlen);
+  RARRAY_PTR(user_data)[1] = LONG2NUM(off+maxlen);
+  return maxlen;
+}
+
+static void tokenizer_mark(langscan_c_tokenizer_t *tokenizer)
+{
+  if (tokenizer == NULL)
+    return;
+  rb_gc_mark((VALUE)langscan_c_tokenizer_get_user_data(tokenizer));
+}
+
+static void tokenizer_free(langscan_c_tokenizer_t *tokenizer)
+{
+  if (tokenizer == NULL)
+    return;
+  langscan_c_free_tokenizer(tokenizer);
+}
+
+static VALUE tokenizer_s_allocate(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, tokenizer_mark, tokenizer_free, NULL);
+}
+
+static VALUE tokenizer_initialize(VALUE self, VALUE user_data)
+{
+  VALUE tmp;
+  user_read_t user_read;
+  langscan_c_tokenizer_t *tokenizer;
+  Data_Get_Struct(self, langscan_c_tokenizer_t, tokenizer);
+  StringValue(user_data);
+  user_read = user_read_str;
+  user_data = rb_ary_new3(2, rb_str_new4(user_data), INT2FIX(0));
+  RBASIC(user_data)->klass = 0;
+  DATA_PTR(self) = langscan_c_make_tokenizer(user_read, (void *)user_data);
+  return self;
+}
+
+static VALUE tokenizer_get_token(VALUE self)
+{
+  langscan_c_tokenizer_t *tokenizer;
+  langscan_c_token_t token;
+  Data_Get_Struct(self, langscan_c_tokenizer_t, tokenizer);
+  if (tokenizer == NULL) { return Qnil; }
+  token = langscan_c_get_token(tokenizer);
+  if (token == langscan_c_eof) {
+    DATA_PTR(self) = NULL;
+    langscan_c_free_tokenizer(tokenizer);
+    return Qnil;
+  }
+  return rb_ary_new3(8,
+    token_symbol_list[token],
+    rb_str_new(langscan_c_curtoken_text(tokenizer), langscan_c_curtoken_leng(tokenizer)),
+    INT2NUM(langscan_c_curtoken_beg_lineno(tokenizer)),
+    INT2NUM(langscan_c_curtoken_beg_columnno(tokenizer)),
+    INT2NUM(langscan_c_curtoken_beg_byteno(tokenizer)),
+    INT2NUM(langscan_c_curtoken_end_lineno(tokenizer)),
+    INT2NUM(langscan_c_curtoken_end_columnno(tokenizer)),
+    INT2NUM(langscan_c_curtoken_end_byteno(tokenizer)));
+}
+
+static VALUE tokenizer_close(VALUE self)
+{
+  langscan_c_tokenizer_t *tokenizer;
+  Data_Get_Struct(self, langscan_c_tokenizer_t, tokenizer);
+  if (tokenizer == NULL) { return Qnil; }
+  DATA_PTR(self) = NULL;
+  langscan_c_free_tokenizer(tokenizer);
+  return Qnil;
+}
+
+void Init_c()
+{
+  VALUE LangScan = rb_define_module("LangScan");
+  VALUE LangScan_C = rb_define_module_under(LangScan, "C");
+  VALUE Tokenizer = rb_define_class_under(LangScan_C, "Tokenizer", rb_cData);
+  langscan_c_token_t token_id;
+
+  token_id = 0;
+  token_symbol_list[token_id++] = Qnil;
+#define LANGSCAN_C_TOKEN(token) token_symbol_list[token_id++] = ID2SYM(rb_intern(#token));
+  LANGSCAN_C_TOKEN_LIST
+#undef LANGSCAN_C_TOKEN
+
+  rb_define_alloc_func(Tokenizer, tokenizer_s_allocate);
+  rb_define_method(Tokenizer, "initialize", tokenizer_initialize, 1);
+  rb_define_method(Tokenizer, "get_token", tokenizer_get_token, 0);
+  rb_define_method(Tokenizer, "close", tokenizer_close, 0);
+}