langscan 1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. data/AUTHORS.txt +19 -0
  2. data/History.txt +126 -0
  3. data/Manifest.txt +167 -0
  4. data/README.rdoc +89 -0
  5. data/Rakefile +40 -0
  6. data/ext/langscan/_make_c.rb +20 -0
  7. data/ext/langscan/_make_h.rb +30 -0
  8. data/ext/langscan/_template.c +134 -0
  9. data/ext/langscan/_template.h +53 -0
  10. data/ext/langscan/c/c/Makefile +157 -0
  11. data/ext/langscan/c/c/c.c +134 -0
  12. data/ext/langscan/c/c/c.h +66 -0
  13. data/ext/langscan/c/c/ctok.c +4622 -0
  14. data/ext/langscan/c/c/ctok.l +212 -0
  15. data/ext/langscan/c/c/extconf.rb +3 -0
  16. data/ext/langscan/c/c/modulename.txt +1 -0
  17. data/ext/langscan/c/c/tokenlist.txt +13 -0
  18. data/ext/langscan/csharp/csharp/Makefile +157 -0
  19. data/ext/langscan/csharp/csharp/csharp.c +134 -0
  20. data/ext/langscan/csharp/csharp/csharp.h +65 -0
  21. data/ext/langscan/csharp/csharp/csharptok.c +2965 -0
  22. data/ext/langscan/csharp/csharp/csharptok.l +200 -0
  23. data/ext/langscan/csharp/csharp/extconf.rb +3 -0
  24. data/ext/langscan/csharp/csharp/modulename.txt +1 -0
  25. data/ext/langscan/csharp/csharp/tokenlist.txt +12 -0
  26. data/ext/langscan/d/d/Makefile +157 -0
  27. data/ext/langscan/d/d/d.c +134 -0
  28. data/ext/langscan/d/d/d.h +64 -0
  29. data/ext/langscan/d/d/dtok.c +5461 -0
  30. data/ext/langscan/d/d/dtok.l +282 -0
  31. data/ext/langscan/d/d/extconf.rb +3 -0
  32. data/ext/langscan/d/d/modulename.txt +1 -0
  33. data/ext/langscan/d/d/tokenlist.txt +11 -0
  34. data/ext/langscan/elisp/elisp/Makefile +157 -0
  35. data/ext/langscan/elisp/elisp/elisp.c +134 -0
  36. data/ext/langscan/elisp/elisp/elisp.h +62 -0
  37. data/ext/langscan/elisp/elisp/elisptok.c +2101 -0
  38. data/ext/langscan/elisp/elisp/elisptok.l +151 -0
  39. data/ext/langscan/elisp/elisp/extconf.rb +3 -0
  40. data/ext/langscan/elisp/elisp/modulename.txt +1 -0
  41. data/ext/langscan/elisp/elisp/tokenlist.txt +9 -0
  42. data/ext/langscan/java/java/Makefile +157 -0
  43. data/ext/langscan/java/java/extconf.rb +3 -0
  44. data/ext/langscan/java/java/java.c +134 -0
  45. data/ext/langscan/java/java/java.h +64 -0
  46. data/ext/langscan/java/java/javatok.c +2090 -0
  47. data/ext/langscan/java/java/javatok.l +155 -0
  48. data/ext/langscan/java/java/modulename.txt +1 -0
  49. data/ext/langscan/java/java/tokenlist.txt +11 -0
  50. data/ext/langscan/javascript/javascript/Makefile +157 -0
  51. data/ext/langscan/javascript/javascript/extconf.rb +3 -0
  52. data/ext/langscan/javascript/javascript/javascript.c +134 -0
  53. data/ext/langscan/javascript/javascript/javascript.h +63 -0
  54. data/ext/langscan/javascript/javascript/javascripttok.c +2051 -0
  55. data/ext/langscan/javascript/javascript/javascripttok.l +147 -0
  56. data/ext/langscan/javascript/javascript/modulename.txt +1 -0
  57. data/ext/langscan/javascript/javascript/tokenlist.txt +10 -0
  58. data/ext/langscan/pairmatcher/pairmatcher/Makefile +157 -0
  59. data/ext/langscan/pairmatcher/pairmatcher/extconf.rb +3 -0
  60. data/ext/langscan/pairmatcher/pairmatcher/pairmatcher.c +890 -0
  61. data/ext/langscan/php/php/Makefile +157 -0
  62. data/ext/langscan/php/php/extconf.rb +3 -0
  63. data/ext/langscan/php/php/modulename.txt +1 -0
  64. data/ext/langscan/php/php/php.c +134 -0
  65. data/ext/langscan/php/php/php.h +64 -0
  66. data/ext/langscan/php/php/phptok.c +2406 -0
  67. data/ext/langscan/php/php/phptok.l +212 -0
  68. data/ext/langscan/php/php/tokenlist.txt +11 -0
  69. data/ext/langscan/post-distclean.rb +21 -0
  70. data/ext/langscan/pre-config.rb +57 -0
  71. data/ext/langscan/python/python/Makefile +157 -0
  72. data/ext/langscan/python/python/extconf.rb +3 -0
  73. data/ext/langscan/python/python/modulename.txt +1 -0
  74. data/ext/langscan/python/python/python.c +134 -0
  75. data/ext/langscan/python/python/python.h +61 -0
  76. data/ext/langscan/python/python/pythontok.c +2102 -0
  77. data/ext/langscan/python/python/pythontok.l +155 -0
  78. data/ext/langscan/python/python/tokenlist.txt +8 -0
  79. data/ext/langscan/ruby/compat/ripper/Makefile +158 -0
  80. data/ext/langscan/ruby/compat/ripper/depend +1 -0
  81. data/ext/langscan/ruby/compat/ripper/extconf.rb +4 -0
  82. data/ext/langscan/ruby/compat/ripper/include/eventids1.c +251 -0
  83. data/ext/langscan/ruby/compat/ripper/include/eventids2.c +277 -0
  84. data/ext/langscan/ruby/compat/ripper/include/lex.c +138 -0
  85. data/ext/langscan/ruby/compat/ripper/ripper.c +14420 -0
  86. data/ext/langscan/scheme/scheme/Makefile +157 -0
  87. data/ext/langscan/scheme/scheme/extconf.rb +3 -0
  88. data/ext/langscan/scheme/scheme/modulename.txt +1 -0
  89. data/ext/langscan/scheme/scheme/scheme.c +134 -0
  90. data/ext/langscan/scheme/scheme/scheme.h +60 -0
  91. data/ext/langscan/scheme/scheme/schemetok.c +2447 -0
  92. data/ext/langscan/scheme/scheme/schemetok.l +177 -0
  93. data/ext/langscan/scheme/scheme/tokenlist.txt +7 -0
  94. data/ext/langscan/sh/sh/Makefile +157 -0
  95. data/ext/langscan/sh/sh/extconf.rb +3 -0
  96. data/ext/langscan/sh/sh/modulename.txt +1 -0
  97. data/ext/langscan/sh/sh/sh.c +134 -0
  98. data/ext/langscan/sh/sh/sh.h +61 -0
  99. data/ext/langscan/sh/sh/shtok.c +2470 -0
  100. data/ext/langscan/sh/sh/shtok.l +325 -0
  101. data/ext/langscan/sh/sh/tokenlist.txt +8 -0
  102. data/lib/langscan.rb +124 -0
  103. data/lib/langscan/_common.rb +50 -0
  104. data/lib/langscan/_easyscanner.rb +78 -0
  105. data/lib/langscan/_pairmatcher.rb +46 -0
  106. data/lib/langscan/_type.rb +125 -0
  107. data/lib/langscan/autoconf.rb +51 -0
  108. data/lib/langscan/automake.rb +51 -0
  109. data/lib/langscan/brainfuck.rb +48 -0
  110. data/lib/langscan/c.rb +144 -0
  111. data/lib/langscan/csharp.rb +101 -0
  112. data/lib/langscan/css.rb +109 -0
  113. data/lib/langscan/d.rb +201 -0
  114. data/lib/langscan/eiffel.rb +167 -0
  115. data/lib/langscan/elisp.rb +132 -0
  116. data/lib/langscan/io.rb +84 -0
  117. data/lib/langscan/java.rb +95 -0
  118. data/lib/langscan/javascript.rb +97 -0
  119. data/lib/langscan/lua.rb +116 -0
  120. data/lib/langscan/ocaml.rb +298 -0
  121. data/lib/langscan/ocaml/camlexer.ml +28 -0
  122. data/lib/langscan/ocaml/lexer.mll +230 -0
  123. data/lib/langscan/ocaml/types.ml +36 -0
  124. data/lib/langscan/perl.rb +87 -0
  125. data/lib/langscan/perl/tokenizer.pl +231 -0
  126. data/lib/langscan/php.rb +80 -0
  127. data/lib/langscan/python.rb +101 -0
  128. data/lib/langscan/rpmspec.rb +71 -0
  129. data/lib/langscan/ruby.rb +164 -0
  130. data/lib/langscan/ruby/compat/README +5 -0
  131. data/lib/langscan/ruby/compat/ripper.rb +4 -0
  132. data/lib/langscan/ruby/compat/ripper/core.rb +918 -0
  133. data/lib/langscan/ruby/compat/ripper/filter.rb +70 -0
  134. data/lib/langscan/ruby/compat/ripper/lexer.rb +179 -0
  135. data/lib/langscan/ruby/compat/ripper/sexp.rb +100 -0
  136. data/lib/langscan/scheme.rb +160 -0
  137. data/lib/langscan/sh.rb +116 -0
  138. data/lib/langscan/text.rb +37 -0
  139. data/metaconfig +2 -0
  140. data/script/console +10 -0
  141. data/script/destroy +14 -0
  142. data/script/generate +14 -0
  143. data/script/makemanifest.rb +21 -0
  144. data/setup.rb +1604 -0
  145. data/tasks/extconf.rake +13 -0
  146. data/tasks/extconf/langscan.rake +42 -0
  147. data/test/langscan/brainfuck/test/test_scan.rb +55 -0
  148. data/test/langscan/c/test/test_scan.rb +216 -0
  149. data/test/langscan/c/test/test_token.rb +41 -0
  150. data/test/langscan/csharp/test/test_scan.rb +157 -0
  151. data/test/langscan/css/test/test_css.rb +79 -0
  152. data/test/langscan/d/test/test_scan.rb +233 -0
  153. data/test/langscan/d/test/test_token.rb +205 -0
  154. data/test/langscan/eiffel/test/test_eiffel.rb +95 -0
  155. data/test/langscan/elisp/test/test_elisp.rb +177 -0
  156. data/test/langscan/io/test/test_io.rb +79 -0
  157. data/test/langscan/java/test/test_java.rb +74 -0
  158. data/test/langscan/javascript/test/test_javascript.rb +39 -0
  159. data/test/langscan/lua/test/test_lua.rb +69 -0
  160. data/test/langscan/ocaml/test/test_ocaml.rb +161 -0
  161. data/test/langscan/php/test/test_scan.rb +138 -0
  162. data/test/langscan/python/test/test_scan.rb +105 -0
  163. data/test/langscan/rpmspec/test/test_rpmspec.rb +51 -0
  164. data/test/langscan/ruby/test/test_scan.rb +71 -0
  165. data/test/langscan/scheme/test/test_scan.rb +198 -0
  166. data/test/test_helper.rb +7 -0
  167. data/test/test_langscan.rb +123 -0
  168. metadata +296 -0
@@ -0,0 +1,151 @@
1
+ /*
2
+ * elisptok.l - a lex rule for EmacsLisp.
3
+ *
4
+ * Copyright (C) 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
5
+ * All rights reserved.
6
+ * This is free software with ABSOLUTELY NO WARRANTY.
7
+ *
8
+ * You can redistribute it and/or modify it under the terms of
9
+ * the GNU General Public License version 2.
10
+ */
11
+
12
+ %option reentrant
13
+ %option prefix="langscan_elisp_lex_"
14
+ %option noyywrap
15
+ %option nodefault
16
+
17
+ space [ \t]+
18
+ newline \r\n|\r|\n
19
+ escseq \\.
20
+
21
+ %{
22
+
23
+ #include "elisp.h"
24
+
25
+ #define YY_EXTRA_TYPE langscan_elisp_lex_extra_t *
26
+
27
+ #if YY_NULL != 0
28
+ #error "YY_NULL is not 0."
29
+ #endif
30
+
31
+ #define YY_DECL langscan_elisp_token_t langscan_elisp_lex_lex(yyscan_t yyscanner)
32
+
33
+ #define YY_INPUT(buf,result,max_size) \
34
+ if (!yyextra->eof) { \
35
+ result = yyextra->user_read(&(yyextra->user_data), (buf), (max_size)); \
36
+ if (result == 0) \
37
+ yyextra->eof = 1; \
38
+ }
39
+
40
+ #define UPD update_pos(yyextra, yytext, yyleng)
41
+ static void update_pos(langscan_elisp_lex_extra_t *, char *, int);
42
+
43
+ #define report(token) \
44
+ do { \
45
+ yyextra->text = yytext; \
46
+ yyextra->leng = yyleng; \
47
+ return langscan_elisp_##token; \
48
+ } while (0)
49
+
50
+
51
+ %}
52
+
53
+ %%
54
+
55
+ {space} { UPD; report(space); }
56
+ {newline} { UPD; report(space); }
57
+ (,@|[,`\'])(,@|[,`\']|{space})* { UPD; report(quote_chars); }
58
+ \?[ \t\n] { UPD; report(character); }
59
+ \?(\\[CMHsA]-|\\^)*(\\x?[\\da-fA-F]+|\\.|.) { UPD; report(character); }
60
+ [+\-]?[0-9]*\.[0-9]+(e[+\-]?[0-9]+)? { UPD; report(floating); }
61
+ [+\-]?[0-9]+e[+\-]?[0-9]+ { UPD; report(floating); }
62
+ [+\-]?[0-9]+\.? { UPD; report(integer); }
63
+ ([0-9A-Za-z\+\-\*\/_~!@\$%\^&=:<>\{\}]|{escseq})+ { UPD; report(ident); }
64
+ \"([^\\\"]|{escseq}|\\\n)*\" { UPD; report(string); }
65
+ ;.* { UPD; report(comment); }
66
+ . { UPD; report(punct); }
67
+
68
+ %%
69
+
70
+ static void update_pos(
71
+ langscan_elisp_lex_extra_t *extra,
72
+ char *text,
73
+ int leng)
74
+ {
75
+ int i, j;
76
+ extra->beg_byteno = extra->end_byteno;
77
+ extra->beg_lineno = extra->end_lineno;
78
+ extra->beg_columnno = extra->end_columnno;
79
+ j = 0;
80
+ for (i = 0; i < leng; i++) {
81
+ if (text[i] == '\n') {
82
+ extra->end_lineno++;
83
+ j = i + 1;
84
+ extra->end_columnno = 0;
85
+ }
86
+ }
87
+ extra->end_columnno += leng - j;
88
+ extra->end_byteno += leng;
89
+ }
90
+
91
+ langscan_elisp_tokenizer_t *langscan_elisp_make_tokenizer(
92
+ size_t (*user_read)(void **user_data_p, char *buf, size_t maxlen),
93
+ void *user_data)
94
+ {
95
+ langscan_elisp_tokenizer_t *tokenizer;
96
+ langscan_elisp_lex_extra_t *extra;
97
+ tokenizer = (langscan_elisp_tokenizer_t *)malloc(sizeof(langscan_elisp_tokenizer_t));
98
+ if (tokenizer == NULL)
99
+ return NULL;
100
+ extra = (langscan_elisp_lex_extra_t *)malloc(sizeof(langscan_elisp_lex_extra_t));
101
+ if (extra == NULL)
102
+ return NULL;
103
+ extra->user_read = user_read;
104
+ extra->user_data = user_data;
105
+ extra->beg_lineno = 1;
106
+ extra->beg_columnno = 0;
107
+ extra->beg_byteno = 0;
108
+ extra->end_lineno = 1;
109
+ extra->end_columnno = 0;
110
+ extra->end_byteno = 0;
111
+ extra->eof = 0;
112
+ tokenizer->extra = extra;
113
+ langscan_elisp_lex_lex_init(&tokenizer->scanner);
114
+ langscan_elisp_lex_set_extra(extra, tokenizer->scanner);
115
+ return tokenizer;
116
+ }
117
+
118
+ langscan_elisp_token_t langscan_elisp_get_token(langscan_elisp_tokenizer_t *tokenizer)
119
+ {
120
+ return langscan_elisp_lex_lex(tokenizer->scanner);
121
+ }
122
+
123
+ void langscan_elisp_free_tokenizer(langscan_elisp_tokenizer_t *tokenizer)
124
+ {
125
+ langscan_elisp_lex_extra_t *extra = langscan_elisp_lex_get_extra(tokenizer->scanner);
126
+ free((void *)extra);
127
+ langscan_elisp_lex_lex_destroy(tokenizer->scanner);
128
+ free((void *)tokenizer);
129
+ }
130
+
131
+ user_read_t langscan_elisp_tokenizer_get_user_read(langscan_elisp_tokenizer_t *tokenizer)
132
+ {
133
+ return tokenizer->extra->user_read;
134
+ }
135
+
136
+ void *langscan_elisp_tokenizer_get_user_data(langscan_elisp_tokenizer_t *tokenizer)
137
+ {
138
+ return tokenizer->extra->user_data;
139
+ }
140
+
141
+ const char *langscan_elisp_token_name(langscan_elisp_token_t token)
142
+ {
143
+ static char *token_names[] = {
144
+ "*eof*",
145
+ #define LANGSCAN_ELISP_TOKEN(name) #name,
146
+ LANGSCAN_ELISP_TOKEN_LIST
147
+ #undef LANGSCAN_ELISP_TOKEN
148
+ };
149
+
150
+ return token_names[token];
151
+ }
@@ -0,0 +1,3 @@
1
+ require "mkmf"
2
+ dir_config( 'include' )
3
+ create_makefile('langscan/elisp/elisp')
@@ -0,0 +1 @@
1
+ EmacsLisp
@@ -0,0 +1,9 @@
1
+ space
2
+ comment
3
+ punct
4
+ integer
5
+ floating
6
+ string
7
+ ident
8
+ character
9
+ quote_chars
@@ -0,0 +1,157 @@
1
+
2
+ SHELL = /bin/sh
3
+
4
+ #### Start of system configuration section. ####
5
+
6
+ srcdir = /Users/osuka/devel/git/langscan/ext/langscan/java/java
7
+ topdir = /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin10.0
8
+ hdrdir = $(topdir)
9
+ VPATH = $(srcdir):$(topdir):$(hdrdir)
10
+ exec_prefix = $(prefix)
11
+ prefix = $(DESTDIR)/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr
12
+ sharedstatedir = $(prefix)/com
13
+ mandir = $(DESTDIR)/usr/share/man
14
+ psdir = $(docdir)
15
+ oldincludedir = $(DESTDIR)/usr/include
16
+ localedir = $(datarootdir)/locale
17
+ bindir = $(exec_prefix)/bin
18
+ libexecdir = $(exec_prefix)/libexec
19
+ sitedir = $(DESTDIR)/Library/Ruby/Site
20
+ htmldir = $(docdir)
21
+ vendorarchdir = $(vendorlibdir)/$(sitearch)
22
+ includedir = $(prefix)/include
23
+ infodir = $(DESTDIR)/usr/share/info
24
+ vendorlibdir = $(vendordir)/$(ruby_version)
25
+ sysconfdir = $(prefix)/etc
26
+ libdir = $(exec_prefix)/lib
27
+ sbindir = $(exec_prefix)/sbin
28
+ rubylibdir = $(libdir)/ruby/$(ruby_version)
29
+ docdir = $(datarootdir)/doc/$(PACKAGE)
30
+ dvidir = $(docdir)
31
+ vendordir = $(libdir)/ruby/vendor_ruby
32
+ datarootdir = $(prefix)/share
33
+ pdfdir = $(docdir)
34
+ archdir = $(rubylibdir)/$(arch)
35
+ sitearchdir = $(sitelibdir)/$(sitearch)
36
+ datadir = $(datarootdir)
37
+ localstatedir = $(prefix)/var
38
+ sitelibdir = $(sitedir)/$(ruby_version)
39
+
40
+ CC = gcc
41
+ LIBRUBY = $(LIBRUBY_SO)
42
+ LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
43
+ LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
44
+ LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)
45
+
46
+ RUBY_EXTCONF_H =
47
+ CFLAGS = -fno-common -arch i386 -arch x86_64 -g -Os -pipe -fno-common -DENABLE_DTRACE -fno-common -pipe -fno-common $(cflags)
48
+ INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
49
+ DEFS =
50
+ CPPFLAGS = -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE $(DEFS) $(cppflags)
51
+ CXXFLAGS = $(CFLAGS)
52
+ ldflags = -L. -arch i386 -arch x86_64
53
+ dldflags =
54
+ archflag =
55
+ DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
56
+ LDSHARED = cc -arch i386 -arch x86_64 -pipe -bundle -undefined dynamic_lookup
57
+ AR = ar
58
+ EXEEXT =
59
+
60
+ RUBY_INSTALL_NAME = ruby
61
+ RUBY_SO_NAME = ruby
62
+ arch = universal-darwin10.0
63
+ sitearch = universal-darwin10.0
64
+ ruby_version = 1.8
65
+ ruby = /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/bin/ruby
66
+ RUBY = $(ruby)
67
+ RM = rm -f
68
+ MAKEDIRS = mkdir -p
69
+ INSTALL = /usr/bin/install -c
70
+ INSTALL_PROG = $(INSTALL) -m 0755
71
+ INSTALL_DATA = $(INSTALL) -m 644
72
+ COPY = cp
73
+
74
+ #### End of system configuration section. ####
75
+
76
+ preload =
77
+
78
+ libpath = . $(libdir)
79
+ LIBPATH = -L. -L$(libdir)
80
+ DEFFILE =
81
+
82
+ CLEANFILES = mkmf.log
83
+ DISTCLEANFILES =
84
+
85
+ extout =
86
+ extout_prefix =
87
+ target_prefix = /langscan/java
88
+ LOCAL_LIBS =
89
+ LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl
90
+ SRCS = java.c javatok.c
91
+ OBJS = java.o javatok.o
92
+ TARGET = java
93
+ DLLIB = $(TARGET).bundle
94
+ EXTSTATIC =
95
+ STATIC_LIB =
96
+
97
+ BINDIR = $(bindir)
98
+ RUBYCOMMONDIR = $(sitedir)$(target_prefix)
99
+ RUBYLIBDIR = $(sitelibdir)$(target_prefix)
100
+ RUBYARCHDIR = $(sitearchdir)$(target_prefix)
101
+
102
+ TARGET_SO = $(DLLIB)
103
+ CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map
104
+ CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak
105
+
106
+ all: $(DLLIB)
107
+ static: $(STATIC_LIB)
108
+
109
+ clean:
110
+ @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
111
+
112
+ distclean: clean
113
+ @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
114
+ @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
115
+
116
+ realclean: distclean
117
+ install: install-so install-rb
118
+
119
+ install-so: $(RUBYARCHDIR)
120
+ install-so: $(RUBYARCHDIR)/$(DLLIB)
121
+ $(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
122
+ $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
123
+ install-rb: pre-install-rb install-rb-default
124
+ install-rb-default: pre-install-rb-default
125
+ pre-install-rb: Makefile
126
+ pre-install-rb-default: Makefile
127
+ $(RUBYARCHDIR):
128
+ $(MAKEDIRS) $@
129
+
130
+ site-install: site-install-so site-install-rb
131
+ site-install-so: install-so
132
+ site-install-rb: install-rb
133
+
134
+ .SUFFIXES: .c .m .cc .cxx .cpp .C .o
135
+
136
+ .cc.o:
137
+ $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
138
+
139
+ .cxx.o:
140
+ $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
141
+
142
+ .cpp.o:
143
+ $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
144
+
145
+ .C.o:
146
+ $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
147
+
148
+ .c.o:
149
+ $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
150
+
151
+ $(DLLIB): $(OBJS) Makefile
152
+ @-$(RM) $@
153
+ $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
154
+
155
+
156
+
157
+ $(OBJS): ruby.h defines.h
@@ -0,0 +1,3 @@
1
+ require "mkmf"
2
+ dir_config( 'include' )
3
+ create_makefile('langscan/java/java')
@@ -0,0 +1,134 @@
1
+ /* -*- mode: C; indent-tabs-mode: nil; c-basic-offset: 2 c-style: "BSD" -*- */
2
+ /*
3
+ * _template.c - a template file for LangScan modules
4
+ *
5
+ * Copyright (C) 2004-2005 Akira Tanaka <akr@m17n.org>
6
+ * All rights reserved.
7
+ * This is free software with ABSOLUTELY NO WARRANTY.
8
+ *
9
+ * You can redistribute it and/or modify it under the terms of
10
+ * the GNU General Public License version 2.
11
+ */
12
+
13
+
14
+ #include <ruby.h>
15
+ #include "java.h"
16
+
17
+ static VALUE token_symbol_list[
18
+ #define LANGSCAN_JAVA_TOKEN(token) 1 +
19
+ 1 + LANGSCAN_JAVA_TOKEN_LIST 0
20
+ #undef LANGSCAN_JAVA_TOKEN
21
+ ];
22
+
23
+ #ifndef RSTRING_PTR
24
+ # define RSTRING_PTR(str) (RSTRING(str)->ptr)
25
+ #endif
26
+ #ifndef RSTRING_LEN
27
+ # define RSTRING_LEN(str) (RSTRING(str)->len)
28
+ #endif
29
+
30
+ #ifndef RARRAY_PTR
31
+ # define RARRAY_PTR(str) (RARRAY(str)->ptr)
32
+ #endif
33
+ #ifndef RARRAY_LEN
34
+ # define RARRAY_LEN(str) (RARRAY(str)->len)
35
+ #endif
36
+
37
+ static size_t user_read_str(void **user_data_p, char *buf, size_t maxlen)
38
+ {
39
+ VALUE user_data = (VALUE)*user_data_p;
40
+ VALUE user_str = RARRAY_PTR(user_data)[0];
41
+ VALUE user_off = RARRAY_PTR(user_data)[1];
42
+ long off = NUM2LONG(user_off);
43
+ if (RSTRING_LEN(user_str)-off < maxlen) {
44
+ maxlen = RSTRING_LEN(user_str)-off;
45
+ }
46
+ memcpy(buf, RSTRING_PTR(user_str)+off, maxlen);
47
+ RARRAY_PTR(user_data)[1] = LONG2NUM(off+maxlen);
48
+ return maxlen;
49
+ }
50
+
51
+ static void tokenizer_mark(langscan_java_tokenizer_t *tokenizer)
52
+ {
53
+ if (tokenizer == NULL)
54
+ return;
55
+ rb_gc_mark((VALUE)langscan_java_tokenizer_get_user_data(tokenizer));
56
+ }
57
+
58
+ static void tokenizer_free(langscan_java_tokenizer_t *tokenizer)
59
+ {
60
+ if (tokenizer == NULL)
61
+ return;
62
+ langscan_java_free_tokenizer(tokenizer);
63
+ }
64
+
65
+ static VALUE tokenizer_s_allocate(VALUE klass)
66
+ {
67
+ return Data_Wrap_Struct(klass, tokenizer_mark, tokenizer_free, NULL);
68
+ }
69
+
70
+ static VALUE tokenizer_initialize(VALUE self, VALUE user_data)
71
+ {
72
+ VALUE tmp;
73
+ user_read_t user_read;
74
+ langscan_java_tokenizer_t *tokenizer;
75
+ Data_Get_Struct(self, langscan_java_tokenizer_t, tokenizer);
76
+ StringValue(user_data);
77
+ user_read = user_read_str;
78
+ user_data = rb_ary_new3(2, rb_str_new4(user_data), INT2FIX(0));
79
+ RBASIC(user_data)->klass = 0;
80
+ DATA_PTR(self) = langscan_java_make_tokenizer(user_read, (void *)user_data);
81
+ return self;
82
+ }
83
+
84
+ static VALUE tokenizer_get_token(VALUE self)
85
+ {
86
+ langscan_java_tokenizer_t *tokenizer;
87
+ langscan_java_token_t token;
88
+ Data_Get_Struct(self, langscan_java_tokenizer_t, tokenizer);
89
+ if (tokenizer == NULL) { return Qnil; }
90
+ token = langscan_java_get_token(tokenizer);
91
+ if (token == langscan_java_eof) {
92
+ DATA_PTR(self) = NULL;
93
+ langscan_java_free_tokenizer(tokenizer);
94
+ return Qnil;
95
+ }
96
+ return rb_ary_new3(8,
97
+ token_symbol_list[token],
98
+ rb_str_new(langscan_java_curtoken_text(tokenizer), langscan_java_curtoken_leng(tokenizer)),
99
+ INT2NUM(langscan_java_curtoken_beg_lineno(tokenizer)),
100
+ INT2NUM(langscan_java_curtoken_beg_columnno(tokenizer)),
101
+ INT2NUM(langscan_java_curtoken_beg_byteno(tokenizer)),
102
+ INT2NUM(langscan_java_curtoken_end_lineno(tokenizer)),
103
+ INT2NUM(langscan_java_curtoken_end_columnno(tokenizer)),
104
+ INT2NUM(langscan_java_curtoken_end_byteno(tokenizer)));
105
+ }
106
+
107
+ static VALUE tokenizer_close(VALUE self)
108
+ {
109
+ langscan_java_tokenizer_t *tokenizer;
110
+ Data_Get_Struct(self, langscan_java_tokenizer_t, tokenizer);
111
+ if (tokenizer == NULL) { return Qnil; }
112
+ DATA_PTR(self) = NULL;
113
+ langscan_java_free_tokenizer(tokenizer);
114
+ return Qnil;
115
+ }
116
+
117
+ void Init_java()
118
+ {
119
+ VALUE LangScan = rb_define_module("LangScan");
120
+ VALUE LangScan_JAVA = rb_define_module_under(LangScan, "Java");
121
+ VALUE Tokenizer = rb_define_class_under(LangScan_JAVA, "Tokenizer", rb_cData);
122
+ langscan_java_token_t token_id;
123
+
124
+ token_id = 0;
125
+ token_symbol_list[token_id++] = Qnil;
126
+ #define LANGSCAN_JAVA_TOKEN(token) token_symbol_list[token_id++] = ID2SYM(rb_intern(#token));
127
+ LANGSCAN_JAVA_TOKEN_LIST
128
+ #undef LANGSCAN_JAVA_TOKEN
129
+
130
+ rb_define_alloc_func(Tokenizer, tokenizer_s_allocate);
131
+ rb_define_method(Tokenizer, "initialize", tokenizer_initialize, 1);
132
+ rb_define_method(Tokenizer, "get_token", tokenizer_get_token, 0);
133
+ rb_define_method(Tokenizer, "close", tokenizer_close, 0);
134
+ }