langscan 1.2-x86-mswin32-60

data/ext/langscan/php/php/phptok.l

@@ -0,0 +1,212 @@
+/*
+ * phptok.l - a lex rule for PHP
+ * 
+ * Copyright (C) 2005 MATSUNO Tokuhiro <tokuhirom at yahoo.co.jp>
+ *     All rights reserved.
+ *     This is free software with ABSOLUTELY NO WARRANTY.
+ * 
+ * You can redistribute it and/or modify it under the terms of 
+ * the GNU General Public License version 2.
+ */
+
+%option reentrant
+%option prefix="langscan_php_lex_"
+%option noyywrap
+%option nodefault
+
+%x CLASSDEF
+%x FUNDEF
+%x SCRIPT
+%x HEREDOC
+%x HEREDOCBEG
+
+newline         (\r\n|\r|\n)
+escseq          \\.
+string          '([^\\\']|{escseq})*'|\"([^\\\"]|{escseq})*\"
+integer         0x[0-9a-fA-F]+|[0-9]+
+pointfloat      [0-9]*\.[0-9]+|[0-9]+\.[0-9]*
+exponent        [eE][+\-][0-9]+
+ident		[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
+floatnumber     {pointfloat}{exponent}?|[0-9]+{exponent}
+space		[ \t\r\n]
+
+slash           \/
+star            \*
+nonstar         [^\*]
+nonslashstar    [^\/\*]
+commentcontent  {star}+{nonslashstar}{nonstar}*
+mullinecomment  {slash}{star}{nonstar}*{commentcontent}*{star}+{slash}
+
+%{
+
+#include "php.h"
+
+#define YY_EXTRA_TYPE langscan_php_lex_extra_t *
+
+#if YY_NULL != 0
+#error "YY_NULL is not 0."
+#endif
+
+#define YY_DECL langscan_php_token_t langscan_php_lex_lex(yyscan_t yyscanner)
+
+#define YY_INPUT(buf,result,max_size) \
+  if (!yyextra->eof) { \
+    result = yyextra->user_read(&(yyextra->user_data), (buf), (max_size)); \
+    if (result == 0) \
+      yyextra->eof = 1; \
+  }
+
+#define UPD update_pos(yyextra, yytext, yyleng)
+static void update_pos(langscan_php_lex_extra_t *, char *, int);
+
+#define report(token) \
+  do { \
+    yyextra->text = yytext; \
+    yyextra->leng = yyleng; \
+    return langscan_php_##token; \
+  } while (0)
+
+char *heredocident=NULL;
+int heredocidentlen=0;
+%}
+
+
+%%
+<INITIAL>"<script"{space}+language{space}*={space}*(php|\"php\"|\'php\'){space}*> { UPD; BEGIN(SCRIPT); report(ident);} 
+<INITIAL><%=?|"<?"=?     { UPD; BEGIN(SCRIPT); report(ident);} 
+<INITIAL>"<?php"{space}  { UPD; BEGIN(SCRIPT); report(ident);} 
+<INITIAL>([^<]|<[^?s%])+ { UPD; report(comment);}
+<INITIAL>.               { UPD; report(comment);}
+
+<HEREDOC>{ident};?{newline}  {
+	int identlen = strpbrk(yytext, ";\r\n")-yytext;
+	if (heredocidentlen==identlen && !strncmp(heredocident, yytext, identlen)) {
+		free(heredocident);
+		heredocident = NULL;
+		BEGIN(SCRIPT);
+		yyless(0);
+	} else {
+		UPD;
+		report(string);
+	}
+}
+<HEREDOC>[^\r\n]*{newline} { UPD; report(string);}
+
+<HEREDOCBEG>[ \t]*    { UPD; report(space); }
+<HEREDOCBEG>{ident}   { UPD; heredocident=strdup(yytext); heredocidentlen=yyleng; report(ident); }
+<HEREDOCBEG>{newline} { UPD;
+	if(heredocident) {
+		BEGIN(HEREDOC);
+		report(space);
+	} else { /* invalid here-document */
+		BEGIN(SCRIPT);
+		report(space);
+	}
+}
+
+<SCRIPT>class                                  { UPD; BEGIN(CLASSDEF); report(keyword); }
+<SCRIPT>function                               { UPD; BEGIN(FUNDEF); report(keyword); }
+<CLASSDEF>{ident}                              { UPD; BEGIN(SCRIPT); report(classdef); }
+<FUNDEF>{ident}                                { UPD; BEGIN(SCRIPT); report(fundef); }
+<SCRIPT><<<                                    { UPD; BEGIN(HEREDOCBEG); report(punct); }
+<SCRIPT>::|<>|=>|"++"|--|"||"|&&|AND|XOR|<<|>> { UPD; report(punct); }
+<SCRIPT,FUNDEF>[={}+\-;:,.\[\]()|^&/*%!~$<>?@] { UPD; report(punct); }
+<SCRIPT>(==|!=|<<|>>)=                         { UPD; report(punct); }
+<SCRIPT>[<>+\-!*/.%&|^]=                       { UPD; report(punct); }
+<SCRIPT>[?%]>|"</script"{space}*">"            { UPD; BEGIN(INITIAL); report(ident);}
+<SCRIPT,CLASSDEF,FUNDEF>{mullinecomment}       { UPD; report(comment);}
+<SCRIPT,CLASSDEF,FUNDEF>[ \t\f]+               { UPD; report(space); }
+<SCRIPT,CLASSDEF,FUNDEF>{newline}              { UPD; report(space); }
+<SCRIPT,CLASSDEF,FUNDEF>(#|"//")([^?%\r\n]|[?%][^>])*{newline}? { UPD; report(comment); }
+<SCRIPT>{string}                     { UPD; report(string); }
+<SCRIPT>{integer}                    { UPD; report(integer); }
+<SCRIPT>{floatnumber}                { UPD; report(floating); }
+<SCRIPT>{ident}{space}*\(            { yyless(strpbrk(yytext, " \t\b\f\r\n(")-yytext); UPD; report(funcall); }
+<SCRIPT>\$?{ident}                   { UPD; report(ident); }
+<SCRIPT,HEREDOCBEG,FUNDEF,CLASSDEF>. { UPD; fprintf(stderr, "Token Error : %s\n", yytext); report(punct); }
+
+%%
+
+static void update_pos(
+  langscan_php_lex_extra_t *extra,
+  char *text,
+  int leng)
+{
+  int i, j;
+  extra->beg_byteno = extra->end_byteno;
+  extra->beg_lineno = extra->end_lineno;
+  extra->beg_columnno = extra->end_columnno;
+  j = 0;
+  for (i = 0; i < leng; i++) {
+    if (text[i] == '\r' || (i == 0 || text[i - 1] != '\r') && text[i] == '\n') {
+      extra->end_lineno++;
+      j = i + 1;
+      extra->end_columnno = 0;
+    }
+  }
+  extra->end_columnno += leng - j;
+  extra->end_byteno += leng;
+}
+
+langscan_php_tokenizer_t *langscan_php_make_tokenizer(
+  size_t (*user_read)(void **user_data_p, char *buf, size_t maxlen),
+  void *user_data)
+{
+  langscan_php_tokenizer_t *tokenizer;
+  langscan_php_lex_extra_t *extra;
+  tokenizer = (langscan_php_tokenizer_t *)malloc(sizeof(langscan_php_tokenizer_t));
+  if (tokenizer == NULL)
+    return NULL;
+  extra = (langscan_php_lex_extra_t *)malloc(sizeof(langscan_php_lex_extra_t));
+  if (extra == NULL)
+    return NULL;
+  extra->user_read = user_read;
+  extra->user_data = user_data;
+  extra->beg_lineno = 1;
+  extra->beg_columnno = 0;
+  extra->beg_byteno = 0;
+  extra->end_lineno = 1;
+  extra->end_columnno = 0;
+  extra->end_byteno = 0;
+  extra->eof = 0;
+  tokenizer->extra = extra;
+  langscan_php_lex_lex_init(&tokenizer->scanner);
+  langscan_php_lex_set_extra(extra, tokenizer->scanner);
+  return tokenizer;
+}
+
+langscan_php_token_t langscan_php_get_token(langscan_php_tokenizer_t *tokenizer) 
+{
+  return langscan_php_lex_lex(tokenizer->scanner);
+}
+
+void langscan_php_free_tokenizer(langscan_php_tokenizer_t *tokenizer) 
+{
+  langscan_php_lex_extra_t *extra = langscan_php_lex_get_extra(tokenizer->scanner);
+  free((void *)extra);
+  langscan_php_lex_lex_destroy(tokenizer->scanner);
+  free((void *)tokenizer);
+  free(heredocident);
+}
+
+user_read_t langscan_php_tokenizer_get_user_read(langscan_php_tokenizer_t *tokenizer)
+{
+  return tokenizer->extra->user_read;
+}
+
+void *langscan_php_tokenizer_get_user_data(langscan_php_tokenizer_t *tokenizer)
+{
+  return tokenizer->extra->user_data;
+}
+
+const char *langscan_php_token_name(langscan_php_token_t token)
+{
+  static char *token_names[] = {
+    "*eof*",
+#define LANGSCAN_PHP_TOKEN(name) #name,
+    LANGSCAN_PHP_TOKEN_LIST
+#undef LANGSCAN_PHP_TOKEN
+  };
+
+  return token_names[token];
+}

data/ext/langscan/php/php/tokenlist.txt

@@ -0,0 +1,11 @@
+integer
+floating
+string
+ident
+keyword
+fundef
+funcall
+classdef
+punct
+comment
+space

data/ext/langscan/post-distclean.rb

@@ -0,0 +1,21 @@
+require "fileutils"
+
+def remove(iPath)
+	if ( File.exist?( iPath ) )
+		File.delete( iPath )
+	end
+end
+
+srcdirectories.each do |sub_name|
+	path = File.join( curr_srcdir, sub_name, sub_name )
+
+	flex_file = ( sub_name + "tok.l" )
+	flex_file = File.join( path, flex_file )
+	next unless ( File.file?( flex_file ) )
+	flex_outh = File.basename( flex_file, ".l" ) << ".h"
+	flex_outc = File.basename( flex_file, ".l" ) << ".c"
+	[flex_outh, flex_outc, sub_name + ".h", sub_name + ".c", "extconf.rb"].each do |x|
+		x = File.join( path, x )
+		remove( x )
+	end
+end

data/ext/langscan/pre-config.rb

@@ -0,0 +1,57 @@
+require "fileutils"
+
+r = system( "flex -V" )
+unless ( r )
+	raise "not available flex command."
+end
+
+ruby_exe = "ruby"
+flex_exe = "flex"
+nounistd = ( "yes" == config('nounistd') ? '--nounistd' : '' )
+
+
+srcdirectories.each do |sub_name|
+	path = File.join( curr_srcdir, sub_name, sub_name )
+
+	flex_file = ( sub_name + "tok.l" )
+	flex_file = File.join( path, flex_file )
+	next unless ( File.file?( flex_file ) )
+
+	flex_out = File.basename( flex_file, ".l" ) << ".c"
+	flex_out = File.join( path, flex_out )
+	dist = flex_out + ".dist"
+
+	module_name = File.read( File.join( path, "modulename.txt" ) )
+
+	r = system( %Q;#{flex_exe} #{nounistd} -L -t "#{flex_file}" > "#{flex_out}"; )
+	if ( r && File.exist?( dist ) )
+		FileUtils.copy( dist, flex_out )
+	end
+
+	# cmp -s javatok.c javatok.c.dist
+	if ( File.exist?( dist ) )
+		s = File.read( flex_out ).gsub( /\s+/m, " " )
+		d = File.read( dist     ).gsub( /\s+/m, " " )
+		if ( s != d )
+			FileUtils.copy( flex_out, dist )
+		end
+	end
+
+	mh = File.join( curr_srcdir, "_make_h.rb" )
+	tp = File.join( curr_srcdir, "_template.h" )
+	tl = File.join( curr_srcdir, sub_name, sub_name, "tokenlist.txt" )
+	r = system( %Q;#{ruby_exe} "#{mh}" "#{tp}" "#{path}" "#{tl}" #{sub_name} #{module_name}; )
+	raise "fail: _make_h.rb #{sub_name}." unless (r)
+	mc = File.join( curr_srcdir, "_make_c.rb" )
+	tp = File.join( curr_srcdir, "_template.c" )
+	r = system( %Q;#{ruby_exe} "#{mc}" "#{tp}" "#{path}" #{sub_name} #{module_name}; )
+	raise "fail: _make_c.rb #{sub_name}." unless (r)
+
+	open( File.join( path, "extconf.rb" ), "w" ) do |h|
+		h.write( <<-EOS )
+require "mkmf"
+dir_config( 'include' )
+create_makefile('langscan/#{sub_name}/#{sub_name}')
+		EOS
+	end
+end

data/ext/langscan/python/python/Makefile

@@ -0,0 +1,188 @@
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = C:/usr/ruby/lib/ruby/1.8/i386-mswin32
+hdrdir = $(topdir)
+VPATH = $(srcdir);$(topdir);$(hdrdir)
+
+DESTDIR = C:
+exec_prefix = $(prefix)
+prefix = $(DESTDIR)/usr/ruby
+sharedstatedir = $(DESTDIR)/etc
+mandir = $(prefix)/man
+oldincludedir = $(DESTDIR)/usr/include
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+sitedir = $(prefix)/lib/ruby/site_ruby
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+includedir = $(prefix)/include
+infodir = $(prefix)/info
+vendorlibdir = $(vendordir)/$(ruby_version)
+sysconfdir = $(prefix)/etc
+libdir = $(exec_prefix)/lib
+sbindir = $(exec_prefix)/sbin
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+vendordir = $(prefix)/lib/ruby/vendor_ruby
+archdir = $(rubylibdir)/$(arch)
+sitearchdir = $(sitelibdir)/$(sitearch)
+datadir = $(prefix)/share
+localstatedir = $(DESTDIR)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+
+CC = cl -nologo
+LIBRUBY = $(RUBY_SO_NAME).lib
+LIBRUBY_A = $(RUBY_SO_NAME)-static.lib
+LIBRUBYARG_SHARED = $(LIBRUBY)
+LIBRUBYARG_STATIC = $(LIBRUBY_A)
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -MT -Zi  -O2b2xg- -G6 
+INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =     
+CXXFLAGS = $(CFLAGS) 
+ldflags  = 
+dldflags = -link -incremental:no -debug -opt:ref -opt:icf -dll $(LIBPATH)
+archflag = 
+DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
+LDSHARED = cl -nologo -LD
+AR = lib -nologo
+EXEEXT = .exe
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = msvcrt-ruby18
+arch = i386-mswin32
+sitearch = i386-msvcrt
+ruby_version = 1.8
+ruby = C:/usr/ruby/bin/ruby
+RUBY = $(ruby:/=\)
+RM = $(RUBY) -run -e rm -- -f
+MAKEDIRS = @$(RUBY) -run -e mkdir -- -p
+INSTALL = @$(RUBY) -run -e install -- -vp
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 0644
+COPY = copy > nul
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -libpath:"." -libpath:"$(libdir)"
+DEFFILE = $(TARGET)-$(arch).def
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = vc*.pdb $(DEFFILE)
+
+extout = 
+extout_prefix = 
+target_prefix = /langscan/python
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  oldnames.lib user32.lib advapi32.lib shell32.lib ws2_32.lib  
+SRCS = python.c pythontok.c
+OBJS = python.obj pythontok.obj
+TARGET = python
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR = C:/devel/gem/git/langscan/pkg/langscan-1.2.gem.build/lib$(target_prefix)
+RUBYARCHDIR = C:/devel/gem/git/langscan/pkg/langscan-1.2.gem.build/lib$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).so $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.obj *.lib *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS:/=\) $(CLEANOBJS:/=\) $(CLEANFILES:/=\)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES:/=\)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB:/=\) $(RUBYARCHDIR:/=\)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .obj
+
+{$(hdrdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cc{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cc.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cxx{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cxx.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(topdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(srcdir)}.cpp{}.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+.cpp.obj:
+	$(CXX) $(INCFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -Tp$(<:\=/)
+
+{$(hdrdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+{$(topdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+{$(srcdir)}.c{}.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+.c.obj:
+	$(CC) $(INCFLAGS) $(CFLAGS) $(CPPFLAGS) -c -Tc$(<:\=/)
+
+$(DLLIB): $(DEFFILE) $(OBJS) Makefile
+	@-$(RM) $@
+	$(LDSHARED) -Fe$(@) $(OBJS) $(LIBS) $(LOCAL_LIBS) $(DLDFLAGS) -implib:$(*F:.so=)-$(arch).lib -pdb:$(*F:.so=)-$(arch).pdb -def:$(DEFFILE)
+
+
+
+$(DEFFILE): 
+	$(RUBY) -e "puts 'EXPORTS', 'Init_$(TARGET)'"  > $@
+
+$(OBJS): {.;$(VPATH)}ruby.h {.;$(VPATH)}defines.h