langscan 1.2

data/AUTHORS.txt

@@ -0,0 +1,19 @@
+= Authors
+== Maintainor:
+* Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
+
+== LangScan modules written by:
+* Satoru Takabayashi <satoru at namazu.org>
+* Tanaka Akira <akr at m17n.org>
+* Keisuke Nishida <knishida at open-cobol.org>
+* Yoshinori K. Okuji <okuji at enbug.org>
+* MATSUNO Tokuhiro <tokuhirom at yahoo.co.jp>
+* Tatsuhiko Miyagawa <miyagawa at bulknews.net>
+* Kouichirou Eto <2005 at eto.com>
+* Soutaro Matsumoto <matsumoto at soutaro.com>
+* shinichro.h <hamaji at nii.ac.jp>
+* Yoshinori KUNIGA  <kuniga at users.sourceforge.net>
+
+== Gem version:
+* mallowlabs <mallowlabs at gmail.com>
+

data/History.txt

@@ -0,0 +1,126 @@
+=== 2010-08-01 mallowlabs <mallowlabs@gmail.com>
+
+	* langscan: Gem install support
+
+=== 2007-01-15  Tanaka Akira  <akr@fsij.org>
+
+	* langscan/_pairmatcher.rb: new method:
+	  LangScan::PairMatcher.fragmentsorter.
+
+	* langscan/c.rb: reorder the scan result.
+
+	* langscan/csharp.rb: ditto.
+
+	* langscan/d.rb: ditto.
+
+	* langscan/elisp.rb: ditto.
+
+	* langscan/java.rb: ditto.
+
+	* langscan/javascript.rb: ditto.
+
+	* langscan/scheme.rb: ditto.
+
+=== 2006-12-22  Tanaka Akira  <akr@fsij.org>
+
+	* langscan/c/ctok.l: permit invalid escape sequences such as \(.
+
+=== 2006-09-18  Tanaka Akira  <akr@fsij.org>
+
+	* langscan/_template.c: use RARRAY_PTR and RARRAY_LEN if available.
+
+	* langscan/pairmatcher/pairmatcher.c: use RARRAY_PTR, RARRAY_LEN and
+	  SYMBOL_P if available.  Don't modify internal of an array directly.
+
+=== 2006-09-02  Tanaka Akira  <akr@fsij.org>
+
+	* langscan/_template.c: use RSTRING_PTR and RSTRING_LEN if available.
+
+=== 2006-02-20  Tanaka Akira  <akr@m17n.org>
+
+	* langscan/pairmatcher/pairmatcher.c: use RSTRUCT_PTR if available.
+
+=== 2006-01-08  Tanaka Akira  <akr@m17n.org>
+
+	* langscan/pairmatcher/pairmatcher.c (pairmatcher_initialize):
+	  fix a GC problem.
+
+===	2006-01-05  Tanaka Akira  <akr@m17n.org>
+
+	* autogen.sh: use $_.scan instead of scan because it is removed
+	  at latest ruby-1.9.
+
+	* acinclude.m4 (RUBY_CFLAGS): add CCDLFLAGS because
+	  -fPIC is required on x86_64.
+
+=== 2005-10-31  Kenichi Ishibashi  <bashi@dream.ie.ariake-nct.ac.jp>
+
+	* configure.ac: Bumped version number to 1.2
+	* langscan.rb: Ditto.
+
+	* AUTHORS: Added `shinichiro.h <hamaji at nii.ac.jp>'.
+
+=== 2005-10-29  Shinichiro Hamaji  <hamaji@nii.ac.jp>
+
+	* langscan/eiffle*: add Eiffel support
+
+=== 2005-10-27  Shinichiro Hamaji  <hamaji@nii.ac.jp>
+
+	* langscan/lua*: add Lua support
+	* langscan/ocaml*: detect fun* around in comment
+	* langscan/io*: detect punct more precisely
+
+=== 2005-10-26  Shinichiro Hamaji  <hamaji@nii.ac.jp>
+
+	* langscan/ocaml*: modify OCaml support to detech fundef & funcall
+	* langscan/d/test-d.rb: change TestScan to TestScanD
+
+=== 2005-10-25  Shinichiro Hamaji  <hamaji@nii.ac.jp>
+
+	* langscan/io*: add Io support
+	* langscan/d*: add D support
+
+=== 2005-10-10  Kenichi Ishibashi  <bashi@dream.ie.ariake-nct.ac.jp>
+
+	* langscan/ruby/compat/Makefile.am: Fix a bug that Ruby support
+	doesn't	work on ruby 1.8.
+	* langscan/ruby/compat/ripper/Makefile.am: Ditto.
+
+	* README: Added examples.
+	* langscan/*.rb: Added comments for rdoc.
+	* Makefile.am: Added target `rdoc'.
+
+=== 2005-09-26  Kenichi Ishibashi  <bashi@dream.ie.ariake-nct.ac.jp>
+
+	* langscan: Version 1.1 released.
+
+=== 2005-09-22  Kenichi Ishibashi  <bashi@dream.ie.ariake-nct.ac.jp>
+
+	* configure.ac: Bumped version number to 1.1.
+	* langscan.rb: Ditto.
+
+	* langscan/ruby/compat/Makefile.am: EXTRA_DIST is available whenever
+	RUBY_RIPPER_AVAILABLE is true or not. [Gonzui-devel 375].
+
+	* tests/Makefile.am: Removed `pairmatcher.rb' and added `pairmatcher2.rb'.
+	* tests/pairmatcher.rb: Removed.
+	* tests/pairmatcher2.rb: New file. [Gonzui-devel 373].
+
+	* langscan/ruby/test/test-scan.rb: Renamed `TestScan' to
+	`TestRuby'. [Gonzui-devel 373].
+
+=== 2005-09-19  Satoru Takabayashi  <satoru@namazu.org>
+
+	* tests/test-util.rb: Removed.
+
+	* langscan/ocaml.rb: Require langscan/_common for
+	LangScan::register.
+
+=== 2005-09-16  Kenichi Ishibashi  <bashi at dream.ie.ariake-nct.ac.jp>
+
+	* langscan: version 1.0 released.
+
+	* langscan.rb: Added `VERSION'.
+	* configure.ac: Bumped version number to 1.0.
+	* langscan: Split off from gonzui repository.
+

data/Manifest.txt

@@ -0,0 +1,167 @@
+AUTHORS.txt
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+ext/langscan/_make_c.rb
+ext/langscan/_make_h.rb
+ext/langscan/_template.c
+ext/langscan/_template.h
+ext/langscan/c/c/Makefile
+ext/langscan/c/c/c.c
+ext/langscan/c/c/c.h
+ext/langscan/c/c/ctok.c
+ext/langscan/c/c/ctok.l
+ext/langscan/c/c/extconf.rb
+ext/langscan/c/c/modulename.txt
+ext/langscan/c/c/tokenlist.txt
+ext/langscan/csharp/csharp/Makefile
+ext/langscan/csharp/csharp/csharp.c
+ext/langscan/csharp/csharp/csharp.h
+ext/langscan/csharp/csharp/csharptok.c
+ext/langscan/csharp/csharp/csharptok.l
+ext/langscan/csharp/csharp/extconf.rb
+ext/langscan/csharp/csharp/modulename.txt
+ext/langscan/csharp/csharp/tokenlist.txt
+ext/langscan/d/d/Makefile
+ext/langscan/d/d/d.c
+ext/langscan/d/d/d.h
+ext/langscan/d/d/dtok.c
+ext/langscan/d/d/dtok.l
+ext/langscan/d/d/extconf.rb
+ext/langscan/d/d/modulename.txt
+ext/langscan/d/d/tokenlist.txt
+ext/langscan/elisp/elisp/Makefile
+ext/langscan/elisp/elisp/elisp.c
+ext/langscan/elisp/elisp/elisp.h
+ext/langscan/elisp/elisp/elisptok.c
+ext/langscan/elisp/elisp/elisptok.l
+ext/langscan/elisp/elisp/extconf.rb
+ext/langscan/elisp/elisp/modulename.txt
+ext/langscan/elisp/elisp/tokenlist.txt
+ext/langscan/java/java/Makefile
+ext/langscan/java/java/extconf.rb
+ext/langscan/java/java/java.c
+ext/langscan/java/java/java.h
+ext/langscan/java/java/javatok.c
+ext/langscan/java/java/javatok.l
+ext/langscan/java/java/modulename.txt
+ext/langscan/java/java/tokenlist.txt
+ext/langscan/javascript/javascript/Makefile
+ext/langscan/javascript/javascript/extconf.rb
+ext/langscan/javascript/javascript/javascript.c
+ext/langscan/javascript/javascript/javascript.h
+ext/langscan/javascript/javascript/javascripttok.c
+ext/langscan/javascript/javascript/javascripttok.l
+ext/langscan/javascript/javascript/modulename.txt
+ext/langscan/javascript/javascript/tokenlist.txt
+ext/langscan/pairmatcher/pairmatcher/Makefile
+ext/langscan/pairmatcher/pairmatcher/extconf.rb
+ext/langscan/pairmatcher/pairmatcher/pairmatcher.c
+ext/langscan/php/php/Makefile
+ext/langscan/php/php/extconf.rb
+ext/langscan/php/php/modulename.txt
+ext/langscan/php/php/php.c
+ext/langscan/php/php/php.h
+ext/langscan/php/php/phptok.c
+ext/langscan/php/php/phptok.l
+ext/langscan/php/php/tokenlist.txt
+ext/langscan/post-distclean.rb
+ext/langscan/pre-config.rb
+ext/langscan/python/python/Makefile
+ext/langscan/python/python/extconf.rb
+ext/langscan/python/python/modulename.txt
+ext/langscan/python/python/python.c
+ext/langscan/python/python/python.h
+ext/langscan/python/python/pythontok.c
+ext/langscan/python/python/pythontok.l
+ext/langscan/python/python/tokenlist.txt
+ext/langscan/ruby/compat/ripper/Makefile
+ext/langscan/ruby/compat/ripper/depend
+ext/langscan/ruby/compat/ripper/extconf.rb
+ext/langscan/ruby/compat/ripper/include/eventids1.c
+ext/langscan/ruby/compat/ripper/include/eventids2.c
+ext/langscan/ruby/compat/ripper/include/lex.c
+ext/langscan/ruby/compat/ripper/ripper.c
+ext/langscan/scheme/scheme/Makefile
+ext/langscan/scheme/scheme/extconf.rb
+ext/langscan/scheme/scheme/modulename.txt
+ext/langscan/scheme/scheme/scheme.c
+ext/langscan/scheme/scheme/scheme.h
+ext/langscan/scheme/scheme/schemetok.c
+ext/langscan/scheme/scheme/schemetok.l
+ext/langscan/scheme/scheme/tokenlist.txt
+ext/langscan/sh/sh/Makefile
+ext/langscan/sh/sh/extconf.rb
+ext/langscan/sh/sh/modulename.txt
+ext/langscan/sh/sh/sh.c
+ext/langscan/sh/sh/sh.h
+ext/langscan/sh/sh/shtok.c
+ext/langscan/sh/sh/shtok.l
+ext/langscan/sh/sh/tokenlist.txt
+lib/langscan.rb
+lib/langscan/_common.rb
+lib/langscan/_easyscanner.rb
+lib/langscan/_pairmatcher.rb
+lib/langscan/_type.rb
+lib/langscan/autoconf.rb
+lib/langscan/automake.rb
+lib/langscan/brainfuck.rb
+lib/langscan/c.rb
+lib/langscan/csharp.rb
+lib/langscan/css.rb
+lib/langscan/d.rb
+lib/langscan/eiffel.rb
+lib/langscan/elisp.rb
+lib/langscan/io.rb
+lib/langscan/java.rb
+lib/langscan/javascript.rb
+lib/langscan/lua.rb
+lib/langscan/ocaml.rb
+lib/langscan/ocaml/camlexer.ml
+lib/langscan/ocaml/lexer.mll
+lib/langscan/ocaml/types.ml
+lib/langscan/perl.rb
+lib/langscan/perl/tokenizer.pl
+lib/langscan/php.rb
+lib/langscan/python.rb
+lib/langscan/rpmspec.rb
+lib/langscan/ruby.rb
+lib/langscan/ruby/compat/README
+lib/langscan/ruby/compat/ripper.rb
+lib/langscan/ruby/compat/ripper/core.rb
+lib/langscan/ruby/compat/ripper/filter.rb
+lib/langscan/ruby/compat/ripper/lexer.rb
+lib/langscan/ruby/compat/ripper/sexp.rb
+lib/langscan/scheme.rb
+lib/langscan/sh.rb
+lib/langscan/text.rb
+metaconfig
+script/console
+script/destroy
+script/generate
+script/makemanifest.rb
+setup.rb
+tasks/extconf.rake
+tasks/extconf/langscan.rake
+test/langscan/brainfuck/test/test_scan.rb
+test/langscan/c/test/test_scan.rb
+test/langscan/c/test/test_token.rb
+test/langscan/csharp/test/test_scan.rb
+test/langscan/css/test/test_css.rb
+test/langscan/d/test/test_scan.rb
+test/langscan/d/test/test_token.rb
+test/langscan/eiffel/test/test_eiffel.rb
+test/langscan/elisp/test/test_elisp.rb
+test/langscan/io/test/test_io.rb
+test/langscan/java/test/test_java.rb
+test/langscan/javascript/test/test_javascript.rb
+test/langscan/lua/test/test_lua.rb
+test/langscan/ocaml/test/test_ocaml.rb
+test/langscan/php/test/test_scan.rb
+test/langscan/python/test/test_scan.rb
+test/langscan/rpmspec/test/test_rpmspec.rb
+test/langscan/ruby/test/test_scan.rb
+test/langscan/scheme/test/test_scan.rb
+test/test_helper.rb
+test/test_langscan.rb

data/README.rdoc

@@ -0,0 +1,89 @@
+= LangScan
+
+* http://gonzui.sourceforge.net/langscan/index.html.en
+* Gem version
+ * http://github.com/mallowlabs/langscan
+
+== DESCRIPTION:
+
+program analyzer for source code search engine
+
+== FEATURES/PROBLEMS:
+
+* Support Languages
+ * C
+ * C++
+ * C#
+ * Java
+ * JavaScript
+ * Ruby
+ * Python
+ * PHP
+ * Perl
+ * Objective Caml
+ * Brainfuck
+ * CSS
+ * Shell Script
+ * Emacs Lisp
+ * Scheme
+ * Plain Text
+ * Io
+ * Lua
+ * Eiffel
+
+== SYNOPSIS:
+=== Example1
+  % cat -n hello.c
+       1  #include <stdio.h>
+       2
+       3  int main(int argc, char **argv)
+       4  {
+       5    printf("Hello World.\n");
+       6    return 0;
+       7  }
+  % ruby -rlangscan/c -e 'LangScan::C.scan(ARGF.readlines.join) {|f| p f}' < hello.c
+  #<LangScan::Fragment space " " 1 (8)>
+  #<LangScan::Fragment ident "stdio" 1 (10)>
+  #<LangScan::Fragment punct "." 1 (15)>
+  #<LangScan::Fragment space "\n" 2 (19)>
+  #<LangScan::Fragment space " " 3 (23)>
+  #<LangScan::Fragment preproc_beg "#" 1 (0)>
+  #<LangScan::Fragment ident "include" 1 (1)>
+  ...
+
+=== Example2
+  % cat -n display_fundef.rb
+     1  require 'langscan'
+     2
+     3  # display all function definitions.
+     4  ARGV.each {|arg|
+     5    next unless scanner = LangScan.choose(arg)
+     6    scanner.scan(File.open(arg).readlines.join) {|f|
+     7      if f.type == :fundef
+     8        puts f.text
+     9      end
+    10    }
+    11  }
+  % ruby display_fundef.rb hello.c
+  main
+
+== REQUIREMENTS:
+
+* Ruby 1.8.2+
+* Perl + PPI module (for source codes written in Perl)
+* OCaml (for source codes written in OCaml)
+
+== INSTALL:
+
+* sudo gem install langscan
+
+== LICENSE:
+Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org>
+    All rights reserved.
+    This is free software with ABSOLUTELY NO WARRANTY.
+
+You can redistribute it and/or modify it under the terms of
+the GNU General Public License version 2.
+
+== AUTHORS:
+see 'AUTHORS.txt'

data/Rakefile

@@ -0,0 +1,40 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/langscan'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'langscan' do
+  self.developer 'mallowlabs', 'mallowlabs@gmail.com'
+#  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]
+
+namespace :setup do
+  desc "setup.rb config"
+  task :config do
+    ruby "setup.rb config"
+  end
+
+  desc "setup.rb distclean"
+  task :distclean do
+    ruby "setup.rb distclean"
+  end
+end
+
+

data/ext/langscan/_make_c.rb

@@ -0,0 +1,20 @@
+require 'fileutils'
+raise unless ARGV.length == 4
+template_file_name = ARGV.shift
+output_directory = ARGV.shift
+lang_abbrev = ARGV.shift
+module_name   = ARGV.shift
+
+template = File.read(template_file_name)
+template.gsub!(/LANGNAME/, lang_abbrev.upcase)
+template.gsub!(/langname/, lang_abbrev.downcase)
+template.gsub!(/LangName/, module_name)
+
+c_file_name = File.join( output_directory, lang_abbrev + ".c" )
+FileUtils.rm_f(c_file_name)
+File.open(c_file_name, "w") {|f|
+#  f.printf("/* %s generated automatically from %s */\n",
+#           c_file_name, template_file_name)
+  f.print template
+}
+#File.chmod(0444, c_file_name)