chrislo-sourceclassifier 0.2.0

data/Manifest

@@ -0,0 +1,325 @@
+examples/example.rb
+lib/sourceclassifier.rb
+lib/trainer.rb
+Rakefile
+README.textile
+sourceclassifier.gemspec
+sources/gcc/ackermann.gcc-2.gcc
+sources/gcc/ackermann.gcc-3.gcc
+sources/gcc/ary.gcc
+sources/gcc/binarytrees.gcc
+sources/gcc/binarytrees.gcc-2.gcc
+sources/gcc/binarytrees.gcc-3.gcc
+sources/gcc/chameneos.gcc
+sources/gcc/chameneos.gcc-2.gcc
+sources/gcc/chameneos.gcc-3.gcc
+sources/gcc/chameneosredux.gcc
+sources/gcc/echo.gcc
+sources/gcc/except.gcc
+sources/gcc/fannkuch.gcc
+sources/gcc/fannkuch.gcc-2.gcc
+sources/gcc/fasta.gcc
+sources/gcc/fasta.gcc-2.gcc
+sources/gcc/fasta.gcc-3.gcc
+sources/gcc/fasta.gcc-4.gcc
+sources/gcc/fibo.gcc
+sources/gcc/harmonic.gcc
+sources/gcc/hash.gcc
+sources/gcc/hash2.gcc
+sources/gcc/heapsort.gcc
+sources/gcc/hello.gcc-2.gcc
+sources/gcc/hello.gcc-3.gcc
+sources/gcc/knucleotide.gcc
+sources/gcc/knucleotide.gcc-2.gcc
+sources/gcc/lists.gcc
+sources/gcc/magicsquares.gcc
+sources/gcc/mandelbrot.gcc
+sources/gcc/mandelbrot.gcc-2.gcc
+sources/gcc/mandelbrot.gcc-3.gcc
+sources/gcc/mandelbrot.gcc-4.gcc
+sources/gcc/matrix.gcc
+sources/gcc/message.gcc-2.gcc
+sources/gcc/message.gcc-3.gcc
+sources/gcc/methcall.gcc
+sources/gcc/moments.gcc
+sources/gcc/nbody.gcc
+sources/gcc/nbody.gcc-2.gcc
+sources/gcc/nbody.gcc-3.gcc
+sources/gcc/nestedloop.gcc
+sources/gcc/nsieve.gcc
+sources/gcc/nsievebits.gcc-2.gcc
+sources/gcc/nsievebits.gcc-3.gcc
+sources/gcc/objinst.gcc
+sources/gcc/partialsums.gcc
+sources/gcc/partialsums.gcc-3.gcc
+sources/gcc/partialsums.gcc-4.gcc
+sources/gcc/pidigits.gcc
+sources/gcc/primes.gcc
+sources/gcc/prodcons.gcc
+sources/gcc/random.gcc
+sources/gcc/raytracer.gcc
+sources/gcc/raytracer.gcc-2.gcc
+sources/gcc/recursive.gcc
+sources/gcc/recursive.gcc-2.gcc
+sources/gcc/regexdna.gcc-2.gcc
+sources/gcc/regexmatch.gcc
+sources/gcc/revcomp.gcc
+sources/gcc/revcomp.gcc-2.gcc
+sources/gcc/revcomp.gcc-4.gcc
+sources/gcc/reversefile.gcc
+sources/gcc/reversefile.gcc-2.gcc
+sources/gcc/sieve.gcc
+sources/gcc/spectralnorm.gcc
+sources/gcc/spellcheck.gcc
+sources/gcc/spellcheck.gcc-2.gcc
+sources/gcc/strcat.gcc
+sources/gcc/sumcol.gcc-2.gcc
+sources/gcc/sumcol.gcc-3.gcc
+sources/gcc/sumcol.gcc-4.gcc
+sources/gcc/sumcol.gcc-5.gcc
+sources/gcc/takfp.gcc
+sources/gcc/tcp-stream.gcc
+sources/gcc/tcpecho.gcc
+sources/gcc/tcpecho.gcc-2.gcc
+sources/gcc/tcprequest.gcc
+sources/gcc/threadring.gcc
+sources/gcc/wc.gcc
+sources/gcc/wc.gcc-2.gcc
+sources/gcc/wc.gcc-3.gcc
+sources/gcc/wordfreq.gcc-2.gcc
+sources/perl/ackermann.perl
+sources/perl/ackermann.perl-2.perl
+sources/perl/ackermann.perl-3.perl
+sources/perl/ackermann.perl-4.perl
+sources/perl/ary.perl
+sources/perl/ary.perl-2.perl
+sources/perl/binarytrees.perl
+sources/perl/binarytrees.perl-2.perl
+sources/perl/chameneos.perl
+sources/perl/echo.perl
+sources/perl/except.perl
+sources/perl/fannkuch.perl
+sources/perl/fannkuch.perl-2.perl
+sources/perl/fasta.perl
+sources/perl/fasta.perl-2.perl
+sources/perl/fasta.perl-4.perl
+sources/perl/fibo.perl
+sources/perl/fibo.perl-2.perl
+sources/perl/fibo.perl-3.perl
+sources/perl/harmonic.perl
+sources/perl/hash.perl
+sources/perl/hash.perl-2.perl
+sources/perl/hash.perl-3.perl
+sources/perl/hash2.perl
+sources/perl/heapsort.perl
+sources/perl/heapsort.perl-2.perl
+sources/perl/hello.perl
+sources/perl/knucleotide.perl-2.perl
+sources/perl/lists.perl
+sources/perl/mandelbrot.perl-2.perl
+sources/perl/matrix.perl
+sources/perl/matrix.perl-2.perl
+sources/perl/matrix.perl-3.perl
+sources/perl/message.perl
+sources/perl/methcall.perl
+sources/perl/moments.perl
+sources/perl/nbody.perl
+sources/perl/nestedloop.perl
+sources/perl/nsieve.perl-2.perl
+sources/perl/nsieve.perl-4.perl
+sources/perl/nsievebits.perl
+sources/perl/objinst.perl
+sources/perl/partialsums.perl-3.perl
+sources/perl/pidigits.perl
+sources/perl/pidigits.perl-2.perl
+sources/perl/process.perl
+sources/perl/prodcons.perl
+sources/perl/random.perl-4.perl
+sources/perl/recursive.perl-2.perl
+sources/perl/regexdna.perl
+sources/perl/regexdna.perl-2.perl
+sources/perl/regexdna.perl-3.perl
+sources/perl/regexdna.perl-4.perl
+sources/perl/regexdna.perl-5.perl
+sources/perl/regexdna.perl-6.perl
+sources/perl/regexmatch.perl
+sources/perl/revcomp.perl-2.perl
+sources/perl/reversefile.perl
+sources/perl/reversefile.perl-2.perl
+sources/perl/reversefile.perl-3.perl
+sources/perl/sieve.perl
+sources/perl/spectralnorm.perl-2.perl
+sources/perl/spellcheck.perl
+sources/perl/strcat.perl
+sources/perl/strcat.perl-2.perl
+sources/perl/sumcol.perl
+sources/perl/takfp.perl
+sources/perl/takfp.perl-3.perl
+sources/perl/tcpecho.perl
+sources/perl/tcprequest.perl
+sources/perl/tcpstream.perl
+sources/perl/threadring.perl
+sources/perl/threadring.perl-2.perl
+sources/perl/wc.perl
+sources/perl/wc.perl-2.perl
+sources/perl/wordfreq.perl
+sources/perl/wordfreq.perl-3.perl
+sources/perl/wordfreq.perl3.perl
+sources/python/ackermann.python
+sources/python/ary.python
+sources/python/binarytrees.python
+sources/python/binarytrees.python-3.python
+sources/python/chameneos.python-6.python
+sources/python/chameneosredux.python
+sources/python/chameneosredux.python-2.python
+sources/python/dispatch.python
+sources/python/dispatch.python-2.python
+sources/python/echo.python
+sources/python/except.python
+sources/python/fannkuch.python
+sources/python/fannkuch.python-2.python
+sources/python/fasta.python-2.python
+sources/python/fibo.python
+sources/python/fibo.python-2.python
+sources/python/fibo.python-3.python
+sources/python/harmonic.python-2.python
+sources/python/hash.python
+sources/python/hash.python-2.python
+sources/python/hash2.python
+sources/python/heapsort.python-3.python
+sources/python/hello.python
+sources/python/implicitode.python
+sources/python/knucleotide.python
+sources/python/lists.python
+sources/python/magicsquares.python
+sources/python/mandelbrot.python
+sources/python/mandelbrot.python-2.python
+sources/python/mandelbrot.python-3.python
+sources/python/matrix.python
+sources/python/matrix.python-2.python
+sources/python/message.python
+sources/python/message.python-2.python
+sources/python/message.python-3.python
+sources/python/meteor.python
+sources/python/meteor.python-2.python
+sources/python/methcall.python
+sources/python/moments.python
+sources/python/nbody.python
+sources/python/nbody.python-2.python
+sources/python/nestedloop.python
+sources/python/nsieve.python
+sources/python/nsieve.python-2.python
+sources/python/nsieve.python-4.python
+sources/python/nsievebits.python
+sources/python/nsievebits.python-2.python
+sources/python/objinst.python
+sources/python/partialsums.python
+sources/python/partialsums.python-2.python
+sources/python/partialsums.python-3.python
+sources/python/pidigits.python
+sources/python/pidigits.python-3.python
+sources/python/pidigits.python-4.python
+sources/python/process.python
+sources/python/process.python-2.python
+sources/python/prodcons.python
+sources/python/prodcons.python-2.python
+sources/python/random.python
+sources/python/raytracer.python
+sources/python/recursive.python
+sources/python/regexdna.python
+sources/python/regexdna.python-2.python
+sources/python/regexmatch.python
+sources/python/revcomp.python-3.python
+sources/python/reversefile.python
+sources/python/reversefile.python-2.python
+sources/python/reversefile.python-3.python
+sources/python/sieve.python
+sources/python/spectralnorm.python-2.python
+sources/python/spellcheck.python
+sources/python/strcat.python
+sources/python/strcat.python-2.python
+sources/python/sumcol.python-2.python
+sources/python/sumcol.python-3.python
+sources/python/takfp.python
+sources/python/tcpecho.python
+sources/python/tcprequest.python
+sources/python/tcpstream.python
+sources/python/threadring.python
+sources/python/threadring.python-2.python
+sources/python/threadring.python-3.python
+sources/python/wc.python-2.python
+sources/python/wordfreq.python
+sources/python/wordfreq.python-2.python
+sources/python/wordfreq.python-3.python
+sources/python/wordfreq.python-4.python
+sources/python/wordfreq.python-5.python
+sources/ruby/ackermann.ruby
+sources/ruby/ackermann.ruby-5.ruby
+sources/ruby/ary.ruby
+sources/ruby/binarytrees.ruby-2.ruby
+sources/ruby/chameneos.ruby-2.ruby
+sources/ruby/dispatch.ruby
+sources/ruby/echo.ruby
+sources/ruby/except.ruby
+sources/ruby/except.ruby-2.ruby
+sources/ruby/fannkuch.ruby
+sources/ruby/fasta.ruby
+sources/ruby/fibo.ruby
+sources/ruby/harmonic.ruby-2.ruby
+sources/ruby/hash.ruby
+sources/ruby/hash2.ruby
+sources/ruby/heapsort.ruby
+sources/ruby/hello.ruby
+sources/ruby/knucleotide.ruby-2.ruby
+sources/ruby/lists.ruby
+sources/ruby/mandelbrot.ruby-3.ruby
+sources/ruby/matrix.ruby
+sources/ruby/matrix.ruby-2.ruby
+sources/ruby/message.ruby
+sources/ruby/message.ruby-2.ruby
+sources/ruby/meteor.ruby
+sources/ruby/meteor.ruby-2.ruby
+sources/ruby/methcall.ruby
+sources/ruby/methcall.ruby-2.ruby
+sources/ruby/moments.ruby
+sources/ruby/nbody.ruby-2.ruby
+sources/ruby/nestedloop.ruby
+sources/ruby/nsieve.ruby
+sources/ruby/nsieve.ruby-2.ruby
+sources/ruby/nsievebits.ruby-2.ruby
+sources/ruby/objinst.ruby
+sources/ruby/partialsums.ruby
+sources/ruby/pidigits.ruby
+sources/ruby/pidigits.ruby-2.ruby
+sources/ruby/prodcons.ruby
+sources/ruby/random.ruby
+sources/ruby/recursive.ruby-2.ruby
+sources/ruby/regexdna.ruby
+sources/ruby/regexdna.ruby-2.ruby
+sources/ruby/regexmatch.ruby
+sources/ruby/revcomp.ruby
+sources/ruby/reversefile.ruby
+sources/ruby/sieve.ruby
+sources/ruby/spectralnorm.ruby
+sources/ruby/spellcheck.ruby
+sources/ruby/spellcheck.ruby-2.ruby
+sources/ruby/strcat.ruby
+sources/ruby/strcat.ruby-2.ruby
+sources/ruby/sumcol.ruby
+sources/ruby/sumcol.ruby-2.ruby
+sources/ruby/takfp.ruby
+sources/ruby/tcpecho.ruby
+sources/ruby/tcprequest.ruby
+sources/ruby/tcpstream.ruby
+sources/ruby/threadring.ruby
+sources/ruby/threadring.ruby-2.ruby
+sources/ruby/wc.ruby
+sources/ruby/wordfreq.ruby
+sources/ruby/wordfreq.ruby2.ruby
+test/fixtures/sources/gcc/ackermann.gcc-2.gcc
+test/fixtures/sources/python/ackermann.python
+test/fixtures/sources/ruby/ackermann.ruby
+test/test_source_classifier.rb
+test/test_trainer.rb
+trainer.bin
+Manifest

data/README.textile

@@ -0,0 +1,51 @@
+h1. SourceClassifier
+
+Source classifier identifies programming language using a Bayesian classifier trained on a corpus generated from the "Computer Language Benchmarks Game":http://shootout.alioth.debian.org/ . It is written in Ruby and availabe as a gem. To train the classifier to identify new languages download the sources from github.
+
+h2. Usage
+
+First install the gem using github as a source
+
+  $ gem sources -a http://gems.github.com 
+  $ sudo gem install chrislo-sourceclassifier
+
+Then, to use
+
+<pre>
+  require 'rubygems'
+  require 'sourceclassifier'
+  
+  s = SourceClassifier.new
+  
+  ruby_text = <<EOT
+  def my_sorting_function(a)
+    a.sort
+  end
+  EOT
+  
+  c_text = <<EOT
+  #include <unistd.h>
+  
+  int main() {
+    write(1, "hello world\n", 12);
+    return(0);
+  }
+  EOT
+  
+  s.identify(ruby_text) #=> Ruby
+  s.identify(c_text) #=> Gcc
+</pre>
+
+h2. Training
+
+Download the sources from github and in the directory run the training rake test
+
+  $ rake train
+
+In the ./sources directory are subdirectories for each language you wish to be able to identify. Each subdirectory contains examples of programs written in that language. The name of the directory is significant - it is the value returned by the SourceClassifier.identify() method.
+
+The rake task populate can be used to build these subdirectories from a checkout of the "computer language shootout sources":http://alioth.debian.org/scm/?group_id=30402 but you are free to train the classifier using any available examples.
+
+h2. Acknowledgments
+
+This library depends heavily on the great "Classifier":http://classifier.rubyforge.org/ gem by Lucas Carlson and David Fayram II.

data/Rakefile

@@ -0,0 +1,64 @@
+require 'rubygems'
+
+require 'rake'
+require 'rake/testtask'
+require 'find'
+require 'fileutils'
+require 'echoe'
+
+Echoe.new('sourceclassifier', '0.2.0') do |p|
+  p.description    = "Determine the programming language used in a sample"
+  p.url            = "http://github.com/chrislo/sourceclassifier/tree/master"
+  p.author         = "Chris Lowis"
+  p.email          = "chris.lowis@gmail.com"
+  p.ignore_pattern = ["tmp/*", "script/*", "sources/*"]
+  p.runtime_dependencies = ["classifier"]
+  p.development_dependencies = []
+end
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+
+module Find
+  def match(*paths)
+    matched = []
+    find(*paths) { |path| matched << path if yield path }
+    return matched
+  end
+  module_function :match
+end	
+
+SHOOTOUT_CVS_ROOT = '/Users/chris/tmp/shootout-scm-2008-08-24/'
+
+task :default => [:test_units]
+
+desc "Run basic tests"
+Rake::TestTask.new("test_units") { |t|
+  t.pattern = 'test/test_*.rb'
+  t.verbose = false
+  t.warning = false
+}
+
+desc "Populate training data directories"
+task :populate do
+  languages = %w[ruby python gcc perl java javascript]
+
+  languages.each do |language|
+    # create directories
+    target_dir = "./sources/#{language}"
+    FileUtils.mkdir_p target_dir
+
+    # copy source files to appropriate directories
+    Find.match(SHOOTOUT_CVS_ROOT) do |file| 
+      this_ext = File.extname(file).downcase
+      if this_ext == ".#{language}"
+        FileUtils.cp file, "#{target_dir}/#{File.basename(file)}"
+      end
+    end
+  end
+end
+
+desc "Train using training data directory"
+task :train do
+  require './lib/trainer.rb'
+  t = Trainer.new('./sources','./')
+end

data/examples/example.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'sourceclassifier'
+
+s = SourceClassifier.new
+
+ruby_text = <<EOT
+def my_sorting_function(a)
+  a.sort
+end
+EOT
+
+c_text = <<EOT
+#include <unistd.h>
+
+int main() {
+  write(1, "hello world\n", 12);
+  return(0);
+}
+EOT
+
+s.identify(ruby_text) #=> Ruby
+s.identify(c_text) #=> Gcc