corpus 0.0.1

data/README.rdoc

@@ -0,0 +1,47 @@
+= Corpus -- a collection of open source software projects
+
+== Table of Contents
+1. Introduction
+
+== 1. Introduction
+{corpus}[https://bitbucket.org/martinvelez/corpus] downloads a collection of open source 
+software projects.
+
+* Downloads the Ubuntu source distribution package,
+* Downloads the Most Watched projects for a given language from Github.
+
+== 2. Dependencies
+
+* {Ruby 1.9.3}[http://www.ruby-lang.org/en/downloads/] or greater
+* {Unix Tools}
+  * curl
+  * wget
+  * sqlite3
+
+== 3. Installation
+
+=== Rubygems:
+You might need to use sudo.
+ gem install corpus
+
+=== Not Rubygems:
+1. Download corpus[http://bitbucket.org/martinvelez/corpus/downloads]
+2. Executable is inside the bin directory
+
+== 4. Usage
+List all available tasks
+ corpus -T
+
+== 5. Development
+
+Author:: {Martin Velez}[http://www.martinvelez.com]
+Copyright:: Copyright (C) 2012 {Martin Velez}[http://www.martinvelez.com]
+License:: GPL[http://www.gnu.org/copyleft/gpl.html]
+
+=== Source
+Bitbucket[https://bitbucket.org/martinvelez/corpus/src] is hosting this code.
+ http://bitbucket.org/martinvelez/corpus/src
+
+=== Issues and Bug Reports
+Provide feedback, get help, request features, and reports bugs here:
+ https://bitbucket.org/martinvelez/corpus/issues?status=new?status=open

data/bin/corpus

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'rake'
+
+$download_dir = 'downloads'
+$ubuntu_dir = File.expand_path(File.join($download_dir,'ubuntu'))
+$github_dir = File.expand_path(File.join($download_dir,'github'))
+$sql = {:ubuntu => 'ubuntu.db',:github => 'github.db'}
+$cloc = "perl #{File.expand_path(File.join(File.dirname(__FILE__),"..","ext","cloc-1.56.pl"))}"
+
+Rake.application.init('corpus')
+
+# START TASKS DEFINITIONS
+# -----------------------
+# Crawl specific code repositories
+directory $github_dir
+directory $ubuntu_dir
+
+namespace :crawl do
+
+	# Ubuntu source distribution package can be download in multiple ISO files
+	desc "Download Ubuntu source distribution package"
+	task :ubuntu, [:download_dir] do |t, args|
+		urls_file = File.expand_path(File.join(File.dirname(__FILE__),"..","conf","ubuntu_urls.txt"))	
+		args.with_defaults(:download_dir => $ubuntu_dir)
+
+		puts "Downloading [Ubuntu source distribution package]..."	
+		puts "download_dir = #{args.download_dir}"
+		puts "urls_file = #{urls_file}"
+
+		# TODO: killing/interrupting this Ruby process does not kill wget processes	
+		File.open(urls_file).each_line do |url|
+			`wget -c --directory-prefix=#{args.download_dir} #{url}`
+			puts "ERROR: #{url}" if $? != 0
+		end
+	end
+
+	# Github lists the Most Watched projects for a given language.
+	# The list is paginated into 10 webpages.
+	desc "Downloads the Most Watched projects for a given language from Github"	
+	task :github, [:lang,:download_dir] => $github_dir do |t, args|
+		lang = args.lang || "Java"
+		dd = args.download_dir || File.join($github_dir, lang)
+		repos = []
+
+		for i in 1..10
+			#TODO: use URI to encode "C++" into html entities
+			o = `curl https://github.com/languages/#{lang}/most_watched?page=#{i}`
+			# <a href="/user/repo">Repo</a> => "/user/repo"
+			o.each_line {|l| repos.push(l[/"(.*)"/,1]) if l =~ /\s<a href="\/\w+\// } 
+			repos.pop # TODO: improve regex to exclude pagination bar
+		end
+
+		curr_dir = Dir.pwd
+		Dir.mkdir(dd) if not(File.exists?(dd) and File.directory?(dd))
+		chdir(dd)
+		repos.each do |r|
+			repo_name = r[/\/(.*)/,1][/\/(.*)/,1]
+			o = `git clone https://github.com#{r} 2>&1`
+			if o =~ /fatal/ # downloaded already attempted
+				chdir(repo_name)
+				puts "git-pulling [#{repo_name}]"
+				out = `git pull origin master 2>&1` # try master branch
+				output = `git pull 2>&1` if out =~ /fatal/ # no master branch 
+				chdir(dd)	
+				if output =~ /You asked me to pull/ # git pull failed
+					File.rename(repo_name,"errors") # Deleting can be unsafe
+					`git clone https://github.com#{r} 2>&1` # Try one last time.	
+				end
+			else
+				puts o
+			end
+		end
+		chdir(curr_dir)
+	end # task
+
+end # namespace
+
+
+# Database functions
+namespace :db do
+
+	# Create database files	
+	namespace :create do 
+		desc "Counts LOC from projects downloaded from github"
+		task :github => $sql[:github]
+		desc "Counts LOC from projects downloaded from ubuntu"
+		task :ubuntu => $sql[:ubuntu]
+	end
+
+	# Show programming language LOC counts
+	namespace :show do
+		desc "print table showing LOC for a given database"
+		task :github => $sql[:github] do
+			output = `sqlite3 #{$sql[:github]} \
+				'select language, SUM(nCode) as total from t GROUP BY language ORDER BY total DESC'` 
+			table = output.split("\n").map{|s| s.split("|")}
+			col_width = table.transpose.map{|col| col.map{|cell| cell.to_s.length}.max}
+			table.each{|row| puts row.zip(col_width).map{|lang,count| lang.to_s.ljust(count)}.join(" ")}	
+		end
+	end
+
+	desc "deletes database FILE"
+	task :clobber, [:file] { |t, args| rm $sql[args.file.to_sym] }
+
+end #namespace
+
+# Private task
+file $sql[:github] do
+	repos = Dir.glob("#{$github_dir}/*/*") # downloads/github/LANGUAGE/repo
+	reports = []
+	project = repos.pop
+
+	# First cloc call creates database, subsequent calls append to database	
+	`#{$cloc} --sql 1 --sql-project #{File.basename(project)} #{project} | sqlite3 #{$sql[:github]}`
+	repos.each do |r|
+		`#{$cloc} --sql 1 --sql-project #{File.basename(r)} --sql-append #{r} | sqlite3 #{$sql[:github]}`
+	end
+end
+
+file $sql[:ubuntu] do
+	# get isos
+	# extract
+	# pass to cloc
+end
+
+task :default do
+	puts "usage: '#{Rake.application.name} -T' to get list of all available commands."
+end
+# ---------------------
+# END TASKS DEFINITIONS
+
+Rake.application.top_level

data/conf/github_languages.txt

@@ -0,0 +1,10 @@
+JavaScript
+Ruby
+Python
+Shell
+Java
+PHP
+C
+C++
+Perl
+Objective-C

data/conf/ubuntu_urls.txt

@@ -0,0 +1,3 @@
+http://cdimage.ubuntu.com/source/current/source/precise-src-0.iso
+http://cdimage.ubuntu.com/source/current/source/precise-src-2.iso
+http://cdimage.ubuntu.com/source/current/source/precise-src-3.iso