diggit 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5edef2423b0ffdd802d4c7f78b60fbf466878577
-  data.tar.gz: fd9005d6f5acc9f4f0fc4f84677ed87bff59dddd
+  metadata.gz: 03b553461539e07d43eefe6c94a760b44bdf8940
+  data.tar.gz: a0d58fbe18d396043c68242b916d4d76ca43e45c
 SHA512:
-  metadata.gz: 900953e9d6b740ffcce9d600b8935a70d23d5a91c81156db6820581460e880c897158e37202ac7da6a555aa0e0f9237c7f54bd0599471af423292894a4e3e343
-  data.tar.gz: 05cbb988a51d483787f68155f296693b080c7c5c285390401b369260cd1c366d254eaf2783d27ce2c484d190196da2587f756647fc2d1f479481f88dc03673c6
+  metadata.gz: cc71247842cc94ee9f4cfbab0399e0a49c47503f0b2bc964a7d2216a1fef26fb8798f38d5b2dd6a6ba1a4ef5bc90c82e3e2eddfde17673c6536949ef86cb456e
+  data.tar.gz: 4eea49b3b12d22ea172942431a4e38a057aa75ea61264d1edb70002e150fb6fe71ec90fbfbbd36c2dc7127c23fd2484b4c058a060488bcf97761dc3243bb35f4

data/README.md

@@ -16,7 +16,7 @@ The diggit tool is in the lib folder. Don't hesitate to create a link to diggit.
 
 ## Configuration
 
-The diggit tool is designed to help you analyze software repositories. Firstly you have to create a new folder in which you launch the `diggit init` command. This way, the folder becomes a diggit folder in which you can configure repositories and analyses.
+The diggit tool is designed to help you analyze software repositories. Firstly you have to create a new folder in which you launch the `dgit init` command. This way, the folder becomes a diggit folder in which you can configure repositories and analyses.
 
 ### Setting-up the repositories
 
@@ -38,7 +38,7 @@ A join is performed after all analyses of all repositories have been performed.
 
 Once diggit is configured you can perform the analyses. First you have to perform the clone by using `dgit perform clones`. Then you can launch the analyses by using `dgit perform analyses`. Finally, the joins are executed via the command `dgit perform joins`.
 
-At all time, you can check the status of your diggit folder by using `diggit status`. If you want more info on the status of a given repository, you can use the `dgit sources info https://github.com/jrfaller/diggit.git` command.
+At all time, you can check the status of your diggit folder by using `dgit status`. If you want more info on the status of a given repository, you can use the `dgit sources info https://github.com/jrfaller/diggit.git` command.
 
 ## Cleaning up
 

data/includes/addons/db.rb

@@ -1,6 +1,10 @@
 # encoding: utf-8
 require 'mongo'
 
+# A MongoDB addon for Diggit. The name of this addon is :db. This addon might use an :mongo hash in the global options. In this
+# hash, the :database key allows to configure the name of the database.
+# @!attribute [r] db
+# 	@return [Mongo::DB] the mongo database object.
 class Db < Diggit::Addon
 
 	DEFAULT_DB = 'diggit'

data/includes/addons/output.rb

@@ -1,5 +1,13 @@
 # encoding: utf-8
 
+# A output addon for Diggit. The name of the addon is :output, and can be reached in the
+# addons hash. This addon might use an :output hash in the global options. In this hash, the
+# :out key allows to configure the name of the output folder and :tmp the name of the temporary output
+# folder.
+# @!attribute [r] out
+# 	@return [String] the absolute path of the output folder.
+# @!attribute [r] tmp
+# 	@return [String] the absolute path of the temporary output folder.
 class Output < Diggit::Addon
 
 	attr_reader :out, :tmp

data/includes/analyses/diff_stats.rb

@@ -2,7 +2,7 @@
 
 class DiffStats < Diggit::Analysis
 
-	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascrip" ]
+	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascript" ]
 
 	def run
 		col = @addons[:db].db['diffstats']

data/includes/joins/diff_size.rb

@@ -1,18 +1,67 @@
 # encoding: utf-8
+require "rinruby"
 
 class DiffSize < Diggit::Join
 
-	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascrip" ]
+	SAMPLE_SIZE = 10
 
 	def run
-		col = @addons[:db].db['diffstats']
-    @sources.each do |source|
-      diffs = col.count({:query => {"source" => source}})
-      puts "#{source}: #{diffs}"
-    end
+		@sources.map{ |s| sample(s) }
+	end
+
+	def sample(src)
+		R.src = src[:url]
+		R.size = SAMPLE_SIZE
+		R.eval <<-EOS
+			library(RMongo)
+
+			srcQuery <- function(src) {
+				return(paste('{"source": "', src, '"}', sep=""))
+			}
+
+			noZeros <- function(data) {
+				return(subset(data,changes>0))
+			}
+
+			rubify <- function(data) {
+				return(cbind(data$old_path, data$old_commit, data$new_path, data$new_commit))
+			}
+
+			db <- mongoDbConnect('diffstats')
+			d <- noZeros(dbGetQuery(db, 'diffstats', srcQuery(src), skip=0, limit=0))
+
+			q <- quantile(d$changes)
+
+			s <- subset(d,changes > 0 & changes <= q[3])
+			m <- subset(d,changes > q[3] & changes <= q[4])
+			b <- subset(d,changes > q[4])
+
+			rs <- s[sample(nrow(s), size),]
+			rm <- m[sample(nrow(m), size),]
+			rb <- b[sample(nrow(b), size),]
+
+			rs_ruby <- rubify(rs)
+			rm_ruby <- rubify(rm)
+			rb_ruby <- rubify(rb)
+		EOS
+		write_sample(src, R.rs_ruby, 'small')
+		write_sample(src, R.rm_ruby, 'medium')
+		write_sample(src, R.rb_ruby, 'big')
+	end
+
+	def write_sample(src, sample, type)
+		prefix = "#{@addons[:output].out}/#{src[:id]}/#{type}"
+		n = 1
+		sample.row_vectors.each do |row|
+			FileUtils.mkdir_p("#{prefix}/#{n}")
+			File.write("#{prefix}/#{n}/src_#{row[1]}_#{row[0].gsub(/\//,'_')}", `git -C #{src[:folder]} show #{row[1]}:#{row[0]}`)
+			File.write("#{prefix}/#{n}/dst_#{row[3]}_#{row[2].gsub(/\//,'_')}", `git -C #{src[:folder]} show #{row[3]}:#{row[2]}`)
+			n += 1
+		end
 	end
 
   def clean
+		@sources.each{ |s| FileUtils.rm_rf("#{@addons[:output].out}/#{s[:id]}") }
   end
 
 end

data/lib/diggit_cli.rb

@@ -16,9 +16,9 @@ module Diggit
 			{ name: e.class.name, message: e.to_s, backtrace: e.backtrace }
 		end
 
-		def class_exist?(class_name)
-			obj = Object::const_get(class_name)
-			return obj.is_a?(Class)
+		def plugin_ok?(name, type)
+			obj = Object::const_get(name)
+			return obj < type
 		rescue NameError
 			return false
 		end
@@ -79,7 +79,7 @@ module Diggit
 
 			desc "errors", "Display informations on all source that have encountered an error."
 			def errors
-				diggit.sources.get_all(nil, {error: true}).each{|s| invoke :info, [s[:url]]}
+				diggit.sources.get_all(nil, {error: true}).each{|s| info s[:url] }
 			end
 
 			desc 'import [FILE]', "Import a list of sources from a file (one URL per line)."
@@ -105,10 +105,10 @@ module Diggit
 			desc "add [ADDON*]", "Add the provided addons to the list of active addons."
 			def add(*addons)
 				addons.each do |a|
-					if class_exist?(a)
+					if plugin_ok?(a, Addon)
 						diggit.config.add_addon(a)
 					else
-						say_status(ERROR, "addon #{a} not found", :red)
+						say_status(ERROR, "error loading addon #{a}", :red)
 					end
 				end
 			end
@@ -127,10 +127,10 @@ module Diggit
 			desc "add [JOIN*]", "Add the provided joins to the list of active joins."
 			def add(*joins)
 				joins.each do |j|
-					if class_exist?(j)
+					if plugin_ok?(j, Join)
 						diggit.config.add_join(j)
 					else
-						say_status(ERROR, "join #{j} not found", :red)
+						say_status(ERROR, "error loading join #{j}", :red)
 					end
 				end
 			end
@@ -149,10 +149,10 @@ module Diggit
 			desc "add [ANALYSIS*]", "Add the provided analyses to the list of active analyses."
 			def add(*analyses)
 				analyses.each do |a|
-					if class_exist?(a)
+					if plugin_ok?(a, Analysis)
 						diggit.config.add_analysis(a)
 					else
-						say_status(ERROR, "analysis #{a} not found", :red)
+						say_status(ERROR, "error loading analysis #{a}", :red)
 					end
 				end
 			end
@@ -172,7 +172,11 @@ module Diggit
 			def clones(*source_defs)
 				diggit.sources.get_all(source_defs, {state: :new}).each do |s|
 					begin
-						Rugged::Repository::clone_at(s[:url], s[:folder])
+						if File.exist?(s[:folder])
+							Rugged::Repository::new(s[:folder])
+						else
+							Rugged::Repository::clone_at(s[:url], s[:folder])
+						end
 					rescue => e
 						s[:log][:error] = dump_error(e)
 						say_status(ERROR, "error cloning #{s[:url]}", :red)

data/lib/diggit_core.rb

@@ -3,11 +3,10 @@
 
 require 'rugged'
 require 'oj'
-require 'singleton'
 
 module Diggit
 
-	DIGGIT_RC = '.dgitrrc'
+	DIGGIT_RC = '.dgitrc'
 	DIGGIT_LOG = '.dgitlog'
 	DIGGIT_SOURCES = '.dgitsources'
 
@@ -15,19 +14,53 @@ module Diggit
 	INCLUDES_FOLDER = 'includes'
 	DIGGIT_FOLDER = ".diggit"
 
+	# Base class for Diggit addons.
+	# @abstract Subclass and override name to implement custom Diggit addons.
+	#
+	# @!attribute [r] options
+	# 	@return [Hash] the Diggit options.
 	class Addon
 
+		# Create a new addon.
+		#
+		# @param options [Hash] a hash containing the Diggit options.
 		def initialize(options)
 			@options = options
 		end
 
+		# Returns the name of the addon.
+		# @abstract The name must be overrided.
+		#
+		# @return [Symbol] the name of the addon.
 		def name
+			raise NoMethodError.new "Subclass responsability"
 		end
 
 	end
 
+	# Base class for Diggit analyses. Diggit analyses are applied on each source that has been succesfully cloned. They can access the Diggit addons through the addons attribute.
+	# @see Addon
+	# @abstract Subclass and override run and clean to implement
+	# 	a custom analysis class.
+	# @!attribute [r] source
+	# 	@return [String] the URL of the source to be analyzed.
+	# @!attribute [r] repo
+	# 	@return [Rugged::Repository] the Rugged Repository object corresponding to the source.
+	# @!attribute [r] options
+	# 	@return [Hash] a hash containing the Diggit options.
+	# @!attribute [r] addons
+	# 	@return [Hash{Symbol => Addon}] a hash containing the loaded Diggit addons, indexed by names.
+	# @!attribute globs
+	# 	@return [Hash] a hash shared between all analyses of a source.
 	class Analysis
 
+		# Create a new analysis.
+		#
+		# @param source [String] the URL of the source to be analyzed.
+		# @param repo [Rugged::Repository] the Rugged Repository object corresponding to the source.
+		# @param options [Hash] a hash containing the Diggit options.
+		# @param addons [Hash{Symbol => Addon}] a hash containing the loaded Diggit addons, indexed by names.
+		# @param globs [Hash] a hash shared between all analyses of a source.
 		def initialize(source, repo, options, addons, globs)
 			@source = source
 			@repo = repo
@@ -36,17 +69,39 @@ module Diggit
 			@globs = globs
 		end
 
+		# Run the analysis.
+		# @abstract
 		def run
 			raise NoMethodError.new "Subclass responsability"
 		end
 
+		# Clean the data produced by the analysis.
+		# @abstract
 		def clean
 			raise NoMethodError.new "Subclass responsability"
 		end
 
 	end
 
+	# Base class for Diggit joins. Joins are applied after each source have been analyzed. They can access the Diggit addons through the addons attribute.
+	# @see Addon
+	# @abstract Subclass and override cleand and run to implement custom Diggit join.
+	# @!attribute [r] sources
+	# 	@return [Array]  an array containing the finished sources.
+	# @!attribute [r] options
+	# 	@return [Hash] a hash containing the Diggit options.
+	# @!attribute [r] addons
+	# 	@return [Hash{Symbol => Addon}] a hash containing the loaded Diggit addons, indexed by names.
+	# @!attribute globs
+	# 	@return [Hash] a hash shared between all analyses of a source.
 	class Join
+
+		# Create a new join.
+		#
+		# @param sources [Array] an array containing the finished sources.
+		# @param options [Hash] a hash containing the Diggit options.
+		# @param addons [Hash{Symbol => Addon}] a hash containing the loaded Diggit addons, indexed by names.
+		# @param globs [Hash] a hash shared between all analyses of a source.
 		def initialize(sources, options, addons, globs)
 			@sources = sources
 			@options = options
@@ -54,10 +109,14 @@ module Diggit
 			@globs = globs
 		end
 
+		# Run the join
+		# @abstract
 		def run
 			raise NoMethodError.new "Subclass responsability"
 		end
 
+		# Clean the data produced by the join
+		# @abstract
 		def clean
 			raise NoMethodError.new "Subclass responsability"
 		end
@@ -194,9 +253,9 @@ module Diggit
 
 		def url(source_def)
 			url = source_def
-			if /\d+/.match(source_def)
+			if /^\d+$/.match(source_def)
 				idx = source_def.to_i - 1
-				raise "Wrong source identifier" if idx < 0 || idx >= @sources.size
+				raise "Wrong source identifier: #{source_def}" if idx < 0 || idx >= @sources.size
 				url = @sources[source_def.to_i - 1]
 			end
 			url
@@ -207,11 +266,15 @@ module Diggit
 		end
 
 		def hash(url)
-			{url: url, folder: folder(url), log: @log.log(url)}
+			{url: url, id: id(url), folder: folder(url), log: @log.log(url)}
+		end
+
+		def id(url)
+			url.gsub(/[^[\w-]]+/, "_")
 		end
 
 		def folder(url)
-				File.expand_path(url.gsub(/[^[\w-]]+/, "_"), SOURCES_FOLDER)
+			File.expand_path(id(url), SOURCES_FOLDER)
 		end
 
 	end
@@ -269,10 +332,10 @@ module Diggit
 			Dir["#{global}/**/*.rb"].each{ |f| require f }
 
 			home = File.expand_path(INCLUDES_FOLDER,File.expand_path(DIGGIT_FOLDER,Dir.home))
-			Dir["#{home}/**/*.rb"].each{ |f| require f }
+			Dir["#{home}/**{,/*/**}/*.rb"].each{ |f| require f }
 
 			local = File.expand_path(INCLUDES_FOLDER)
-			Dir["#{local}/**/*.rb"].each{ |f| require f }
+			Dir["#{local}/**{,/*/**}/*.rb"].each{ |f| require f }
 		end
 
 	end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: diggit
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Jean-Rémy Falleri
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-27 00:00:00.000000000 Z
+date: 2014-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rugged
@@ -53,6 +53,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +81,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 description: |
   The Diggit repository analysis tool is a neat swiss knife to enable the analysis of many Git repositories.
 email: jr.falleri@gmail.com