diggit 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a4b7487548c7f0524510efe878c636c54b58c304
+  data.tar.gz: 481b02f7281fa616641b4106a36baa3f3a4b6b6c
+SHA512:
+  metadata.gz: 1a6c92ce2456aff993a743a14a8f71752fdf0f27fae13dafb2e7f58ec1f53795c53381f883b0ce3478380d10799888ef5de5c3908a856bf401a9a27e1fb18280
+  data.tar.gz: bc12d9e29df011d9675fae2392aeee0282ac9cd3571a1e7f216f3545e29a74c55dbc846f9ddc7caf0afb41802333b6d4c632f93847eee3c096f900b6beeaf29f

data/LICENSE

@@ -0,0 +1,165 @@
+GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,45 @@
+# Diggit
+
+A ruby tool to analyse Git repositories
+
+# Installation
+
+Clone diggit using the following command.
+
+```
+git clone https://github.com/jrfaller/diggit.git
+```
+
+The diggit tool is in the lib folder. Don't hesitate to create a link to diggit.rb to be able to launch it in any repository.
+
+# Usage
+
+## Configuration
+
+The diggit tool is designed to help you analyze software repositories. Firstly you have to create a new folder in which you launch the `diggit init` command. This way, the folder becomes a diggit folder in which you can configure repositories and analyses.
+
+### Setting-up the repositories
+
+You can add some repositories to be analyzed with the following command: `dgit sources add https://github.com/jrfaller/diggit.git`.
+
+### Using addons
+
+Addons add features the the diggit tool: for instance capability of writing to a mondodb database, etc. To enable addons for your current diggit folder you can use the following command: `dgit addons add TestAddon`.
+
+### Setting-up analyses
+
+An analysis is applied to each repository. You can configure the analyses to be performed with the following command: `dgit analyses add TestAnalysis`. Analyses are performed in the order they have been added.
+
+### Setting-up joins
+
+A join is performed after all analyses of all repositories have been performed. You can configure the joins to be performed with the following command: `dgit joins add TestJoin`. Joins are performed in the order they have been added.
+
+## Running analyses
+
+Once diggit is configured you can perform the analyses. First you have to perform the clone by using `dgit perform clones`. Then you can launch the analyses by using `dgit perform analyses`. Finally, the joins are executed via the command `dgit perform joins`.
+
+At all time, you can check the status of your diggit folder by using `diggit status`. If you want more info on the status of a given repository, you can use the `dgit sources info https://github.com/jrfaller/diggit.git` command.
+
+## Cleaning up
+
+If something is going error, you can always delete the results of the joins by using the command `dgit clean joins` and of the analysis with the command `dgit clean analyses`.

data/bin/dgit

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/diggit_cli.rb'
+
+Diggit::Cli::DiggitCli.start(ARGV)

data/includes/addons/db.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require 'mongo'
+
+class Db < Diggit::Addon
+
+	DEFAULT_DB = 'diggit'
+
+	attr_reader :db
+
+	def initialize(*args)
+		super
+		client = Mongo::MongoClient.new
+		database = DEFAULT_DB
+		database = @options[:mongo][:database] if @options.has_key?(:mongo) && @options[:mongo].has_key?(:database)
+		@db = client[database]
+	end
+
+	def name
+		:db
+	end
+
+end

data/includes/addons/output.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+class Output < Diggit::Addon
+
+	attr_reader :out, :tmp
+
+	DEFAULT_OUT = 'out'
+	DEFAULT_TMP = 'tmp'
+
+	def initialize(*args)
+		super
+
+		out = DEFAULT_OUT
+		out = @options[:output][:out] if @options.has_key?(:output) && @options[:output].has_key?(:out)
+		tmp = DEFAULT_TMP
+		tmp = @options[:output][:tmp] if @options.has_key?(:output) && @options[:output].has_key?(:tmp)
+
+		@out = File.absolute_path(out)
+		@tmp = File.absolute_path(tmp)
+
+		FileUtils.mkdir_p(@out) unless File.exist?(@out)
+		FileUtils.mkdir_p(@tmp) unless File.exist?(@tmp)
+	end
+
+	def name
+		:output
+	end
+
+end

data/includes/addons/test.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+class TestAddon < Diggit::Addon
+
+	def initialize(*args)
+		super
+	end
+
+	def name
+		:test
+	end
+
+end

data/includes/analyses/authors.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+class AuthorAnalysis < Diggit::Analysis
+
+	def run
+		walker = Rugged::Walker.new(@repo)
+		walker.sorting(Rugged::SORT_DATE)
+		walker.push(@repo.last_commit)
+		authors = walker.collect{ |c| c.author[:name] }.uniq
+		puts "Authors : #{authors}"
+	end
+end

data/includes/analyses/cloc.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+require 'yaml'
+
+class ClocAnalysis < Diggit::Analysis
+
+	def run
+		cloc = `cloc . --progress-rate=0 --quiet --yaml`
+		unless cloc.empty?
+			yaml = YAML.load(cloc.lines[2..-1].join)
+			yaml.delete('header')
+			output = { source: @source, cloc: yaml }
+			col = @addons[:db].db['cloc']
+			col.insert(output)
+		end
+	end
+
+	def clean(source)
+		@addons[:db].db['cloc'].remove({source: source})
+	end
+
+end

data/includes/analyses/diff_stats.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+class DiffStats < Diggit::Analysis
+
+	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascrip" ]
+
+	def run
+		col = @addons[:db].db['diffstats']
+
+		walker = Rugged::Walker.new(@repo)
+		ref = @repo.references["refs/heads/master"]
+		walker.push(ref.target_id)
+		walker.each do |commit|
+			parent1 = commit.parents[0]
+			if parent1
+				diff1 = parent1.diff(commit)
+				diff1.each_delta do |delta|
+					old_path = delta.old_file[:path]
+					new_path = delta.new_file[:path]
+					old_ext = File.extname(old_path).downcase
+					new_ext = File.extname(new_path).downcase
+					if delta.status == :modified && old_ext.eql?(new_ext) && ACCEPTED_EXTENSIONS.include?(old_ext)
+						sha_old = delta.old_file[:oid]
+						sha_new = delta.new_file[:oid]
+						patch = @repo.lookup(sha_new).diff(@repo.lookup(sha_old))
+						changes = patch.changes
+						edit = {source: @source, old_path: old_path, new_path: new_path, old_commit: commit.oid, new_commit: parent1.oid, changes: changes}
+						col.insert(edit)
+					end
+				end
+			end
+		end
+	end
+
+  def clean
+    @addons[:db].db['diffstats'].remove({source: @source})
+  end
+
+end

data/includes/analyses/metadata.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+
+class MetadataAnalysis < Diggit::Analysis
+
+	def run
+		# Importing tags
+		tags = @addons[:db].db['tags']
+		@repo.tags.each do |t|
+			tag = { source: @source, name: t.name, target: t.target.oid }
+			tags.insert(tag)
+		end
+
+		# Importing branches
+		branches = @addons[:db].db['branches']
+		@repo.branches.each do |b|
+			branch = { source: @source, name: b.name, target: b.target.oid }
+			branches.insert(branch)
+		end
+
+		# Importing commits
+		commits = @addons[:db].db['commits']
+		walker = Rugged::Walker.new(@repo)
+		walker.sorting(Rugged::SORT_DATE)
+		walker.push(@repo.last_commit)
+		walker.each do |c|
+			commit = {
+				source: @source, oid: c.oid, message: c.message, author: c.author,
+				committer: c.committer, parent_ids: c.parent_ids, time: c.time
+			}
+			commits.insert(commit)
+		end
+	end
+
+	def clean
+		@addons[:db].db['tags'].remove({source: @source})
+		@addons[:db].db['branches'].remove({source: @source})
+		@addons[:db].db['commits'].remove({source: @source})
+	end
+
+end

data/includes/analyses/pom.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+
+class PomAnalysis < Diggit::Analysis
+
+	def run
+		pom_files = Dir['**/pom.xml']
+		pom_files.each do |file|
+			puts file
+			git = `git --no-pager log --pretty=%H%n%an --name-status #{file}`.lines
+			history = []
+			git.each_slice(4) { |slice| history << [slice[0].strip, slice[1].strip, slice[3].split("\t")[0].strip] }
+			puts history.inspect
+		end
+	end
+
+end

data/includes/analyses/test.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+class TestAnalysis < Diggit::Analysis
+
+	def run
+		puts "TestAnalysis performed"
+	end
+
+	def clean
+		puts "TestAnalysis cleaned on #{@source}"
+	end
+
+end

data/includes/joins/diff_size.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+class DiffSize < Diggit::Join
+
+	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascrip" ]
+
+	def run
+		col = @addons[:db].db['diffstats']
+    @sources.each do |source|
+      diffs = col.count({:query => {"source" => source}})
+      puts "#{source}: #{diffs}"
+    end
+	end
+
+  def clean
+  end
+
+end

data/includes/joins/test.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+class TestJoin < Diggit::Join
+
+	def run
+		p "TestJoin performed"
+	end
+
+	def clean
+		p "TestJoin cleaned"
+	end
+
+end

data/lib/diggit_cli.rb

@@ -0,0 +1,309 @@
+require 'thor'
+require 'fileutils'
+
+require_relative 'diggit_core'
+
+module Diggit
+
+	module Utils
+
+		def diggit
+			@diggit = Diggit.new if @diggit.nil?
+			return @diggit
+		end
+
+		def dump_error(e)
+			{ name: e.class.name, message: e.to_s, backtrace: e.backtrace }
+		end
+
+		def class_exist?(class_name)
+			obj = Object::const_get(class_name)
+			return obj.is_a?(Class)
+		rescue NameError
+			return false
+		end
+
+		def source_color(source_hash)
+			if source_hash[:log][:error].empty?
+				return :blue
+			else
+				return :red
+			end
+		end
+
+		def status_color(status)
+			if status == DONE
+				return :green
+			else
+				return :red
+			end
+		end
+
+	end
+
+	module Cli
+
+		DONE = '[done]'
+		WARNING = '[warning]'
+		ERROR = '[error]'
+		INFO = '[info]'
+
+		class SourcesCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc 'list', "Display the list of sources."
+			def list
+				idx = 1
+				diggit.sources.hashes.each do |s|
+					say_status("[#{s[:log][:state]}]", "#{idx}: #{s[:url]}", source_color(s))
+					idx += 1
+				end
+				errors = diggit.sources.errors.size
+				status = (errors== 0 && DONE) || ERROR
+				say_status(status, "listed #{diggit.sources.size} sources including #{errors} errors", status_color(status))
+			end
+
+			desc 'info [SOURCE_DEF]', "Display informations on the provided source definition (either source URL or id)."
+			def info(source_def)
+				s = diggit.sources.get(source_def)
+				say_status("[#{s[:log][:state]}]", "#{s[:url]}", source_color(s))
+				say_status('[folder]', "#{s[:folder]}", :blue)
+				unless s[:log][:error].empty?
+					say_status(ERROR, "#{s[:log][:error][:name]}", :red)
+					say_status('[message]', "#{s[:log][:error][:message]}", :red)
+					say_status('[backtrace]', "", :red)
+					say(s[:log][:error][:backtrace].join("\n"))
+				end
+			end
+
+			desc "errors", "Display informations on all source that have encountered an error."
+			def errors
+				diggit.sources.get_all(nil, {error: true}).each{|s| invoke :info, [s[:url]]}
+			end
+
+			desc 'import [FILE]', "Import a list of sources from a file (one URL per line)."
+			def import(urls_file)
+				IO.readlines(urls_file).each{ |line| diggit.sources.add(line.strip) }
+			end
+
+			desc "add [URL*]", "Add the provided urls to the list of sources."
+			def add(*urls)
+				urls.each{ |u| diggit.sources.add(u) }
+			end
+
+			desc "rem [SOURCE_DEF*]", "Remove the sources correspondign to the provided source definitions (id or URL) from the list of sources."
+			def rem(*sources_defs)
+				sources_defs.each { |s| diggit.sources.rem(s) }
+			end
+		end
+
+		class AddonsCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc "add [ADDON*]", "Add the provided addons to the list of active addons."
+			def add(*addons)
+				addons.each do |a|
+					if class_exist?(a)
+						diggit.config.add_addon(a)
+					else
+						say_status(ERROR, "addon #{a} not found", :red)
+					end
+				end
+			end
+
+			desc "rem [ADDON*]", "Remove the provided addons from the list of active addons."
+			def rem(*addons)
+				addons.each{ |a| diggit.config.rem_addon(a) }
+			end
+
+		end
+
+		class JoinsCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc "add [JOIN*]", "Add the provided joins to the list of active joins."
+			def add(*joins)
+				joins.each do |j|
+					if class_exist?(j)
+						diggit.config.add_join(j)
+					else
+						say_status(ERROR, "join #{j} not found", :red)
+					end
+				end
+			end
+
+			desc "rem [JOIN*]", "Remove the provided joins from the list of active joins."
+			def rem(*joins)
+				joins.each{ |j| diggit.config.rem_join(j) }
+			end
+
+		end
+
+		class AnalysesCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc "add [ANALYSIS*]", "Add the provided analyses to the list of active analyses."
+			def add(*analyses)
+				analyses.each do |a|
+					if class_exist?(a)
+						diggit.config.add_analysis(a)
+					else
+						say_status(ERROR, "analysis #{a} not found", :red)
+					end
+				end
+			end
+
+			desc "rem [ANALYSIS*]", "Remove the provided analyses from the list of active analyses."
+			def rem(*analyses)
+				analyses.each{ |a| diggit.config.rem_analysis(a) }
+			end
+
+		end
+
+		class PerformCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc "clones [SOURCE_DEFS*]", "Clone the sources corresponding to the provided source definitions (id or URL). Clone all sources if no source definitions are provided."
+			def clones(*source_defs)
+				diggit.sources.get_all(source_defs, {state: :new}).each do |s|
+					begin
+						Rugged::Repository::clone_at(s[:url], s[:folder])
+					rescue => e
+						s[:log][:error] = dump_error(e)
+						say_status(ERROR, "error cloning #{s[:url]}", :red)
+					else
+						s[:log][:state] = :cloned
+						s[:log][:error] = {}
+						say_status(DONE, "#{s[:url]} cloned", :blue)
+					ensure
+						diggit.sources.update(s)
+					end
+				end
+			end
+
+			desc "analyses [SOURCE_DEFS*]", "Perform the configured analyses to the sources corresponding to the provided source definitions (id or URL). Analyze all sources if no source definitions are provided."
+			def analyses(*source_defs)
+				addons = diggit.config.load_addons
+				diggit.sources.get_all(source_defs, {state: :cloned}).each do |s|
+					FileUtils.cd(s[:folder])
+					globs = {}
+					performed_analyses = []
+					begin
+						repo = Rugged::Repository.new('.')
+						diggit.config.load_analyses(s[:url], repo, addons, globs).each do |a|
+							performed_analyses << a.class.to_s
+							a.run
+						end
+					rescue => e
+						s[:log][:error] = dump_error(e)
+						s[:log][:analyses] = performed_analyses[1..-2]
+						say_status(ERROR, "error performing #{performed_analyses.last} on #{s[:url]}", :red)
+					else
+						s[:log][:analyses] = performed_analyses
+						s[:log][:state] = :finished
+						s[:log][:error] = {}
+						say_status(DONE, "source #{s[:url]} analyzed", :blue)
+					ensure
+						FileUtils.cd(diggit.root)
+						diggit.sources.update(s)
+					end
+				end
+			end
+
+			desc "joins", "Perform the configured joins."
+			def joins
+				addons = diggit.config.load_addons
+				globs = {}
+				diggit.config.load_joins(diggit.sources.get_all([], {state: :finished, error: false}), addons, globs).each{ |j| j.run }
+				say_status(DONE, "joins performed", :blue)
+			end
+
+		end
+
+		class CleanCli < Thor
+			include Thor::Actions
+			include Utils
+
+			desc "analyses", "Clean the configured analyzes on the provided source definitions (id or URL). Clean all sources if no source definitions are provided."
+			def analyses(*source_defs)
+				addons = diggit.config.load_addons
+				diggit.sources.get_all(source_defs, {state: :finished}).each do |s|
+					globs = {}
+					diggit.config.load_analyses(s[:url], nil, addons, globs).each{ |a| a.clean}
+					s[:log][:state] = :cloned
+					s[:log][:analyses] = []
+					s[:log][:error] = {}
+					diggit.sources.update(s)
+					say_status(DONE, "cleaned analyses on #{s[:url]}", :blue)
+				end
+			end
+
+			desc "joins", "Clean the configured joins."
+			def joins
+				addons = diggit.config.load_addons
+				globs = {}
+				diggit.config.load_joins(diggit.sources.get_all([], {state: :finished, error: false}), addons, globs).each{ |j| j.clean }
+			end
+
+		end
+
+		class DiggitCli < Thor
+			include Thor::Actions
+			include Utils
+
+			def initialize(*args)
+				super
+				cmd = args[2][:current_command].name
+				unless 'init'.eql?(cmd) || 'help'.eql?(cmd) || File.exist?(DIGGIT_RC)
+					say_status(ERROR, "this is not a diggit directory", :red)
+					exit
+				end
+				diggit
+			end
+
+			desc "init", "Initialize the current folder as a diggit folder."
+			def init
+				FileUtils.touch(DIGGIT_SOURCES)
+				Oj.to_file(DIGGIT_LOG, {})
+				Oj.to_file(DIGGIT_RC, { addons: [], analyses: [], joins: [], options: {} })
+				say_status(DONE, "folder initialized")
+			end
+
+			desc 'status', "Display the status of the current diggit folder."
+			def status
+				color = (diggit.sources.get_all(nil, {error: true}).size > 0 && :red) || :blue
+				say_status('[sources]', "#{diggit.sources.get_all([], {state: :new}).size} new (#{diggit.sources.get_all([], {state: :new, error: true}).size} errors), #{diggit.sources.get_all([], {state: :cloned}).size} cloned (#{diggit.sources.get_all([], {state: :cloned, error: true}).size} errors), #{diggit.sources.get_all([], {state: :finished}).size} finished", color)
+				say_status('[addons]', "#{diggit.config.addons.join(', ')}", :blue)
+				say_status('[analyses]', "#{diggit.config.analyses.join(', ')}", :blue)
+				say_status('[joins]', "#{diggit.config.joins.join(', ')}", :blue)
+				say_status('[options]', "#{diggit.config.options}", :blue)
+			end
+
+			desc "sources SUBCOMMAND ...ARGS", "manage sources for the current diggit folder."
+			subcommand "sources", SourcesCli
+
+			desc "joins SUBCOMMAND ...ARGS", "manage joins for the current diggit folder."
+			subcommand "joins", JoinsCli
+
+			desc "analyses SUBCOMMAND ...ARGS", "manage analyses for the current diggit folder."
+			subcommand "analyses", AnalysesCli
+
+			desc "addons SUBCOMMAND ...ARGS", "manage addons for the current diggit folder."
+			subcommand "addons", AddonsCli
+
+			desc "perform SUBCOMMAND ...ARGS", "perform actions in the current diggit folder."
+			subcommand "perform", PerformCli
+
+			desc "clean SUBCOMMAND ...ARGS", "clean the current diggit folder."
+			subcommand "clean", CleanCli
+
+		end
+
+	end
+
+end

data/lib/diggit_core.rb

@@ -0,0 +1,273 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'rugged'
+require 'oj'
+require 'singleton'
+
+module Diggit
+
+	DIGGIT_RC = '.dgitrrc'
+	DIGGIT_LOG = '.dgitlog'
+	DIGGIT_SOURCES = '.dgitsources'
+
+	SOURCES_FOLDER = 'sources'
+	INCLUDES_FOLDER = 'includes'
+
+	class Addon
+
+		def initialize(options)
+			@options = options
+		end
+
+		def name
+		end
+
+	end
+
+	class Analysis
+
+		def initialize(source, repo, options, addons, globs)
+			@source = source
+			@repo = repo
+			@options = options
+			@addons = addons
+			@globs = globs
+		end
+
+		def run
+			raise NoMethodError.new "Subclass responsability"
+		end
+
+		def clean
+			raise NoMethodError.new "Subclass responsability"
+		end
+
+	end
+
+	class Join
+		def initialize(sources, options, addons, globs)
+			@sources = sources
+			@options = options
+			@addons = addons
+			@globs = globs
+		end
+
+		def run
+			raise NoMethodError.new "Subclass responsability"
+		end
+
+		def clean
+			raise NoMethodError.new "Subclass responsability"
+		end
+
+	end
+
+	class Config
+
+		def initialize
+			@config = Oj.load_file(DIGGIT_RC)
+		end
+
+		def save
+			Oj.to_file(DIGGIT_RC, @config)
+		end
+
+		def analyses
+			@config[:analyses]
+		end
+
+		def add_analysis(analysis)
+			analyses << analysis unless analyses.include?(analysis)
+			save
+		end
+
+		def rem_analysis(analysis)
+			analyses.delete(analysis)
+			save
+		end
+
+		def load_analyses(source, repo, addons, globs)
+			analyses.map{ |a| Object::const_get(a).new(source, repo, options, addons, globs) }
+		end
+
+		def addons
+			return @config[:addons]
+		end
+
+		def add_addon(addon)
+			addons << addon unless addons.include?(addon)
+			save
+		end
+
+		def rem_addon(addon)
+			addons.delete(addon)
+			save
+		end
+
+		def load_addons
+			result = {}
+			addons.each do |a|
+				obj = Object::const_get(a).new(options)
+				result[obj.name] = obj
+			end
+			return result
+		end
+
+		def joins
+			return @config[:joins]
+		end
+
+		def add_join(join)
+			joins << join unless joins.include?(join)
+			save
+		end
+
+		def rem_join(join)
+			joins.delete(join)
+			save
+		end
+
+		def load_joins(finished_sources, addons, globs)
+			return joins.map{ |j| Object::const_get(j).new(finished_sources, options, addons, globs) }
+		end
+
+		def options
+			return @config[:options]
+		end
+
+	end
+
+	class Sources
+
+		def initialize
+			@log = Log.new
+			@sources = []
+			IO.readlines(DIGGIT_SOURCES).each{ |line| @sources << line.strip }
+		end
+
+		def size
+			return @sources.size
+		end
+
+		def save
+			File.open(DIGGIT_SOURCES, "w") do |f|
+				@sources.each{ |s| f.puts(s) }
+			end
+		end
+
+		def add(url)
+			unless @sources.include?(url)
+				@sources << url
+				@log.init(url)
+				save
+			end
+		end
+
+		def rem(source_def)
+			url = url(source_def)
+			@sources.delete(url)
+			@log.rem(url)
+			save
+		end
+
+		def get(source_def)
+			hash(url(source_def))
+		end
+
+		def get_all(source_defs, filter={})
+			sources = []
+			if source_defs.nil? || source_defs.empty?
+				sources = hashes
+			else
+				sources = source_defs.map{ |d| hash(d) }
+			end
+			sources = sources.select{ |s| s[:log][:state] == filter[:state] } if (filter.has_key?(:state))
+			sources = sources.select{ |s| s[:log][:error].empty? != filter[:error] } if (filter.has_key?(:error))
+			return sources
+		end
+
+		def update(source_hash)
+			@log.update(source_hash)
+		end
+
+		def url(source_def)
+			url = source_def
+			if /\d+/.match(source_def)
+				idx = source_def.to_i - 1
+				raise "Wrong source identifier" if idx < 0 || idx >= @sources.size
+				url = @sources[source_def.to_i - 1]
+			end
+			url
+		end
+
+		def hashes
+			@sources.map{ |s| hash(s) }
+		end
+
+		def hash(url)
+			{url: url, folder: folder(url), log: @log.log(url)}
+		end
+
+		def folder(url)
+				File.expand_path(url.gsub(/[^[\w-]]+/, "_"), SOURCES_FOLDER)
+		end
+
+	end
+
+	class Log
+
+		def initialize
+			@log = Oj.load_file(DIGGIT_LOG)
+		end
+
+		def save
+			Oj.to_file(DIGGIT_LOG, @log)
+		end
+
+		def init(url)
+			unless @log.has_key?(url)
+				@log[url] = default_log
+				save
+			end
+		end
+
+		def update(hash)
+			@log[hash[:url]] = hash[:log]
+			save
+		end
+
+		def rem(url)
+			@log.delete(url)
+			save
+		end
+
+		def log(url)
+			return @log[url]
+		end
+
+		def default_log
+			return {state: :new, error: [], analyses: []}
+		end
+
+	end
+
+	class Diggit
+		attr_accessor :sources, :config, :root
+
+		def initialize(*args)
+			super
+			@root = FileUtils.pwd
+			@sources = Sources.new
+			@config = Config.new
+			load_plugins
+		end
+
+		def load_plugins
+			globals = File.expand_path(INCLUDES_FOLDER,File.expand_path('..',File.dirname(File.realpath(__FILE__))))
+			Dir.glob("#{globals}/**/*.rb").each{ |f| require f }
+		end
+
+	end
+
+end

data/spec/diggit_spec.rb

@@ -0,0 +1,129 @@
+require 'spec_helper'
+require 'fileutils'
+require 'oj'
+
+FileUtils.rm_rf('tmp')
+FileUtils.mkdir('tmp')
+FileUtils.cd('tmp')
+
+TEST_URL = "https://github.com/jrfaller/test-git"
+TEST_FOLDER = "https_github_com_jrfaller_test-git"
+WRONG_URL = "foo"
+
+RSpec.describe Diggit::Cli::DiggitCli do
+
+	it "should init a diggit folder" do
+		result = capture(:stdout) { Diggit::Cli::DiggitCli.start(["init"]) }
+		expect(result).to include("folder initialized")
+		expect(sources).to be_empty
+		expect(log).to be_empty
+		expect(config).to eq({analyses: [], addons: [], joins: [],  options: {}})
+	end
+
+	it "should add an url" do
+		Diggit::Cli::DiggitCli.start(["sources", "add", TEST_URL])
+		expect(sources).to include(TEST_URL)
+	end
+
+	it "should add another url" do
+		Diggit::Cli::DiggitCli.start(["sources", "add", WRONG_URL])
+		expect(sources).to include(WRONG_URL)
+	end
+
+	it "should display the status" do
+		result = capture(:stdout) { Diggit::Cli::DiggitCli.start(["status"]) }
+		expect(result).to include("2 new (0 errors)")
+	end
+
+	it "should add an analysis" do
+		Diggit::Cli::DiggitCli.start(["analyses", "add", "TestAnalysis"])
+		expect(config[:analyses]).to eq(["TestAnalysis"])
+	end
+
+	it "should add an addon" do
+		Diggit::Cli::DiggitCli.start(["addons", "add", "TestAddon"])
+		expect(config[:addons]).to eq(["TestAddon"])
+	end
+
+	it "should add a join" do
+		Diggit::Cli::DiggitCli.start(["joins", "add", "TestJoin"])
+		expect(config[:joins]).to eq(["TestJoin"])
+	end
+
+	it "should perform clones on all urls, handling errors" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["perform", "clones"]) }
+		expect(results).to include("#{TEST_URL} cloned")
+		expect(log[TEST_URL]).to include(state: :cloned, error: {})
+		expect(File.exist?(File.expand_path(TEST_FOLDER,Diggit::SOURCES_FOLDER))).to be true
+		expect(results).to include("error cloning foo")
+		expect(log[WRONG_URL][:state]).to eq(:new)
+		expect(log[WRONG_URL][:error]).to include(name: "Rugged::NetworkError")
+	end
+
+	it "should display info on a regular url" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["sources", "info", TEST_URL]) }
+		expect(results).to include("cloned")
+		expect(results).to include(TEST_URL)
+	end
+
+	it "should display info and error on an url in error" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["sources", "info", WRONG_URL]) }
+		expect(results).to include("new")
+		expect(results).to include(WRONG_URL)
+		expect(results).to include("error")
+		expect(results).to include("Rugged::NetworkError")
+	end
+
+	it "should display all errors" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["sources", "errors"]) }
+		expect(results).to include("new")
+		expect(results).to include(WRONG_URL)
+		expect(results).to include("error")
+		expect(results).to include("Rugged::NetworkError")
+	end
+
+	it "should remove urls" do
+		Diggit::Cli::DiggitCli.start(["sources", "rem", WRONG_URL])
+		expect(sources).to_not include(WRONG_URL)
+	end
+
+	it "should perform analyses" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["perform", "analyses"]) }
+		expect(results).to include("TestAnalysis performed")
+		expect(results).to include("source #{TEST_URL} analyzed")
+		expect(log[TEST_URL][:state]).to eq(:finished)
+	end
+
+	it "should perform joins" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["perform", "joins"]) }
+		expect(results).to include("TestJoin performed")
+		expect(results).to include("joins performed")
+	end
+
+	it "should clean joins" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["clean", "joins"]) }
+		expect(results).to include("TestJoin cleaned")
+	end
+
+	it "should clean analyses" do
+		results = capture(:stdout) { Diggit::Cli::DiggitCli.start(["clean", "analyses"]) }
+		expect(results).to include("TestAnalysis cleaned on #{TEST_URL}")
+		expect(log[TEST_URL][:state]).to eq(:cloned)
+	end
+
+	it "should remove a join" do
+		Diggit::Cli::DiggitCli.start(["joins", "rem", "TestJoin"])
+		expect(config[:addons]).not_to include("TestJoin")
+	end
+
+	it "should remove an analysis" do
+		Diggit::Cli::DiggitCli.start(["analyses", "rem", "TestAnalysis"])
+		expect(config[:analyses]).not_to include("TestAnalysis")
+	end
+
+	it "should remove an addon" do
+		Diggit::Cli::DiggitCli.start(["addons", "rem", "TestAddon"])
+		expect(config[:addons]).not_to include("TestAddon")
+	end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,31 @@
+require 'oj'
+
+require_relative('../lib/diggit_cli')
+
+$0 = "grit"
+ARGV.clear
+
+def config
+	return Oj.load_file(Diggit::DIGGIT_RC)
+end
+
+def log
+	return Oj.load_file(Diggit::DIGGIT_LOG)
+end
+
+def sources
+	return IO.readlines(Diggit::DIGGIT_SOURCES).map{ |line| line.strip }
+end
+
+def capture(stream)
+  begin
+    stream = stream.to_s
+    eval "$#{stream} = StringIO.new"
+    yield
+    result = eval("$#{stream}").string
+  ensure
+    eval("$#{stream} = #{stream.upcase}")
+  end
+
+  result
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: diggit
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jean-Rémy Falleri
+- Matthieu Foucault
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+description: |
+  The Diggit repository analysis tool is a neat swiss knife to enable the analysis of many Git repositories.
+email: jr.falleri@gmail.com
+executables:
+- dgit
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/dgit
+- includes/addons/db.rb
+- includes/addons/output.rb
+- includes/addons/test.rb
+- includes/analyses/authors.rb
+- includes/analyses/cloc.rb
+- includes/analyses/diff_stats.rb
+- includes/analyses/metadata.rb
+- includes/analyses/pom.rb
+- includes/analyses/test.rb
+- includes/joins/diff_size.rb
+- includes/joins/test.rb
+- lib/diggit_cli.rb
+- lib/diggit_core.rb
+- spec/diggit_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jrfaller/diggit
+licenses:
+- LGPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A Git repository analysis tool.
+test_files: []