diggit 1.0.3 → 2.0.0

data/includes/addons/db.rb

@@ -1,26 +0,0 @@
-# encoding: utf-8
-require 'mongo'
-
-# A MongoDB addon for Diggit. The name of this addon is :db. This addon might use an :mongo hash in the global options. In this
-# hash, the :database key allows to configure the name of the database.
-# @!attribute [r] db
-# 	@return [Mongo::DB] the mongo database object.
-class Db < Diggit::Addon
-
-	DEFAULT_DB = 'diggit'
-
-	attr_reader :db
-
-	def initialize(*args)
-		super
-		client = Mongo::MongoClient.new
-		database = DEFAULT_DB
-		database = @options[:mongo][:database] if @options.has_key?(:mongo) && @options[:mongo].has_key?(:database)
-		@db = client[database]
-	end
-
-	def name
-		:db
-	end
-
-end

data/includes/addons/output.rb

@@ -1,37 +0,0 @@
-# encoding: utf-8
-
-# A output addon for Diggit. The name of the addon is :output, and can be reached in the
-# addons hash. This addon might use an :output hash in the global options. In this hash, the
-# :out key allows to configure the name of the output folder and :tmp the name of the temporary output
-# folder.
-# @!attribute [r] out
-# 	@return [String] the absolute path of the output folder.
-# @!attribute [r] tmp
-# 	@return [String] the absolute path of the temporary output folder.
-class Output < Diggit::Addon
-
-	attr_reader :out, :tmp
-
-	DEFAULT_OUT = 'out'
-	DEFAULT_TMP = 'tmp'
-
-	def initialize(*args)
-		super
-
-		out = DEFAULT_OUT
-		out = @options[:output][:out] if @options.has_key?(:output) && @options[:output].has_key?(:out)
-		tmp = DEFAULT_TMP
-		tmp = @options[:output][:tmp] if @options.has_key?(:output) && @options[:output].has_key?(:tmp)
-
-		@out = File.absolute_path(out)
-		@tmp = File.absolute_path(tmp)
-
-		FileUtils.mkdir_p(@out) unless File.exist?(@out)
-		FileUtils.mkdir_p(@tmp) unless File.exist?(@tmp)
-	end
-
-	def name
-		:output
-	end
-
-end

data/includes/addons/sources_options.rb

@@ -1,21 +0,0 @@
-# encoding: utf-8
-
-class SourcesOptions < Diggit::Addon
-
-	SOURCES_OPTIONS_FILE = ".dgitsources-options"
-
-	def initialize(*args)
-		super
-		@sources_options = {}
-		@sources_options = Oj.load_file(SOURCES_OPTIONS_FILE) if File.exists? SOURCES_OPTIONS_FILE
-	end
-
-	def name
-		:sources_options
-	end
-
-	def method_missing(meth, *args, &block)
-		@sources_options.send meth, *args, &block
-	end
-
-end

data/includes/analyses/authors.rb

@@ -1,12 +0,0 @@
-# encoding: utf-8
-
-class AuthorAnalysis < Diggit::Analysis
-
-	def run
-		walker = Rugged::Walker.new(@repo)
-		walker.sorting(Rugged::SORT_DATE)
-		walker.push(@repo.last_commit)
-		authors = walker.collect{ |c| c.author[:name] }.uniq
-		puts "Authors : #{authors}"
-	end
-end

data/includes/analyses/cloc.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-
-require 'yaml'
-
-class ClocAnalysis < Diggit::Analysis
-
-	def run
-		cloc = `cloc . --progress-rate=0 --quiet --yaml`
-		unless cloc.empty?
-			yaml = YAML.load(cloc.lines[2..-1].join)
-			yaml.delete('header')
-			output = { source: @source, cloc: yaml }
-			col = @addons[:db].db['cloc']
-			col.insert(output)
-		end
-	end
-
-	def clean(source)
-		@addons[:db].db['cloc'].remove({source: source})
-	end
-
-end
-
-
-class ClocPerFileAnalysis < Diggit::Analysis
-
-	def run
-		commit_oid = "HEAD"
-		commit_oid = @addons[:sources_options][@source]["cloc-commit-id"] if @addons.has_key?(:sources_options) && @addons[:sources_options].has_key?(@source) && @addons[:sources_options][@source].has_key?("cloc-commit-id")
-		@repo.checkout(commit_oid, {:strategy=>[:force,:remove_untracked]})
-		cloc = `cloc . --progress-rate=0 --quiet --by-file --yaml --script-lang=Python,python`
-		unless cloc.empty?
-			yaml = YAML.load(cloc.lines[2..-1].join)
-			yaml.delete('header')
-			yaml.delete('SUM')
-			cloc_a = []
-			yaml.each do |key, value|
-				# transform the hash so the filenames are not keys anymore (as they may contain a '.' it is incompatible with mongo)
-				path = key.gsub(/^\.\//, '') # remove the './' at the start of filenames
-				cloc_a << value.merge({:path => path})
-			end
-			output = { source: @source, commit_oid: commit_oid.to_s, cloc: cloc_a }
-			col = @addons[:db].db['cloc-file']
-			col.insert(output)
-		end
-	end
-
-	def clean
-		@addons[:db].db['cloc-file'].remove({source: @source})
-	end
-end

data/includes/analyses/diff_stats.rb

@@ -1,39 +0,0 @@
-# encoding: utf-8
-
-class DiffStats < Diggit::Analysis
-
-	ACCEPTED_EXTENSIONS = [".java", ".c", ".h", ".js", ".javascript" ]
-
-	def run
-		col = @addons[:db].db['diffstats']
-
-		walker = Rugged::Walker.new(@repo)
-		ref = @repo.references["refs/heads/master"]
-		walker.push(ref.target_id)
-		walker.each do |commit|
-			parent1 = commit.parents[0]
-			if parent1
-				diff1 = parent1.diff(commit)
-				diff1.each_delta do |delta|
-					old_path = delta.old_file[:path]
-					new_path = delta.new_file[:path]
-					old_ext = File.extname(old_path).downcase
-					new_ext = File.extname(new_path).downcase
-					if delta.status == :modified && old_ext.eql?(new_ext) && ACCEPTED_EXTENSIONS.include?(old_ext)
-						sha_old = delta.old_file[:oid]
-						sha_new = delta.new_file[:oid]
-						patch = @repo.lookup(sha_new).diff(@repo.lookup(sha_old))
-						changes = patch.changes
-						edit = {source: @source, old_path: old_path, new_path: new_path, old_commit: commit.oid, new_commit: parent1.oid, changes: changes}
-						col.insert(edit)
-					end
-				end
-			end
-		end
-	end
-
-  def clean
-    @addons[:db].db['diffstats'].remove({source: @source})
-  end
-
-end

data/includes/analyses/metadata.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-
-class MetadataAnalysis < Diggit::Analysis
-
-	def run
-		# Importing tags
-		tags = @addons[:db].db['tags']
-		@repo.tags.each do |t|
-			tag = { source: @source, name: t.name, target: t.target.oid }
-			tags.insert(tag)
-		end
-
-		# Importing branches
-		branches = @addons[:db].db['branches']
-		@repo.branches.each do |b|
-			branch = { source: @source, name: b.name, target: b.target.oid }
-			branches.insert(branch)
-		end
-
-		# Importing commits
-		commits = @addons[:db].db['commits']
-		walker = Rugged::Walker.new(@repo)
-		walker.sorting(Rugged::SORT_DATE)
-		walker.push(@repo.last_commit)
-		walker.each do |c|
-			commit = {
-				source: @source, oid: c.oid, message: c.message, author: c.author,
-				committer: c.committer, parent_ids: c.parent_ids, time: c.time
-			}
-			commits.insert(commit)
-		end
-	end
-
-	def clean
-		@addons[:db].db['tags'].remove({source: @source})
-		@addons[:db].db['branches'].remove({source: @source})
-		@addons[:db].db['commits'].remove({source: @source})
-	end
-
-end

data/includes/analyses/pom.rb

@@ -1,16 +0,0 @@
-# encoding: utf-8
-
-class PomAnalysis < Diggit::Analysis
-
-	def run
-		pom_files = Dir['**/pom.xml']
-		pom_files.each do |file|
-			puts file
-			git = `git --no-pager log --pretty=%H%n%an --name-status #{file}`.lines
-			history = []
-			git.each_slice(4) { |slice| history << [slice[0].strip, slice[1].strip, slice[3].split("\t")[0].strip] }
-			puts history.inspect
-		end
-	end
-
-end

data/includes/joins/diff_size.rb

@@ -1,67 +0,0 @@
-# encoding: utf-8
-require "rinruby"
-
-class DiffSize < Diggit::Join
-
-	SAMPLE_SIZE = 10
-
-	def run
-		@sources.map{ |s| sample(s) }
-	end
-
-	def sample(src)
-		R.src = src[:url]
-		R.size = SAMPLE_SIZE
-		R.eval <<-EOS
-			library(RMongo)
-
-			srcQuery <- function(src) {
-				return(paste('{"source": "', src, '"}', sep=""))
-			}
-
-			noZeros <- function(data) {
-				return(subset(data,changes>0))
-			}
-
-			rubify <- function(data) {
-				return(cbind(data$old_path, data$old_commit, data$new_path, data$new_commit))
-			}
-
-			db <- mongoDbConnect('diffstats')
-			d <- noZeros(dbGetQuery(db, 'diffstats', srcQuery(src), skip=0, limit=0))
-
-			q <- quantile(d$changes)
-
-			s <- subset(d,changes > 0 & changes <= q[3])
-			m <- subset(d,changes > q[3] & changes <= q[4])
-			b <- subset(d,changes > q[4])
-
-			rs <- s[sample(nrow(s), size),]
-			rm <- m[sample(nrow(m), size),]
-			rb <- b[sample(nrow(b), size),]
-
-			rs_ruby <- rubify(rs)
-			rm_ruby <- rubify(rm)
-			rb_ruby <- rubify(rb)
-		EOS
-		write_sample(src, R.rs_ruby, 'small')
-		write_sample(src, R.rm_ruby, 'medium')
-		write_sample(src, R.rb_ruby, 'big')
-	end
-
-	def write_sample(src, sample, type)
-		prefix = "#{@addons[:output].out}/#{src[:id]}/#{type}"
-		n = 1
-		sample.row_vectors.each do |row|
-			FileUtils.mkdir_p("#{prefix}/#{n}")
-			File.write("#{prefix}/#{n}/src_#{row[1]}_#{row[0].gsub(/\//,'_')}", `git -C #{src[:folder]} show #{row[1]}:#{row[0]}`)
-			File.write("#{prefix}/#{n}/dst_#{row[3]}_#{row[2].gsub(/\//,'_')}", `git -C #{src[:folder]} show #{row[3]}:#{row[2]}`)
-			n += 1
-		end
-	end
-
-  def clean
-		@sources.each{ |s| FileUtils.rm_rf("#{@addons[:output].out}/#{s[:id]}") }
-  end
-
-end

data/includes/joins/test.rb

@@ -1,13 +0,0 @@
-# encoding: utf-8
-
-class TestJoin < Diggit::Join
-
-	def run
-		p "TestJoin performed"
-	end
-
-	def clean
-		p "TestJoin cleaned"
-	end
-
-end

data/lib/diggit_cli.rb

@@ -1,334 +0,0 @@
-require 'thor'
-require 'fileutils'
-
-require_relative 'diggit_core'
-
-module Diggit
-
-	module Utils
-
-		DONE = '[done]'
-		WARNING = '[warning]'
-		ERROR = '[error]'
-		INFO = '[info]'
-
-		def diggit
-			@diggit = Diggit.new if @diggit.nil?
-			return @diggit
-		end
-
-		def dump_error(e)
-			{ name: e.class.name, message: e.to_s, backtrace: e.backtrace }
-		end
-
-		def plugin_ok?(name, type)
-			obj = Object::const_get(name)
-			return obj < type
-		rescue NameError
-			return false
-		end
-
-		def source_color(source_hash)
-			if source_hash[:log][:error].empty?
-				return :blue
-			else
-				return :red
-			end
-		end
-
-		def status_color(status)
-			if status == DONE
-				return :green
-			else
-				return :red
-			end
-		end
-
-		def say_error(msg)
-			say_status(ERROR, msg, :red)
-		end
-
-		def say_info(msg)
-			say_status(INFO, msg, :blue)
-		end
-
-		def say_done(msg)
-			say_status(DONE, msg, :green)
-		end
-
-	end
-
-	module Cli
-
-		class SourcesCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc 'list', "Display the list of sources."
-			def list
-				idx = 1
-				diggit.sources.hashes.each do |s|
-					say_status("[#{s[:log][:state]}]", "#{idx}: #{s[:url]}", source_color(s))
-					idx += 1
-				end
-				errors = diggit.sources.get_all(nil, {error: true}).size
-				status = (errors== 0 && DONE) || ERROR
-				say_status(status, "listed #{diggit.sources.size} sources including #{errors} errors", status_color(status))
-			end
-
-			desc 'info [SOURCE_DEF]', "Display informations on the provided source definition (either source URL or id)."
-			def info(source_def)
-				s = diggit.sources.get(source_def)
-				say_status("[#{s[:log][:state]}]", "#{s[:url]}", source_color(s))
-				say_status('[folder]', "#{s[:folder]}", :blue)
-				unless s[:log][:error].empty?
-					say_error("#{s[:log][:error][:name]}")
-					say_status('[message]', "#{s[:log][:error][:message]}", :red)
-					say_status('[backtrace]', "", :red)
-					say(s[:log][:error][:backtrace].join("\n"))
-				end
-			end
-
-			desc "errors", "Display informations on all source that have encountered an error."
-			def errors
-				diggit.sources.get_all(nil, {error: true}).each{|s| info s[:url] }
-			end
-
-			desc 'import [FILE]', "Import a list of sources from a file (one URL per line)."
-			def import(urls_file)
-				IO.readlines(urls_file).each{ |line| diggit.sources.add(line.strip) }
-			end
-
-			desc "add [URL*]", "Add the provided urls to the list of sources."
-			def add(*urls)
-				urls.each{ |u| diggit.sources.add(u) }
-			end
-
-			desc "rem [SOURCE_DEF*]", "Remove the sources correspondign to the provided source definitions (id or URL) from the list of sources."
-			def rem(*sources_defs)
-				sources_defs.each { |s| diggit.sources.rem(s) }
-			end
-		end
-
-		class AddonsCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc "add [ADDON*]", "Add the provided addons to the list of active addons."
-			def add(*addons)
-				addons.each do |a|
-					if plugin_ok?(a, Addon)
-						diggit.config.add_addon(a)
-					else
-						say_error("error loading addon #{a}")
-					end
-				end
-			end
-
-			desc "rem [ADDON*]", "Remove the provided addons from the list of active addons."
-			def rem(*addons)
-				addons.each{ |a| diggit.config.rem_addon(a) }
-			end
-
-		end
-
-		class JoinsCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc "add [JOIN*]", "Add the provided joins to the list of active joins."
-			def add(*joins)
-				joins.each do |j|
-					if plugin_ok?(j, Join)
-						diggit.config.add_join(j)
-					else
-						say_error("error loading join #{j}")
-					end
-				end
-			end
-
-			desc "rem [JOIN*]", "Remove the provided joins from the list of active joins."
-			def rem(*joins)
-				joins.each{ |j| diggit.config.rem_join(j) }
-			end
-
-		end
-
-		class AnalysesCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc "add [ANALYSIS*]", "Add the provided analyses to the list of active analyses."
-			def add(*analyses)
-				analyses.each do |a|
-					if plugin_ok?(a, Analysis)
-						diggit.config.add_analysis(a)
-					else
-						say_error("error loading analysis #{a}")
-					end
-				end
-			end
-
-			desc "rem [ANALYSIS*]", "Remove the provided analyses from the list of active analyses."
-			def rem(*analyses)
-				analyses.each{ |a| diggit.config.rem_analysis(a) }
-			end
-
-		end
-
-		class PerformCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc "clones [SOURCE_DEFS*]", "Clone the sources corresponding to the provided source definitions (id or URL). Clone all sources if no source definitions are provided."
-			def clones(*source_defs)
-				diggit.sources.get_all(source_defs, {state: :new}).each do |s|
-					begin
-						if File.exist?(s[:folder])
-							Rugged::Repository::new(s[:folder])
-						else
-							Rugged::Repository::clone_at(s[:url], s[:folder], {
-  								transfer_progress: lambda { |total_objects, indexed_objects, received_objects, local_objects, total_deltas, indexed_deltas, received_bytes|
-    								msg = "Clone of #{s[:url]} in progress : #{received_objects}/#{total_objects} objects received.\r"
-    								$stderr.print msg
-    								@last_message_length = msg.length
-							  }
-							})
-							print " " * @last_message_length + "\r" # clean the line used to output transfer progress
-						end
-					rescue => e
-						s[:log][:error] = dump_error(e)
-						say_error("error cloning #{s[:url]}")
-					else
-						s[:log][:state] = :cloned
-						s[:log][:error] = {}
-						say_done("#{s[:url]} cloned")
-					ensure
-						diggit.sources.update(s)
-					end
-				end
-			end
-
-			desc "analyses [SOURCE_DEFS*]", "Perform the configured analyses to the sources corresponding to the provided source definitions (id or URL). Analyze all sources if no source definitions are provided."
-			def analyses(*source_defs)
-				addons = diggit.config.load_addons
-				diggit.sources.get_all(source_defs, {state: :cloned}).each do |s|
-					FileUtils.cd(s[:folder])
-					globs = {}
-					performed_analyses = []
-					begin
-						repo = Rugged::Repository.new('.')
-						diggit.config.load_analyses(s[:url], repo, addons, globs).each do |a|
-							performed_analyses << a.class.to_s
-							a.run
-						end
-					rescue => e
-						s[:log][:error] = dump_error(e)
-						s[:log][:analyses] = performed_analyses[1..-2]
-						say_error("error performing #{performed_analyses.last} on #{s[:url]}")
-					else
-						s[:log][:analyses] = performed_analyses
-						s[:log][:state] = :finished
-						s[:log][:error] = {}
-						say_done("source #{s[:url]} analyzed")
-					ensure
-						FileUtils.cd(diggit.root)
-						diggit.sources.update(s)
-					end
-				end
-			end
-
-			desc "joins", "Perform the configured joins."
-			def joins
-				addons = diggit.config.load_addons
-				globs = {}
-				diggit.config.load_joins(diggit.sources.get_all([], {state: :finished, error: false}), addons, globs).each{ |j| j.run }
-				say_done("joins performed")
-			end
-
-		end
-
-		class CleanCli < Thor
-			include Thor::Actions
-			include Utils
-
-			desc "analyses", "Clean the configured analyzes on the provided source definitions (id or URL). Clean all sources if no source definitions are provided."
-			def analyses(*source_defs)
-				addons = diggit.config.load_addons
-				diggit.sources.get_all(source_defs, {state: :finished}).each do |s|
-					globs = {}
-					diggit.config.load_analyses(s[:url], nil, addons, globs).each{ |a| a.clean}
-					s[:log][:state] = :cloned
-					s[:log][:analyses] = []
-					s[:log][:error] = {}
-					diggit.sources.update(s)
-					say_done("cleaned analyses on #{s[:url]}")
-				end
-			end
-
-			desc "joins", "Clean the configured joins."
-			def joins
-				addons = diggit.config.load_addons
-				globs = {}
-				diggit.config.load_joins(diggit.sources.get_all([], {state: :finished, error: false}), addons, globs).each{ |j| j.clean }
-			end
-
-		end
-
-		class DiggitCli < Thor
-			include Thor::Actions
-			include Utils
-
-			def initialize(*args)
-				super
-				cmd = args[2][:current_command].name
-				unless 'init'.eql?(cmd) || 'help'.eql?(cmd)
-					unless File.exist?(DIGGIT_RC)
-						say_error("this is not a diggit directory")
-					else
-						diggit
-					end
-				end
-			end
-
-			desc "init", "Initialize the current folder as a diggit folder."
-			def init
-				FileUtils.touch(DIGGIT_SOURCES)
-				Oj.to_file(DIGGIT_LOG, {})
-				Oj.to_file(DIGGIT_RC, { addons: [], analyses: [], joins: [], options: {} })
-				say_done("Diggit folder successfully initialized")
-			end
-
-			desc 'status', "Display the status of the current diggit folder."
-			def status
-				color = (diggit.sources.get_all(nil, {error: true}).size > 0 && :red) || :blue
-				say_status('[sources]', "#{diggit.sources.get_all([], {state: :new}).size} new (#{diggit.sources.get_all([], {state: :new, error: true}).size} errors), #{diggit.sources.get_all([], {state: :cloned}).size} cloned (#{diggit.sources.get_all([], {state: :cloned, error: true}).size} errors), #{diggit.sources.get_all([], {state: :finished}).size} finished", color)
-				say_status('[addons]', "#{diggit.config.addons.join(', ')}", :blue)
-				say_status('[analyses]', "#{diggit.config.analyses.join(', ')}", :blue)
-				say_status('[joins]', "#{diggit.config.joins.join(', ')}", :blue)
-				say_status('[options]', "#{diggit.config.options}", :blue)
-			end
-
-			desc "sources SUBCOMMAND ...ARGS", "manage sources for the current diggit folder."
-			subcommand "sources", SourcesCli
-
-			desc "joins SUBCOMMAND ...ARGS", "manage joins for the current diggit folder."
-			subcommand "joins", JoinsCli
-
-			desc "analyses SUBCOMMAND ...ARGS", "manage analyses for the current diggit folder."
-			subcommand "analyses", AnalysesCli
-
-			desc "addons SUBCOMMAND ...ARGS", "manage addons for the current diggit folder."
-			subcommand "addons", AddonsCli
-
-			desc "perform SUBCOMMAND ...ARGS", "perform actions in the current diggit folder."
-			subcommand "perform", PerformCli
-
-			desc "clean SUBCOMMAND ...ARGS", "clean the current diggit folder."
-			subcommand "clean", CleanCli
-
-		end
-
-	end
-
-end