diggit 1.0.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdc49eaf85ab5ea705046d562d66d05aec279b07
-  data.tar.gz: 8046df957be6fcc9a9296256c9bee91562f8ee7e
+  metadata.gz: ab5b0bd0d5959100238fb8bdcd2d6471a660bc94
+  data.tar.gz: 41bcf9e0cea8e6110faded4f51b37caa8475adf2
 SHA512:
-  metadata.gz: 6b42a77ecfb10bc8f95e952d58f1a8bae42395b017038fea82fd5240a2f238645ec9007172dfdc54671e945674bc90bcc96946fadae62238a2781cc9a39c7d52
-  data.tar.gz: f2626d93e3f2e1179d10f16f5cfcb3ddfcb292f99614518925e60349e1c5814bdf721beed37cb6851822ddba08ba6e5595de7a0fd50855f62f5faa3d48cc7247
+  metadata.gz: 2d11da8a99ed2706e882bbdaf9a95e011f46fb3f3e1941c6728311de15b7796ab624ce62ac0220e778094f6a8f29e0b798e79efa6c5916de8cce471a8f9b0b8f
+  data.tar.gz: f7272bb6679af26829ee4ccbc1b013f2f6d6c1e2026b01e0501335a0cd726e82b117c42fc8f2a28fbc24bae2ef988c86b37fd54f06581f3b78520cee010850a1

data/README.md

@@ -1,6 +1,6 @@
 # Diggit
 
-A ruby tool to analyse Git repositories
+A ruby tool to analyze Git repositories
 
 # Installation
 
@@ -36,10 +36,6 @@ A join is performed after all analyses of all repositories have been performed.
 
 ## Running analyses
 
-Once diggit is configured you can perform the analyses. First, you have to clone the repositories by using `dgit perform clones`. Then you can launch the analyses by using `dgit perform analyses`. Finally, the joins are executed via the command `dgit perform joins`.
+Once diggit is configured you can perform the analyses. First, you have to clone the repositories by using `dgit clones perform`. Then you can launch the analyses by using `dgit analyses perform`. Finally, the joins are executed via the command `dgit joins perform`. You can use the `mode` option to handle the cleaning of joins or analyses.
 
-At all time, you can check the status of your diggit folder by using `dgit status`. If you want more info on the status of a given repository, you can use the `dgit sources info https://github.com/jrfaller/diggit.git` command.
-
-## Cleaning up
-
-If something is going wrong, you can always delete the results of the joins by using the command `dgit clean joins` and of the analysis with the command `dgit clean analyses`.
+At all time, you can check the status of your diggit folder by using `dgit status`.

data/bin/dgit

@@ -1,6 +1,191 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 
-require_relative '../lib/diggit_cli.rb'
+require 'gli'
+require_relative '../lib/dgit'
 
-Diggit::Cli::DiggitCli.start(ARGV)
+include GLI::App
+
+program_desc 'A git repository analysis tool.'
+
+version Diggit::VERSION
+
+subcommand_option_handling :normal
+arguments :strict
+
+desc 'Init a diggit folder.'
+skips_pre
+command :init do |c|
+	c.action do |_global_options, _options, _args|
+		Diggit::Dig.init_dir
+		Log.ok "Diggit folder initialized."
+	end
+end
+
+desc 'Display the status of the diggit folder.'
+command :status do |c|
+	c.action do |_global_options, _options, _args|
+		Log.info "Config"
+		Log.info "======"
+		Log.info "- Analyses: #{Diggit::Dig.it.config.get_analyses.join(', ')}"
+		Log.info "- Joins: #{Diggit::Dig.it.config.get_joins.join(', ')}"
+		Log.info ""
+		Log.info "Journal"
+		Log.info "======="
+		Log.info "- New sources:"
+		Log.indent do
+			Log.ok "* Ok: #{Diggit::Dig.it.journal.sources_by_state(:new).size}"
+			Log.error "* Error: #{Diggit::Dig.it.journal.sources_by_state(:new, true).size}"
+		end
+		Log.info "- Cloned sources:"
+		Log.indent do
+			Log.ok "* Ok: #{Diggit::Dig.it.journal.sources_by_state(:cloned).size}"
+			Log.error "* Error: #{Diggit::Dig.it.journal.sources_by_state(:cloned, true).size}"
+		end
+	end
+end
+
+desc 'Manage the sources of the diggit folder.'
+command :sources do |c|
+	c.desc 'List the sources.'
+	c.command :list do |list|
+		list.action do |_global_options, _options, _args|
+			sources = Diggit::Dig.it.journal.sources
+			sources.each_index do |idx|
+				msg = "#{idx}	#{sources[idx].url} (#{sources[idx].state})"
+				sources[idx].error? ? Log.error(msg) : Log.ok(msg)
+			end
+		end
+	end
+	c.desc 'Add a source.'
+	c.arg_name 'url'
+	c.command :add do |add|
+		add.action do |_global_options, _options, args|
+			Diggit::Dig.it.journal.add_source args[0]
+		end
+	end
+	c.desc 'Import sources from a file.'
+	c.arg_name 'file'
+	c.command :import do |import|
+		import.action do |_global_options, _options, args|
+			File.open(args[0]).each { |line| Diggit::Dig.it.journal.add_source(line) }
+		end
+	end
+	c.default_command :list
+end
+
+desc 'Manage the joins of the diggit folder.'
+command :joins do |c|
+	c.desc 'List the joins'
+	c.command :list do |list|
+		list.action do |_global_options, _options, _args|
+			Diggit::Dig.it.config.get_joins.each { |a| Log.info a.name }
+		end
+	end
+	c.desc 'Add add join.'
+	c.arg_name 'name'
+	c.command :add do |add|
+		add.action do |_global_options, _options, args|
+			Diggit::Dig.it.config.add_join args[0]
+		end
+	end
+	c.desc 'Perform joins.'
+	c.command :perform do |perform|
+		perform.flag [:s, :sources], desc: "list of sources", type: Array, default_value: []
+		perform.flag [:a, :analyses], desc: "list of analyses", type: Array, default_value: []
+		perform.flag [:m, :mode], desc: "running mode",
+				must_match: { "run" => :run, "clean" => :clean, "rerun" => :rerun }, default_value: :run
+		perform.action do |_global_options, options, _args|
+			Diggit::Dig.it.join(options[:s], options[:a], options[:m])
+		end
+	end
+	c.default_command :list
+end
+
+desc 'Manage the analyses of the diggit folder.'
+command :analyses do |c|
+	c.desc 'List the analyses'
+	c.command :list do |list|
+		list.action do |_global_options, _options, _args|
+			Diggit::Dig.it.config.get_analyses.each { |a| Log.info a.name }
+		end
+	end
+	c.desc 'Add an analysis.'
+	c.arg_name 'name'
+	c.command :add do |add|
+		add.action do |_global_options, _options, args|
+			Diggit::Dig.it.config.add_analysis args[0]
+		end
+	end
+	c.desc 'Perform analyses.'
+	c.command :perform do |perform|
+		perform.flag [:s, :sources], desc: "list of sources", type: Array, default_value: []
+		perform.flag [:a, :analyses], desc: "list of analyses", type: Array, default_value: []
+		perform.flag [:m, :mode], desc: "running mode",
+				must_match: { "run" => :run, "clean" => :clean, "rerun" => :rerun }, default_value: :run
+		perform.action do |_global_options, options, _args|
+			Diggit::Dig.it.analyze(options[:s], options[:a], options[:m])
+		end
+	end
+	c.default_command :list
+end
+
+desc 'Manage clone actions.'
+command :clone do |c|
+	c.desc 'Perform the clones.'
+	c.command :perform do |perform|
+		perform.flag [:s, :sources], desc: "list of sources", type: Array, default_value: []
+		perform.action do |_global_options, options, _args|
+			Diggit::Dig.it.clone(*options[:s])
+		end
+	end
+	c.default_command :perform
+end
+
+desc 'Display errors.'
+command :errors do |c|
+	c.desc 'Display the list of errors.'
+	c.command :list do |list|
+		list.action do |_global_options, _options, _args|
+			sources = Diggit::Dig.it.journal.sources
+			sources.each_index do |idx|
+				msg = "#{idx}	#{sources[idx].url} (#{sources[idx].state})"
+				Log.error msg if sources[idx].error?
+			end
+		end
+	end
+	c.desc 'Display a source error.'
+	c.arg_name 'id'
+	c.command :show do |show|
+		show.action do |_global_options, _options, args|
+			source = Diggit::Dig.it.journal.sources_by_ids(args[0].to_i)[0]
+			Log.ok "Error summary for source #{args[0]}"
+			error = source.error
+			Log.info "URL: #{source.url}"
+			Log.info "State: #{source.state}"
+			Log.info "Error:"
+			Log.indent do
+				Log.error error[:message]
+				Log.info error[:backtrace].join("\n")
+			end
+		end
+	end
+	c.default_command :list
+end
+
+pre do |_global, _command, _options, _args|
+	Diggit::Dig.init
+end
+
+post do |_global, _command, _options, _args|
+		# Post logic here, skips_post to skip commands
+end
+
+on_error do |exception|
+	Log.error "Error running diggit."
+	Log.error exception.message
+	Log.info exception.backtrace.join("\n")
+	false
+end
+
+exit run(ARGV)

data/lib/dgit.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+require_relative "dgit/core"
+require_relative "dgit/plugins"
+require_relative "dgit/version"
+require_relative "dgit/formatador"

data/lib/dgit/core.rb

@@ -0,0 +1,479 @@
+# encoding: utf-8
+
+require 'oj'
+require 'rugged'
+require 'singleton'
+require_relative 'formatador'
+
+class String
+	def underscore
+		gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+				.gsub(/([a-z\d])([A-Z])/, '\1_\2'). tr("-", "_").downcase
+	end
+
+	def camel_case
+		return self if self !~ /_/ && self =~ /[A-Z]+.*/
+		split('_').map(&:capitalize).join
+	end
+
+	def id
+		gsub(/[^[\w-]]+/, "_")
+	end
+end
+
+module Diggit
+	class Source
+		attr_reader :url, :repository
+		attr_accessor :entry
+
+		def initialize(url)
+			@url = url
+			@entry = Journal.new_source_entry
+			@repository = nil
+		end
+
+		def id
+			@url.id
+		end
+
+		def folder
+			Dig.it.file_path("sources/#{id}")
+		end
+
+		def error?
+			!(@entry[:last_error].nil? || @entry[:last_error].empty?)
+		end
+
+		def error
+			@entry[:last_error]
+		end
+
+		def error=(error)
+			@entry[:last_error] = error
+		end
+
+		def state
+			@entry[:state]
+		end
+
+		def state=(state)
+			@entry[:state] = state
+		end
+
+		def new?
+			@entry[:state] == :new
+		end
+
+		def cloned?
+			@entry[:state] == :cloned
+		end
+
+		def add_performed_analysis(name)
+			@entry[:performed_analyses] << name
+		end
+
+		def analysis_performed?(name)
+			@entry[:performed_analyses].include?(name)
+		end
+
+		def analyses_performed?(*names)
+			(names - @entry[:performed_analyses]).empty?
+		end
+
+		def del_performed_analysis(name)
+			@entry[:performed_analyses].delete_if { |a| a == name }
+		end
+
+		def add_ongoing_analysis(name)
+			@entry[:ongoing_analyses].include?(name)
+		end
+
+		def del_ongoing_analysis(name)
+			@entry[:ongoing_analyses].delete_if { |a| a == name }
+		end
+
+		def analysis_ongoing?(name)
+			@entry[:ongoing_analyses].include?(name)
+		end
+
+		def analyses_ongoing?(*names)
+			(names - @entry[:ongoing_analyses]).empty?
+		end
+
+		def analysis?(name)
+			analysis_ongoing?(name) || analysis_performed?(name)
+		end
+
+		def del_analysis(name)
+			del_ongoing_analysis(name)
+			del_performed_analysis(name)
+		end
+
+		def clone
+			if File.exist?(folder)
+				Rugged::Repository.new(folder)
+			else
+				Rugged::Repository.clone_at(url, folder)
+			end
+			self.state = :cloned
+		rescue => e
+			Log.error "Error cloning #{url}."
+			self.error = Journal.dump_error(e)
+		end
+
+		def load_repository
+			fail "Source not cloned #{url}." if new?
+			@repository = Rugged::Repository.new(folder)
+		end
+	end
+
+	class Journal
+		def initialize(hash)
+			@sources = {}
+			@workspace = {}
+			hash[:urls].each do |u|
+				s = Source.new(u)
+				s.entry = hash[:sources][u] if !hash[:sources].nil? && hash[:sources].key?(u)
+				@sources[u] = s
+			end
+			@workspace = hash[:workspace]
+			@workspace = Journal.new_workspace_entry if hash[:workspace].nil? || hash[:workspace].empty?
+		end
+
+		def sources
+			@sources.values
+		end
+
+		def sources_by_state(state, error = false)
+			@sources.select { |_u, s| s.state == state && s.error? == error }.values
+		end
+
+		def sources_by_ids(*ids)
+			return sources if ids.empty?
+			source_array = sources
+			result = []
+			ids.each do |id|
+				fail "No such source index #{id}." if id >= source_array.length
+				result << source_array[id]
+			end
+			result
+		end
+
+		def update_source(source)
+			fail "No such source #{source}." unless @sources.key?(source.url)
+			@sources[source.url] = source
+			Dig.it.save_journal
+		end
+
+		def add_source(url)
+			@sources[url] = Source.new(url) unless @sources.key?(url)
+			Dig.it.save_journal
+		end
+
+		def join?(name)
+			@workspace[:performed_joins].include?(name)
+		end
+
+		def add_join(name)
+			@workspace[:performed_joins] << name
+			Dig.it.save_journal
+		end
+
+		def join_error?
+			!@workspace[:last_error].nil?
+		end
+
+		def join_error
+			@workspace[:last_error]
+		end
+
+		def join_error=(error)
+			@workspace[:last_error] = Journal.dump_error(error)
+			Dig.it.save_journal
+		end
+
+		def to_hash
+			entry_hash = {}
+			@sources.each { |entry| entry_hash[entry[0]] = entry[1].entry }
+			{ urls: @sources.keys, sources: entry_hash, workspace: @workspace }
+		end
+
+		def self.new_source_entry
+			{ state: :new, performed_analyses: [], error_analyses: [], ongoing_analyses: [], last_error: {} }
+		end
+
+		def self.new_workspace_entry
+			{ performed_joins: [], last_error: {} }
+		end
+
+		def self.dump_error(error)
+			{ name: error.class.name, message: error.to_s, backtrace: error.backtrace }
+		end
+	end
+
+	class Config
+		attr_reader :analyses, :joins
+
+		def initialize(hash)
+			@analyses = []
+			@joins = []
+			hash[:analyses].each { |a| load_analysis(a) }
+			hash[:joins].each { |j| load_join(j) }
+		end
+
+		def to_hash
+			{ analyses: @analyses.map(&:name), joins: @joins.map(&:name) }
+		end
+
+		def add_analysis(name)
+			load_analysis(name) unless @analyses.map(&:name).include?(name)
+			Dig.it.save_config
+		end
+
+		def load_analysis(name)
+			@analyses << Dig.it.plugin_loader.load_plugin(name, :analysis)
+		end
+
+		def del_analysis(name)
+			@analyses.delete_if { |a| a.name == name }
+			Dig.it.save_config
+		end
+
+		def get_analyses(*names)
+			return analyses if names.empty?
+			analyses.select { |a| names.include?(a.name) }
+		end
+
+		def add_join(name)
+			load_join(name) unless @joins.map(&:name).include?(name)
+			Dig.it.save_config
+		end
+
+		def load_join(name)
+			@joins << Dig.it.plugin_loader.load_plugin(name, :join)
+		end
+
+		def del_join(name)
+			@joins.delete_if { |j| j.name == name }
+			Dig.it.save_config
+		end
+
+		def get_joins(*names)
+			return joins if names.empty?
+			joins.select { |j| joins.include?(j.name) }
+		end
+
+		def self.empty_config
+			{ analyses: [], joins: [] }
+		end
+	end
+
+	class PluginLoader
+		include Singleton
+
+		PLUGINS_TYPES = [:addon, :analysis, :join]
+
+		def load_plugin(name, type, instance = false)
+			plugin = search_plugin(name, type)
+			if plugin
+				if instance
+					return plugin.new(Dig.it.options)
+				else
+					return plugin
+				end
+			else
+				fail "Plugin #{name} not found."
+			end
+		end
+
+		def search_plugin(name, type)
+			plugin = nil
+			if @plugins.key?(name)
+				plugin = @plugins[name]
+			else
+				fail "Unknown plugin type #{type}." unless PLUGINS_TYPES.include?(type)
+				if load_file(name, type)
+					plugin = Object.const_get(name.camel_case)
+					base_class = Object.const_get("Diggit::#{type.to_s.camel_case}")
+					if plugin < base_class
+						@plugins[name] = plugin
+					else
+						fail "Plugin #{name} not of kind #{type}."
+					end
+				end
+			end
+			plugin
+		end
+
+		def load_file(name, type)
+			f_glob = PluginLoader.plugin_path(name, type, File.expand_path('../..', File.dirname(File.realpath(__FILE__))))
+			f_home = PluginLoader.plugin_path(name, type, File.expand_path(Dig::DGIT_FOLDER, Dir.home))
+			f_local = PluginLoader.plugin_path(name, type, Dig.it.folder)
+			found = true
+			if File.exist?(f_local)
+				require f_local
+			elsif File.exist?(f_home)
+				require f_home
+			elsif File.exist?(f_glob)
+				require f_glob
+			else
+				found = false
+			end
+			found
+		end
+
+		def self.plugin_path(name, type, root)
+			File.expand_path("#{name}.rb", File.expand_path(type.to_s, File.expand_path('plugins', root)))
+		end
+
+		def initialize
+			@plugins = {}
+		end
+	end
+
+	class Dig
+		DGIT_FOLDER = ".dgit"
+		DGIT_SOURCES = "sources"
+		DGIT_CONFIG = "config"
+		DGIT_OPTIONS = "options"
+		DGIT_JOURNAL = "journal"
+
+		attr_reader :config, :options, :journal, :plugin_loader, :folder
+
+		@diggit = nil
+
+		def self.it
+			fail "Diggit has not been initialized." if @diggit.nil?
+			@diggit
+		end
+
+		def self.init(folder = '.')
+			@diggit = Dig.new(folder)
+			@diggit.load_options
+			@diggit.load_config
+			@diggit.load_journal
+			@diggit
+		end
+
+		def self.init_dir(folder = '.')
+			dgit_folder = File.expand_path(DGIT_FOLDER, folder)
+			FileUtils.mkdir(dgit_folder)
+			Oj.to_file(File.expand_path(DGIT_CONFIG, dgit_folder), Config.empty_config)
+			Oj.to_file(File.expand_path(DGIT_OPTIONS, dgit_folder), {})
+			FileUtils.touch(File.expand_path(DGIT_SOURCES, dgit_folder))
+			Oj.to_file(File.expand_path(DGIT_JOURNAL, dgit_folder), {})
+			FileUtils.mkdir(File.expand_path('sources', folder))
+		end
+
+		def initialize(folder)
+			fail "Folder #{folder} is not a diggit folder." unless File.exist?(File.expand_path(DGIT_FOLDER, folder))
+			@plugin_loader = PluginLoader.instance
+			@folder = folder
+		end
+
+		def load_journal
+			url_array = []
+			IO.readlines(config_path(DGIT_SOURCES)).each { |l| url_array << l.strip }
+			saved_hash = Oj.load_file(config_path(DGIT_JOURNAL))
+			hash = { urls: url_array, sources: saved_hash[:sources], workspace: saved_hash[:workspace] }
+			@journal = Journal.new(hash)
+		end
+
+		def save_journal
+			hash = @journal.to_hash
+			File.open(config_path(DGIT_SOURCES), "w") { |f| hash[:urls].each { |u| f.puts(u) } }
+			Oj.to_file(config_path(DGIT_JOURNAL), { sources: hash[:sources], workspace: hash[:workspace] })
+		end
+
+		def load_options
+			@options = Oj.load_file(config_path(DGIT_OPTIONS))
+		end
+
+		def save_options
+			Oj.to_file(config_path(DGIT_OPTIONS), options)
+		end
+
+		def load_config
+			@config = Config.new(Oj.load_file(config_path(DGIT_CONFIG)))
+		end
+
+		def save_config
+			config_hash = @config.to_hash
+			Oj.to_file(config_path(DGIT_CONFIG), config_hash)
+		end
+
+		def clone(*source_ids)
+			@journal.sources_by_ids(*source_ids).select(&:new?).each(&:clone)
+		ensure
+			save_journal
+		end
+
+		def analyze(source_ids = [], analyses = [], mode = :run)
+			@journal.sources_by_ids(*source_ids).select(&:cloned?).each do |s|
+				@config.get_analyses(*analyses).each do |klass|
+					a = klass.new(@options)
+					s.load_repository
+					a.source = s
+					clean_analysis(s, a) if clean_mode?(mode) && s.analysis?(a.name)
+					run_analysis(s, a) if run_mode?(mode) && !s.analysis_performed?(a.name)
+				end
+			end
+		end
+
+		def clean_mode?(mode)
+			mode == :rerun || mode == :clean
+		end
+
+		def run_mode?(mode)
+			mode == :rerun || mode == :run
+		end
+
+		def clean_analysis(s, a)
+			a.clean
+			s.del_analysis(a.name)
+		ensure
+			save_journal
+		end
+
+		def run_analysis(s, a)
+			s.add_ongoing_analysis(a.name)
+			a.run
+			s.del_ongoing_analysis(a.name)
+			s.add_performed_analysis(a.name)
+		rescue => e
+			Log.error "Error applying analysis #{a.name} on #{s.url}"
+			s.error = Journal.dump_error(e)
+		ensure
+			save_journal
+		end
+
+		def join(source_ids = [], joins = [], mode = :run)
+			@config.get_joins(*joins).each do |klass|
+				j = klass.new(@options)
+				j.clean if clean_mode?(mode)
+				source_array = @journal.sources_by_ids(*source_ids)
+						.select { |s| s.cloned? && s.analyses_performed?(*klass.required_analyses) }
+				run_join(j, source_array) if run_mode?(mode) && !source_array.empty?
+			end
+		end
+
+		def run_join(j, source_array)
+			j.sources = source_array
+			j.run
+			@journal.add_join(j.name)
+		rescue => e
+			Log.error "Error applying join #{j.name}"
+			@journal.join_error = e
+		ensure
+			save_journal
+		end
+
+		def config_path(name)
+			File.expand_path(name, File.expand_path(DGIT_FOLDER, @folder))
+		end
+
+		def file_path(name)
+			File.expand_path(name, @folder)
+		end
+	end
+end