repo_miner 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bc0a4630da25d7d089012de9d1e954f4c8b3521
-  data.tar.gz: 87d1fa6593047e0a52e0dd460d8c32caafc51aee
+  metadata.gz: f8e005fe499b1896489c3f607ac01c8865dd5bba
+  data.tar.gz: daa6c769d3dd9112c1e4266e2d3f32583292d0ea
 SHA512:
-  metadata.gz: dd822f039d5b4e025e03e61e3fc08158b4ec8d93ee3e088a3c6c2a726d430d989661262966d65f3169ee62c24596cb70d7611ab91c85fa14f454cc5931514b57
-  data.tar.gz: 945d1f2af6d73d6db682206062ef049a599be14876ba74515767967066b38afa6d5c449591194c768f7d49ad3463cadb57ced6c41de526a3e889bd37b38b9f2c
+  metadata.gz: b9fb75bda604333a55de49e7bd1e92d8bc728b8ab0b73139a7ea01bf76e289444d0ed223948c2f78890d93f9fc9a06905664518d550e834609f84f56fc5e23f4
+  data.tar.gz: b85aabbad7cbb291cee1b4d56c57b68ed69f874ff7b5bdd7fd2daef6375b3201bd84cc99905796281aae394c381f1cbf6cd33d3145749eb0a92a29f132f97ff9

data/README.md

@@ -29,7 +29,23 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Make a new repository:
+
+```ruby
+repository = RepoMiner::Repository.new('/path/to/git/repo')
+```
+
+Analyse all commits for a given branch:
+
+```ruby
+commits = repository.analyse('master')
+```
+
+See mined dependency data for a given commit:
+
+```ruby
+commits.last #=> RepoMiner::Commit:0x007fd87fdf1150(message: "Fixes 1597", sha: c656e48ada19c6c83f7705893f0a73cfc1844abf, data: {:email=>{:committer=>"andrewnez@gmail.com", :author=>"andrewnez@gmail.com"}, :dependencies=>{:added_manifests=>[], :modified_manifests=>[{:path=>"Gemfile", :platform=>"rubygems", :added_dependencies=>[], :modified_dependencies=>[], :removed_dependencies=>[{:name=>"sass", :requirement=>"= 3.4.24", :type=>:runtime}]}, {:path=>"Gemfile.lock", :platform=>"rubygems", :added_dependencies=>[{:name=>"sass-listen", :requirement=>"4.0.0", :type=>"runtime"}], :modified_dependencies=>[{:name=>"commonmarker", :requirement=>"0.16.8", :type=>"runtime", :previous_requirement=>"0.16.7"}, {:name=>"gitlab", :requirement=>"4.2.0", :type=>"runtime", :previous_requirement=>"4.1.0"}, {:name=>"rack-cors", :requirement=>"1.0.0", :type=>"runtime", :previous_requirement=>"0.4.1"}, {:name=>"sass", :requirement=>"3.5.1", :type=>"runtime", :previous_requirement=>"3.4.24"}, {:name=>"sassc", :requirement=>"1.11.4", :type=>"runtime", :previous_requirement=>"1.11.2"}], :removed_dependencies=>[]}], :removed_manifests=>[]}})
+```
 
 ## Development
 

data/lib/repo_miner/commit.rb

@@ -1,18 +1,26 @@
 module RepoMiner
   class Commit
     attr_reader :repository
-    attr_reader :commit
+    attr_reader :rugged_commit
     attr_accessor :data
 
-    def initialize(repository, commit)
+    def initialize(repository, rugged_commit)
       @repository = repository
-      @commit = commit
+      @rugged_commit = rugged_commit
       @data = {}
     end
 
+    def message
+      rugged_commit.message.strip
+    end
+
+    def sha
+      rugged_commit.oid
+    end
+
     def analyse
-      # for every miner (except Base) analyse commit
       Miners::Email.new.analyse(self)
+      Miners::Dependencies.new.analyse(self)
 
       self
     end
@@ -20,5 +28,25 @@ module RepoMiner
     def add_data(key, miner_data)
       data[key] = miner_data
     end
+
+    def content_before(file_path)
+      content_for_commit(rugged_commit.parents[0], file_path)
+    end
+
+    def content_after(file_path)
+      content_for_commit(rugged_commit, file_path)
+    end
+
+    def inspect
+      "RepoMiner::Commit:#{"0x00%x" % (object_id << 1)}(message: #{message}, sha: #{sha}, data: #{data})"
+    end
+
+    private
+
+    def content_for_commit(rugged_commit, file_path)
+      path = rugged_commit.tree.path(file_path)
+      blob = repository.rugged_repository.lookup(path[:oid])
+      blob.content
+    end
   end
 end

data/lib/repo_miner/miners/dependencies.rb

@@ -0,0 +1,182 @@
+require 'bibliothecary'
+
+module RepoMiner
+  module Miners
+    class Dependencies
+      def analyse(commit)
+        all_paths = blob_paths(commit.rugged_commit)
+
+        added_paths = all_paths.select{|path| path[:status] == :added }.map{|path| path[:path] }
+        modified_paths = all_paths.select{|path| path[:status] == :modified }.map{|path| path[:path] }
+        removed_paths = all_paths.select{|path| path[:status] == :deleted }.map{|path| path[:path] }
+
+        added_manifest_paths = Bibliothecary.identify_manifests(added_paths)
+        modified_manifest_paths = Bibliothecary.identify_manifests(modified_paths)
+        removed_manifest_paths = Bibliothecary.identify_manifests(removed_paths)
+
+        # don't both analysing commits where no dependency files touched
+        return nil if added_manifest_paths.empty? && modified_manifest_paths.empty? && removed_manifest_paths.empty?
+
+        # Added manifest files
+        added_manifests = []
+        added_manifest_paths.each do |manifest_path|
+          manifest = Bibliothecary.analyse_file(manifest_path, commit.content_after(manifest_path))
+
+          new_manifest = manifest.first
+
+          if new_manifest
+            dependencies = new_manifest[:dependencies]
+            added = dependencies.map{|d| d[:name] }
+
+            added.map! do |dep_name|
+              dep = dependencies.find{|d| d[:name] == dep_name }
+              {
+                name: dep_name,
+                requirement: dep[:requirement],
+                type: dep[:type]
+              }
+            end
+
+            added_manifests << {
+              path: manifest_path,
+              platform: manifest[0][:platform],
+              added_dependencies: added,
+              modified_dependencies: [],
+              removed_dependencies: []
+            }
+          end
+        end
+
+        # Modified manifest files
+        modified_manifests = []
+        modified_manifest_paths.each do |manifest_path|
+          before_manifest = Bibliothecary.analyse_file(manifest_path, commit.content_before(manifest_path))
+          before_modified_manifest = before_manifest.first
+
+          after_manifest = Bibliothecary.analyse_file(manifest_path, commit.content_after(manifest_path))
+          after_modified_manifest = after_manifest.first
+
+          if before_modified_manifest && after_modified_manifest
+            before_dependencies = before_modified_manifest[:dependencies]
+            after_dependencies = after_modified_manifest[:dependencies]
+
+            added_dependency_names = after_dependencies.map{|d| d[:name] } - before_dependencies.map{|d| d[:name] }
+            removed_dependency_names = before_dependencies.map{|d| d[:name] } - after_dependencies.map{|d| d[:name] }
+
+            potentially_modified_dependency_names = after_dependencies.map{|d| d[:name] } - added_dependency_names - removed_dependency_names
+            modified_dependency_names = potentially_modified_dependency_names.select do |name|
+              after = after_dependencies.find{|d| d[:name] == name }
+              before = before_dependencies.find{|d| d[:name] == name }
+              (after[:requirement] != before[:requirement]) || (after[:type] != before[:type])
+            end
+
+            # added_dependencies
+            added_dependencies = added_dependency_names.map do |dep_name|
+              dep = after_dependencies.find{|d| d[:name] == dep_name }
+              {
+                name: dep_name,
+                requirement: dep[:requirement],
+                type: dep[:type]
+              }
+            end
+
+            # modified_dependencies
+            modified_dependencies = modified_dependency_names.map do |dep_name|
+              after = after_dependencies.find{|d| d[:name] == dep_name }
+              before = before_dependencies.find{|d| d[:name] == dep_name }
+              dep_hash = {
+                name: dep_name,
+                requirement: after[:requirement],
+                type: after[:type]
+              }
+              dep_hash[:previous_requirement] = before[:requirement] if after[:requirement] != before[:requirement]
+              dep_hash[:previous_type] = before[:type] if after[:type] != before[:type]
+              dep_hash
+            end
+
+            # removed_dependencies
+            removed_dependencies = removed_dependency_names.map do |dep_name|
+              dep = before_dependencies.find{|d| d[:name] == dep_name }
+              {
+                name: dep_name,
+                requirement: dep[:requirement],
+                type: dep[:type]
+              }
+            end
+
+            modified_manifests << {
+              path: manifest_path,
+              platform: after_modified_manifest[:platform],
+              added_dependencies: added_dependencies,
+              modified_dependencies: modified_dependencies,
+              removed_dependencies: removed_dependencies
+            }
+          end
+        end
+
+        # Removed manifest files
+        removed_manifests = []
+        removed_manifest_paths.each do |manifest_path|
+          manifest = Bibliothecary.analyse_file(manifest_path, commit.content_before(manifest_path))
+
+          removed_manifest = manifest.first
+
+          if removed_manifest
+            dependencies = removed_manifest[:dependencies]
+            removed = dependencies.map{|d| d[:name] }
+
+            removed.map! do |dep_name|
+              dep = dependencies.find{|d| d[:name] == dep_name }
+              {
+                name: dep_name,
+                requirement: dep[:requirement],
+                type: dep[:type]
+              }
+            end
+
+            removed_manifests << {
+              path: manifest_path,
+              platform: manifest[0][:platform],
+              added_dependencies: [],
+              modified_dependencies: [],
+              removed_dependencies: removed
+            }
+          end
+        end
+
+        data = {
+          added_manifests: added_manifests,
+          modified_manifests: modified_manifests,
+          removed_manifests: removed_manifests
+        }
+
+        commit.add_data(:dependencies, data)
+      end
+
+      private
+
+      def blob_paths(commit)
+        paths = []
+
+        if commit.parents.count == 0 # initial commit
+          commit.tree.walk_blobs(:postorder) do |root, entry|
+            paths << {
+              status: :added,
+              path: "#{root}#{entry[:name]}"
+            }
+          end
+        else
+          diffs = commit.parents[0].diff(commit)
+
+          diffs.each_delta do |delta|
+            paths << {
+              status: delta.status,
+              path: delta.new_file[:path]
+            }
+          end
+        end
+        paths
+      end
+    end
+  end
+end

data/lib/repo_miner/miners/email.rb

@@ -3,17 +3,14 @@ module RepoMiner
     class Email
       def analyse(commit)
         # analyse commit
-        committer_email = commit.commit.committer[:email]
-        author_email = commit.commit.author[:email]
+        committer_email = commit.rugged_commit.committer[:email]
+        author_email = commit.rugged_commit.author[:email]
 
         # attach mined info to commit
         commit.add_data(:email, {
           committer: committer_email,
           author: author_email
         })
-
-        # return the commit
-        commit
       end
     end
   end

data/lib/repo_miner/repository.rb

@@ -7,14 +7,14 @@ module RepoMiner
       @repo_path = repo_path
     end
 
-    def repository
+    def rugged_repository
       @repository ||= Rugged::Repository.new(repo_path)
     end
 
     def walk(branch)
-      @walker = Rugged::Walker.new(repository)
+      @walker = Rugged::Walker.new(rugged_repository)
       @walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
-      @walker.push(repository.branches[branch].target_id)
+      @walker.push(rugged_repository.branches[branch].target_id)
       @walker
     end
 

data/lib/repo_miner/version.rb

@@ -1,3 +1,3 @@
 module RepoMiner
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/repo_miner.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = "Mine repositories for interesting changes over time"
   spec.homepage      = "https://github.com/librariesio/repo_miner"
-  spec.license       = "APGL-3.0"
+  spec.license       = "AGPL-3.0"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rugged"
+  spec.add_dependency "bibliothecary"
 
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: repo_miner
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-13 00:00:00.000000000 Z
+date: 2017-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rugged
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bibliothecary
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -86,13 +100,14 @@ files:
 - bin/setup
 - lib/repo_miner.rb
 - lib/repo_miner/commit.rb
+- lib/repo_miner/miners/dependencies.rb
 - lib/repo_miner/miners/email.rb
 - lib/repo_miner/repository.rb
 - lib/repo_miner/version.rb
 - repo_miner.gemspec
 homepage: https://github.com/librariesio/repo_miner
 licenses:
-- APGL-3.0
+- AGPL-3.0
 metadata: {}
 post_install_message: 
 rdoc_options: []