autoproj-stats 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bddd0ba07ea493877bd8a87c2e34bd1b413e976c
+  data.tar.gz: 5204ab40ecfe8b5fe447f61ea8ad0d3d16cfa846
+SHA512:
+  metadata.gz: 4402d0f9d750e8d0f8e7393773585551863648d3c54a13642b572baeacc21bce628f341d50979deec37e4300b69fadbdf8e1a08f37e6f98f2ba486385dec9ad0
+  data.tar.gz: b2ede3bcb23664ed294b431da28ef2f7b444c1309250ab0390e1939ac3cff4f678b9adc09f1c2414e350f23252e2b28357555cfd7f77ee711e988811411ebe58

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.2
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in autoproj-stats.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Sylvain Joyeux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+# Autoproj::Stats
+
+An Autoproj plugin to generate authorship/copyright statistics
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'autoproj-stats'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install autoproj-stats
+
+## Usage
+
+The plugin can be accessed from the autoproj command line with
+
+~~~
+autoproj help stats
+~~~
+
+In its simplest form, it's going to compute authorship information for all
+supported VCS types in the autoproj installation, and display summary
+information at the end.
+
+~~~
+autoproj stats
+~~~
+
+The statistics can be saved in a text file with --save
+
+~~~
+autoproj stats --save=stats.txt
+~~~
+
+The first thing you may want to improve is sanitizing author name(s). This is
+done by providing a YAML config file with an 'aliases' section. The method is
+usually to comb through the list of names in the summary generated by autoproj
+stats and replace dummy names by actual ones.
+
+~~~
+aliases:
+    Name As Reported By Autoproj Stats: Actual Name
+~~~
+
+This config file (let's call it stats.yml) can then be passed to autoproj stats
+with the --config option
+
+~~~
+autoproj stats --config=stats.yml
+~~~
+
+Then, you probably want to add affiliation information (a.k.a. copyright
+assignation). The simplest is to provide an institution/copyright holder name
+per author name:
+
+~~~
+affiliations:
+    Author Name: Company Name
+~~~
+
+However, some authors change affiliations over time, in which case you provide a
+list of company and start dates:
+
+~~~
+affiliations:
+    Author Name:
+    - [Company1, 2008-02-01]
+    - [Company2, 2012-03-01]
+~~~
+
+Finally, you can provide per-package license information. No way to give
+fine-grained stuff there.
+
+~~~
+licenses:
+    package_name: License Name
+~~~
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/doudou/autoproj-stats.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/autoproj-stats.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'autoproj/stats/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "autoproj-stats"
+  spec.version       = Autoproj::Stats::VERSION
+  spec.authors       = ["Sylvain Joyeux"]
+  spec.email         = ["sylvain.joyeux@m4x.org"]
+
+  spec.summary       = "authorship and copyright statistics for an autoproj workspace"
+  spec.description   =<<-EOD
+This autoproj plugin adds the 'stats' subcommand to autoproj, which allows to
+compute per-package and aggregated statistics information about authorship, for
+all packages within an autoproj workspace
+EOD
+  spec.homepage      = "https://github.com/doudou/autoproj-stats"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "autoproj", "~> 2.0"
+  spec.add_dependency "tty-table"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", '>= 5.0', '~> 5.0'
+end

data/lib/autoproj-stats.rb

@@ -0,0 +1,13 @@
+require 'autoproj/cli/stats'
+
+class Autoproj::CLI::Main
+    desc 'stats [PKG]', 'compute ownership information about the given package(s)'
+    option 'config', desc: 'configuration file that specifies name mappings and copyright info'
+    option 'parallel', desc: 'compute stats with that many parallel processes',
+        type: :numeric, default: 1
+    option 'save', desc: 'save in the provided file instead of outputting on STDOUT'
+    def stats(*packages)
+        run_autoproj_cli(:stats, :Stats, Hash[], *Array(packages))
+    end
+end
+

data/lib/autoproj/cli/stats.rb

@@ -0,0 +1,134 @@
+require 'autoproj'
+require 'autoproj/cli/inspection_tool'
+require 'autoproj/ops/stats'
+require 'concurrent'
+
+require 'tty-table'
+
+module Autoproj
+    module CLI
+        class Stats < InspectionTool
+            def run(user_selection, options = Hash.new)
+                initialize_and_load
+                source_packages, * =
+                    finalize_setup(user_selection,
+                                   ignore_non_imported_packages: true)
+                source_packages = source_packages.map { |pkg_name| ws.manifest.find_autobuild_package(pkg_name) }
+
+                config = YAML.load(File.read(options[:config]))
+                excluded = (config['exclude'] || Set.new).to_set
+                sanitizer = Autoproj::Stats::Sanitizer.new
+                if options[:config]
+                    sanitizer.load(options[:config])
+                end
+
+                total_sloc = 0
+                overall_per_author = Hash.new
+                overall_per_copyright = Hash.new
+                overall_copyright_per_license = Hash.new
+
+                ops = Ops::Stats.new(sanitizer: sanitizer)
+                stats = ops.process(source_packages, parallel: options[:parallel])
+                per_package_table = TTY::Table.new
+                stats.each do |pkg, pkg_stats|
+                    next if excluded.include?(pkg.name)
+                    row_count = [1, pkg_stats.authors.size, pkg_stats.copyright.size].max
+
+                    # Compute per-package info
+                    sloc = pkg_stats.sloc
+                    author_info    = line_count_summary(pkg_stats.authors, sloc)
+                    copyright_info = line_count_summary(pkg_stats.copyright, sloc)
+                    license = sanitizer.license_of(pkg)
+
+                    # Compute aggregated info
+                    total_sloc += sloc
+                    overall_per_author.merge!(pkg_stats.authors) { |_, v1, v2| v1 + v2 }
+                    overall_per_copyright.merge!(pkg_stats.copyright) { |_, v1, v2| v1 + v2 }
+                    copyrights_for_this_license =
+                        (overall_copyright_per_license[license || "Unknown"] ||= Hash.new)
+                    copyrights_for_this_license.merge!(pkg_stats.copyright) { |_, v1, v2| v1 + v2 }
+
+                    Array.new(row_count).zip([pkg.name], [sloc], [license], *author_info, *copyright_info) do |line|
+                        per_package_table << line[1..-1].map { |v| v || '' }
+                    end
+                end
+
+                no_license = stats.find_all do |pkg, pkg_stats|
+                    !sanitizer.license_of(pkg)
+                end
+
+                copyright_per_license_table = TTY::Table.new
+                overall_copyright_per_license.sort_by(&:first).each do |license_name, copyrights|
+                    first_col = Array.new(copyrights.size)
+                    first_col[0] = license_name
+                    copyright_info = line_count_summary(copyrights, total_sloc)
+                    first_col.zip(*copyright_info) do |line|
+                        copyright_per_license_table << line.map { |v| v || "" }
+                    end
+                end
+
+                no_stats = (source_packages.to_set - stats.keys.to_set)
+                puts "could not compute stats for #{no_stats.size} packages: #{no_stats.map(&:name).sort.join(", ")}"
+                puts "#{no_license.size} packages without known license: #{no_license.map { |pkg, _| pkg.name }.sort.join(", ")}"
+                puts "#{stats.size} Packages"
+                puts "#{total_sloc} SLOC counted"
+                puts "#{overall_per_author.size} unique authors"
+
+                begin
+                    io = if path = options[:save]
+                             File.open(path, 'w')
+                         else
+                             STDOUT
+                         end
+
+                    io.puts "== Overall stats per author (sorted by contribution)"
+                    author_names, *author_info = line_count_summary(overall_per_author, total_sloc)
+                    io.puts TTY::Table.new(author_names.zip(*author_info)).render(:ascii)
+                    io.puts
+                    io.puts "== Overall stats per author (sorted alphabetically)"
+                    author_names, *author_info = line_count_summary(overall_per_author, total_sloc, &:first)
+                    io.puts TTY::Table.new(author_names.zip(*author_info)).render(:ascii)
+                    io.puts
+                    io.puts "== Overall stats per copyright"
+                    copyright_names, *copyright_info = line_count_summary(overall_per_copyright, total_sloc)
+                    io.puts TTY::Table.new(copyright_names.zip(*copyright_info)).render(:ascii)
+                    io.puts
+                    io.puts "== Breakdown of copyright per license"
+                    io.puts copyright_per_license_table.render(:ascii)
+                    io.puts
+                    io.puts "== Per-package stats"
+                    io.puts per_package_table.render(:ascii)
+                ensure
+                    if io && io != STDOUT
+                        io.close
+                    end
+                end
+            end
+
+            def line_count_summary(count_per_category, sloc, reverse: true, &block)
+                ordered =
+                    if block_given?
+                        count_per_category.sort_by(&block)
+                    else
+                        count_per_category.sort_by { |_, count| count }
+                    end
+                ordered = ordered.reverse if reverse
+                names  = ordered.map(&:first)
+                counts = ordered.map(&:last)
+                ratios = count_to_ratios(counts, sloc)
+                return names, counts, ratios
+            end
+
+            def count_to_ratios(counts, total)
+                counts.map do |v|
+                    ratio = (Float(v) / total)
+                    if ratio < 0.01
+                        "< 1%"
+                    else
+                        "#{Integer(ratio * 100)}%"
+                    end
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/ops/stats.rb

@@ -0,0 +1,55 @@
+require 'autoproj/stats'
+
+module Autoproj
+    module Ops
+        # Computes per-package statistics
+        class Stats
+            attr_reader :sanitizer
+
+            def initialize(sanitizer: Autoproj::Stats::Sanitizer.new)
+                @sanitizer = sanitizer
+            end
+
+            def process(packages, parallel: 1)
+                executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
+                result = Hash.new
+
+                futures = packages.map do |pkg|
+                    [pkg, Concurrent::Future.execute(executor: executor) { compute_package_stats(pkg) }]
+                end
+                futures.inject(Hash.new) do |h, (pkg, future)|
+                    if stats = future.value
+                        h[pkg] = stats
+                    elsif future.reason.kind_of?(Exception)
+                        raise future.reason
+                    else
+                        pkg.error "%s: failed, #{future.reason}"
+                    end
+                    h
+                end
+
+            ensure
+                executor.shutdown
+                executor.wait_for_termination
+            end
+
+            def compute_package_stats(pkg)
+                if pkg.importer
+                    if stats_generator = find_generator_for_importer(pkg.importer)
+                        stats_generator.(pkg)
+                    else
+                        Autoproj.warn "no stats generator for #{pkg.name} (#{pkg.importer.class})"
+                    end
+                else
+                    Autoproj.warn "no importer for #{pkg.name}"
+                end
+            end
+
+            def find_generator_for_importer(importer)
+                if importer.class == Autobuild::Git
+                    return Autoproj::Stats::Git.new(sanitizer: sanitizer)
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/stats.rb

@@ -0,0 +1,5 @@
+require 'autoproj/stats/version'
+require 'autoproj/stats/sanitizer'
+require 'autoproj/stats/sloc_counter'
+require 'autoproj/stats/package_stats'
+require 'autoproj/stats/git'

data/lib/autoproj/stats/git.rb

@@ -0,0 +1,62 @@
+module Autoproj
+    module Stats
+        # Statistic generator for git-based packages
+        class Git
+            attr_reader :sloc_counter
+            attr_reader :sanitizer
+            def initialize(sloc_counter: SLOCCounter.new, sanitizer: Sanitizer.new)
+                @sloc_counter = sloc_counter
+                @sanitizer = sanitizer
+            end
+
+            def call(pkg)
+                pkg.progress_start "processing %s", done_message: "processed %s" do
+                    importer = pkg.importer
+                    entries = importer.run_git(pkg, 'ls-tree', '-r', 'HEAD', encoding: 'UTF-8')
+
+                    progress_counter = 0
+                    entries.inject(PackageStats.new) do |aggregate, line|
+                        mode, type, sha, path = line.split(/\s+/)
+                        if file_counter = sloc_counter.find_counter_for_path(path)
+                            aggregate += process_file(pkg, path, sloc: file_counter)
+                        else
+                            pkg.warn "%s: no SLOC count counter available for #{path}"
+                        end
+                        progress_counter += 1
+                        pkg.progress "processing %s (#{progress_counter}/#{entries.size})"
+                        aggregate
+                    end
+                end
+            end
+
+            def process_file(pkg, path, sloc: proc { true })
+                blamed = pkg.importer.run_git(pkg, 'blame', '-w', '-C', '-C', '-C', '-M', '--minimal', 'HEAD', '--', path, encoding: 'UTF-8')
+
+                authors = Hash.new(0)
+                copyrights = Hash.new(0)
+
+                line_matcher = Regexp.new(/^\^?[0-9a-f]+.*\((.*) (\d{4})-(\d{2})-(\d{2}).*\d+\) (.*)$/)
+                blamed.each do |line|
+                    line = line.encode('UTF-8', invalid: :replace, undef: :replace)
+                    if m = line_matcher.match(line)
+                        _, name, y, m, d, code = *m
+                        code = code.strip
+                        next if code.empty? || !sloc.(code)
+
+                        name = sanitizer.sanitize_author_name(name.strip)
+                        copyright = sanitizer.compute_copyright_of(
+                            name, Date.new(y.to_i, m.to_i, d.to_i))
+                        authors[name] += 1
+                        copyrights[copyright] += 1
+                    else
+                        raise "cannot match #{line}"
+                    end
+                end
+
+                sloc = authors.values.inject(0, &:+)
+                PackageStats.new(sloc, authors, copyrights)
+            end
+        end
+    end
+end
+

data/lib/autoproj/stats/package_stats.rb

@@ -0,0 +1,20 @@
+module Autoproj
+    module Stats
+        class PackageStats
+            attr_reader :sloc
+            attr_reader :authors
+            attr_reader :copyright
+            def initialize(sloc = 0, authors = Hash.new, copyright = Hash.new)
+                @sloc, @authors, @copyright = sloc, authors, copyright
+            end
+
+            def +(other)
+                PackageStats.new(
+                    sloc + other.sloc,
+                    authors.merge(other.authors) { |_, v1, v2| v1 + v2 },
+                    copyright.merge(other.copyright) { |_, v1, v2| v1 + v2 })
+            end
+        end
+    end
+end
+

data/lib/autoproj/stats/sanitizer.rb

@@ -0,0 +1,59 @@
+module Autoproj
+    module Stats
+        # Class passed to the stats generators to sanitize names and compute
+        # copyright
+        class Sanitizer
+            attr_reader :aliases
+            attr_reader :simple_copyrights
+            attr_reader :timeline_affiliations
+            attr_reader :licenses
+
+            def initialize(aliases: Hash.new)
+                @simple_copyrights = Hash.new
+                @timeline_affiliations = Hash.new
+                @aliases = aliases
+            end
+
+            def load(path)
+                config = YAML.load(File.read(path))
+                @aliases = config['aliases']
+                @licenses = config['licenses'] || Hash.new
+                if affiliations = config['affiliations']
+                    affiliations.each do |name, entry|
+                        if entry.respond_to?(:to_str)
+                            simple_copyrights[name] = entry
+                        else
+                            timeline_affiliations[name] = entry.sort_by(&:last)
+                        end
+                    end
+                end
+            end
+
+            def license_of(pkg)
+                licenses[pkg.name]
+            end
+
+            def sanitize_author_name(name)
+                aliases[name] || name
+            end
+
+            def compute_copyright_of(author_name, date)
+                if affiliation = simple_copyrights[author_name]
+                    affiliation
+                elsif timeline = timeline_affiliations[author_name]
+                    timeline.inject("Unknown (#{author_name})") do |aff, (new_aff, start_date)|
+                        if date < start_date
+                            return aff
+                        else
+                            new_aff
+                        end
+                    end
+                else
+                    "Unknown (#{author_name})"
+                end
+            end
+        end
+    end
+end
+
+

data/lib/autoproj/stats/sloc_counter.rb

@@ -0,0 +1,21 @@
+module Autoproj
+    module Stats
+        class SLOCCounter
+            def find_counter_for_path(path)
+                case path
+                when /\.rb$/, /\.orogen$/
+                    lambda { |line| line !~ /^(?:#|end)$/ }
+                when /\.py$/
+                    lambda { |line| line !~ /^#/ }
+                when /\.(?:[ch]pp|cc?|hh?)$/
+                    lambda { |line| line !~ /^(?:[{}]+|\/\/)$/ }
+                when /CMakeLists.txt/, /\.cmake$/
+                    lambda { |line| line !~ /^#/ }
+                when /\.sdf$/
+                    lambda { |line| true }
+                end
+            end
+        end
+    end
+end
+

data/lib/autoproj/stats/version.rb

@@ -0,0 +1,5 @@
+module Autoproj
+    module Stats
+        VERSION = "0.1.0"
+    end
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: autoproj-stats
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sylvain Joyeux
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: autoproj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: tty-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: |
+  This autoproj plugin adds the 'stats' subcommand to autoproj, which allows to
+  compute per-package and aggregated statistics information about authorship, for
+  all packages within an autoproj workspace
+email:
+- sylvain.joyeux@m4x.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- autoproj-stats.gemspec
+- lib/autoproj-stats.rb
+- lib/autoproj/cli/stats.rb
+- lib/autoproj/ops/stats.rb
+- lib/autoproj/stats.rb
+- lib/autoproj/stats/git.rb
+- lib/autoproj/stats/package_stats.rb
+- lib/autoproj/stats/sanitizer.rb
+- lib/autoproj/stats/sloc_counter.rb
+- lib/autoproj/stats/version.rb
+homepage: https://github.com/doudou/autoproj-stats
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.3
+signing_key: 
+specification_version: 4
+summary: authorship and copyright statistics for an autoproj workspace
+test_files: []
+has_rdoc: