usedby 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 51866979c0e578e1c2c4bda5c7f61e79cd9b6bb0a5366eaaf2c66f474b00c6ea
+  data.tar.gz: 54e3aa2e5a415b570ead51e7c004b5a37683f76360c8d084874f0b05812281c7
+SHA512:
+  metadata.gz: da372934168ed4cf0fb96e648c7735631c0068652516fa08142a8666f20be2f57f9a0b35861aec240b6a2b2ee29cc35b5387b3fa9c4a9f152ac2375beadb7b04
+  data.tar.gz: 5ad1483db112e0b5cbf819f095de455dae8f90105c9a93fc85fcaf4d2c5bec296f9a9291519274ed03c54eee80e9dc878575961b638bb4a01288fe012442d8d7

data/CHANGES.md

@@ -0,0 +1,21 @@
+# Change Log
+
+## 0.3.0 (2018/07/20)
+
+**Added**
+
+* Don't include archived repositories.
+
+
+## 0.2.0 (2018/07/03)
+
+**Added**
+
+* Require `GITHUB_ORGANIZATION` to be provided on the command line.
+
+## 0.1.0 (2018/07/03)
+
+**Added**
+
+* Provide initial version of the `organization_gem_dependencies` command line
+  tool.

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2018, AppFolio, Inc.
+Copyright (c) 2018, Bryce Boe
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,99 @@
+# usedby
+
+Figure out where your gems are actually being used!
+Similar to GitHub's "Used By" feature, but for private repos.
+
+This gem installs a command line utility `usedby`, that
+outputs a json file with a reverse dependency tree.
+
+This acts more or less like GitHub's "Used By" feature.
+It currently uses GitHub's code search API which has a few limitations:
+https://help.github.com/en/github/searching-for-information-on-github/searching-code
+
+## Installation
+
+```sh
+gem install usedby
+```
+
+## Usage
+
+```sh
+usedby GITHUB_ORGANIZATION [--direct] [--gems GEM1,GEM2,GEM3]
+```
+
+You will be securely prompted for a [GitHub Personal Access Token](https://github.com/settings/tokens).
+
+For example, running `usedby rails --direct --gems railties,rake` produces output
+like the following:
+
+```json
+{
+  "railties": {
+    "4.0.0.beta": [
+      "routing_concerns/Gemfile.lock"
+    ],
+    "4.0.0": [
+      "prototype-rails/Gemfile.lock"
+    ],
+    "4.2.1": [
+      "rails-perftest/Gemfile.lock"
+    ],
+    "4.2.10": [
+      "rails-docs-server/test/fixtures/releases/v4.2.10/Gemfile.lock"
+    ],
+    "5.1.1": [
+      "actioncable-examples/Gemfile.lock"
+    ],
+    "5.2.1": [
+      "rails_fast_attributes/Gemfile.lock"
+    ],
+    "5.2.2": [
+      "globalid/Gemfile.lock"
+    ],
+    "6.0.1": [
+      "webpacker/Gemfile.lock"
+    ],
+    "6.0.2.1": [
+      "rails-contributors/Gemfile.lock"
+    ],
+    "6.1.0.alpha": [
+      "rails/Gemfile.lock"
+    ]
+  },
+  "rake": {
+    "0.9.2.2": [
+      "commands/Gemfile.lock",
+      "etagger/Gemfile.lock",
+      "routing_concerns/Gemfile.lock"
+    ],
+    "10.1.0": [
+      "prototype-rails/Gemfile.lock"
+    ],
+    "10.4.2": [
+      "jquery-ujs/Gemfile.lock",
+      "rails-perftest/Gemfile.lock"
+    ],
+    "10.5.0": [
+      "rails_fast_attributes/Gemfile.lock",
+      "record_tag_helper/Gemfile.lock"
+    ],
+    "12.0.0": [
+      "actioncable-examples/Gemfile.lock",
+      "rails-docs-server/test/fixtures/releases/v4.2.10/Gemfile.lock",
+      "rails-dom-testing/Gemfile.lock"
+    ],
+    "12.3.2": [
+      "globalid/Gemfile.lock"
+    ],
+    "13.0.0": [
+      "webpacker/Gemfile.lock"
+    ],
+    "13.0.1": [
+      "rails-contributors/Gemfile.lock",
+      "rails/Gemfile.lock"
+    ]
+  }
+}
+
+```

data/bin/usedby

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'usedby'
+exit Usedby::Cli.new.run

data/lib/usedby.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'usedby/cli'
+require 'usedby/version'

data/lib/usedby/cli.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'io/console'
+require 'json'
+require 'optparse'
+
+require 'octokit'
+
+require 'usedby/version_ranges_intersection'
+
+module Usedby
+  # Define the command line interface.
+  class Cli
+    using VersionRangesIntersection
+
+    GEMFILE_LOCK_SEARCH_TERM = 'org:%s filename:Gemfile.lock'
+    GEMSPEC_SEARCH_TERM = 'org:%s extension:gemspec'
+    USAGE = <<~USAGE
+      Usage: usedby [options] GITHUB_ORGANIZATION
+    USAGE
+
+
+    def run
+      parse_options
+      if ARGV.size != 1
+        STDERR.puts USAGE
+        return 1
+      end
+      github_organization = ARGV[0]
+
+      access_token = ENV['GITHUB_ACCESS_TOKEN'] || \
+                     STDIN.getpass('GitHub Personal Access Token: ')
+      github = Octokit::Client.new(access_token: access_token)
+
+      gems = {}
+
+      remote_search(github, github_organization, GEMFILE_LOCK_SEARCH_TERM) do |gemfile_lock|
+        content = remote_file(github, gemfile_lock)
+        next unless content
+        content = Bundler::LockfileParser.new(content)
+        merge!(gems, process_gemfile(content, "#{gemfile_lock.repository.name}/#{gemfile_lock.path}"))
+      end
+
+      remote_search(github, github_organization, GEMSPEC_SEARCH_TERM) do |gemspec|
+        content = remote_file(github, gemspec)
+        next unless content
+        merge!(gems, process_gemspec(content, "#{gemspec.repository.name}/#{gemspec.path}"))
+      end
+
+      output gems
+
+      0
+    end
+
+    private
+
+    def archived_repositories(github, organization)
+      github.organization_repositories(organization)
+      last_response = github.last_response
+
+      repositories = []
+      last_response.data.each do |repository|
+        repositories << repository.name if repository.archived
+      end
+      until last_response.rels[:next].nil?
+        sleep_time = 0
+        begin
+          last_response = last_response.rels[:next].get
+        rescue StandardError
+          sleep_time += 1
+          STDERR.puts "Sleeping #{sleep_time} seconds"
+          sleep(sleep_time)
+          retry
+        end
+        last_response.data.each do |repository|
+          repositories << repository.name if repository.archived
+        end
+      end
+      repositories
+    end
+
+    def build_ignore_paths(ignored_paths, file)
+      File.open(file).each do |line|
+        cleaned = line.strip
+        ignored_paths << cleaned if cleaned != ''
+      end
+    rescue Errno::ENOENT, Errno::EISDIR
+      STDERR.puts "No such file #{file}"
+      exit 1
+    end
+
+    def filtered?(gemfile_path)
+      @options[:ignore_paths].each do |ignore_path|
+        return true if gemfile_path.start_with?(ignore_path)
+      end
+      false
+    end
+
+    def remote_search(github, organization, search_term)
+      archived = archived_repositories(github, organization)
+      github.search_code(search_term % organization, per_page: 1000)
+      last_response = github.last_response
+
+      matches = []
+      last_response.data.items.each do |match|
+        matches << match unless archived.include? match.repository.name
+      end
+      until last_response.rels[:next].nil?
+        sleep_time = 0
+        begin
+          last_response = last_response.rels[:next].get
+        rescue StandardError
+          sleep_time += 1
+          STDERR.puts "Sleeping #{sleep_time} seconds"
+          sleep(sleep_time)
+          retry
+        end
+        last_response.data.items.each do |match|
+          matches << match unless archived.include? match.repository.name
+        end
+      end
+
+      matches.sort_by(&:html_url).each do |match|
+        yield match
+      end
+    end
+
+    def remote_file(github, file)
+      github_path = "#{file.repository.name}/#{file.path}"
+      if filtered?(github_path)
+        STDERR.puts "Skipping #{github_path}"
+        return
+      end
+      STDERR.puts "Processing #{github_path}"
+      sleep_time = 0
+      begin
+        content = Base64.decode64(github.get(file.url).content)
+      rescue StandardError
+        sleep_time += 1
+        STDERR.puts "Sleeping #{sleep_time} seconds"
+        sleep(sleep_time)
+        retry
+      end
+      content
+    end
+
+    def merge!(base, additions)
+      additions.each do |gem, versions|
+        if base.include? gem
+          base_versions = base[gem]
+          versions.each do |version, projects|
+            if base_versions.include? version
+              base_versions[version].concat(projects)
+            else
+              base_versions[version] = projects
+            end
+          end
+        else
+          base[gem] = versions
+        end
+      end
+    end
+
+    def output(gems)
+      sorted_gems = {}
+      gems.sort.each do |gem, versions|
+        sorted_gems[gem] = {}
+        versions.sort.each do |version, projects|
+          sorted_gems[gem][version_ranges_to_s(version)] = projects.sort
+        end
+      end
+      puts JSON.pretty_generate(sorted_gems)
+    end
+
+    def parse_options
+      @options = { direct: false, ignore_paths: [] }
+      OptionParser.new do |config|
+        config.banner = USAGE
+        config.on('-d', '--direct',
+                  'Consider only direct dependencies.') do |direct|
+          @options[:direct] = direct
+        end
+        config.on('-i', '--ignore-file [FILEPATH]',
+                  'Ignore projects included in file.') do |ignore_file|
+
+          build_ignore_paths(@options[:ignore_paths], ignore_file)
+        end
+        config.on('-g', '--gems [GEM1,GEM2,GEM3]',
+                  'Consider only given gems.') do |gems|
+
+          @options[:gems] = gems.split(',')
+        end
+        config.version = Usedby::VERSION
+      end.parse!
+    end
+
+    def process_gemfile(gemfile, project)
+      dependencies = gemfile.dependencies.map { |dependency, _, _| dependency }
+      gems = {}
+
+      gemfile.specs.each do |spec|
+        next if @options[:direct] && !dependencies.include?(spec.name)
+        next if @options[:gems] && !@options[:gems].include?(spec.name)
+        spec_version_ranges = Bundler::VersionRanges.for(Gem::Requirement.new(spec.version))
+        gems[spec.name] = {}
+        gems[spec.name][spec_version_ranges] = [project]
+      end
+      gems
+    end
+
+    # Process dependencies in gemspec according to:
+    # https://guides.rubygems.org/specification-reference/
+    # Sample supported formats:
+    #      s.add_dependency(%q<rspec>.freeze, ["~> 3.2"])
+    #   spec.add_runtime_dependency "multi_json", "~>1.12", ">=1.12.0"
+    # Sample unsupported formats:
+    #      s.add_development_dependency "rake", "~> 10.5" if on_less_than_1_9_3?
+    #      s.add_dependency 'sunspot', Sunspot::VERSION
+    def process_gemspec(content, project)
+      gems = {}
+      dummy_spec = Gem::Specification.new
+      content.each_line do |line|
+        if line =~ /^\s*(\w+)\.add_(development_dependency|runtime_dependency|dependency)\b/
+          spec_name = $1
+          begin
+            eval line.sub(spec_name, 'dummy_spec')
+          rescue => e
+            $stderr.puts e
+            next
+          end
+          dep = dummy_spec.dependencies.last
+          gem_name = dep.name
+          next if @options[:gems] && !@options[:gems].include?(gem_name)
+          gem_version_ranges = Bundler::VersionRanges.for(dep.requirement)
+          gems[gem_name] = {}
+          gems[gem_name][gem_version_ranges] = [project]
+        end
+      end
+      gems
+    end
+
+    # Uses mathematical notation for ranges
+    # The unbounded range is [0, ∞)
+    def version_ranges_to_s(gem_version_ranges)
+      if !gem_version_ranges.kind_of?(Array) || gem_version_ranges.size != 2
+        $stderr.puts "Unknown format for version ranges: #{gem_version_ranges}"
+        return ""
+      end
+
+      # base case: a specific version
+      if gem_version_ranges[0].size == 1
+        range = gem_version_ranges[0][0]
+        if range.left.version == range.right.version
+          return range.left.version.to_s
+        end
+      end
+
+      gem_version_ranges[0] = Bundler::VersionRanges::ReqR.reduce(gem_version_ranges[0])
+
+      arr = []
+      gem_version_ranges[0].each do |reqr|
+        range_begin = reqr.left.inclusive ? "[" : "("
+        range_end = reqr.right.inclusive ? "]" : ")"
+        arr << "#{range_begin}#{reqr.left.version.to_s}, #{reqr.right.version.to_s}#{range_end}"
+      end
+      gem_version_ranges[1].each do |neq|
+        # special case: exclude specific version
+        arr << "!= #{neq.version.to_s}"
+      end
+      arr.join(", ")
+    end
+  end
+end

data/lib/usedby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Usedby
+  VERSION = '0.4.0'
+end

data/lib/usedby/version_ranges_intersection.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module VersionRangesIntersection
+  refine Bundler::VersionRanges::ReqR do
+    # Checks whether two ranges overlap
+    def intersect?(other)
+      case self <=> other
+      when -1
+        return (right <=> other.left) == 1
+      when 1
+        return (other.right <=> left) == 1
+      end
+
+      true
+    end
+
+    # Returns a new range which is the intersection, or else nil
+    def intersection(other)
+      return nil unless intersect?(other)
+
+      inter = clone
+      case inter <=> other
+      when -1
+        inter.left = other.left
+      when 1
+        inter.right = other.right
+      end
+
+      inter
+    end
+  end
+
+  # Class methods need to be refined on the singleton_class
+  refine Bundler::VersionRanges::ReqR.singleton_class do
+    def reduce(reqrs = [])
+      reduced_reqrs = reqrs.clone
+      i = 0
+      while i < reduced_reqrs.size - 1
+        intersecting_range = reduced_reqrs[i].intersection(reduced_reqrs[i + 1])
+        if intersecting_range
+          reduced_reqrs[i + 1] = intersecting_range
+          reduced_reqrs.delete_at(i)
+        else
+          i += 1
+        end
+      end
+
+      reduced_reqrs
+    end
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: usedby
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- AppFolio, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: octokit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: 'usedby is a command line tool to discover all dependents of ruby gems
+  across a github organization.
+
+  '
+email: opensource@appfolio.com
+executables:
+- usedby
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGES.md
+- LICENSE.txt
+- README.md
+- bin/usedby
+- lib/usedby.rb
+- lib/usedby/cli.rb
+- lib/usedby/version.rb
+- lib/usedby/version_ranges_intersection.rb
+homepage: https://github.com/appfolio/usedby
+licenses:
+- BSD-2-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Discover all dependents of ruby gems across a github organization.
+test_files: []