dependagrab 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aece265264bb3c2edca450d0136c6412e1155cefe2f50fdc29b4961bf3041676
+  data.tar.gz: 8ff6d6ca94e374971421ca0f038f65af5a6f9cb33aae83b50b2259ee9ec10298
+SHA512:
+  metadata.gz: b91b6dfcae48f521b5ec19eb53abac3ea0a79d8c12d1df5752dcf25509b1ba645393846407758bf1e537f72c8c7bc638e9da342ce01f5cf93bcaf91de1567781
+  data.tar.gz: 566d298e0c41af8d79bddd9a107f672359c14f9aee053c12787b200af228d26890c88705cb458f743f93d23e1d4a1cea53465ed3dc12e483eafdb798e852bbdb

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+before_install: gem install bundler -v 1.17.2

data/CHANGE_LOG.md

@@ -0,0 +1,3 @@
+# V0.1
+Initial Release
+

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in dependagrab.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,59 @@
+PATH
+  remote: .
+  specs:
+    dependagrab (0.1.0)
+      graphql-client (= 0.17.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
+    graphql (1.13.0)
+    graphql-client (0.17.0)
+      activesupport (>= 3.0)
+      graphql (~> 1.10)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (10.5.0)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.17)
+  dependagrab!
+  pry (~> 0.14)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.17.2

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Dave Elliott
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# Dependagrab
+
+Tool for extracting GitHub dependency warnings and converting it into a ThreadFix compatible file
+
+## Installation
+
+Install it with:
+
+    $ gem install dependagrab
+
+## Usage
+
+    `$ dependagrab --help`
+
+    `$ export GITHUB_API_TOKEN=<TOKEN>`
+    `$ dependagrab DDAZZA/dependagrab`
+
+## Development
+
+```
+$ git clone https://github.com/DDAZZA/dependagrab.git
+$ bundle install
+$ ruby -Ilib ./bin/dependagrab --help
+```
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/DDAZZA/dependagrab.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "dependagrab"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/dependagrab

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "dependagrab/cli"
+
+Dependagrab::CLI.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dependagrab.gemspec

@@ -0,0 +1,46 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "dependagrab/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "dependagrab"
+  spec.version       = Dependagrab::VERSION
+  spec.authors       = ["Dave Elliott"]
+  spec.email         = ["ddazza@gmail.com"]
+
+  spec.summary       = %q{Utility for extracting dependency warnings from GitHub}
+
+  # spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "https://github.com/DDAZZA/dependagrab"
+
+
+  spec.license       = 'MIT'
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "https://github.com/DDAZZA/dependagrab"
+    spec.metadata["changelog_uri"] = "https://github.com/DDAZZA/dependagrab/blob/master/CHange_LOG.md"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib", "static"]
+
+  spec.add_dependency "graphql-client", "0.17.0"
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry", "~> 0.14"
+end

data/lib/dependagrab/cli.rb

@@ -0,0 +1,94 @@
+require 'getoptlong'
+require 'dependagrab'
+
+module Dependagrab
+  require 'dependagrab/console_writer'
+  require 'dependagrab/file_writer'
+
+  class CLI
+    def self.start
+      opts = GetoptLong.new(
+        [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+        [ '--version', '-v', GetoptLong::NO_ARGUMENT ],
+        [ '--output', '-o', GetoptLong::REQUIRED_ARGUMENT ],
+      )
+
+
+      options = {}
+
+      begin
+        opts.each do |opt, arg|
+          case opt
+          when '--help'
+            print_help
+            exit 0
+          when '--version'
+            puts "dependagrab #{Dependagrab::VERSION}"
+            exit 0
+          when '--output'
+            options[:output] = arg
+          end
+        end
+      rescue GetoptLong::Error => e
+        print_help
+        exit 1
+      end
+
+      if ARGV.length != 1
+        STDERR.puts "Missing REPO argument (try --help for usage)"
+        exit 1
+      end
+
+      repo = ARGV.shift
+      _, options[:owner], options[:repo] = repo.split /([\w-]+)\/([\w-]+)$/
+      if  options[:owner].nil? || options[:repo].nil?
+        STDERR.puts "Invalid REPO format"
+        exit 1
+      end
+
+      run(options)
+    end
+
+    private
+
+    def self.run(options)
+      result = Dependagrab::GithubClient.new(options).grab
+
+      if options[:output]
+        begin
+          FileWriter.new(options[:output]).write!(result[:alerts])
+          puts "#{result[:alerts].count} dependency warnings written to '#{options.fetch(:output)}'"
+        rescue => e
+          STDERR.puts "Failed to write file '#{options.fetch(:output)}'"
+          STDERR.puts e.message
+          exit 1
+        end
+      else
+        ConsoleWriter.new.write!(result[:alerts])
+      end
+    end
+
+    def self.print_usage
+      puts "Usage: dependagrab <REPO> [Options]"
+      puts
+    end
+
+    def self.print_help
+      print_usage
+      puts <<-EOF
+<REPO>      GitHub Repository (e.g. DDAZZA/dependagrab)
+
+Options:
+  --output        Destination to write JSON file
+
+Misc Options:
+  -v, --version   Prints version
+  -h, --help      Prints this message
+
+For private repositories you will need to set the GITHUB_API_TOKEN environment variable.
+The GitHub documentation provides steps for setup (https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token).
+      EOF
+    end
+  end
+end
+

data/lib/dependagrab/console_writer.rb

@@ -0,0 +1,23 @@
+module Dependagrab
+
+  # For writing output in a human readable format in the terminal
+  class ConsoleWriter
+
+    def write!(result)
+      puts ["SEVERITY".ljust(8), "PACKAGE".ljust(32), "SUMMARY"].join("\t")
+      puts '-' * 120
+
+      result.each do |line|
+        attr = [
+          line[:severity].ljust(8),
+          "#{line[:package_name]} (#{line[:vulnerable_version_range]})".ljust(32),
+          "#{line[:summary]} (#{line[:ghsa_id]})"
+        ]
+        puts(attr.join("\t"))
+      end
+
+      puts
+      puts "Total: #{result.count}"
+    end
+  end
+end

data/lib/dependagrab/file_writer.rb

@@ -0,0 +1,71 @@
+require 'securerandom'
+
+module Dependagrab
+  # For writing output in to a file in a json format
+  # format is aligned to ThreadFix(https://denimgroup.atlassian.net/wiki/spaces/TDOC/pages/496009270/ThreadFix+File+Format)
+  #
+  class FileWriter
+
+    # Destination to write file
+    attr_accessor :output_file
+
+    def initialize(output_file)
+      @output_file = output_file
+    end
+
+    def write!(result)
+      scan = scan_meta_data
+
+      result.each do |alert|
+        scan[:findings].append(
+          parse_threadfix_finding(alert)
+        )
+      end
+
+      File.open(output_file, "w") do |f|
+        f.write(scan.to_json)
+      end
+    end
+
+
+    private
+
+    def scan_meta_data
+      {
+        id: SecureRandom.uuid,
+        created: Time.now.strftime('%Y-%m-%dT%H:%M:%SZ'),
+        exported: Time.now.strftime('%Y-%m-%dT%H:%M:%SZ'),
+        source: "AppSec Team",
+        collectionType: "DEPENDENCY",
+        findings: [],
+      }
+    end
+
+    # Converts an alert into a ThreadFix finding format
+    #
+    def parse_threadfix_finding(alert)
+      {
+        nativeId: alert[:id],
+        severity: alert[:severity].gsub("MODERATE", "MEDIUM"),
+        nativeSeverity: alert[:severity].gsub("MODERATE", "MEDIUM"),
+        summary: alert[:summary],
+        cvsScore: alert[:cvss],
+        description: alert[:description],
+        dependencyDetails: {
+          library: alert[:package_name],
+          description: alert[:description],
+          reference: alert[:ghsa_id],
+          referenceLink: alert[:permalink],
+          version: alert[:vulnerable_version_range],
+          issueType: "VULNERABILITY",
+        },
+        mappings: [
+          {
+            mappingType: "CWE",
+            value: alert[:cwe][4..],
+          }
+        ]
+      }
+    end
+  end
+end

data/lib/dependagrab/gh_api.rb

@@ -0,0 +1,79 @@
+require "graphql/client"
+require "graphql/client/http"
+
+module Dependagrab
+  module GHAPI
+    GRAPHQL_API = "https://api.github.com/graphql"
+    SCHEMA_PATH = "static/gh_schema.json"
+
+    # Configure GraphQL endpoint using the basic HTTP network adapter.
+    HTTP = GraphQL::Client::HTTP.new(GRAPHQL_API) do
+      def headers(context)
+        # Optionally set any HTTP headers
+        {
+          "User-Agent": "dependagrab #{Dependagrab::VERSION}",
+        }.tap do |h|
+          if context[:api_token]
+            h["Authorization"] = "bearer #{context[:api_token]}"
+          end
+        end
+      end
+    end
+
+    # However, it's smart to dump this to a JSON file and load from disk
+    #
+    # Run it from a script or rake task
+    # GraphQL::Client.dump_schema(GHAPI::HTTP, "gh_schema.json")
+    Schema = GraphQL::Client.load_schema(SCHEMA_PATH)
+
+    Client = GraphQL::Client.new(schema: Schema, execute: HTTP)
+
+    Query = Client.parse <<-'GRAPHQL'
+    query($repo: String!, $owner: String!, $after_cursor: String) {
+     repository(name: $repo, owner: $owner) {
+       vulnerabilityAlerts(first: 20, after: $after_cursor) {
+         pageInfo {
+           endCursor
+           hasNextPage
+         }
+         nodes {
+
+           securityVulnerability {
+             package {
+               name
+               ecosystem
+             }
+             vulnerableVersionRange
+             firstPatchedVersion {
+               identifier
+             }
+             advisory {
+               cvss {
+                 vectorString
+                 score
+               }
+               cwes(first:100) {
+                 edges {
+                   node {
+                     cweId
+                     name
+                   }
+                 }
+               }
+               id
+               ghsaId
+               severity
+               summary
+               permalink
+               description
+             }
+
+           }
+         }
+       }
+     }
+    }
+    GRAPHQL
+  end
+
+end

data/lib/dependagrab/github_client.rb

@@ -0,0 +1,91 @@
+require './lib/dependagrab/gh_api'
+
+module Dependagrab
+  class GithubClient
+
+    attr_reader :repo, :owner, :token
+
+    def initialize(options={})
+      @repo = options.fetch(:repo)
+      @owner = options.fetch(:owner)
+      @token = options[:token] || default_github_token
+    end
+
+    def grab
+      result = {
+        alerts: []
+      }
+      query_variables = { repo: repo, owner: owner }
+
+      has_next_page = true # always run query at least once
+      while(has_next_page)
+        response = GHAPI::Client.query(GHAPI::Query,
+                                      variables: query_variables,
+                                      context: { api_token: token }
+                                     )
+        validate_response!(response)
+
+        response.original_hash['data']['repository']['vulnerabilityAlerts']['nodes'].each do |alert|
+          result[:alerts].append(
+            parse_alert(alert)
+          )
+        end
+
+        # Sets the last position for pagination
+        query_variables[:after_cursor] = response.data.repository.vulnerability_alerts.page_info.end_cursor
+
+        has_next_page = response.data.repository.vulnerability_alerts.page_info.has_next_page
+      end
+
+      result
+    end
+
+    private
+
+
+    def validate_response!(response)
+      if response.errors.messages.any?
+        raise GhApiError.new("GitHub API Error(s): " + response.errors.messages.values.join("\n"))
+      end
+
+      if response.original_hash['errors'] && response.original_hash['errors'].any?
+        raise Error.new(response.original_hash['errors'].map { |e| e['message'] })
+      end
+    end
+
+    # Converts response format to hash
+    #
+    def parse_alert(alert)
+      vuln = alert['securityVulnerability']
+
+      {}.tap do |finding|
+        finding[:id] = vuln['advisory']['id']
+        finding[:ghsa_id] = vuln['advisory']['ghsaId']
+        finding[:package_name] = vuln['package']['name']
+        finding[:ecosystem] = vuln['package']['ecosystem']
+        finding[:severity] = vuln['advisory']['severity']
+        finding[:cvss_vector] = vuln['advisory']['cvss']['vectorString']
+        finding[:cvss] = vuln['advisory']['cvss']['score']
+        finding[:permalink] = vuln['advisory']['permalink']
+        finding[:summary] = vuln['advisory']['summary']
+        finding[:description] = vuln['advisory']['description']
+        finding[:vulnerable_version_range] = vuln['vulnerableVersionRange']
+        finding[:first_patched_version] = vuln['firstPatchedVersion']
+
+        if vuln['advisory']['cwes']['edges'].any?
+          cwe = vuln['advisory']['cwes']['edges'][0]
+          finding[:cwe] = cwe['node']['cweId']
+          finding[:cw_name] = cwe['node']['name']
+        end
+      end
+    end
+
+    def default_github_token
+      begin
+        ENV.fetch("GITHUB_API_TOKEN")
+      rescue
+        raise MissingConfigError.new("GitHub API token is not configured")
+      end
+    end
+  end
+end

data/lib/dependagrab/version.rb

@@ -0,0 +1,3 @@
+module Dependagrab
+  VERSION = "0.1.0"
+end

data/lib/dependagrab.rb

@@ -0,0 +1,8 @@
+module Dependagrab
+  require "dependagrab/version"
+  require "dependagrab/github_client"
+
+  class Error < StandardError; end
+  class MissingConfigError < Dependagrab::Error; end
+  class GhApiError < Dependagrab::Error; end
+end