vcsmap 2.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ce25dd4820391402c9309ce065ee9d349e566985
+  data.tar.gz: 81fbabbbc3ade9495cb3aa3b1307ee70ba1efa8e
+SHA512:
+  metadata.gz: 621f538ba408036582602c9a1e4958e7c2f6f1626cf7f74acaa01d1bd3d0acc147f932cf1a5630bddd5b5112bbf5897879965570b3b723886d26bcfb271ab618
+  data.tar.gz: 68638ec08e026e5584a0dedb0f958b590a420dd383a47e51a06117443789ea1213553a95bf5be497f26ae84e816815fc0bfb21c5f1b702d68e1b46e7498a5d38

data/bin/vcsmap

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.dirname(File.realpath(__FILE__)) + '/../lib')
+
+require 'vcsmap'
+
+cli = Vcsmap::CLI.new(ARGV)
+cli.run

data/lib/vcsmap/cli.rb

@@ -0,0 +1,86 @@
+require 'tty'
+
+module Vcsmap
+  class CLI
+    def initialize(arguments)
+      @command = arguments[0]
+      @plugin = arguments[1]
+      @pages = arguments[2]
+      @output = arguments[3]
+    end
+
+    def run
+      case @command
+      when 'list'
+        list_plugins
+      when 'run'
+        run_plugin(@output === '--no-ascii')
+      when nil
+        abort "vcsmap requires a command. #{usage}"
+      else
+        abort "Command not recognized. #{usage}"
+      end
+    end
+
+    private
+
+    def usage
+      "See http://vcsmap.org or open #{Helpers.project_directory}/README.md for instructions."
+    end
+
+    def list_plugins
+      puts "Available plugins:\n\n"
+      PluginList.render_list
+      exit
+    end
+
+    def run_plugin(no_ascii)
+      begin
+        plugin = PluginList.get_object(@plugin)
+      rescue KeyError
+        abort "Cannot find plugin with name '#{@plugin}'."
+      rescue NameError
+        abort "The plugin '#{@plugin}' has not been implemented yet."
+      end
+
+      unless @pages && (1..100).cover?(@pages.to_i)
+        abort 'Specify a number of pages (1-100) to load after the plugin name (1 page = ~10 results).'
+      end
+
+      puts 'Searching for matching files ...'
+      provider = Vcsmap::Provider::GitHub.new
+      results = provider.search(plugin, @pages.to_i)
+
+      bar = Vcsmap::ProgressBar.new(results.count)
+
+      data = []
+
+      abort "No files were found matching the search string (#{plugin.search_string})." if results.empty?
+
+      results.each do |result|
+        bar.step
+        file = HTTP.follow(true).get(result).body.to_s
+        credentials = plugin.credentials(file)
+        credentials << result.split('/').slice(3, 2).join('/')
+        # TODO: make an object that holds credentials and has empty? and valid? methods.
+        # this object should also be able to filter 'false' creds like localhost and default ones.
+        data << credentials unless credentials[1].nil? || credentials[1].empty?
+      end
+
+      if data.empty?
+        abort "Some files were loaded (#{results.count}), but none of them contained matching credentials. " \
+              'You could try a higher page number.'
+      end
+
+      bar.clear
+
+      csv_writer = Vcsmap::CsvWriter.new(@plugin, plugin.table_header, data)
+      csv_writer.write!
+
+      unless no_ascii
+        table = TTY::Table.new plugin.table_header << 'Repo', data
+        puts table.render(:ascii)
+      end
+    end
+  end
+end

data/lib/vcsmap/csv_writer.rb

@@ -0,0 +1,35 @@
+require 'csv'
+
+module Vcsmap
+  class CsvWriter
+    def initialize(plugin_name, plugin_header, data)
+      @file_path = file_path(plugin_name)
+      @header = header(plugin_header)
+      @data = data
+    end
+
+    def write!
+      puts "Writing CSV to #{Helpers.project_directory}/#{@file_path} ..."
+      CSV.open(@file_path, 'wb', force_quotes: true) do |csv|
+        csv << @header
+        @data.each do |line|
+          csv << line
+        end
+      end
+    end
+
+    private
+
+    def folder
+      'output/'
+    end
+
+    def file_path(plugin_name)
+      folder + "#{DateTime.now.strftime('%Y%m%dT%H%M')}-#{plugin_name}.csv"
+    end
+
+    def header(plugin_header)
+      plugin_header << 'Repo'
+    end
+  end
+end

data/lib/vcsmap/helpers.rb

@@ -0,0 +1,7 @@
+module Vcsmap
+  module Helpers
+    def self.project_directory
+      File.dirname(File.dirname(__FILE__)).to_s
+    end
+  end
+end

data/lib/vcsmap/plugin.rb

@@ -0,0 +1,4 @@
+module Vcsmap
+  module Plugin
+  end
+end

data/lib/vcsmap/plugin_list.rb

@@ -0,0 +1,67 @@
+module Vcsmap
+  class PluginList
+    PLUGINS = {
+      aws: {
+        title: 'AWS access key',
+        description: 'Extracts AWS credentials from config and credentials files.',
+        class_name: 'Vcsmap::Plugin::AwsAccessToken'
+      },
+      wordpress_config: {
+        title: 'Wordpress configuration files',
+        description: 'Extracts database credentials from wp-config.php.',
+        class_name: 'Vcsmap::Plugin::WordpressConfig'
+      },
+      google_oauth: {
+        title: 'Google oAuth tokens',
+        description: 'Extracts oAuth credentials from client_secrets.json.',
+        class_name: 'Vcsmap::Plugin::GoogleOauth'
+      },
+      filezilla_xml: {
+        title: 'Filezilla configuration XML',
+        description: 'Extracts FTP credentials from Filezilla configuration files.',
+        class_name: 'Vcsmap::Plugin::FilezillaXml'
+      },
+      solr_dataconfig: {
+        title: 'Solr dataConfig.xml credentials',
+        description: 'Extracts JdbcDataSource credentials from a Solor dataConfig.xml.',
+        class_name: 'Vcsmap::Plugin::SolrDataconfig'
+      },
+      sublime_github: {
+        title: 'Sublime Text GitHub tokens',
+        description: 'Extracts GitHub tokens from the Sublime Text settings file for GitHub.',
+        class_name: 'Vcsmap::Plugin::GithubSublimesettings'
+      },
+      facebook_secrets: {
+        title: 'Facebook app secret',
+        description: 'Extracts Facebook tokens from fb_client_secrets.json.',
+        class_name: 'Vcsmap::Plugin::FacebookClientSecrets'
+      },
+      instagram_tokens: {
+        title: 'Instagram access tokens',
+        description: 'Extracts Instagram access tokens.',
+        class_name: 'Vcsmap::Plugin::InstagramTokens'
+      }
+    }.freeze
+
+    def self.all
+      PLUGINS.sort
+    end
+
+    def self.find(name)
+      PLUGINS.fetch(name.to_sym)
+    end
+
+    def self.get_object(name)
+      plugin = find(name)
+      Object.const_get(plugin[:class_name]).new
+    end
+
+    def self.render_list
+      all.each do |plugin|
+        puts Pastel.new.green "[#{plugin[0]}] #{plugin[1][:title]}"
+        puts plugin[1][:description]
+        puts
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/aws_access_token.rb

@@ -0,0 +1,25 @@
+module Vcsmap
+  module Plugin
+    class AwsAccessToken < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = ['aws_secret_access_key',
+                          'filename:config',
+                          'filename:credentials'].join('+')
+        @access_key_id_regex = /=\s+(AKIA[0-9A-Z]{16})/
+        @secret_access_key_regex = %r{=\s+([0-9a-zA-Z\/+]{40})}
+      end
+
+      def credentials(file)
+        @access_key_id = capture_match(@access_key_id_regex, file)
+        @secret_access_key = capture_match(@secret_access_key_regex, file)
+        [@access_key_id, @secret_access_key]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        ['Access Key ID', 'Secret Access Key']
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/base_plugin.rb

@@ -0,0 +1,19 @@
+module Vcsmap
+  module Plugin
+    class BasePlugin
+      attr_reader :search_string
+
+      def capture_match(regex, file)
+        match = regex.match(clean_file(file))
+        return '' if match.nil?
+        match.captures.first
+      end
+
+      private
+
+      def clean_file(file)
+        file.scrub
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/facebook_client_secrets.rb

@@ -0,0 +1,23 @@
+module Vcsmap
+  module Plugin
+    class FacebookClientSecrets < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:fb_client_secrets.json+app_secret'
+        @app_id_regex = /(?:\"|')app_id(?:\"|')\:(?:\ |)(?:\"|')(.*?)(?:\"|')/i
+        @app_secret_regex = /(?:\"|')app_secret(?:\"|')\:(?:\ |)(?:\"|')(.*?)(?:\"|')/i
+      end
+
+      def credentials(file)
+        @app_id = capture_match(@app_id_regex, file)
+        @app_secret = capture_match(@app_secret_regex, file)
+        ['oAuth', @app_id, @app_secret]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        %w(Protocol app_id app_secret)
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/filezilla_xml.rb

@@ -0,0 +1,37 @@
+module Vcsmap
+  module Plugin
+    class FilezillaXml < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:filezilla.xml+Pass'
+        @host_regex = /<Host>(.*)<\/Host>/
+        @username_regex = /<User>(.*)<\/User>/
+        @password_regex = /<Pass>(.*)<\/Pass>/
+        @encoded_password_regex = /<Pass encoding="base64">(.*)<\/Pass>/
+        @port_regex = /<Port>(.*)<\/Port>/
+      end
+
+      def credentials(file)
+        @host = capture_match(@host_regex, file)
+        @user = capture_match(@username_regex, file)
+        @pass = find_password(file)
+        @port = capture_match(@port_regex, file)
+        ['FTP', @host, @port, @user, @pass]
+      end
+
+      def table_header
+        %w(Protocol Host Port Username Password)
+      end
+
+      private
+
+      def find_password(file)
+        @pass = capture_match(@password_regex, file)
+        @base64_pass = capture_match(@encoded_password_regex, file)
+
+        return @pass unless @pass.empty?
+        return Base64.decode64(@base64_pass) unless @base64_pass.empty?
+        ''
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/github_sublimesettings.rb

@@ -0,0 +1,21 @@
+module Vcsmap
+  module Plugin
+    class GithubSublimesettings < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:GitHub.sublime-settings+github_token'
+        @token_regex = /(?:\"|')github_token(?:\"|')\:(?:\ |)(?:\"|')(.*?)(?:\"|')/i
+      end
+
+      def credentials(file)
+        @token = capture_match(@token_regex, file)
+        ['GitHub token', @token]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        %w(Protocol Token)
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/google_oauth.rb

@@ -0,0 +1,23 @@
+module Vcsmap
+  module Plugin
+    class GoogleOauth < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:client_secrets.json+.apps.googleusercontent.com'
+        @client_id_regex = /(?:\"|')client_id(?:\"|')\:(?:\ |)(?:\"|')(.*?)(?:\"|')/i
+        @client_secret_regex = /(?:\"|')client_secret(?:\"|')\:(?:\ |)(?:\"|')(.*?)(?:\"|')/i
+      end
+
+      def credentials(file)
+        @client_id = capture_match(@client_id_regex, file)
+        @client_secret = capture_match(@client_secret_regex, file)
+        ['oAuth', @client_id, @client_secret]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        %w(Protocol client_id client_secret)
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/instagram_tokens.rb

@@ -0,0 +1,19 @@
+module Vcsmap
+  module Plugin
+    class InstagramTokens < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'instagram+access_token'
+        @access_token_regex = /([0-9]{9,10}\.[a-f0-9]{7}\.[a-f0-9]{31,32})/
+      end
+
+      def credentials(file)
+        @access_token = capture_match(@access_token_regex, file)
+        ['Instagram', @access_token]
+      end
+
+      def table_header
+        ['Protocol', 'Access Token']
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/solr_dataconfig.rb

@@ -0,0 +1,25 @@
+module Vcsmap
+  module Plugin
+    class SolrDataconfig < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:dataConfig.xml+dataSource+JdbcDataSource+password'
+        @url_regex = /url=\"(jdbc(.*?))\"/
+        @username_regex = /user=\"(.*?)\"/
+        @password_regex = /password=\"(.*?)\"/
+      end
+
+      def credentials(file)
+        @url = capture_match(@url_regex, file)
+        @user = capture_match(@username_regex, file)
+        @pass = capture_match(@password_regex, file)
+        ['JDBC', @url, @user, @pass]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        %w(Protocol URL Username Password)
+      end
+    end
+  end
+end

data/lib/vcsmap/plugins/wordpress_config.rb

@@ -0,0 +1,27 @@
+module Vcsmap
+  module Plugin
+    class WordpressConfig < Vcsmap::Plugin::BasePlugin
+      def initialize
+        @search_string = 'filename:wp-config.php+DB_PASSWORD'
+        @host_regex = /(?:\'|\")DB_HOST(?:\'|\")\,(?:\ |)(?:\'|\")(.*?)(?:\'|\")/i
+        @username_regex = /(?:\'|\")DB_USER(?:\'|\")\,(?:\ |)(?:\'|\")(.*?)(?:\'|\")/i
+        @password_regex = /(?:\'|\")DB_PASSWORD(?:\'|\")\,(?:\ |)(?:\'|\")(.*?)(?:\'|\")/i
+        @database_regex = /(?:\'|\")DB_NAME(?:\'|\")\,(?:\ |)(?:\'|\")(.*?)(?:\'|\")/i
+      end
+
+      def credentials(file)
+        @host = capture_match(@host_regex, file)
+        @user = capture_match(@username_regex, file)
+        @pass = capture_match(@password_regex, file)
+        @database = capture_match(@database_regex, file)
+        ['MySQL', @host, @user, @pass, @database]
+      rescue NoMethodError
+        []
+      end
+
+      def table_header
+        %w(Protocol Host Username Password Database)
+      end
+    end
+  end
+end

data/lib/vcsmap/progress_bar.rb

@@ -0,0 +1,18 @@
+require 'tty'
+
+module Vcsmap
+  class ProgressBar
+    def initialize(count)
+      @bar = TTY::ProgressBar.new("Loading #{count} possible results [:bar]", total: count)
+    end
+
+    def step
+      @bar.advance
+    end
+
+    def clear
+      print "\r"
+      $stdout.flush
+    end
+  end
+end

data/lib/vcsmap/provider.rb

@@ -0,0 +1,4 @@
+module Vcsmap
+  module Provider
+  end
+end

data/lib/vcsmap/providers/github.rb

@@ -0,0 +1,46 @@
+module Vcsmap
+  module Provider
+    class GitHub
+      def initialize
+        @cookie = get_cookie
+      end
+
+      def search(plugin, total_pages)
+        result_urls = []
+        @query = plugin.search_string
+        (1..total_pages).each do |page|
+          result_urls += get_results_from_page(page)
+        end
+        result_urls
+      end
+
+      private
+
+      def get_cookie
+        cookie = ENV['GITHUB_COOKIE']
+        http = HTTP.cookies(user_session: cookie).get('https://github.com/login')
+        abort "[Error] No valid session cookie in ENV['GITHUB_COOKIE']." unless http.status == 302
+        cookie
+      end
+
+      def get_results_from_page(page)
+        url = "https://github.com/search?p=#{page}&o=desc&q=#{@query}&s=indexed&type=Code"
+        html = HTTP.cookies(user_session: @cookie).get(url)
+        parse_response(html.body.to_s)
+      end
+
+      def parse_response(body)
+        doc = Nokogiri::HTML(body)
+        hits = doc.css('p.title a:last')
+        # TODO: use this when there are no file boxes.
+        # hits = doc.css('.code-list-item .full-path a')
+        urls = hits.map { |hit| 'https://github.com' + hit.attr('href') }
+        urls.map { |url| get_raw(url) }
+      end
+
+      def get_raw(url)
+        url.gsub('blob', 'raw')
+      end
+    end
+  end
+end

data/lib/vcsmap.rb

@@ -0,0 +1,30 @@
+require 'http'
+require 'yaml'
+require 'nokogiri'
+
+# TODO: include only if needed
+require_relative 'vcsmap/cli'
+require_relative 'vcsmap/csv_writer'
+require_relative 'vcsmap/progress_bar'
+
+require_relative 'vcsmap/helpers'
+require_relative 'vcsmap/plugin'
+require_relative 'vcsmap/plugin_list'
+require_relative 'vcsmap/provider'
+
+# TODO: work on require_all
+require_relative 'vcsmap/plugins/base_plugin'
+require_relative 'vcsmap/plugins/aws_access_token'
+require_relative 'vcsmap/plugins/facebook_client_secrets'
+require_relative 'vcsmap/plugins/filezilla_xml'
+require_relative 'vcsmap/plugins/github_sublimesettings'
+require_relative 'vcsmap/plugins/google_oauth'
+require_relative 'vcsmap/plugins/instagram_tokens'
+require_relative 'vcsmap/plugins/solr_dataconfig'
+require_relative 'vcsmap/plugins/wordpress_config'
+
+require_relative 'vcsmap/providers/github'
+
+module Vcsmap
+  VERSION = '2.0.1'.freeze
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: vcsmap
+version: !ruby/object:Gem::Version
+  version: 2.0.1
+platform: ruby
+authors:
+- Melvin Lammerts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-24 00:00:00.000000000 Z
+dependencies: []
+description: A plugin-based tool to scan public version control systems for sensitive
+  information.
+email: hi@melvin.sh
+executables:
+- vcsmap
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/vcsmap
+- lib/vcsmap.rb
+- lib/vcsmap/cli.rb
+- lib/vcsmap/csv_writer.rb
+- lib/vcsmap/helpers.rb
+- lib/vcsmap/plugin.rb
+- lib/vcsmap/plugin_list.rb
+- lib/vcsmap/plugins/aws_access_token.rb
+- lib/vcsmap/plugins/base_plugin.rb
+- lib/vcsmap/plugins/facebook_client_secrets.rb
+- lib/vcsmap/plugins/filezilla_xml.rb
+- lib/vcsmap/plugins/github_sublimesettings.rb
+- lib/vcsmap/plugins/google_oauth.rb
+- lib/vcsmap/plugins/instagram_tokens.rb
+- lib/vcsmap/plugins/solr_dataconfig.rb
+- lib/vcsmap/plugins/wordpress_config.rb
+- lib/vcsmap/progress_bar.rb
+- lib/vcsmap/provider.rb
+- lib/vcsmap/providers/github.rb
+homepage: http://vcsmap.org
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Scans public repositories for sensitive information.
+test_files: []