owasp-glue 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f5e7eb728b9c905eb0a61adef5fb2aaf7c427ef4
+  data.tar.gz: 755da45bbc21effa03b77ca3466a8e3286c30790
+SHA512:
+  metadata.gz: e56539ab4dfcbfd383d0d39765dee5e560a723ab49187dd68df7b9dbd20733649d67cab07ab33d6075a01970334b65bdc4ad55a1873a432b13b67c5036aefe45
+  data.tar.gz: bba3c2f4f09ab3735417ff76ab4c3fec5d93a2c5a0e822dad839865cc1f12017e2a7ba9192b1416be5b1c4435aab308ed97efd85340b30f1a6f877efa8acf306

data/CHANGES

@@ -0,0 +1,27 @@
+## 0.9.0
+  * Rename to Glue
+## 0.8.0
+  * Java Tooling
+## 0.6.0
+  * Docker image.
+  * JavaScript tools (retire.js, nodesecurity, eslint)
+  * Java tools (SonarCube, Findbugs)
+## 0.5.3
+  * Checkmarx
+  * JIRA Integration
+  * Search for secrets (a la gitrob)
+  * OWASP dependency-check
+## 0.1.0
+  * Structure for pipeline and initial tool series.
+  * gem packaging
+  * Finding / reporting
+  * docker mounting
+  * Support for task labels.  Run with 'pipeline -l label' and only tasks that identify with that label will run.
+  * Move to only keep options in tracker.
+
+## FUTURE
+  * TODO:  add scap
+  * TODO:  Devtools:  pmd, brakeman, dependency checker, codescan, scanjs, bandit, etc.
+  * TODO:  Misc OS checks
+  * TODO:  Active:  ZAP, gauntlt
+  * TODO:  .NET

data/FEATURES

@@ -0,0 +1,19 @@
+# Capabilities
+* AV
+* FIM
+
+FUTURE:
+* oscap - https://www.redhat.com/archives/open-scap-list/2013-May/msg00007.html
+* pmd
+* brakeman
+* dependency checker
+* codescan
+
+# Images
+* Raw directory on file system.
+* git
+* docker
+
+FUTURE:
+* iso
+* vmdk, vdi, ami

data/README.md

@@ -0,0 +1,117 @@
+<img src="./glue.png" width="120"/>
+
+# Glue
+
+Glue is a framework for running a series of tools.  Generally, it is intended as a backbone for automating a security analysis pipeline of tools.
+
+# Recommended Usage
+
+For those wishing to run Glue, we recommend using the docker image because
+it should have the other tools it uses available already and configured.
+See the documentation for more info.  [Glue Docker Documentation](./DOCKER.md)
+
+For those interested in how to use Glue in a DevOps context, see
+[Glue DevOps Integration Options](./DEVOPS.md)
+
+# Installation
+
+gem install Glue
+
+or
+
+docker run owasp/glue
+
+# Installation for Development
+
+git clone https://github.com/owasp/glue
+cd glue                     -- RVM will set to 2.3.1 with Gemset Glue
+gem install bundler
+bundle install
+
+## Running in Development
+
+cd lib
+../bin/glue -h
+
+# Extending Glue
+
+Glue is intended to be extended through added "tasks".  To add a new tool,
+copy an existing task and tweak to make it work for the tool in question.
+
+# Usage
+
+Glue <options> <target>
+
+## Options
+
+Common options include:
+-d for debug
+-f for format (takes "json", "csv", "jira")
+
+For a full list of options, use `Glue --help` or see the [OPTIONS.md](./OPTIONS.md) file.
+
+## Target
+
+The target can be:
+* Filesystem (which is analyzed in place)
+* Git repo (which is cloned for analysis)
+* Other types of images (.iso, docker, etc. are experimental)
+
+
+# Dependencies
+
+* clamav
+* hashdeep
+* rm (*nix)
+* git
+* mount (*nix)
+* docker
+
+# Development
+
+To run the code, run the following from the root directory:
+>ruby bin/Glue <options> target
+
+To build a gem, just run:
+gem build Glue.gemspec
+
+
+# Integration
+
+## Git Hooks
+
+First, grab the hook from the code.
+```
+meditation:hooks mk$ cp /area53/owasp/Glue/hooks/pre-commit .
+```
+
+Then make it executable.
+```
+meditation:hooks mk$ chmod +x pre-commit
+```
+
+Make sure the shell you are committing in can see docker.
+```
+meditation:hooks mk$ eval "$(docker-machine env default)"
+```
+
+Now go test and make a change and commit a file.
+The result should be that Glue runs against your
+code and will not allow commits unless the results
+are clean. (Which is not necessarily a reasonable
+expectation)
+
+
+# Configuration files
+
+For advanced usage scenarios, you can save your configuration and use it at runtime.
+
+# Authors
+
+Matt Konda
+Alex Lock
+Rafa Perez
+
+# License
+
+Apache 2:  http://www.apache.org/licenses/LICENSE-2.0

data/bin/glue

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+#Adjust path in case called directly and not through gem
+$:.unshift "#{File.expand_path(File.dirname(__FILE__))}/../lib"
+
+require 'glue'
+require 'glue/options'
+require 'glue/version'
+
+#Parse options
+begin
+  options, parser = Glue::Options.parse! ARGV
+rescue OptionParser::ParseError => e
+  $stderr.puts e.message.capitalize
+  $stderr.puts "Please see `glue --help` for valid options"
+  exit -1
+end
+
+#Exit early for these options
+if options[:list_checks] or options[:list_optional_checks]
+  Glue.list_checks options
+  exit
+elsif options[:create_config]
+  glue.dump_config options
+  exit
+elsif options[:show_help]
+  puts parser
+  exit
+elsif options[:show_version]
+  puts "Glue #{Glue::Version}"
+  exit
+end
+
+#Set application path according to the commandline arguments
+unless options[:target]
+  if ARGV[-1].nil?
+    options[:target] = "."
+  else
+    options[:target] = ARGV[-1]
+  end
+end
+
+trap("INT") do
+  $stderr.puts "\nInterrupted - exiting."
+
+  if options[:debug]
+    $stderr.puts caller
+  end
+
+  exit!
+end
+
+if options[:quiet].nil?
+  options[:quiet] = :command_line
+end
+
+begin
+    #Run scan and output a report
+    tracker = Glue.run options.merge(:print_report => true, :quiet => options[:quiet])
+
+    #Return error code if --exit-on-warn is used and warnings were found
+    if options[:exit_on_warn] and not tracker.findings.empty?
+      exit Glue::Warnings_Found_Exit_Code
+    end
+rescue Glue::NoTargetError => e
+  $stderr.puts e.message
+  exit 1
+end

data/lib/glue.rb

@@ -0,0 +1,317 @@
+require 'rubygems'
+require 'yaml'
+require 'set'
+require 'tempfile'
+
+module Glue
+
+  #This exit code is used when warnings are found and the --exit-on-warn
+  #option is set
+  Warnings_Found_Exit_Code = 3
+
+  @debug = false
+  @quiet = false
+  @loaded_dependencies = []
+
+  #Run Glue.
+  #
+  #Options:
+  #
+  #  * :config_file - configuration file
+  #  * :exit_on_warn - return false if warnings found, true otherwise. Not recommended for library use (default: false)
+  #  * :output_files - files for output
+  #  * :output_formats - formats for output (:to_s, :to_tabs, :to_csv, :to_html)
+  #  * :parallel_checks - run checks in parallel (default: true)
+  #  * :print_report - if no output file specified, print to stdout (default: false)
+  #  * :quiet - suppress most messages (default: true)
+  def self.run options
+    options = set_options options
+
+    @quiet = !!options[:quiet]
+    @debug = !!options[:debug]
+
+    if @quiet
+      options[:report_progress] = false
+    end
+
+    unless options[:logfile].nil?
+      if options[:logfile].is_a? File
+        $logfile = options[:logfile]
+      else
+        $logfile = File.open(options[:logfile], 'a')
+      end
+
+      begin
+        scan options
+      ensure
+        $logfile.close unless options[:logfile].is_a? File
+      end
+    end
+  end
+
+  #Sets up options for run, checks given application path
+  def self.set_options options
+    if options.is_a? String
+      options = { :target => options }
+    end
+
+    if options[:quiet] == :command_line
+      command_line = true
+      options.delete :quiet
+    end
+
+    options = default_options.merge(load_options(options[:config_file], options[:quiet])).merge(options)
+
+    if options[:quiet].nil? and not command_line
+      options[:quiet] = true
+    end
+
+    options[:output_format] = get_output_format options
+
+    if options[:appname].nil?
+      path = options[:target]
+      options[:appname] = File.split(path).last
+    end
+    options
+  end
+
+  CONFIG_FILES = [
+    File.expand_path("./config/glue.yml"),
+    File.expand_path("~/.glue/config.yml"),
+    File.expand_path("/etc/glue/config.yml")
+  ]
+
+  #Load options from YAML file
+  def self.load_options custom_location, quiet
+    #Load configuration file
+    if config = config_file(custom_location)
+      options = YAML.load_file config
+
+      if options
+        options.each { |k, v| options[k] = Set.new v if v.is_a? Array }
+
+        # notify if options[:quiet] and quiet is nil||false
+        notify "[Notice] Using configuration in #{config}" unless (options[:quiet] || quiet)
+        options
+      else
+        notify "[Notice] Empty configuration file: #{config}" unless quiet
+        {}
+      end
+    else
+      {}
+    end
+  end
+
+  def self.config_file custom_location = nil
+    supported_locations = [File.expand_path(custom_location || "")] + CONFIG_FILES
+    supported_locations.detect {|f| File.file?(f) }
+  end
+
+  #Default set of options
+  def self.default_options
+    {
+      :parallel_tasks => true,
+      :skip_tasks => Set.new(),
+      :exit_on_warn => true,
+      :output_format => :text,
+      :working_dir => "~/line/tmp/",
+      :zap_host => "http://localhost",
+      :zap_port => "9999",
+      :labels => Set.new() << "filesystem" << "code"     # Defaults to run.
+    }
+  end
+
+  #Determine output formats based on options[:output_formats]
+  #or options[:output_files]
+  def self.get_output_format options
+    if options[:output_file]
+      get_format_from_output_file options[:output_file]
+    elsif options[:output_format]
+      get_format_from_output_format options[:output_format]
+    else
+      begin
+        require 'terminal-table'
+        return [:to_s]
+      rescue LoadError
+        return [:to_json]
+      end
+    end
+  end
+
+  def self.get_format_from_output_format output_format
+    case output_format
+    when :html, :to_html
+      [:to_html]
+    when :csv, :to_csv
+      [:to_csv]
+    when :pdf, :to_pdf
+      [:to_pdf]
+    when :tabs, :to_tabs
+      [:to_tabs]
+    when :json, :to_json
+      [:to_json]
+    when :jira, :to_jira
+      [:to_jira]
+    when :markdown, :to_markdown
+      [:to_markdown]
+    else
+      [:to_s]
+    end
+  end
+  private_class_method :get_format_from_output_format
+
+  def self.get_format_from_output_file output_file
+      case output_file
+      when /\.html$/i
+        :to_html
+      when /\.csv$/i
+        :to_csv
+      when /\.pdf$/i
+        :to_pdf
+      when /\.tabs$/i
+        :to_tabs
+      when /\.json$/i
+        :to_json
+      when /\.md$/i
+        :to_markdown
+      else
+        :to_s
+      end
+  end
+  private_class_method :get_format_from_output_file
+
+  #Output list of tasks (for `-k` option)
+  def self.list_checks options
+    require 'glue/scanner'
+
+    add_external_tasks options
+
+    if options[:list_optional_tasks]
+      $stderr.puts "Optional Tasks:"
+      tasks = Tasks.optional_tasks
+    else
+      $stderr.puts "Available tasks:"
+      tasks = Tasks.tasks
+    end
+
+    format_length = 30
+
+    $stderr.puts "-" * format_length
+    tasks.each do |task|
+      $stderr.printf("%-#{format_length}s\n", task.name)
+    end
+  end
+
+  #Output configuration to YAML
+  def self.dump_config options
+    if options[:create_config].is_a? String
+      file = options[:create_config]
+    else
+      file = nil
+    end
+
+    options.delete :create_config
+
+    options.each do |k,v|
+      if v.is_a? Set
+        options[k] = v.to_a
+      end
+    end
+
+    if file
+      File.open file, "w" do |f|
+        YAML.dump options, f
+      end
+      puts "Output configuration to #{file}"
+    else
+      puts YAML.dump(options)
+    end
+    exit
+  end
+
+  #Run a scan. Generally called from Glue.run instead of directly.
+  def self.scan options
+    #Load scanner
+    notify "Loading scanner..."
+
+    begin
+      require 'glue/scanner'
+      require 'glue/tracker'
+      require 'glue/mounters'
+      require 'glue/filters'
+      require 'glue/reporters'
+
+    rescue LoadError => e
+      $stderr.puts e.message
+      raise NoGlueError, "Cannot find lib/ directory or load the key glue."
+    end
+
+#    debug "API: #{options[:jira_api_url.to_s]}"
+#    debug "Project: #{options[:jira_project.to_s]}"
+#    debug "Cookie: #{options[:jira_cookie.to_s]}"
+
+    add_external_tasks options
+
+    tracker = Tracker.new options
+    debug "Mounting ... #{options[:target]}"
+    # Make the target accessible.
+    target = Glue::Mounters.mount tracker
+
+    #Start scanning
+    scanner = Scanner.new
+    notify "Processing target...#{options[:target]}"
+    scanner.process target, tracker
+
+    # Filter the results (Don't report anything that has been reported before)
+    Glue::Filters.filter tracker
+
+    # Generate Report
+    notify "Generating report...#{options[:output_format]}"
+    Glue::Reporters.run_report tracker
+
+    tracker
+  end
+
+  def self.error message
+    $stderr.puts message
+    $logfile.puts "[#{Time.now}] #{message}" if $logfile
+  end
+
+  def self.warn message
+    $stderr.puts message unless @quiet
+    $logfile.puts "[#{Time.now}] #{message}" if $logfile
+  end
+
+  def self.notify message
+    $stderr.puts message #unless @debug
+    $logfile.puts "[#{Time.now}] #{message}" if $logfile
+  end
+
+  def self.debug message
+    $stderr.puts message if @debug
+    $logfile.puts "[#{Time.now}] #{message}" if $logfile
+  end
+
+  def self.load_glue_dependency name
+    return if @loaded_dependencies.include? name
+
+    begin
+      require name
+    rescue LoadError => e
+      $stderr.puts e.message
+      $stderr.puts "Please install the appropriate dependency."
+      exit! -1
+    end
+  end
+
+  def self.add_external_tasks options
+    options[:additional_tasks_path].each do |path|
+      Glue::Tasks.initialize_tasks path
+    end if options[:additional_tasks_path]
+  end
+
+  class DependencyError < RuntimeError; end
+  class NoGlueError < RuntimeError; end
+  class NoTargetError < RuntimeError; end
+  class JiraConfigError < RuntimeError; end
+end