owasp-pipeline 0.8.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c19bcf63e27cbc2579358e2f581943797d31b056
+  data.tar.gz: 4b1daacd45d7dcb1d5bad0643da0b9c9cf8e93ad
+SHA512:
+  metadata.gz: cf4edf3934d7f2b596bc6baf3c1e85449100026b86aec1fcb690723a8f4cb2f48c2e3a974c7b79ec478384970501c1f8413ed7a4743f1e5087f922cb8571a06e
+  data.tar.gz: c7cdbd7eaf00185447d5fbb2ebf042f4c68cdfa15ef10547bd923b98d8373a736b69b3c48cc030728f05164502852e07a7eaad6a38f7660bd17e7db77c91b1f5

data/CHANGES

@@ -0,0 +1,23 @@
+## 0.6.0
+  * Docker image.
+  * JavaScript tools (retire.js, nodesecurity, eslint)
+  * Java tools (SonarCube, Findbugs)
+## 0.5.3
+  * Checkmarx
+  * JIRA Integration
+  * Search for secrets (a la gitrob) 
+  * OWASP dependency-check
+## 0.1.0
+  * Structure for pipeline and initial tool series.
+  * gem packaging
+  * Finding / reporting
+  * docker mounting
+  * Support for task labels.  Run with 'pipeline -l label' and only tasks that identify with that label will run. 
+  * Move to only keep options in tracker.
+
+## FUTURE
+  * TODO:  add scap
+  * TODO:  Devtools:  pmd, brakeman, dependency checker, codescan, scanjs, bandit, etc.
+  * TODO:  Misc OS checks
+  * TODO:  Active:  ZAP, gauntlt
+  * TODO:  .NET

data/FEATURES

@@ -0,0 +1,19 @@
+# Capabilities
+* AV
+* FIM
+
+FUTURE:
+* oscap - https://www.redhat.com/archives/open-scap-list/2013-May/msg00007.html
+* pmd
+* brakeman
+* dependency checker
+* codescan
+
+# Images
+* Raw directory on file system.
+* git
+* docker
+
+FUTURE:
+* iso
+* vmdk, vdi, ami

data/README.md

@@ -0,0 +1,101 @@
+![Pipeline Logo](https://upload.wikimedia.org/wikipedia/commons/3/37/The_Great_Wave_of_Kanagava.jpg)
+
+# Pipeline
+
+Pipeline is a framework for running a series of tools.  Generally, it is intended as a backbone 
+for automating a security analysis pipeline of tools.
+
+# Recommended Usage
+
+For those wishing to run pipeline, we recommend using the docker image.
+See the documentation for more info.  [Pipeline Docker Documentation](./DOCKER.md)
+
+For those interested in how to use Pipeline in a DevOps context, see
+[Pipeline DevOps Integration Options](./DEVOPS.md)
+
+# Installation
+
+gem install pipeline
+
+# Extending Pipeline
+
+Pipeline is intended to be extended through added "tasks".  To add a new tool, 
+copy an existing task and tweak to make it work for the tool in question.
+
+# Usage
+
+pipeline <options> <target>
+
+## Options
+
+Common options include: 
+-d for debug
+-f for format (takes "json", "csv", "jira")
+
+For a full list of options, use `pipeline --help` or see the [OPTIONS.md](./OPTIONS.md) file.
+
+## Target
+
+The target can be: 
+* Filesystem (which is analyzed in place)
+* Git repo (which is cloned for analysis)
+* Other types of images (.iso, docker, etc. are experimental)
+
+
+# Dependencies
+
+* clamav
+* hashdeep
+* rm (*nix)
+* git
+* mount (*nix)
+* docker
+
+# Development
+
+To run the code, run the following from the root directory: 
+>ruby bin/pipeline <options> target
+
+To build a gem, just run: 
+gem build pipeline.gemspec
+
+
+# Integration
+
+## Git Hooks
+
+First, grab the hook from the code.
+```
+meditation:hooks mk$ cp /area53/owasp/pipeline/hooks/pre-commit .
+```
+
+Then make it executable.
+```
+meditation:hooks mk$ chmod +x pre-commit
+```
+
+Make sure the shell you are committing in can see docker.
+```
+meditation:hooks mk$ eval "$(docker-machine env default)"
+```
+
+Now go test and make a change and commit a file.
+The result should be that pipeline runs against your 
+code and will not allow commits unless the results 
+are clean. (Which is not necessarily a reasonable 
+expectation)
+
+
+# Configuration files
+
+For advanced usage scenarios, you can save your configuration and use it at runtime.
+
+# Authors
+
+Matt Konda
+Alex Lock
+Rafa Perez
+
+# License
+
+Apache 2:  http://www.apache.org/licenses/LICENSE-2.0

data/bin/pipeline

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+#Adjust path in case called directly and not through gem
+$:.unshift "#{File.expand_path(File.dirname(__FILE__))}/../lib"
+
+require 'pipeline'
+require 'pipeline/options'
+require 'pipeline/version'
+
+#Parse options
+begin
+  options, parser = Pipeline::Options.parse! ARGV
+rescue OptionParser::ParseError => e
+  $stderr.puts e.message.capitalize
+  $stderr.puts "Please see `pipeline --help` for valid options"
+  exit -1
+end
+
+#Exit early for these options
+if options[:list_checks] or options[:list_optional_checks]
+  Pipeline.list_checks options
+  exit
+elsif options[:create_config]
+  Pipeline.dump_config options
+  exit
+elsif options[:show_help]
+  puts parser
+  exit
+elsif options[:show_version]
+  puts "Pipeline #{Pipeline::Version}"
+  exit
+end
+
+#Set application path according to the commandline arguments
+unless options[:target]
+  if ARGV[-1].nil?
+    options[:target] = "."
+  else
+    options[:target] = ARGV[-1]
+  end
+end
+
+trap("INT") do
+  $stderr.puts "\nInterrupted - exiting."
+
+  if options[:debug]
+    $stderr.puts caller
+  end
+
+  exit!
+end
+
+if options[:quiet].nil?
+  options[:quiet] = :command_line
+end
+
+begin
+    #Run scan and output a report
+    tracker = Pipeline.run options.merge(:print_report => true, :quiet => options[:quiet])
+
+    #Return error code if --exit-on-warn is used and warnings were found
+    if options[:exit_on_warn] and not tracker.findings.empty?
+      exit Pipeline::Warnings_Found_Exit_Code
+    end
+rescue Pipeline::NoTargetError => e
+  $stderr.puts e.message
+  exit 1
+end

data/lib/pipeline.rb

@@ -0,0 +1,301 @@
+require 'rubygems'
+require 'yaml'
+require 'set'
+require 'tempfile'
+
+module Pipeline
+
+  #This exit code is used when warnings are found and the --exit-on-warn
+  #option is set
+  Warnings_Found_Exit_Code = 3
+
+  @debug = false
+  @quiet = false
+  @loaded_dependencies = []
+
+  #Run Pipeline.
+  #
+  #Options:
+  #
+  #  * :config_file - configuration file
+  #  * :exit_on_warn - return false if warnings found, true otherwise. Not recommended for library use (default: false)
+  #  * :output_files - files for output
+  #  * :output_formats - formats for output (:to_s, :to_tabs, :to_csv, :to_html)
+  #  * :parallel_checks - run checks in parallel (default: true)
+  #  * :print_report - if no output file specified, print to stdout (default: false)
+  #  * :quiet - suppress most messages (default: true)
+  def self.run options
+    options = set_options options
+
+    @quiet = !!options[:quiet]
+    @debug = !!options[:debug]
+
+    if @quiet
+      options[:report_progress] = false
+    end
+
+    scan options
+  end
+
+  #Sets up options for run, checks given application path
+  def self.set_options options
+    if options.is_a? String
+      options = { :target => options }
+    end
+
+    if options[:quiet] == :command_line
+      command_line = true
+      options.delete :quiet
+    end
+
+    options = default_options.merge(load_options(options[:config_file], options[:quiet])).merge(options)
+
+    if options[:quiet].nil? and not command_line
+      options[:quiet] = true
+    end
+
+    options[:output_format] = get_output_format options
+
+    if options[:appname].nil?
+      path = options[:target]
+      options[:appname] = File.split(path).last
+    end
+    options
+  end
+
+  CONFIG_FILES = [
+    File.expand_path("./config/pipeline.yml"),
+    File.expand_path("~/.pipeline/config.yml"),
+    File.expand_path("/etc/pipeline/config.yml")
+  ]
+
+  #Load options from YAML file
+  def self.load_options custom_location, quiet
+    #Load configuration file
+    if config = config_file(custom_location)
+      options = YAML.load_file config
+
+      if options
+        options.each { |k, v| options[k] = Set.new v if v.is_a? Array }
+
+        # notify if options[:quiet] and quiet is nil||false
+        notify "[Notice] Using configuration in #{config}" unless (options[:quiet] || quiet)
+        options
+      else
+        notify "[Notice] Empty configuration file: #{config}" unless quiet
+        {}
+      end
+    else
+      {}
+    end
+  end
+
+  def self.config_file custom_location = nil
+    supported_locations = [File.expand_path(custom_location || "")] + CONFIG_FILES
+    supported_locations.detect {|f| File.file?(f) }
+  end
+
+  #Default set of options
+  def self.default_options
+    {
+      :parallel_tasks => true,
+      :skip_tasks => Set.new(),
+      :exit_on_warn => true,
+      :output_format => :text,
+      :working_dir => "~/line/tmp/",
+      :zap_host => "http://localhost",
+      :zap_port => "9999",
+      :labels => Set.new() << "filesystem" << "code"     # Defaults to run.
+    }
+  end
+
+  #Determine output formats based on options[:output_formats]
+  #or options[:output_files]
+  def self.get_output_format options
+    if options[:output_file]
+      get_format_from_output_file options[:output_file]
+    elsif options[:output_format]
+      get_format_from_output_format options[:output_format]
+    else
+      begin
+        require 'terminal-table'
+        return [:to_s]
+      rescue LoadError
+        return [:to_json]
+      end
+    end
+  end
+
+  def self.get_format_from_output_format output_format
+    case output_format
+    when :html, :to_html
+      [:to_html]
+    when :csv, :to_csv
+      [:to_csv]
+    when :pdf, :to_pdf
+      [:to_pdf]
+    when :tabs, :to_tabs
+      [:to_tabs]
+    when :json, :to_json
+      [:to_json]
+    when :jira, :to_jira
+      [:to_jira]
+    when :markdown, :to_markdown
+      [:to_markdown]
+    else
+      [:to_s]
+    end
+  end
+  private_class_method :get_format_from_output_format
+
+  def self.get_format_from_output_file output_file
+      case output_file
+      when /\.html$/i
+        :to_html
+      when /\.csv$/i
+        :to_csv
+      when /\.pdf$/i
+        :to_pdf
+      when /\.tabs$/i
+        :to_tabs
+      when /\.json$/i
+        :to_json
+      when /\.md$/i
+        :to_markdown
+      else
+        :to_s
+      end
+  end
+  private_class_method :get_format_from_output_file
+
+  #Output list of tasks (for `-k` option)
+  def self.list_checks options
+    require 'pipeline/scanner'
+
+    add_external_tasks options
+
+    if options[:list_optional_tasks]
+      $stderr.puts "Optional Tasks:"
+      tasks = Tasks.optional_tasks
+    else
+      $stderr.puts "Available tasks:"
+      tasks = Tasks.tasks
+    end
+
+    format_length = 30
+
+    $stderr.puts "-" * format_length
+    tasks.each do |task|
+      $stderr.printf("%-#{format_length}s\n", task.name)
+    end
+  end
+
+  #Output configuration to YAML
+  def self.dump_config options
+    if options[:create_config].is_a? String
+      file = options[:create_config]
+    else
+      file = nil
+    end
+
+    options.delete :create_config
+
+    options.each do |k,v|
+      if v.is_a? Set
+        options[k] = v.to_a
+      end
+    end
+
+    if file
+      File.open file, "w" do |f|
+        YAML.dump options, f
+      end
+      puts "Output configuration to #{file}"
+    else
+      puts YAML.dump(options)
+    end
+    exit
+  end
+
+  #Run a scan. Generally called from Pipeline.run instead of directly.
+  def self.scan options
+    #Load scanner
+    notify "Loading scanner..."
+
+    begin
+      require 'pipeline/scanner'
+      require 'pipeline/tracker'
+      require 'pipeline/mounters'
+      require 'pipeline/filters'
+      require 'pipeline/reporters'
+
+    rescue LoadError => e
+      $stderr.puts e.message
+      raise NoPipelineError, "Cannot find lib/ directory or load the key pipeline."
+    end
+
+#    debug "API: #{options[:jira_api_url.to_s]}"
+#    debug "Project: #{options[:jira_project.to_s]}"
+#    debug "Cookie: #{options[:jira_cookie.to_s]}"
+
+    add_external_tasks options
+
+    tracker = Tracker.new options
+    debug "Mounting ... #{options[:target]}"
+    # Make the target accessible.
+    target = Pipeline::Mounters.mount tracker
+
+    #Start scanning
+    scanner = Scanner.new
+    notify "Processing target...#{options[:target]}"
+    scanner.process target, tracker
+
+    # Filter the results (Don't report anything that has been reported before)
+    Pipeline::Filters.filter tracker
+
+    # Generate Report
+    notify "Generating report...#{options[:output_format]}"
+    Pipeline::Reporters.run_report tracker
+
+    tracker
+  end
+
+  def self.error message
+    $stderr.puts message
+  end
+
+  def self.warn message
+    $stderr.puts message unless @quiet
+  end
+
+  def self.notify message
+    $stderr.puts message #unless @debug
+  end
+
+  def self.debug message
+    $stderr.puts message if @debug
+  end
+
+  def self.load_pipeline_dependency name
+    return if @loaded_dependencies.include? name
+
+    begin
+      require name
+    rescue LoadError => e
+      $stderr.puts e.message
+      $stderr.puts "Please install the appropriate dependency."
+      exit! -1
+    end
+  end
+
+  def self.add_external_tasks options
+    options[:additional_tasks_path].each do |path|
+      Pipeline::Tasks.initialize_tasks path
+    end if options[:additional_tasks_path]
+  end
+
+  class DependencyError < RuntimeError; end
+  class NoPipelineError < RuntimeError; end
+  class NoTargetError < RuntimeError; end
+  class JiraConfigError < RuntimeError; end
+end