owasp-pipeline 0.8.3

data/lib/pipeline/tasks/eslint.rb

@@ -0,0 +1,71 @@
+require 'pipeline/tasks/base_task'
+require 'json'
+require 'pipeline/util'
+
+class Pipeline::ESLint < Pipeline::BaseTask
+
+  Pipeline::Tasks.add self
+  include Pipeline::Util
+
+  def initialize(trigger, tracker)
+    super(trigger,tracker)
+    @name = "ESLint/ScanJS"
+    @description = "Source analysis for JavaScript"
+    @stage = :code
+    @labels << "code" << "javascript"
+  end
+
+  def run
+    Pipeline.notify "#{@name}"
+    rootpath = @trigger.path
+    currentpath = File.expand_path File.dirname(__FILE__)
+    Pipeline.debug "ESLint Config Path: #{currentpath}"
+    @result = `eslint -c #{currentpath}/scanjs-eslintrc --no-color --quiet --format json #{rootpath}`
+  end
+
+  def analyze
+    # puts @result
+    begin
+      parsed = JSON.parse(@result)
+      parsed.each do |result|
+        findings = {}
+        prints = []
+        messages = []
+        result['messages'].each do |msg|
+          message = msg['message']
+          findings[message] = {} if findings[message].nil?
+          findings[message][:detail] = msg['ruleId']
+          if messages.include?(message)
+            findings[message][:source] = "#{findings[message][:source]},#{msg['line']}" unless findings[message][:source].include?(",#{msg['line']}")
+          else
+            findings[message][:source] = "#{result['filePath']} Line: #{msg['line']}"
+            messages << message
+          end
+          findings[message][:severity] = severity(msg['severity'].to_s)
+        end
+        findings.each do |key, value|
+          print = fingerprint("#{key}#{value[:detail]}#{value[:source]}#{value[:sev]}")
+          unless prints.include?(print)
+            prints << print
+            report key, value[:detail], value[:source], value[:severity], print
+          end
+        end
+      end
+    rescue Exception => e
+      Pipeline.warn e.message
+      Pipeline.warn e.backtrace
+    end
+  end
+
+  def supported?
+    supported=runsystem(true, "eslint", "-c", "~/.scanjs-eslintrc")
+    if supported =~ /command not found/
+      Pipeline.notify "Install eslint and the scanjs .eslintrc"
+      return false
+    else
+      return true
+    end
+  end
+
+end
+

data/lib/pipeline/tasks/fim.rb

@@ -0,0 +1,61 @@
+# https://github.com/jessek/hashdeep/releases/tag/release-4.4
+
+require 'pipeline/tasks/base_task'
+require 'open3'
+
+class Pipeline::FIM < Pipeline::BaseTask
+
+  Pipeline::Tasks.add self
+
+  def initialize(trigger, tracker)
+    super(trigger,tracker)
+    @name = "FIM"
+    @description = "File integrity monitor"
+    @stage = :file
+    @result = ''
+    @labels << "filesystem"
+  end
+
+  def run
+    Pipeline.notify "#{@name}"
+    rootpath = @trigger.path
+    if File.exists?("/area81/tmp/#{rootpath}/filehash")
+      Pipeline.notify "File Hashes found, comparing to file system"
+      cmd="hashdeep -j99 -r -a -vv -k /area81/tmp/#{rootpath}/filehash #{rootpath}"
+
+      # Ugly stdout parsing
+      r=/(.*): No match/
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+        while line = stdout.gets
+          if line.match r
+            @result << line
+          end
+        end
+      end
+    else
+      Pipeline.notify "No existing baseline - generating initial hashes"
+      cmd="mkdir -p /area81/tmp/#{rootpath}; hashdeep -j99 -r #{rootpath} > /area81/tmp/#{rootpath}/filehash"
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+        while line = stdout.gets
+          puts "."
+          end
+      end
+      @result = ''
+    end
+  end
+
+  def analyze
+    list = @result.split(/\n/)
+    list.each do |v|
+       # v.slice! installdir
+       Pipeline.notify v
+       report "File changed.", v, @name, :low
+    end
+  end
+
+  def supported?
+    # In future, verify tool is available.
+    return true
+  end
+
+end

data/lib/pipeline/tasks/nsp.rb

@@ -0,0 +1,59 @@
+require 'pipeline/tasks/base_task'
+require 'pipeline/util'
+
+class Pipeline::NodeSecurityProject < Pipeline::BaseTask
+
+  Pipeline::Tasks.add self
+  include Pipeline::Util
+
+  def initialize(trigger, tracker)
+    super(trigger, tracker)
+    @name = "NodeSecurityProject"
+    @description = "Node Security Project"
+    @stage = :code
+    @labels << "code"
+  end
+
+  def run
+    Pipeline.notify "#{@name}"
+    rootpath = @trigger.path
+    Dir.chdir("#{rootpath}") do
+      @results = JSON.parse `nsp check --output json 2>&1`
+    end
+  end
+
+  def analyze
+    begin
+      # This block iterates through each package name found and selects the unique nsp advisories
+      # regardless of version, and builds a pipeline finding hash for each unique package/advisory combo.
+      @results.uniq {|finding| finding['module']}.each do |package|
+        @results.select {|f| f['module'] == package['module']}.uniq {|m| m['advisory']}.each do |unique_finding|
+          description = "#{unique_finding['module']} - #{unique_finding['title']}"
+          detail = "Upgrade to versions: #{unique_finding['patched_versions']}\n#{unique_finding['advisory']}"
+          source = {
+            :scanner => 'NodeSecurityProject',
+            :file => "#{unique_finding['module']} - #{unique_finding['vulnerable_versions']}",
+            :line => nil,
+            :code => nil
+          }
+          report description, detail, source, 'medium', fingerprint("#{description}#{detail}#{source}")
+        end
+      end
+    rescue Exception => e
+      Pipeline.warn e.message
+      Pipeline.warn e.backtrace
+    end
+  end
+
+  def supported?
+    supported=runsystem(true, "nsp", "--version")
+    if supported =~ /command not found/
+      Pipeline.notify "Install nodesecurity: 'npm install -g nsp'"
+      return false
+    else
+      return true
+    end
+  end
+
+end
+

data/lib/pipeline/tasks/owasp-dep-check.rb

@@ -0,0 +1,120 @@
+require 'pipeline/tasks/base_task'
+require 'pipeline/util'
+require 'rexml/document'
+require 'rexml/streamlistener'
+include REXML
+
+# SAX Like Parser for OWASP DEP CHECK XML.
+class Pipeline::DepCheckListener
+  include StreamListener
+  
+  def initialize(task)
+    @task = task
+    @count = 0
+    @sw = ""
+    @url = ""
+    @desc = ""
+    @cwe = ""
+    @cvss = ""
+    @name = ""
+    @fingerprint = ""
+  end
+  
+  def tag_start(name, attrs)
+    case name
+    when "vulnerability"
+      @count = @count + 1 
+      # Pipeline.debug "Grabbed #{@count} vulns."
+      @sw = ""
+      @url = ""
+      @desc = ""
+      @cwe = ""
+      @cvss = ""
+      @name = ""
+      @fingerprint = ""
+    end
+  end
+  
+  def tag_end(name)
+    case name
+    when "name"
+      if @text =~ /\D/
+        @name = @text
+      end
+    when "cvssScore"
+      @cvss = @text
+    when "cwe"
+      @cwe = @text
+    when "description"
+      @desc = @text
+    when "vulnerableSoftware"
+      @sw = ""
+    when "software"
+      @sw << ", " << @text
+    when "url"
+      @url << ", " << @text
+    when "vulnerability"
+      detail = @sw + "\n"+ @url
+      description = @desc + "\n" + @cwe
+      @fingerprint = @sw+"-"+@name
+      puts "Fingerprint: #{@fingerprint}"
+      puts "Vuln: #{@name} CVSS: #{@cvss} Description #{description} Detail #{detail}"
+      @task.report @name, description, detail, @cvss, @fingerprint
+    end
+  end
+  
+  def text(text)
+    @text = text
+  end
+end
+
+class Pipeline::OWASPDependencyCheck < Pipeline::BaseTask
+  
+  Pipeline::Tasks.add self
+  include Pipeline::Util
+
+  def initialize(trigger,tracker)
+    super(trigger,tracker)
+    @name = "OWASP Dependency Check"
+    @description = "Dependency analysis for Java and .NET"
+    @stage = :code
+    @labels << "code" << "java" << ".net"
+  end
+  
+  def run
+    Pipeline.notify "#{@name}"
+    rootpath = @trigger.path
+    @result= runsystem(true, "/home/pipe/line/tools/dependency-check/bin/dependency-check.sh", "-a", "pipeline", "-f", "XML", "-out", "#{rootpath}", "-s", "#{rootpath}")
+  end
+
+  def analyze
+    path = @trigger.path + "/dependency-check-report.xml"
+    begin
+      Pipeline.debug "Parsing report #{path}"
+      get_warnings(path)
+    rescue Exception => e
+      Pipeline.notify "Problem running OWASP Dep Check ... skipped."
+      Pipeline.notify e.message
+      raise e
+    end
+  end
+
+  def supported?
+    supported=runsystem(true, "/home/pipe/line/tools//dependency-check/bin/dependency-check.sh", "-v")
+    if supported =~ /command not found/
+      Pipeline.notify "Install dependency-check."
+      return false
+    else
+      return true
+    end
+  end
+
+  def get_warnings(path)
+    listener = Pipeline::DepCheckListener.new(self)
+    parser = Parsers::StreamParser.new(File.new(path), listener)
+    parser.parse
+  end
+end
+
+
+

data/lib/pipeline/tasks/patterns.json

@@ -0,0 +1,394 @@
+[
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A.*_rsa\\z",
+        "caption": "Private SSH key",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A.*_dsa\\z",
+        "caption": "Private SSH key",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A.*_ed25519\\z",
+        "caption": "Private SSH key",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A.*_ecdsa\\z",
+        "caption": "Private SSH key",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "pem",
+        "caption": "Potential cryptographic private key",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "regex",
+        "pattern": "\\Akey(pair)?\\z",
+        "caption": "Potential cryptographic private key",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "pkcs12",
+        "caption": "Potential cryptographic key bundle",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "pfx",
+        "caption": "Potential cryptographic key bundle",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "p12",
+        "caption": "Potential cryptographic key bundle",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "asc",
+        "caption": "Potential cryptographic key bundle",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "otr.private_key",
+        "caption": "Pidgin OTR private key",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?(bash_|zsh_|z)?history\\z",
+        "caption": "Shell command history file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?mysql_history\\z",
+        "caption": "MySQL client command history file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?psql_history\\z",
+        "caption": "PostgreSQL client command history file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?irb_history\\z",
+        "caption": "Ruby IRB console history file",
+        "description": null
+    },
+    {
+        "part": "path",
+        "type": "regex",
+        "pattern": "\\.?purple\\/accounts\\.xml\\z",
+        "caption": "Pidgin chat client account configuration file",
+        "description": null
+    },
+    {
+        "part": "path",
+        "type": "regex",
+        "pattern": "\\.?xchat2?\\/servlist_?\\.conf\\z",
+        "caption": "Hexchat/XChat IRC client server list configuration file",
+        "description": null
+    },
+    {
+        "part": "path",
+        "type": "regex",
+        "pattern": "\\.?irssi\\/config\\z",
+        "caption": "Irssi IRC client configuration file",
+        "description": null
+    },
+    {
+        "part": "path",
+        "type": "regex",
+        "pattern": "\\.?recon-ng\\/keys\\.db\\z",
+        "caption": "Recon-ng web reconnaissance framework API key database",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?dbeaver-data-sources.xml\\z",
+        "caption": "DBeaver SQL database manager configuration file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?muttrc\\z",
+        "caption": "Mutt e-mail client configuration file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?s3cfg\\z",
+        "caption": "S3cmd configuration file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?trc\\z",
+        "caption": "T command-line Twitter client configuration file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "ovpn",
+        "caption": "OpenVPN client configuration file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?gitrobrc\\z",
+        "caption": "Well, this is awkward... Gitrob configuration file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?(bash|zsh)rc\\z",
+        "caption": "Shell configuration file",
+        "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys."
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?(bash_|zsh_)?profile\\z",
+        "caption": "Shell profile configuration file",
+        "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys."
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?(bash_|zsh_)?aliases\\z",
+        "caption": "Shell command alias configuration file",
+        "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys."
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "secret_token.rb",
+        "caption": "Ruby On Rails secret token configuration file",
+        "description": "If the Rails secret token is known, it can allow for remote code execution. (http://www.exploit-db.com/exploits/27527/)"
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "omniauth.rb",
+        "caption": "OmniAuth configuration file",
+        "description": "The OmniAuth configuration file might contain client application secrets."
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "carrierwave.rb",
+        "caption": "Carrierwave configuration file",
+        "description": "Can contain credentials for online storage systems such as Amazon S3 and Google Storage."
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "schema.rb",
+        "caption": "Ruby On Rails database schema file",
+        "description": "Contains information on the database schema of a Ruby On Rails application."
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "database.yml",
+        "caption": "Potential Ruby On Rails database configuration file",
+        "description": "Might contain database credentials."
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "settings.py",
+        "caption": "Django configuration file",
+        "description": "Might contain database credentials, online storage system credentials, secret keys, etc."
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A(.*)?config(\\.inc)?\\.php\\z",
+        "caption": "PHP configuration file",
+        "description": "Might contain credentials and keys."
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "kdb",
+        "caption": "KeePass password manager database file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "agilekeychain",
+        "caption": "1Password password manager database file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "keychain",
+        "caption": "Apple Keychain database file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "regex",
+        "pattern": "\\Akey(store|ring)\\z",
+        "caption": "GNOME Keyring database file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "log",
+        "caption": "Log file",
+        "description": "Log files might contain information such as references to secret HTTP endpoints, session IDs, user information, passwords and API keys."
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "pcap",
+        "caption": "Network traffic capture file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "regex",
+        "pattern": "\\Asql(dump)?\\z",
+        "caption": "SQL dump file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "gnucash",
+        "caption": "GnuCash database file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "backup",
+        "caption": "Contains word: backup",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "dump",
+        "caption": "Contains word: dump",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "password",
+        "caption": "Contains word: password",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "private.*key",
+        "caption": "Contains words: private, key",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml",
+        "caption": "Jenkins publish over SSH plugin file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "credentials.xml",
+        "caption": "Potential Jenkins credentials file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?htpasswd\\z",
+        "caption": "Apache htpasswd file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A\\.?netrc\\z",
+        "caption": "Configuration file for auto-login process",
+        "description": "Might contain username and password."
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "kwallet",
+        "caption": "KDE Wallet Manager database file",
+        "description": null
+    },
+    {
+        "part": "filename",
+        "type": "match",
+        "pattern": "LocalSettings.php",
+        "caption": "Potential MediaWiki configuration file",
+        "description": null
+    },
+    {
+        "part": "extension",
+        "type": "match",
+        "pattern": "tblk",
+        "caption": "Tunnelblick VPN configuration file",
+        "description": null
+    },
+    {
+        "part": "path",
+        "type": "regex",
+        "pattern": "\\A\\.?gem/credentials\\z",
+        "caption": "Rubygems credentials file",
+        "description": "Might contain API key for a rubygems.org account."
+    },
+    {
+        "part": "filename",
+        "type": "regex",
+        "pattern": "\\A*\\.pubxml(\\.user)?\\z",
+        "caption": "Potential MSBuild publish profile",
+        "description": null
+    }
+]