trapline 0.1.0

data/lib/trapline/rule.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Trapline
+  class Rule
+    SEVERITIES = %w[CRITICAL HIGH MEDIUM LOW INFO ERROR WARNING INVENTORY EXPERIMENT].freeze
+    MODES = %w[search taint join extract].freeze
+    PATTERN_KEYS = %w[pattern patterns pattern-either pattern-regex match taint].freeze
+
+    KEY_ORDER = %w[id message severity languages mode options pattern patterns pattern-either
+                   pattern-regex match pattern-sources pattern-propagators pattern-sanitizers
+                   pattern-sinks paths fix fix-regex metadata].freeze
+    LIST_METADATA = %w[cwe cve references owasp].freeze
+
+    attr_reader :data
+
+    def initialize(data = {})
+      @data = stringify(data)
+    end
+
+    def self.parse(yaml)
+      RuleFile.parse(yaml).rules.first || raise(ParseError, "no rule found")
+    end
+
+    %w[id message severity languages mode fix].each do |key|
+      define_method(key) { @data[key] }
+      define_method("#{key}=") { |v| @data[key] = v }
+    end
+
+    def metadata
+      @data["metadata"] ||= {}
+    end
+
+    def taint?
+      @data["mode"] == "taint"
+    end
+
+    def primary_language
+      Array(languages).first
+    end
+
+    def [](key)
+      @data[key.to_s]
+    end
+
+    def []=(key, value)
+      @data[key.to_s] = value
+    end
+
+    def merge_fields(fields)
+      fields = stringify(fields)
+      %w[id message severity fix].each do |k|
+        next unless fields.key?(k)
+
+        v = fields[k].to_s
+        v.strip.empty? ? @data.delete(k) : @data[k] = v
+      end
+      @data["languages"] = split_list(fields["languages"]) if fields.key?("languages")
+
+      (fields["metadata"] || {}).each do |k, v|
+        value = LIST_METADATA.include?(k) ? split_list(v) : v
+        value.nil? || (value.respond_to?(:empty?) && value.empty?) ? metadata.delete(k) : metadata[k] = value
+      end
+      @data.delete("metadata") if metadata.empty?
+      self
+    end
+
+    def to_h
+      known = KEY_ORDER.select { |k| @data.key?(k) }.to_h { |k| [k, @data[k]] }
+      known.merge(@data.except(*KEY_ORDER))
+    end
+
+    def to_yaml
+      RuleFile.new([self]).to_yaml
+    end
+
+    private
+
+    def split_list(value)
+      case value
+      when Array then value.map { |v| v.to_s.strip }.reject(&:empty?)
+      when nil then []
+      else value.to_s.split(/[,\n]/).map(&:strip).reject(&:empty?)
+      end
+    end
+
+    def stringify(obj)
+      case obj
+      when Hash then obj.to_h { |k, v| [k.to_s, stringify(v)] }
+      when Array then obj.map { |v| stringify(v) }
+      else obj
+      end
+    end
+  end
+
+  class RuleFile
+    attr_reader :rules
+
+    def initialize(rules = [])
+      @rules = rules
+    end
+
+    def self.parse(yaml)
+      doc = begin
+        YAML.safe_load(yaml.to_s, aliases: false)
+      rescue Psych::SyntaxError => e
+        raise ParseError.new("invalid YAML: #{e.problem} (line #{e.line}, column #{e.column})",
+                             line: e.line, column: e.column)
+      end
+
+      list = case doc
+             when Hash
+               if doc.key?("rules") then doc["rules"]
+               elsif doc.key?("id") then [doc]
+               else raise ParseError, "the document must have a `rules:` key"
+               end
+             when Array then doc
+             when nil then raise ParseError, "empty document"
+             else raise ParseError, "the document must be a map with `rules:`"
+             end
+      raise ParseError, "`rules` must be a list" unless list.is_a?(Array)
+
+      new(list.map { |r| r.is_a?(Hash) ? Rule.new(r) : raise(ParseError, "each item in `rules` must be a map") })
+    end
+
+    def to_yaml
+      Yaml.dump({ "rules" => rules.map(&:to_h) })
+    end
+  end
+
+  module Yaml
+    module_function
+
+    def dump(obj)
+      ast = Psych::Visitors::YAMLTree.create
+      ast << obj
+      ast.tree.each do |node|
+        next unless node.is_a?(Psych::Nodes::Scalar) && node.value.is_a?(String)
+
+        if node.value.include?("\n") && node.value.lines.none? { |l| l.end_with?(" \n") }
+          node.style = Psych::Nodes::Scalar::LITERAL
+          node.plain = false
+          node.quoted = true
+        elsif node.style == Psych::Nodes::Scalar::DOUBLE_QUOTED && plain_safe?(node.value)
+          # Psych quotes anything that starts with a symbol ($X, (...)); semgrep
+          # patterns are more readable without quotes when YAML allows it.
+          node.style = Psych::Nodes::Scalar::PLAIN
+          node.plain = true
+          node.quoted = false
+        end
+      end
+      ast.tree.yaml(nil, line_width: -1).sub(/\A---\s*\n/, "")
+    end
+
+    def plain_safe?(value)
+      return false if value.empty? || value.include?("\n") || value != value.strip
+      return false if value.match?(/\A[-?:,\[\]{}#&*!|>'"%@`]/) || value.include?(": ") || value.include?(" #")
+
+      YAML.safe_load("k: #{value}") == { "k" => value }
+    rescue Psych::Exception
+      false
+    end
+  end
+end

data/lib/trapline/runtime.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Trapline
+  module Runtime
+    DOTNET = { name: "dotnet", images: %w[mcr.microsoft.com/dotnet/runtime mcr.microsoft.com/dotnet/aspnet] }.freeze
+    RUNTIMES = {
+      "php:php" => { name: "php", images: %w[php] },
+      "python:python" => { name: "python", images: %w[python] },
+      "python:cpython" => { name: "python", images: %w[python] },
+      "nodejs:node.js" => { name: "node", images: %w[node] },
+      "ruby-lang:ruby" => { name: "ruby", images: %w[ruby] },
+      "golang:go" => { name: "golang", images: %w[golang] },
+      "microsoft:.net" => DOTNET,
+      "microsoft:.net_core" => DOTNET,
+      "microsoft:asp.net_core" => { name: "aspnet", images: %w[mcr.microsoft.com/dotnet/aspnet] }
+    }.freeze
+
+    Interval = Struct.new(:low, :high) # [low, high); high nil = unbounded
+
+    module_function
+
+    def runtime_for(product)
+      RUNTIMES[Cve.product_key(product)]
+    end
+
+    # CPE ranges as [major, minor, patch] intervals. Non-numeric versions
+    # (8.2.0rc1, 2.7.x) are skipped.
+    def intervals(ranges)
+      Array(ranges).filter_map do |r|
+        if r["exact"]
+          v = parse(r["exact"]) or next
+          next Interval.new(v, bump(v))
+        end
+
+        low = bound(r["start_including"], r["start_excluding"]) || ([0, 0, 0] unless r["start_including"] || r["start_excluding"])
+        high = bound(r["end_excluding"], r["end_including"])
+        next if low.nil? || ((r["end_excluding"] || r["end_including"]) && high.nil?)
+        next if high && (low <=> high) >= 0
+
+        Interval.new(low, high)
+      end
+    end
+
+    # Half-open bound: `as_is` is used directly, `bumped` is moved one patch up.
+    def bound(as_is, bumped)
+      as_is ? parse(as_is) : bumped && bump(parse(bumped))
+    end
+
+    def parse(version)
+      return nil unless version.to_s.match?(/\A\d+(\.\d+){0,2}\z/)
+
+      parts = version.split(".").map(&:to_i)
+      parts + [0] * (3 - parts.size)
+    end
+
+    def bump(v)
+      v && [v[0], v[1], v[2] + 1]
+    end
+
+    def format(v)
+      v.join(".")
+    end
+
+    # Partial tags (`8`, `8.1`) only match when every version under them is
+    # affected: `php:7.4` is flagged, `php:8.2` is not when 8.2.35 fixed it.
+    def version_regex(intervals)
+      intervals.map { |i| range_regex(i.low, i.high) }.uniq.join("|")
+    end
+
+    def range_regex(low, high)
+      n = low.size
+      a = low[0]
+      d = high&.first
+      return int_range(a, d && d - 1) if n == 1
+      return "#{a}\\.(?:#{range_regex(low.drop(1), high.drop(1))})" if d == a
+
+      alts = []
+      if low.drop(1).all?(&:zero?)
+        first_full = a
+      else
+        alts << "#{a}\\.(?:#{range_regex(low.drop(1), nil)})"
+        first_full = a + 1
+      end
+      last_full = d && d - 1
+      if last_full.nil? || first_full <= last_full
+        alts << "(?:#{int_range(first_full, last_full)})(?:\\.\\d+){0,#{n - 1}}"
+      end
+      alts << "#{d}\\.(?:#{range_regex([0] * (n - 1), high.drop(1))})" if d && !high.drop(1).all?(&:zero?)
+      alts.join("|")
+    end
+
+    def int_range(lo, hi)
+      return (lo..hi).to_a.join("|") if hi
+
+      digits = lo.to_s.size
+      [*(lo..(10**digits - 1)), "\\d{#{digits + 1},}"].join("|")
+    end
+
+    def image_rule(product, rule_id)
+      rt = runtime_for(product) or return nil
+      ivs = intervals(product["versions"])
+      return nil if ivs.empty?
+
+      fixed = ivs.filter_map { |i| i.high && format(i.high) }.uniq
+      { name: rt[:name], images: rt[:images], body: { "pattern-regex" => image_regex(rt[:images], ivs) },
+        test: test_code(rt[:images].first, ivs, rule_id), fixed: fixed }
+    end
+
+    # Matches `<image>:<tag>` as a whole token (not `myorg/php:8.1` nor
+    # `php:8.1.2x`) and Helm's `repository: <image>` followed within a few
+    # lines by `tag: <tag>`. The finding starts at the image name.
+    def image_regex(images, ivs)
+      name = images.map { |i| i.include?("/") ? Regexp.escape(i) : "(?:docker\\.io/)?(?:library/)?#{Regexp.escape(i)}" }
+                   .join("|")
+      tag = "(?:#{version_regex(ivs)})(?:-[\\w.-]+)?(?![\\w.-])"
+      ref = "(?<![\\w./@-])(?:#{name}):#{tag}"
+      helm = "repository:[ \\t]*[\"']?(?:#{name})[\"']?[ \\t]*\\r?\\n" \
+             "(?:(?![ \\t]*(?:repository|tag):)[^\\n]*\\n){0,3}[ \\t]*tag:[ \\t]*[\"']?#{tag}"
+      "#{ref}|#{helm}"
+    end
+
+    def test_code(image, ivs, rule_id)
+      lines = []
+      ivs.first(4).each do |i|
+        lines << "# ruleid: #{rule_id}" << "FROM #{image}:#{format(vulnerable(i))}-alpine"
+        lines << "# ok: #{rule_id}" << "FROM #{image}:#{format(i.high)}-alpine" if i.high
+      end
+      i = ivs.first
+      v = format(vulnerable(i))
+      lines << "" << "# docker-compose.yml, Kubernetes, CI" << "services:" << "  app:"
+      lines << "    # ruleid: #{rule_id}" << "    image: #{image}:#{v}"
+      lines << "    # ok: #{rule_id}" << "    image: myorg/#{image.split('/').last}:#{v}"
+      lines << "containers:" << "  # ruleid: #{rule_id}" << "  - image: \"#{image}:#{v}\""
+      lines << "  # ok: #{rule_id}" << "  - image: \"#{image}:#{format(i.high)}\"" if i.high
+      lines << "" << "# Helm values.yaml" << "image:" << "  # ruleid: #{rule_id}" << "  repository: #{image}"
+      lines << "  pullPolicy: IfNotPresent" << "  tag: \"#{v}\""
+      if i.high
+        lines << "sidecar:" << "  # ok: #{rule_id}" << "  repository: #{image}" << "  tag: \"#{format(i.high)}\""
+      end
+      "#{lines.join("\n")}\n"
+    end
+
+    def vulnerable(interval)
+      high = interval.high
+      high && high[2].positive? ? [*high[0, 2], high[2] - 1] : interval.low
+    end
+  end
+end

data/lib/trapline/store.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Trapline
+  class Store
+    SLUG = %r{\A[A-Za-z0-9][A-Za-z0-9._\-/]*\z}
+    RULE_EXT = %w[.yaml .yml].freeze
+
+    attr_reader :root
+
+    def initialize(root = Trapline.config.rules_path)
+      @root = File.expand_path(root.to_s)
+    end
+
+    def list
+      Dir.glob(File.join(root, "**", "*.{yaml,yml}")).sort.map { |path| summary(slug_for(path), path) }
+    end
+
+    def exists?(slug)
+      !rule_path(slug).nil?
+    end
+
+    def read(slug)
+      path = rule_path(slug) || raise(NotFound, "rule not found: #{slug}")
+      test = test_path(slug)
+      yaml = File.read(path)
+      { slug: slug, yaml: yaml, test_code: test && File.read(test), test_language: test && test_language(test, yaml),
+        updated_at: File.mtime(path).utc.iso8601 }
+    end
+
+    def write(slug, yaml, test_code: nil, language: nil)
+      check_slug!(slug)
+      path = rule_path(slug) || File.join(root, "#{slug}.yaml")
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, yaml.end_with?("\n") ? yaml : "#{yaml}\n")
+
+      unless test_code.to_s.strip.empty?
+        ext = Languages.extension_for(language || safe_language(yaml))
+        test_file = File.join(File.dirname(path), "#{File.basename(slug)}.#{ext}")
+        (test_paths(slug) - [test_file]).each { |p| File.delete(p) }
+        File.write(test_file, test_code)
+      end
+      read(slug)
+    end
+
+    def delete(slug)
+      path = rule_path(slug) || raise(NotFound, "rule not found: #{slug}")
+      File.delete(path)
+      test_paths(slug).each { |p| File.delete(p) }
+      true
+    end
+
+    # Registry-style ids are named after their last segment, like the
+    # semgrep/opengrep rules repos.
+    def suggest_slug(id)
+      name = id.to_s.split(".").reject(&:empty?).last.to_s
+      base = name.downcase.gsub(%r{[^a-z0-9_/-]+}, "-").gsub(%r{\A[-/]+|[-/]+\z}, "")
+      base = "rule" if base.empty?
+      (1..).lazy.map { |n| n == 1 ? base : "#{base}-#{n}" }.find { |slug| !exists?(slug) }
+    end
+
+    private
+
+    def summary(slug, path)
+      rule = RuleFile.parse(File.read(path)).rules.first
+      meta = rule&.data&.dig("metadata") || {}
+      { slug: slug, id: rule&.id, message: rule&.message.to_s.lines.first.to_s.strip,
+        severity: rule&.severity, languages: Array(rule&.languages),
+        cve: Array(meta["cve"]), cwe: Array(meta["cwe"]), has_test: !test_path(slug).nil?,
+        updated_at: File.mtime(path).utc.iso8601 }
+    rescue ParseError => e
+      { slug: slug, id: nil, error: e.message, updated_at: File.mtime(path).utc.iso8601 }
+    end
+
+    def slug_for(path)
+      path.delete_prefix("#{root}/").sub(/\.ya?ml\z/, "")
+    end
+
+    def check_slug!(slug)
+      raise InvalidSlug, "invalid name: #{slug.inspect}" unless slug.to_s.match?(SLUG) && !slug.include?("..")
+    end
+
+    def rule_path(slug)
+      check_slug!(slug)
+      RULE_EXT.map { |e| File.join(root, "#{slug}#{e}") }.find { |p| File.file?(p) }
+    end
+
+    def test_paths(slug)
+      check_slug!(slug)
+      Dir.glob(File.join(root, "#{slug}.*")).reject { |p| RULE_EXT.include?(File.extname(p)) || p.end_with?(".fixed") }
+    end
+
+    def test_path(slug)
+      test_paths(slug).first
+    end
+
+    # generic and regex share .txt: the rule's own language wins.
+    def test_language(test, yaml)
+      ext = File.extname(test).delete_prefix(".").downcase
+      own = Languages.find(safe_language(yaml))
+      return own.key if own&.extension == ext
+
+      Languages.from_extension(ext)&.key
+    end
+
+    def safe_language(yaml)
+      RuleFile.parse(yaml).rules.first&.primary_language
+    rescue ParseError
+      nil
+    end
+  end
+end

data/lib/trapline/suite.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Trapline
+  class Suite
+    Result = Struct.new(:slug, :status, :passed, :total, :errors, :failures, keyword_init: true)
+
+    def initialize(store: Trapline.store, engine: Trapline.engine)
+      @store = store
+      @engine = engine
+    end
+
+    def run
+      raise Error, "no engine found: install opengrep or semgrep" unless @engine
+
+      @store.list.map { |entry| run_one(entry) }
+    end
+
+    private
+
+    def run_one(entry)
+      slug = entry[:slug]
+      return result(slug, "error", errors: [entry[:error]]) if entry[:error]
+
+      rule = @store.read(slug)
+      validation = Validator.new.validate(rule[:yaml])
+      return result(slug, "error", errors: validation.errors.map { |i| "#{i.path}: #{i.message}" }) unless validation.valid?
+      return result(slug, "skipped") unless rule[:test_code]
+
+      res = @engine.run(rule[:yaml], code: rule[:test_code], language: rule[:test_language])
+      ann = res.annotations || { passed: 0, total: 0, results: [] }
+      failures = ann[:results].select { |r| r[:status] == "fail" }.map { |r| r[:reason] }
+      status = if res.errors.any? then "error"
+               elsif ann[:total].zero? then "no-annotations"
+               elsif failures.any? then "fail"
+               else "pass"
+               end
+      result(slug, status, passed: ann[:passed], total: ann[:total], errors: res.errors, failures: failures)
+    end
+
+    def result(slug, status, passed: 0, total: 0, errors: [], failures: [])
+      Result.new(slug: slug, status: status, passed: passed, total: total, errors: errors, failures: failures)
+    end
+  end
+end

data/lib/trapline/templates/cwe/cwe-22.yml

@@ -0,0 +1,167 @@
+cwe: CWE-22
+name: "Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')"
+title: Path traversal
+owasp: ["A01:2021 - Broken Access Control"]
+keywords: ["path traversal", "directory traversal", "../", "arbitrary file read", "arbitrary files"]
+message: >-
+  Untrusted input is used to build a file path, which can allow reading or
+  writing files outside the intended directory. Normalize the path and check
+  that it stays inside an allowed base directory.
+languages:
+  ruby:
+    rule: |
+      mode: taint
+      pattern-sources:
+        - pattern: params
+        - pattern: cookies
+      pattern-sanitizers:
+        - pattern: File.basename(...)
+      pattern-sinks:
+        - patterns:
+            - pattern-either:
+                - pattern: File.read($P, ...)
+                - pattern: File.open($P, ...)
+                - pattern: File.readlines($P, ...)
+                - pattern: File.write($P, ...)
+                - pattern: IO.read($P, ...)
+                - pattern: send_file($P, ...)
+                - pattern: Rails.root.join(..., $P, ...)
+            - focus-metavariable: $P
+    test: |
+      class FilesController < ApplicationController
+        def show
+          # ruleid: __RULE_ID__
+          send_file Rails.root.join("uploads", params[:name])
+          # ruleid: __RULE_ID__
+          data = File.read("/var/data/#{params[:name]}")
+          # ok: __RULE_ID__
+          data = File.read("/var/data/#{File.basename(params[:name])}")
+        end
+      end
+  python:
+    rule: |
+      mode: taint
+      pattern-sources:
+        - pattern: flask.request.$ANY
+        - patterns:
+            - pattern-inside: |
+                def $VIEW(request, ...):
+                  ...
+            - pattern: request.$ANY
+      pattern-sanitizers:
+        - pattern: os.path.basename(...)
+        - pattern: werkzeug.utils.secure_filename(...)
+      pattern-sinks:
+        - pattern: open(...)
+        - pattern: flask.send_file(...)
+        - pattern: os.remove(...)
+        - pattern: shutil.rmtree(...)
+    test: |
+      import os
+      from flask import request, send_file
+      from werkzeug.utils import secure_filename
+
+      def download():
+          name = request.args.get("file")
+          # ruleid: __RULE_ID__
+          return send_file(os.path.join("/srv/files", name))
+
+      def download_safe():
+          name = secure_filename(request.args.get("file"))
+          # ok: __RULE_ID__
+          return send_file(os.path.join("/srv/files", name))
+  javascript:
+    rule: |
+      mode: taint
+      pattern-sources:
+        - pattern: $REQ.query
+        - pattern: $REQ.body
+        - pattern: $REQ.params
+      pattern-sanitizers:
+        - pattern: path.basename(...)
+      pattern-sinks:
+        - patterns:
+            - pattern-either:
+                - pattern: fs.$FN($P, ...)
+                - pattern: $RES.sendFile($P, ...)
+                - pattern: $RES.download($P, ...)
+            - focus-metavariable: $P
+    test: |
+      const fs = require("fs");
+      const path = require("path");
+
+      app.get("/file", (req, res) => {
+        // ruleid: __RULE_ID__
+        fs.readFile("/srv/files/" + req.query.name, (e, d) => res.send(d));
+        // ok: __RULE_ID__
+        fs.readFile(path.join("/srv/files", path.basename(req.query.name)), (e, d) => res.send(d));
+      });
+  java:
+    rule: |
+      mode: taint
+      pattern-sources:
+        - pattern: (javax.servlet.http.HttpServletRequest $REQ).$GET(...)
+        - pattern: (jakarta.servlet.http.HttpServletRequest $REQ).$GET(...)
+        - patterns:
+            - pattern-inside: |
+                $RET $METHOD(..., @RequestParam $T $P, ...) { ... }
+            - pattern: $P
+      pattern-sanitizers:
+        - pattern: FilenameUtils.getName(...)
+        - pattern: (java.nio.file.Path $P).getFileName()
+      pattern-sinks:
+        - pattern: new File(...)
+        - pattern: new FileInputStream(...)
+        - pattern: new FileOutputStream(...)
+        - pattern: new FileReader(...)
+        - pattern: Paths.get(...)
+        - pattern: Path.of(...)
+    test: |
+      import javax.servlet.http.HttpServletRequest;
+      import java.io.File;
+      import java.io.FileInputStream;
+      import org.apache.commons.io.FilenameUtils;
+
+      public class DownloadServlet {
+          public void doGet(HttpServletRequest request) throws Exception {
+              String name = request.getParameter("file");
+              // ruleid: __RULE_ID__
+              FileInputStream in = new FileInputStream("/srv/files/" + name);
+              // ok: __RULE_ID__
+              File safe = new File("/srv/files/" + FilenameUtils.getName(name));
+          }
+      }
+  go:
+    rule: |
+      mode: taint
+      pattern-sources:
+        - pattern: '($R : *http.Request).URL.Query().Get(...)'
+        - pattern: '($R : *http.Request).FormValue(...)'
+      pattern-sanitizers:
+        - pattern: filepath.Base(...)
+      pattern-sinks:
+        - pattern: os.Open(...)
+        - pattern: os.OpenFile(...)
+        - pattern: os.ReadFile(...)
+        - pattern: ioutil.ReadFile(...)
+        - pattern: os.Remove(...)
+        - patterns:
+            - pattern: http.ServeFile($W, $REQ, $P)
+            - focus-metavariable: $P
+    test: |
+      package main
+
+      import (
+      	"net/http"
+      	"os"
+      	"path/filepath"
+      )
+
+      func download(w http.ResponseWriter, r *http.Request) {
+      	name := r.URL.Query().Get("file")
+      	// ruleid: __RULE_ID__
+      	data, _ := os.ReadFile("/srv/files/" + name)
+      	// ok: __RULE_ID__
+      	safe, _ := os.ReadFile(filepath.Join("/srv/files", filepath.Base(name)))
+      	w.Write(append(data, safe...))
+      }