codeclimate-fede 0.85.21

data/lib/cc/analyzer/container/result.rb

@@ -0,0 +1,74 @@
+module CC
+  module Analyzer
+    class Container
+      class Result
+        attr_reader \
+          :container_name,
+          :duration,
+          :exit_status,
+          :output_byte_count,
+          :stderr,
+          :stdout
+
+        def initialize(
+          container_name: "",
+          duration: 0,
+          exit_status: 0,
+          maximum_output_exceeded: false,
+          output_byte_count: 0,
+          skipped: false,
+          stderr: "",
+          stdout: "",
+          timed_out: false
+        )
+          @container_name = container_name
+          @duration = duration
+          @exit_status = exit_status
+          @maximum_output_exceeded = maximum_output_exceeded
+          @output_byte_count = output_byte_count
+          @skipped = skipped
+          @stderr = stderr
+          @stdout = stdout
+          @timed_out = timed_out
+        end
+
+        def self.skipped(ex)
+          new(
+            exit_status: 0,
+            skipped: true,
+            stderr: ex.message,
+          )
+        end
+
+        def merge_from_exception(ex)
+          self.exit_status = 99
+          self.stderr = ex.message
+          self
+        end
+
+        def timed_out?
+          @timed_out
+        end
+
+        def maximum_output_exceeded?
+          @maximum_output_exceeded
+        end
+
+        def errored?
+          timed_out? ||
+            maximum_output_exceeded? ||
+            exit_status.nil? ||
+            exit_status.nonzero?
+        end
+
+        def skipped?
+          @skipped
+        end
+
+        private
+
+        attr_writer :exit_status, :stderr
+      end
+    end
+  end
+end

data/lib/cc/analyzer/container.rb

@@ -0,0 +1,208 @@
+require "posix/spawn"
+require "thread"
+
+require "cc/analyzer/container/result"
+
+module CC
+  module Analyzer
+    #
+    # Running an abstract docker container
+    #
+    # Input:
+    #   - image
+    #   - name
+    #   - command (Optional)
+    #
+    # Output:
+    #   - Result
+    #     - exit_status
+    #     - timed_out?
+    #     - duration
+    #     - maximum_output_exceeded?
+    #     - output_byte_count
+    #     - stderr
+    #
+    # Never raises (unless broken)
+    #
+    class Container
+      DEFAULT_TIMEOUT = 15 * 60 # 15m
+      DEFAULT_MAXIMUM_OUTPUT_BYTES = 500_000_000
+
+      def initialize(image:, name:, command: nil)
+        @image = image
+        @name = name
+        @command = command
+        @timed_out = false
+        @maximum_output_exceeded = false
+        @stdout_io = StringIO.new
+        @stderr_io = StringIO.new
+        @output_byte_count = 0
+        @counter_mutex = Mutex.new
+
+        # By default accumulate and include stdout in result
+        @output_delimeter = "\n"
+        @on_output = ->(output) { @stdout_io.puts(output) }
+      end
+
+      def on_output(delimeter = "\n", &block)
+        @output_delimeter = delimeter
+        @on_output = block
+      end
+
+      def run(options = [])
+        started = Time.now
+
+        command = docker_run_command(options)
+        Analyzer.logger.debug("docker run: #{command.inspect}")
+        pid, _, out, err = POSIX::Spawn.popen4(*command)
+
+        @t_out = read_stdout(out)
+        @t_err = read_stderr(err)
+        t_timeout = timeout_thread
+
+        # blocks until the engine stops. this is put in a thread so that we can
+        # explicitly abort it as part of #stop. otherwise a run-away container
+        # could still block here forever if the docker-kill/wait is not
+        # successful. there may still be stdout in flight if it was being
+        # produced more quickly than consumed.
+        @t_wait = Thread.new { _, @status = Process.waitpid2(pid) }
+        @t_wait.join
+
+        # blocks until all readers are done. they're still governed by the
+        # timeout thread at this point. if we hit the timeout while processing
+        # output, the threads will be Thread#killed as part of #stop and this
+        # will unblock with the correct value in @timed_out
+        [@t_out, @t_err].each(&:join)
+
+        duration =
+          if @timed_out
+            timeout * 1000
+          else
+            ((Time.now - started) * 1000).round
+          end
+
+        Result.new(
+          container_name: @name,
+          duration: duration,
+          exit_status: @status&.exitstatus,
+          maximum_output_exceeded: @maximum_output_exceeded,
+          output_byte_count: output_byte_count,
+          stderr: @stderr_io.string,
+          stdout: @stdout_io.string,
+          timed_out: @timed_out,
+        )
+      ensure
+        kill_reader_threads
+        t_timeout&.kill
+      end
+
+      def stop(message = nil)
+        reap_running_container(message)
+        kill_reader_threads
+        kill_wait_thread
+      end
+
+      private
+
+      attr_reader :output_byte_count, :counter_mutex
+
+      def docker_run_command(options)
+        [
+          "docker", "run",
+          "--name", @name,
+          options,
+          @image,
+          @command
+        ].flatten.compact
+      end
+
+      def read_stdout(out)
+        Thread.new do
+          out.each_line(@output_delimeter) do |chunk|
+            output = chunk.chomp(@output_delimeter)
+
+            Analyzer.logger.debug("engine stdout: #{output}")
+            @on_output.call(output)
+            check_output_bytes(output.bytesize)
+          end
+        ensure
+          out.close
+        end
+      end
+
+      def read_stderr(err)
+        Thread.new do
+          err.each_line do |line|
+            Analyzer.logger.debug("engine stderr: #{line.chomp}")
+            @stderr_io.write(line)
+            check_output_bytes(line.bytesize)
+          end
+        ensure
+          err.close
+        end
+      end
+
+      def timeout_thread
+        Thread.new do
+          # Doing one long `sleep timeout` seems to fail sometimes, so
+          # we do a series of short timeouts before exiting
+          start_time = Time.now
+          loop do
+            sleep 10
+            duration = Time.now - start_time
+            break if duration >= timeout
+          end
+
+          @timed_out = true
+          stop("timed out")
+        end.run
+      end
+
+      def check_output_bytes(last_read_byte_count)
+        counter_mutex.synchronize do
+          @output_byte_count += last_read_byte_count
+        end
+
+        if output_byte_count > maximum_output_bytes
+          @maximum_output_exceeded = true
+          stop("maximum output exceeded")
+        end
+      end
+
+      def kill_reader_threads
+        @t_out&.kill
+        @t_err&.kill
+      end
+
+      def kill_wait_thread
+        @t_wait&.kill
+      end
+
+      def reap_running_container(message)
+        Analyzer.logger.warn("killing container name=#{@name} message=#{message.inspect}")
+        POSIX::Spawn::Child.new("docker", "kill", @name, timeout: 2.minutes)
+        POSIX::Spawn::Child.new("docker", "wait", @name, timeout: 2.minutes)
+      rescue POSIX::Spawn::TimeoutExceeded
+        Analyzer.logger.error("unable to kill container name=#{@name} message=#{message.inspect}")
+        Analyzer.statsd.increment("container.zombie")
+        Analyzer.statsd.increment("container.zombie.#{metric_name}") if metric_name
+      end
+
+      def timeout
+        ENV.fetch("CONTAINER_TIMEOUT_SECONDS", DEFAULT_TIMEOUT).to_i
+      end
+
+      def maximum_output_bytes
+        ENV.fetch("CONTAINER_MAXIMUM_OUTPUT_BYTES", DEFAULT_MAXIMUM_OUTPUT_BYTES).to_i
+      end
+
+      def metric_name
+        if /^cc-engines-(?<engine>[^-]+)-(?<channel>[^-]+)-/ =~ @name
+          "engine.#{engine}.#{channel}"
+        elsif /^builder-(?<action>[^-]+)-/ =~ @name
+          "builder.#{action}"
+        end
+      end
+    end
+  end
+end

data/lib/cc/analyzer/container_listener.rb

@@ -0,0 +1,9 @@
+module CC
+  module Analyzer
+    class ContainerListener
+      def started(_engine, _details); end
+
+      def finished(_engine, _details, _result); end
+    end
+  end
+end

data/lib/cc/analyzer/engine.rb

@@ -0,0 +1,125 @@
+require "securerandom"
+
+module CC
+  module Analyzer
+    #
+    # Running specifically an Engine container
+    #
+    # Input:
+    #   - name
+    #   - metadata
+    #     - image
+    #     - command (optional)
+    #   - config (becomes /config.json)
+    #   - label
+    #   - io (to write filtered, validated output)
+    #
+    # Output:
+    #   - Container::Result
+    #
+    class Engine
+      Error = Class.new(StandardError)
+
+      def initialize(name, metadata, config, label)
+        @name = name
+        @metadata = metadata
+        @config = config
+        @label = label.to_s
+        @error = nil
+      end
+
+      def run(io)
+        write_config_file
+
+        container = Container.new(
+          image: metadata.fetch("image"),
+          command: metadata["command"],
+          name: container_name,
+        )
+
+        container.on_output("\0") do |output|
+          handle_output(container, io, output)
+        end
+
+        container.run(container_options).tap do |result|
+          result.merge_from_exception(error) if error.present?
+        end
+      ensure
+        delete_config_file
+      end
+
+      private
+
+      attr_reader :name, :metadata
+      attr_accessor :error
+
+      def handle_output(container, io, raw_output)
+        output = EngineOutput.new(name, raw_output)
+
+        return if output_filter.filter?(output)
+
+        unless output.valid?
+          self.error = Error.new("engine produced invalid output: #{output.error}")
+          container.stop("output invalid")
+        end
+
+        unless io.write(output_overrider.apply(output).to_json)
+          self.error = Error.new("#{io.class}#write returned false, indicating an error")
+          container.stop("output error")
+        end
+      end
+
+      def qualified_name
+        "#{name}:#{@config.fetch("channel", "stable")}"
+      end
+
+      def container_options
+        options = [
+          "--cap-drop", "all",
+          "--label", "com.codeclimate.label=#{@label}",
+          "--log-driver", "none",
+          "--memory-swap", "-1",
+          "--net", "none",
+          "--rm",
+          "--volume", "#{code.host_path}:/code:ro",
+          "--volume", "#{config_file.host_path}:/config.json:ro",
+          "--user", "9000:9000"
+        ]
+        if (memory = metadata["memory"]).present?
+          options.concat(["--memory", memory.to_s])
+        end
+        options
+      end
+
+      def container_name
+        @container_name ||= "cc-engines-#{qualified_name.tr(":", "-")}-#{SecureRandom.uuid}"
+      end
+
+      def write_config_file
+        @config["debug"] = ENV["CODECLIMATE_DEBUG"]
+        Analyzer.logger.debug "/config.json content: #{@config.inspect}"
+        config_file.write(@config.to_json)
+      end
+
+      def delete_config_file
+        config_file.delete if config_file.file?
+      end
+
+      def code
+        @code ||= MountedPath.code
+      end
+
+      def config_file
+        @config_file ||= MountedPath.tmp.join(SecureRandom.uuid)
+      end
+
+      def output_filter
+        @output_filter ||= EngineOutputFilter.new(@config)
+      end
+
+      def output_overrider
+        @output_overrider ||= EngineOutputOverrider.new(@config)
+      end
+    end
+  end
+end

data/lib/cc/analyzer/engine_output.rb

@@ -0,0 +1,74 @@
+module CC
+  module Analyzer
+    class EngineOutput
+      delegate :blank?, to: :raw_output
+
+      def initialize(name, raw_output)
+        @name = name
+        @raw_output = raw_output
+      end
+
+      def issue?
+        valid_with_type?("issue")
+      end
+
+      def measurement?
+        valid_with_type?("measurement")
+      end
+
+      def as_issue
+        Issue.new(name, raw_output)
+      end
+
+      def to_json
+        if issue?
+          as_issue.to_json
+        elsif measurement?
+          Measurement.new(name, raw_output).to_json
+        end
+      end
+
+      def valid?
+        valid_json? && validator && validator.valid?
+      end
+
+      def error
+        if !valid_json?
+          { message: "Invalid JSON", output: raw_output }
+        elsif !validator.present?
+          { message: "Unsupported document type", output: raw_output }
+        else
+          validator.error
+        end
+      end
+
+      private
+
+      attr_accessor :name, :raw_output
+
+      def valid_json?
+        parsed_output.present?
+      end
+
+      def valid_with_type?(type)
+        parsed_output &&
+          parsed_output["type"].present? &&
+          parsed_output["type"].downcase == type
+      end
+
+      def parsed_output
+        @parsed_output ||= JSON.parse(raw_output)
+      rescue JSON::ParserError
+        nil
+      end
+
+      def validator
+        if issue?
+          IssueValidator.new(parsed_output)
+        elsif measurement?
+          MeasurementValidator.new(parsed_output)
+        end
+      end
+    end
+  end
+end

data/lib/cc/analyzer/engine_output_filter.rb

@@ -0,0 +1,36 @@
+module CC
+  module Analyzer
+    class EngineOutputFilter
+      ISSUE_TYPE = "issue".freeze
+
+      def initialize(config = {})
+        @config = config
+      end
+
+      def filter?(output)
+        output.blank? || (output.issue? && ignore_issue?(output.as_issue))
+      end
+
+      private
+
+      def ignore_issue?(issue)
+        check_disabled?(issue) || ignore_fingerprint?(issue)
+      end
+
+      def check_disabled?(issue)
+        !check_config(issue.check_name).fetch("enabled", true)
+      end
+
+      def ignore_fingerprint?(issue)
+        @config.fetch("exclude_fingerprints", []).include?(issue.fingerprint)
+      rescue SourceExtractor::InvalidLocation
+        false
+      end
+
+      def check_config(check_name)
+        @checks ||= @config.fetch("checks", {})
+        @checks.fetch(check_name, {})
+      end
+    end
+  end
+end

data/lib/cc/analyzer/engine_output_overrider.rb

@@ -0,0 +1,31 @@
+module CC
+  module Analyzer
+    class EngineOutputOverrider
+      def initialize(config = {})
+        @config = config
+      end
+
+      def apply(output)
+        if output.issue?
+          override_severity(output.as_issue.as_json)
+        else
+          output
+        end
+      end
+
+      private
+
+      attr_reader :config
+
+      def override_severity(issue)
+        issue.merge(override("severity"))
+      end
+
+      def override(name)
+        config.
+          fetch("issue_override", {}).
+          slice(name)
+      end
+    end
+  end
+end

data/lib/cc/analyzer/filesystem.rb

@@ -0,0 +1,50 @@
+module CC
+  module Analyzer
+    class Filesystem
+      attr_reader :root
+
+      def initialize(root)
+        @root = root
+      end
+
+      def exist?(path)
+        File.exist?(path_for(path))
+      end
+
+      def source_buffer_for(path)
+        SourceBuffer.new(path, read_path(path))
+      end
+
+      def read_path(path)
+        File.read(path_for(path))
+      end
+
+      def write_path(path, content)
+        File.write(path_for(path), content)
+        File.chown(root_uid, root_gid, path_for(path))
+      end
+
+      def ls
+        Dir.entries(root).reject { |entry| [".", ".."].include?(entry) }
+      end
+
+      private
+
+      def path_for(path)
+        File.join(root, path)
+      end
+
+      def root_uid
+        root_stat.uid
+      end
+
+      def root_gid
+        root_stat.gid
+      end
+
+      def root_stat
+        @root_stat ||= File.stat(root)
+      end
+    end
+  end
+end

data/lib/cc/analyzer/formatters/formatter.rb

@@ -0,0 +1,53 @@
+module CC
+  module Analyzer
+    module Formatters
+      class Formatter
+        def initialize(filesystem, output = $stdout)
+          @filesystem = filesystem
+          @output = output
+        end
+
+        def write(data)
+          json = JSON.parse(data)
+          json["engine_name"] = current_engine.name
+
+          case json["type"].downcase
+          when "issue"
+            issues << json
+          when "warning"
+            warnings << json
+          when "measurement"
+            measurements << json
+          else
+            raise "Invalid type found: #{json["type"]}"
+          end
+        end
+
+        def started
+        end
+
+        def engine_running(engine)
+          @current_engine = engine
+          yield
+        ensure
+          @current_engine = nil
+        end
+
+        def finished
+        end
+
+        def close
+        end
+
+        def failed(_output)
+        end
+
+        InvalidFormatterError = Class.new(StandardError)
+
+        private
+
+        attr_reader :current_engine
+      end
+    end
+  end
+end