s3-antivirus 0.1.0

data/lib/s3_antivirus/scan.rb

@@ -0,0 +1,142 @@
+require "aws-sdk-sqs"
+require "json"
+require "securerandom"
+require "syslog/logger"
+require "uri"
+require "yaml"
+
+module S3Antivirus
+  class Scan
+    extend Memoist
+    include AwsServices
+    include Logger
+    include Conf
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def run
+      logger.info "Polling SQS queue for S3 antivirus findings. Started #{Time.now}..."
+      return if @options[:noop]
+
+      poller = Aws::SQS::QueuePoller.new(conf['queue'])
+      poller.poll do |msg|
+        begin
+          body = JSON.parse(msg.body)
+
+          logger.debug "body #{body}"
+          if body.key?('Records')
+            body['Records'].each do |record|
+              # Instance variables change on each iteration
+              @s3_record = S3Record.new(record)
+              @notifier = Notifier.new(@s3_record)
+              @tagger = Tagger.new(@s3_record)
+
+              logger.info("Checking #{@s3_record.human_key}")
+              within_size_limit = check_file_size
+              next unless within_size_limit # continue to next file because oversized
+              filename = "/tmp/#{SecureRandom.uuid}"
+              success = download_file(filename)
+              next unless success # continue to next file because unable to download
+              scan_file(filename)
+            end
+          end
+        rescue Exception => e
+          logger.error "message failed: #{e.inspect} #{msg.inspect}"
+          raise e
+        end
+        # Sometimes stdout doesnt get shown but the message is processed file.
+        # Know this because the SNS message gets sent.
+        # Even this $stdout.flush doesn't seem to be enough.
+        $stdout.flush
+      end
+    end
+
+  private
+    def scan_file(filename)
+      logger.info "Scanning #{human_key}..."
+      sh("clamdscan #{filename}")
+      exitstatus = $?.exitstatus
+      if exitstatus == 0 # No virus found.
+        clean_file_found
+      elsif exitstatus == 1 # Virus(es) found.
+        virus_found
+      else # An error occured.
+        raise "#{human_key} could not be scanned, clamdscan exit status was #{exitstatus}, retry"
+      end
+    ensure
+      system("rm #{filename}")
+    end
+
+    def clean_file_found
+      if conf['tag_files']
+        logger.info "#{human_key} is clean (tagging)"
+        @tagger.tag("clean");
+      else
+        logger.info "#{human_key} is clean"
+      end
+      if conf['report_clean']
+        @notifier.notify(status: "clean", action: "no");
+      end
+    end
+
+    def virus_found
+      if conf['delete']
+        logger.info "#{human_key} is infected (deleting)"
+        s3.delete_object(
+          bucket: @s3_record.bucket,
+          key: @s3_record.key,
+        )
+        @notifier.notify(status: "infected", action: "delete");
+      elsif conf['tag_files']
+        logger.info "#{human_key} is infected (tagging)"
+        @tagger.tag("infected");
+        @notifier.notify(status: "infected", action: "tag");
+      else
+        logger.info "#{human_key} is infected"
+        @notifier.notify(status: "infected", action: "no");
+      end
+    end
+
+    def human_key
+      @s3_record.human_key
+    end
+
+    def download_file(filename)
+      logger.info "Downloading #{@s3_record.human_key} to #{filename}..."
+      params = {
+        response_target: filename,
+        bucket: @s3_record.bucket,
+        key: @s3_record.key,
+      }
+      params[:version_id] = @s3_record.version if @s3_record.version
+
+      begin
+        s3.get_object(params)
+      rescue Aws::S3::Errors::NoSuchKey
+        logger.info "#{@s3_record.human_key} no longer exist, skipping."
+        return false
+      end
+      true # successful download
+    end
+
+    def check_file_size
+      within_size_limit = true
+      if @s3_record.oversized?
+        logger.info "#{@s3_record.human_key} is too large. Bigger than half of the EBS volume, skipping."
+        if conf['tag_files']
+          @tagger.tag("oversized")
+        end
+        @notifier.notify(status: "oversized", action: "no")
+        within_size_limit = false
+      end
+      within_size_limit
+    end
+
+    def sh(command)
+      logger.info("=> #{command}")
+      system(command)
+    end
+  end
+end

data/lib/s3_antivirus/tagger.rb

@@ -0,0 +1,28 @@
+module S3Antivirus
+  class Tagger
+    include AwsServices
+    include Conf
+
+    def initialize(s3_record)
+      @s3_record = s3_record
+      @bucket, @key, @version = s3_record.bucket, s3_record.key, s3_record.version
+      @tag_key = conf['tag_key']
+    end
+
+    # Different tag values:
+    #
+    #   clean
+    #   inflected
+    #   oversized
+    #
+    def tag(value)
+      params = {
+        bucket: @bucket,
+        key: @key,
+        tagging: {tag_set: [{key: @tag_key, value: value}]}
+      }
+      params[:version_id] = @version if @version
+      s3.put_object_tagging(params)
+    end
+  end
+end

data/lib/s3_antivirus/tee.rb

@@ -0,0 +1,32 @@
+require "syslog/logger"
+
+module S3Antivirus
+  class Tee
+    extend Memoist
+
+    def initialize(path, options={})
+      @path, @options = path, options
+    end
+
+    def method_missing(name, message, &block)
+      if logger.respond_to?(name)
+        # Interesting note about level mapping: http://bit.ly/2PP5Y6Z
+        #   Messages from Ruby applications are not considered as critical as messages
+        #   from other system daemons using syslog(3), so most messages are reduced by one level.
+        #
+        # Will mimic behavior for puts stdout.
+        # Interestingly, the default logger.level is ::Logger::DEBUG which is 0.
+        # So can't check against that, so will check against the method name.
+        puts message unless name == :debug
+        logger.send(name, message, &block)
+      else
+        super
+      end
+    end
+
+    def logger
+      Syslog::Logger.new(@path)
+    end
+    memoize :logger
+  end
+end

data/lib/s3_antivirus/version.rb

@@ -0,0 +1,3 @@
+module S3Antivirus
+  VERSION = "0.1.0"
+end

data/lib/s3_antivirus.rb

@@ -0,0 +1,13 @@
+$stdout.sync = true unless ENV["S3_ANTIVIRUS_STDOUT_SYNC"] == "0"
+
+$:.unshift(File.expand_path("../", __FILE__))
+require "memoist"
+require "rainbow/ext/string"
+require "s3_antivirus/version"
+
+require "s3_antivirus/autoloader"
+S3Antivirus::Autoloader.setup
+
+module S3Antivirus
+  class Error < StandardError; end
+end

data/s3-antivirus.gemspec

@@ -0,0 +1,49 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "s3_antivirus/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3-antivirus"
+  spec.version       = S3Antivirus::VERSION
+  spec.authors       = ["Tung Nguyen"]
+  spec.email         = ["tongueroo@gmail.com"]
+  spec.summary       = "Detects if files uploaded to s3 contain a virus with ClamAV and auto-deletes or tags them"
+  spec.homepage      = "https://github.com/tongueroo/s3-antivirus"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "aws-sdk-s3"
+  spec.add_dependency "aws-sdk-sns"
+  spec.add_dependency "aws-sdk-sqs"
+  spec.add_dependency "memoist"
+  spec.add_dependency "rainbow"
+  spec.add_dependency "recursive-open-struct"
+  spec.add_dependency "thor"
+  spec.add_dependency "zeitwerk"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "cli_markdown"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    # spec.metadata["allowed_push_host"] = ""
+
+    spec.metadata["homepage_uri"] = spec.homepage
+
+    # spec.metadata["source_code_uri"] = ""
+    # spec.metadata["changelog_uri"] = ""
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+end

data/spec/fixtures/home/.s3-antivirus.conf

@@ -0,0 +1,8 @@
+delete: true
+report_clean: true
+tag_files: true
+tag_key: clamav-status
+region: us-west-2
+queue: https://sqs.us-west-2.amazonaws.com/112233445566/s3-ScanQueue-1BRQO2VOPIUM8
+topic: arn:aws:sns:us-west-2:112233445566:s3-FindingsTopic-J1E10FB4G914
+volume_size: 8

data/spec/fixtures/sqs-event.json

@@ -0,0 +1,38 @@
+{
+  "Records": [
+    {
+      "eventVersion": "2.1",
+      "eventSource": "aws:s3",
+      "awsRegion": "us-west-2",
+      "eventTime": "2019-12-14T22:56:24.066Z",
+      "eventName": "ObjectCreated:Put",
+      "userIdentity": {
+        "principalId": "AWS:AIDAJTCD6O457QEXAMPLE"
+      },
+      "requestParameters": {
+        "sourceIPAddress": "52.33.148.151"
+      },
+      "responseElements": {
+        "x-amz-request-id": "C6044CD6DEXAMPLE",
+        "x-amz-id-2": "V1wa7UVBCDXAFHPwoS6+I02H4idFtO4A5omkL2AeCfTsbz8/JWMVYgGzdE5+uhe1kWYYAEXAMPLE"
+      },
+      "s3": {
+        "s3SchemaVersion": "1.0",
+        "configurationId": "s3-antivirus",
+        "bucket": {
+          "name": "fake-bucket",
+          "ownerIdentity": {
+            "principalId": "AYC6O1EXAMPLE"
+          },
+          "arn": "arn:aws:s3:::fake-bucket"
+        },
+        "object": {
+          "key": "a.txt",
+          "size": 2,
+          "eTag": "60b725f10c9c85c70d97880dfEXAMPLE",
+          "sequencer": "005DF568980EXAMPLE"
+        }
+      }
+    }
+  ]
+}

data/spec/lib/cli_spec.rb

@@ -0,0 +1,8 @@
+describe S3Antivirus::CLI do
+  describe "s3-antivirus" do
+    it "scan" do
+      out = execute("exe/s3-antivirus scan --noop")
+      expect(out).to include("Polling SQS queue")
+    end
+  end
+end

data/spec/lib/scan_spec.rb

@@ -0,0 +1,36 @@
+require "aws-sdk-sqs"
+require "recursive-open-struct"
+require "ostruct"
+require "json"
+class Aws::SQS::QueuePoller
+  @@msg = nil
+  def self.msg=(val)
+    @@msg = val
+  end
+
+  def poll
+    yield(@@msg)
+  end
+end
+
+describe S3Antivirus::Scan do
+  before(:each) do
+    Aws::SQS::QueuePoller.msg = RecursiveOpenStruct.new(body: body)
+  end
+
+  # Pretty causal test. Mocked out most of the methods.
+  let(:null) { double(:null).as_null_object }
+  let(:scan) do
+    scan = S3Antivirus::Scan.new
+    allow(scan).to receive(:download_file).and_return(true)
+    allow(scan).to receive(:scan_file)
+    scan
+  end
+  let(:body) { IO.read("spec/fixtures/sqs-event.json") }
+
+  describe "Scan" do
+    it "run" do
+      scan.run
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,32 @@
+ENV["S3_ANTIVIRUS_TEST"] = "1"
+# Ensures aws api never called. Fixture home folder does not contain ~/.aws/credentials
+ENV['HOME'] = File.join(Dir.pwd,'spec/fixtures/home')
+ENV['AWS_REGION'] = "us-west-2"
+
+# CodeClimate test coverage: https://docs.codeclimate.com/docs/configuring-test-coverage
+# require 'simplecov'
+# SimpleCov.start
+
+require "pp"
+require "byebug"
+root = File.expand_path("../", File.dirname(__FILE__))
+require "#{root}/lib/s3-antivirus"
+
+module Helper
+  def execute(cmd)
+    puts "Running: #{cmd}" if show_command?
+    out = `#{cmd}`
+    puts out if show_command?
+    out
+  end
+
+  # Added SHOW_COMMAND because DEBUG is also used by other libraries like
+  # bundler and it shows its internal debugging logging also.
+  def show_command?
+    ENV['DEBUG'] || ENV['SHOW_COMMAND']
+  end
+end
+
+RSpec.configure do |c|
+  c.include Helper
+end

metadata

@@ -0,0 +1,284 @@
+--- !ruby/object:Gem::Specification
+name: s3-antivirus
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tung Nguyen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sns
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: memoist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: recursive-open-struct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cli_markdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- tongueroo@gmail.com
+executables:
+- s3-antivirus
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/s3-antivirus
+- lib/s3-antivirus.rb
+- lib/s3_antivirus.rb
+- lib/s3_antivirus/autoloader.rb
+- lib/s3_antivirus/aws_services.rb
+- lib/s3_antivirus/cli.rb
+- lib/s3_antivirus/command.rb
+- lib/s3_antivirus/completer.rb
+- lib/s3_antivirus/completer/script.rb
+- lib/s3_antivirus/completer/script.sh
+- lib/s3_antivirus/conf.rb
+- lib/s3_antivirus/config.rb
+- lib/s3_antivirus/help.rb
+- lib/s3_antivirus/help/completion.md
+- lib/s3_antivirus/help/completion_script.md
+- lib/s3_antivirus/help/scan.md
+- lib/s3_antivirus/logger.rb
+- lib/s3_antivirus/notifier.rb
+- lib/s3_antivirus/s3_record.rb
+- lib/s3_antivirus/scan.rb
+- lib/s3_antivirus/tagger.rb
+- lib/s3_antivirus/tee.rb
+- lib/s3_antivirus/version.rb
+- s3-antivirus.gemspec
+- spec/fixtures/home/.s3-antivirus.conf
+- spec/fixtures/sqs-event.json
+- spec/lib/cli_spec.rb
+- spec/lib/scan_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/tongueroo/s3-antivirus
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/tongueroo/s3-antivirus
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Detects if files uploaded to s3 contain a virus with ClamAV and auto-deletes
+  or tags them
+test_files:
+- spec/fixtures/home/.s3-antivirus.conf
+- spec/fixtures/sqs-event.json
+- spec/lib/cli_spec.rb
+- spec/lib/scan_spec.rb
+- spec/spec_helper.rb