s3-antivirus 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 627d0930ad4358631d348d641ab1d99de75f00760b01ab9d29a418705e4a3202
+  data.tar.gz: 3942b5a0a8d18cf3682edd447383942e6c9bcae864b9d872d8352bed64ad5039
+SHA512:
+  metadata.gz: d725114194aeaaddb285f91eda29e8f9a9ee9ab8d8bcad6ea8fda3618c9effe84861e8b4b2d130c0c671b26d5da1f43d88e33f7abf677708f37f01f357c93e79
+  data.tar.gz: fb6793d2c93008cef13d75093acee30f19af07b4add6bdcf7c02e8d67317fa21ccbf6f05f7bb25180b5129337e0a5deb45581c04ad5c3f8a8cae5b64b137e2a0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+_yardoc
+coverage
+doc/
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper
+--color
+--format documentation

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
+
+## [0.1.0]
+- Initial release.

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem dependencies in s3-antivirus.gemspec
+gemspec
+
+gem "codeclimate-test-reporter", group: :test, require: nil

data/Guardfile

@@ -0,0 +1,19 @@
+guard "bundler", cmd: "bundle" do
+  watch("Gemfile")
+  watch(/^.+\.gemspec/)
+end
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2019 Tung Nguyen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# S3 AntiVirus with ClamAV
+
+[![Gem Version](https://badge.fury.io/rb/s3-antivirus.png)](http://badge.fury.io/rb/s3-antivirus)
+
+[![BoltOps Badge](https://img.boltops.com/boltops/badges/boltops-badge.png)](https://www.boltops.com)
+
+Detects if files uploaded to s3 contain a virus with ClamAV and auto-deletes or tags them.  Works by processing an SQS Queue that contain messages from [S3 Bucket Event Notifications](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/enable-event-notifications.html).
+
+## Usage
+
+    s3-antivirus scan
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem "s3-antivirus"
+
+And then execute:
+
+    bundle
+
+Or install it yourself as:
+
+    gem install s3-antivirus
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am "Add some feature"`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+task default: :spec
+
+RSpec::Core::RakeTask.new
+
+require_relative "lib/s3-antivirus"
+require "cli_markdown"
+desc "Generates cli reference docs as markdown"
+task :docs do
+  mkdir_p "docs/_includes"
+  CliMarkdown::Creator.create_all(cli_class: S3Antivirus::CLI, cli_name: "s3-antivirus")
+end

data/exe/s3-antivirus

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+# Trap ^C
+Signal.trap("INT") {
+  puts "\nCtrl-C detected. Exiting..."
+  sleep 0.1
+  exit
+}
+
+$:.unshift(File.expand_path("../../lib", __FILE__))
+require "s3-antivirus"
+require "s3_antivirus/cli"
+
+S3Antivirus::CLI.start(ARGV)

data/lib/s3-antivirus.rb

@@ -0,0 +1 @@
+require_relative "s3_antivirus"

data/lib/s3_antivirus/autoloader.rb

@@ -0,0 +1,22 @@
+require "zeitwerk"
+
+module S3Antivirus
+  class Autoloader
+    class Inflector < Zeitwerk::Inflector
+      def camelize(basename, _abspath)
+        map = { cli: "CLI", version: "VERSION" }
+        map[basename.to_sym] || super
+      end
+    end
+
+    class << self
+      def setup
+        loader = Zeitwerk::Loader.new
+        loader.inflector = Inflector.new
+        loader.push_dir(File.dirname(__dir__)) # lib
+        loader.ignore("#{File.dirname(__dir__)}/s3-antivirus.rb")
+        loader.setup
+      end
+    end
+  end
+end

data/lib/s3_antivirus/aws_services.rb

@@ -0,0 +1,18 @@
+require "aws-sdk-s3"
+require "aws-sdk-sns"
+
+module S3Antivirus
+  module AwsServices
+    extend Memoist
+
+    def s3
+      Aws::S3::Client.new
+    end
+    memoize :s3
+
+    def sns
+      Aws::SNS::Client.new
+    end
+    memoize :sns
+  end
+end

data/lib/s3_antivirus/cli.rb

@@ -0,0 +1,29 @@
+module S3Antivirus
+  class CLI < Command
+    class_option :verbose, type: :boolean
+    class_option :noop, type: :boolean
+
+    desc "scan", "Poll SQS queue for s3 virus findings."
+    long_desc Help.text(:scan)
+    def scan
+      Scan.new(options).run
+    end
+
+    desc "completion *PARAMS", "Prints words for auto-completion."
+    long_desc Help.text(:completion)
+    def completion(*params)
+      Completer.new(CLI, *params).run
+    end
+
+    desc "completion_script", "Generates a script that can be eval to setup auto-completion."
+    long_desc Help.text(:completion_script)
+    def completion_script
+      Completer::Script.generate
+    end
+
+    desc "version", "prints version"
+    def version
+      puts VERSION
+    end
+  end
+end

data/lib/s3_antivirus/command.rb

@@ -0,0 +1,82 @@
+require "thor"
+
+# Override thor's long_desc identation behavior
+# https://github.com/erikhuda/thor/issues/398
+class Thor
+  module Shell
+    class Basic
+      def print_wrapped(message, options = {})
+        message = "\n#{message}" unless message[0] == "\n"
+        stdout.puts message
+      end
+    end
+  end
+end
+
+module S3Antivirus
+  class Command < Thor
+    class << self
+      def dispatch(m, args, options, config)
+        # Allow calling for help via:
+        #   s3-antivirus command help
+        #   s3-antivirus command -h
+        #   s3-antivirus command --help
+        #   s3-antivirus command -D
+        #
+        # as well thor's normal way:
+        #
+        #   s3-antivirus help command
+        help_flags = Thor::HELP_MAPPINGS + ["help"]
+        if args.length > 1 && !(args & help_flags).empty?
+          args -= help_flags
+          args.insert(-2, "help")
+        end
+
+        #   s3-antivirus version
+        #   s3-antivirus --version
+        #   s3-antivirus -v
+        version_flags = ["--version", "-v"]
+        if args.length == 1 && !(args & version_flags).empty?
+          args = ["version"]
+        end
+
+        super
+      end
+
+      # Override command_help to include the description at the top of the
+      # long_description.
+      def command_help(shell, command_name)
+        meth = normalize_command_name(command_name)
+        command = all_commands[meth]
+        alter_command_description(command)
+        super
+      end
+
+      def alter_command_description(command)
+        return unless command
+
+        # Add description to beginning of long_description
+        long_desc = if command.long_description
+            "#{command.description}\n\n#{command.long_description}"
+          else
+            command.description
+          end
+
+        # add reference url to end of the long_description
+        unless website.empty?
+          full_command = [command.ancestor_name, command.name].compact.join('-')
+          url = "#{website}/reference/s3-antivirus-#{full_command}"
+          long_desc += "\n\nHelp also available at: #{url}"
+        end
+
+        command.long_description = long_desc
+      end
+      private :alter_command_description
+
+      # meant to be overriden
+      def website
+        ""
+      end
+    end
+  end
+end

data/lib/s3_antivirus/completer/script.rb

@@ -0,0 +1,6 @@
+class S3Antivirus::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/s3_antivirus/completer/script.sh

@@ -0,0 +1,10 @@
+_s3-antivirus() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(s3-antivirus completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _s3-antivirus s3-antivirus

data/lib/s3_antivirus/completer.rb

@@ -0,0 +1,159 @@
+=begin
+Code Explanation:
+
+There are 3 types of things to auto-complete:
+
+  1. command: the command itself
+  2. parameters: command parameters.
+  3. options: command options
+
+Here's an example:
+
+  mycli hello name --from me
+
+  * command: hello
+  * parameters: name
+  * option: --from
+
+When command parameters are done processing, the remaining completion words will be options.  We can tell that the command params are completed based on the method arity.
+
+## Arity
+
+For example, say you had a method for a CLI command with the following form:
+
+  ufo scale service count --cluster development
+
+It's equivalent ruby method:
+
+  scale(service, count) = has an arity of 2
+
+So typing:
+
+  ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
+
+So the completion should only show options, something like this:
+
+  --noop --verbose --cluster
+
+## Splat Arguments
+
+When the ruby method has a splat argument, it's arity is negative.  Here are some example methods and their arities.
+
+   ship(service) = 1
+   scale(service, count) = 2
+   ships(*services) = -1
+   foo(example, *rest) = -2
+
+Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
+
+Here are some test cases, hit TAB after typing the command:
+
+  s3-antivirus completion
+  s3-antivirus completion hello
+  s3-antivirus completion hello name
+  s3-antivirus completion hello name --
+  s3-antivirus completion hello name --noop
+
+  s3-antivirus completion
+  s3-antivirus completion sub:goodbye
+  s3-antivirus completion sub:goodbye name
+
+## Subcommands and Thor::Group Registered Commands
+
+Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
+
+  * regular command: ufo ship
+  * subcommand: ufo docker
+  * Thor::Group command: ufo init
+
+Auto-completion accounts for each of these type of commands.
+=end
+module S3Antivirus
+  class Completer
+    def initialize(command_class, *params)
+      @params = params
+      @current_command = @params[0]
+      @command_class = command_class # CLI initiall
+    end
+
+    def run
+      if subcommand?(@current_command)
+        subcommand_class = @command_class.subcommand_classes[@current_command]
+        @params.shift # destructive
+        Completer.new(subcommand_class, *@params).run # recursively use subcommand
+        return
+      end
+
+      # full command has been found!
+      unless found?(@current_command)
+        puts all_commands
+        return
+      end
+
+      # will only get to here if command aws found (above)
+      arity = @command_class.instance_method(@current_command).arity.abs
+      if @params.size > arity or thor_group_command?
+        puts options_completion
+      else
+        puts params_completion
+      end
+    end
+
+    def subcommand?(command)
+      @command_class.subcommands.include?(command)
+    end
+
+    # hacky way to detect that command is a registered Thor::Group command
+    def thor_group_command?
+      command_params(raw=true) == [[:rest, :args]]
+    end
+
+    def found?(command)
+      public_methods = @command_class.public_instance_methods(false)
+      command && public_methods.include?(command.to_sym)
+    end
+
+    # all top-level commands
+    def all_commands
+      commands = @command_class.all_commands.reject do |k,v|
+        v.is_a?(Thor::HiddenCommand)
+      end
+      commands.keys
+    end
+
+    def command_params(raw=false)
+      params = @command_class.instance_method(@current_command).parameters
+      # Example:
+      # >> Sub.instance_method(:goodbye).parameters
+      # => [[:req, :name]]
+      # >>
+      raw ? params : params.map!(&:last)
+    end
+
+    def params_completion
+      offset = @params.size - 1
+      offset_params = command_params[offset..-1]
+      command_params[offset..-1].first
+    end
+
+    def options_completion
+      used = ARGV.select { |a| a.include?('--') } # so we can remove used options
+
+      method_options = @command_class.all_commands[@current_command].options.keys
+      class_options = @command_class.class_options.keys
+
+      all_options = method_options + class_options + ['help']
+
+      all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
+      filtered_options = all_options - used
+      filtered_options.uniq
+    end
+
+    # Useful for debugging. Using puts messes up completion.
+    def log(msg)
+      File.open("/tmp/complete.log", "a") do |file|
+        file.puts(msg)
+      end
+    end
+  end
+end

data/lib/s3_antivirus/conf.rb

@@ -0,0 +1,11 @@
+module S3Antivirus
+  module Conf
+    extend Memoist
+
+    def conf
+      conf = Config.new
+      conf.data
+    end
+    memoize :conf
+  end
+end

data/lib/s3_antivirus/config.rb

@@ -0,0 +1,29 @@
+module S3Antivirus
+  class Config
+    include Logger
+
+    attr_reader :data
+    def initialize(path=nil)
+      @data = load(path)
+    end
+
+    def load(path)
+      YAML.load_file(lookup_path(path))
+    end
+
+    def lookup_path(path=nil)
+      paths = [
+        path,
+        "./s3-antivirus.conf",
+        "#{ENV['HOME']}/.s3-antivirus.conf",
+        "/etc/s3-antivirus.conf"
+      ].compact
+      found = paths.find { |p| File.exist?(p) }
+      unless found
+        logger.fatal("FATAL: unable to find the s3-antivirus.conf file. Paths considered: #{paths}")
+        exit 1
+      end
+      found
+    end
+  end
+end

data/lib/s3_antivirus/help/completion.md

@@ -0,0 +1,20 @@
+## Examples
+
+    s3-antivirus completion
+
+Prints words for TAB auto-completion.
+
+    s3-antivirus completion
+    s3-antivirus completion hello
+    s3-antivirus completion hello name
+
+To enable, TAB auto-completion add the following to your profile:
+
+    eval $(s3-antivirus completion_script)
+
+Auto-completion example usage:
+
+    s3-antivirus [TAB]
+    s3-antivirus hello [TAB]
+    s3-antivirus hello name [TAB]
+    s3-antivirus hello name --[TAB]

data/lib/s3_antivirus/help/completion_script.md

@@ -0,0 +1,3 @@
+To use, add the following to your `~/.bashrc` or `~/.profile`
+
+    eval $(s3-antivirus completion_script)

data/lib/s3_antivirus/help/scan.md

@@ -0,0 +1,24 @@
+## Example
+
+    $ s3-antivirus scan
+    Polling SQS queue for S3 antivirus findings. Started 2019-12-15 04:07:09 +0000...
+    Checking s3://test-bucket/eicar.txt
+    Downloading s3://test-bucket/eicar.txt to /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b...
+    Scanning s3://test-bucket/eicar.txt...
+    => clamdscan /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b
+    /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b: Eicar-Test-Signature FOUND
+
+    ----------- SCAN SUMMARY -----------
+    Infected files: 1
+    Time: 0.001 sec (0 m 0 s)
+    s3://test-bucket/eicar.txt is infected (deleting)
+    Checking s3://test-bucket/a.txt
+    Downloading s3://test-bucket/a.txt to /tmp/56da4b41-a672-42d4-a766-1727f5dc256d...
+    Scanning s3://test-bucket/a.txt...
+    => clamdscan /tmp/56da4b41-a672-42d4-a766-1727f5dc256d
+    /tmp/56da4b41-a672-42d4-a766-1727f5dc256d: OK
+
+    ----------- SCAN SUMMARY -----------
+    Infected files: 0
+    Time: 0.001 sec (0 m 0 s)
+    s3://test-bucket/a.txt is clean (tagging)

data/lib/s3_antivirus/help.rb

@@ -0,0 +1,9 @@
+module S3Antivirus::Help
+  class << self
+    def text(namespaced_command)
+      path = namespaced_command.to_s.gsub(':','/')
+      path = File.expand_path("../help/#{path}.md", __FILE__)
+      IO.read(path) if File.exist?(path)
+    end
+  end
+end

data/lib/s3_antivirus/logger.rb

@@ -0,0 +1,7 @@
+module S3Antivirus
+  module Logger
+    def logger
+      $logger ||= Tee.new("s3-antivirus")
+    end
+  end
+end

data/lib/s3_antivirus/notifier.rb

@@ -0,0 +1,35 @@
+module S3Antivirus
+  class Notifier
+    include AwsServices
+    include Conf
+
+    def initialize(s3_record)
+      @s3_record = s3_record
+      @bucket, @key, @version = s3_record.bucket, s3_record.key, s3_record.version
+    end
+
+    def notify(status:, action:)
+      data = {
+        action: action,
+        bucket: @bucket,
+        key: @key,
+        status: status,
+      }
+      data[:version] = @version if @version
+      message_attributes = data.inject({}) do |result, (k,v)|
+        result.merge(
+          k => {
+            data_type: "String",
+            string_value: v
+          }
+        )
+      end
+      sns.publish(
+        topic_arn: conf['topic'],
+        message: "#{@s3_record.human_key} is #{status}, #{action} action executed",
+        subject: "s3-antivirus s3://#{@bucket}",
+        message_attributes: message_attributes
+      )
+    end
+  end
+end

data/lib/s3_antivirus/s3_record.rb

@@ -0,0 +1,39 @@
+module S3Antivirus
+  class S3Record
+    include Conf
+
+    def initialize(record)
+      @record = record # record data from SQS event structure
+    end
+
+    def human_key
+      text = "s3://#{bucket}/#{key}"
+      text += " (version: #{version})" if version
+      text
+    end
+
+    def bucket
+      @record['s3']['bucket']['name']
+    end
+
+    def key
+      URI.decode(@record['s3']['object']['key']).gsub('+', ' ')
+    end
+
+    def version
+      @record['s3']['object']['versionId']
+    end
+
+    def oversized?
+      size > max_size
+    end
+
+    def size
+      @record['s3']['object']['size']
+    end
+
+    def max_size
+      conf['volume_size'] * 1073741824 / 2 # in bytes
+    end
+  end
+end