s3-antivirus 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 627d0930ad4358631d348d641ab1d99de75f00760b01ab9d29a418705e4a3202
4
+ data.tar.gz: 3942b5a0a8d18cf3682edd447383942e6c9bcae864b9d872d8352bed64ad5039
5
+ SHA512:
6
+ metadata.gz: d725114194aeaaddb285f91eda29e8f9a9ee9ab8d8bcad6ea8fda3618c9effe84861e8b4b2d130c0c671b26d5da1f43d88e33f7abf677708f37f01f357c93e79
7
+ data.tar.gz: fb6793d2c93008cef13d75093acee30f19af07b4add6bdcf7c02e8d67317fa21ccbf6f05f7bb25180b5129337e0a5deb45581c04ad5c3f8a8cae5b64b137e2a0
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ _yardoc
7
+ coverage
8
+ doc/
9
+ InstalledFiles
10
+ lib/bundler/man
11
+ pkg
12
+ rdoc
13
+ spec/reports
14
+ test/tmp
15
+ test/version_tmp
16
+ tmp
17
+ Gemfile.lock
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --require spec_helper
2
+ --color
3
+ --format documentation
data/CHANGELOG.md ADDED
@@ -0,0 +1,7 @@
1
+ # Change Log
2
+
3
+ All notable changes to this project will be documented in this file.
4
+ This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
5
+
6
+ ## [0.1.0]
7
+ - Initial release.
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem dependencies in s3-antivirus.gemspec
4
+ gemspec
5
+
6
+ gem "codeclimate-test-reporter", group: :test, require: nil
data/Guardfile ADDED
@@ -0,0 +1,19 @@
1
+ guard "bundler", cmd: "bundle" do
2
+ watch("Gemfile")
3
+ watch(/^.+\.gemspec/)
4
+ end
5
+
6
+ guard :rspec, cmd: "bundle exec rspec" do
7
+ require "guard/rspec/dsl"
8
+ dsl = Guard::RSpec::Dsl.new(self)
9
+
10
+ # RSpec files
11
+ rspec = dsl.rspec
12
+ watch(rspec.spec_helper) { rspec.spec_dir }
13
+ watch(rspec.spec_support) { rspec.spec_dir }
14
+ watch(rspec.spec_files)
15
+
16
+ # Ruby files
17
+ ruby = dsl.ruby
18
+ dsl.watch_spec_files_for(ruby.lib_files)
19
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2019 Tung Nguyen
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,33 @@
1
+ # S3 AntiVirus with ClamAV
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/s3-antivirus.png)](http://badge.fury.io/rb/s3-antivirus)
4
+
5
+ [![BoltOps Badge](https://img.boltops.com/boltops/badges/boltops-badge.png)](https://www.boltops.com)
6
+
7
+ Detects if files uploaded to s3 contain a virus with ClamAV and auto-deletes or tags them. Works by processing an SQS Queue that contain messages from [S3 Bucket Event Notifications](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/enable-event-notifications.html).
8
+
9
+ ## Usage
10
+
11
+ s3-antivirus scan
12
+
13
+ ## Installation
14
+
15
+ Add this line to your application's Gemfile:
16
+
17
+ gem "s3-antivirus"
18
+
19
+ And then execute:
20
+
21
+ bundle
22
+
23
+ Or install it yourself as:
24
+
25
+ gem install s3-antivirus
26
+
27
+ ## Contributing
28
+
29
+ 1. Fork it
30
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
31
+ 3. Commit your changes (`git commit -am "Add some feature"`)
32
+ 4. Push to the branch (`git push origin my-new-feature`)
33
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,14 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ task default: :spec
5
+
6
+ RSpec::Core::RakeTask.new
7
+
8
+ require_relative "lib/s3-antivirus"
9
+ require "cli_markdown"
10
+ desc "Generates cli reference docs as markdown"
11
+ task :docs do
12
+ mkdir_p "docs/_includes"
13
+ CliMarkdown::Creator.create_all(cli_class: S3Antivirus::CLI, cli_name: "s3-antivirus")
14
+ end
data/exe/s3-antivirus ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Trap ^C
4
+ Signal.trap("INT") {
5
+ puts "\nCtrl-C detected. Exiting..."
6
+ sleep 0.1
7
+ exit
8
+ }
9
+
10
+ $:.unshift(File.expand_path("../../lib", __FILE__))
11
+ require "s3-antivirus"
12
+ require "s3_antivirus/cli"
13
+
14
+ S3Antivirus::CLI.start(ARGV)
@@ -0,0 +1 @@
1
+ require_relative "s3_antivirus"
@@ -0,0 +1,22 @@
1
+ require "zeitwerk"
2
+
3
+ module S3Antivirus
4
+ class Autoloader
5
+ class Inflector < Zeitwerk::Inflector
6
+ def camelize(basename, _abspath)
7
+ map = { cli: "CLI", version: "VERSION" }
8
+ map[basename.to_sym] || super
9
+ end
10
+ end
11
+
12
+ class << self
13
+ def setup
14
+ loader = Zeitwerk::Loader.new
15
+ loader.inflector = Inflector.new
16
+ loader.push_dir(File.dirname(__dir__)) # lib
17
+ loader.ignore("#{File.dirname(__dir__)}/s3-antivirus.rb")
18
+ loader.setup
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,18 @@
1
+ require "aws-sdk-s3"
2
+ require "aws-sdk-sns"
3
+
4
+ module S3Antivirus
5
+ module AwsServices
6
+ extend Memoist
7
+
8
+ def s3
9
+ Aws::S3::Client.new
10
+ end
11
+ memoize :s3
12
+
13
+ def sns
14
+ Aws::SNS::Client.new
15
+ end
16
+ memoize :sns
17
+ end
18
+ end
@@ -0,0 +1,29 @@
1
+ module S3Antivirus
2
+ class CLI < Command
3
+ class_option :verbose, type: :boolean
4
+ class_option :noop, type: :boolean
5
+
6
+ desc "scan", "Poll SQS queue for s3 virus findings."
7
+ long_desc Help.text(:scan)
8
+ def scan
9
+ Scan.new(options).run
10
+ end
11
+
12
+ desc "completion *PARAMS", "Prints words for auto-completion."
13
+ long_desc Help.text(:completion)
14
+ def completion(*params)
15
+ Completer.new(CLI, *params).run
16
+ end
17
+
18
+ desc "completion_script", "Generates a script that can be eval to setup auto-completion."
19
+ long_desc Help.text(:completion_script)
20
+ def completion_script
21
+ Completer::Script.generate
22
+ end
23
+
24
+ desc "version", "prints version"
25
+ def version
26
+ puts VERSION
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,82 @@
1
+ require "thor"
2
+
3
+ # Override thor's long_desc identation behavior
4
+ # https://github.com/erikhuda/thor/issues/398
5
+ class Thor
6
+ module Shell
7
+ class Basic
8
+ def print_wrapped(message, options = {})
9
+ message = "\n#{message}" unless message[0] == "\n"
10
+ stdout.puts message
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+ module S3Antivirus
17
+ class Command < Thor
18
+ class << self
19
+ def dispatch(m, args, options, config)
20
+ # Allow calling for help via:
21
+ # s3-antivirus command help
22
+ # s3-antivirus command -h
23
+ # s3-antivirus command --help
24
+ # s3-antivirus command -D
25
+ #
26
+ # as well thor's normal way:
27
+ #
28
+ # s3-antivirus help command
29
+ help_flags = Thor::HELP_MAPPINGS + ["help"]
30
+ if args.length > 1 && !(args & help_flags).empty?
31
+ args -= help_flags
32
+ args.insert(-2, "help")
33
+ end
34
+
35
+ # s3-antivirus version
36
+ # s3-antivirus --version
37
+ # s3-antivirus -v
38
+ version_flags = ["--version", "-v"]
39
+ if args.length == 1 && !(args & version_flags).empty?
40
+ args = ["version"]
41
+ end
42
+
43
+ super
44
+ end
45
+
46
+ # Override command_help to include the description at the top of the
47
+ # long_description.
48
+ def command_help(shell, command_name)
49
+ meth = normalize_command_name(command_name)
50
+ command = all_commands[meth]
51
+ alter_command_description(command)
52
+ super
53
+ end
54
+
55
+ def alter_command_description(command)
56
+ return unless command
57
+
58
+ # Add description to beginning of long_description
59
+ long_desc = if command.long_description
60
+ "#{command.description}\n\n#{command.long_description}"
61
+ else
62
+ command.description
63
+ end
64
+
65
+ # add reference url to end of the long_description
66
+ unless website.empty?
67
+ full_command = [command.ancestor_name, command.name].compact.join('-')
68
+ url = "#{website}/reference/s3-antivirus-#{full_command}"
69
+ long_desc += "\n\nHelp also available at: #{url}"
70
+ end
71
+
72
+ command.long_description = long_desc
73
+ end
74
+ private :alter_command_description
75
+
76
+ # meant to be overriden
77
+ def website
78
+ ""
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,6 @@
1
+ class S3Antivirus::Completer::Script
2
+ def self.generate
3
+ bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
4
+ puts "source #{bash_script}"
5
+ end
6
+ end
@@ -0,0 +1,10 @@
1
+ _s3-antivirus() {
2
+ COMPREPLY=()
3
+ local word="${COMP_WORDS[COMP_CWORD]}"
4
+ local words=("${COMP_WORDS[@]}")
5
+ unset words[0]
6
+ local completion=$(s3-antivirus completion ${words[@]})
7
+ COMPREPLY=( $(compgen -W "$completion" -- "$word") )
8
+ }
9
+
10
+ complete -F _s3-antivirus s3-antivirus
@@ -0,0 +1,159 @@
1
+ =begin
2
+ Code Explanation:
3
+
4
+ There are 3 types of things to auto-complete:
5
+
6
+ 1. command: the command itself
7
+ 2. parameters: command parameters.
8
+ 3. options: command options
9
+
10
+ Here's an example:
11
+
12
+ mycli hello name --from me
13
+
14
+ * command: hello
15
+ * parameters: name
16
+ * option: --from
17
+
18
+ When command parameters are done processing, the remaining completion words will be options. We can tell that the command params are completed based on the method arity.
19
+
20
+ ## Arity
21
+
22
+ For example, say you had a method for a CLI command with the following form:
23
+
24
+ ufo scale service count --cluster development
25
+
26
+ It's equivalent ruby method:
27
+
28
+ scale(service, count) = has an arity of 2
29
+
30
+ So typing:
31
+
32
+ ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
33
+
34
+ So the completion should only show options, something like this:
35
+
36
+ --noop --verbose --cluster
37
+
38
+ ## Splat Arguments
39
+
40
+ When the ruby method has a splat argument, it's arity is negative. Here are some example methods and their arities.
41
+
42
+ ship(service) = 1
43
+ scale(service, count) = 2
44
+ ships(*services) = -1
45
+ foo(example, *rest) = -2
46
+
47
+ Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
48
+
49
+ Here are some test cases, hit TAB after typing the command:
50
+
51
+ s3-antivirus completion
52
+ s3-antivirus completion hello
53
+ s3-antivirus completion hello name
54
+ s3-antivirus completion hello name --
55
+ s3-antivirus completion hello name --noop
56
+
57
+ s3-antivirus completion
58
+ s3-antivirus completion sub:goodbye
59
+ s3-antivirus completion sub:goodbye name
60
+
61
+ ## Subcommands and Thor::Group Registered Commands
62
+
63
+ Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
64
+
65
+ * regular command: ufo ship
66
+ * subcommand: ufo docker
67
+ * Thor::Group command: ufo init
68
+
69
+ Auto-completion accounts for each of these type of commands.
70
+ =end
71
+ module S3Antivirus
72
+ class Completer
73
+ def initialize(command_class, *params)
74
+ @params = params
75
+ @current_command = @params[0]
76
+ @command_class = command_class # CLI initiall
77
+ end
78
+
79
+ def run
80
+ if subcommand?(@current_command)
81
+ subcommand_class = @command_class.subcommand_classes[@current_command]
82
+ @params.shift # destructive
83
+ Completer.new(subcommand_class, *@params).run # recursively use subcommand
84
+ return
85
+ end
86
+
87
+ # full command has been found!
88
+ unless found?(@current_command)
89
+ puts all_commands
90
+ return
91
+ end
92
+
93
+ # will only get to here if command aws found (above)
94
+ arity = @command_class.instance_method(@current_command).arity.abs
95
+ if @params.size > arity or thor_group_command?
96
+ puts options_completion
97
+ else
98
+ puts params_completion
99
+ end
100
+ end
101
+
102
+ def subcommand?(command)
103
+ @command_class.subcommands.include?(command)
104
+ end
105
+
106
+ # hacky way to detect that command is a registered Thor::Group command
107
+ def thor_group_command?
108
+ command_params(raw=true) == [[:rest, :args]]
109
+ end
110
+
111
+ def found?(command)
112
+ public_methods = @command_class.public_instance_methods(false)
113
+ command && public_methods.include?(command.to_sym)
114
+ end
115
+
116
+ # all top-level commands
117
+ def all_commands
118
+ commands = @command_class.all_commands.reject do |k,v|
119
+ v.is_a?(Thor::HiddenCommand)
120
+ end
121
+ commands.keys
122
+ end
123
+
124
+ def command_params(raw=false)
125
+ params = @command_class.instance_method(@current_command).parameters
126
+ # Example:
127
+ # >> Sub.instance_method(:goodbye).parameters
128
+ # => [[:req, :name]]
129
+ # >>
130
+ raw ? params : params.map!(&:last)
131
+ end
132
+
133
+ def params_completion
134
+ offset = @params.size - 1
135
+ offset_params = command_params[offset..-1]
136
+ command_params[offset..-1].first
137
+ end
138
+
139
+ def options_completion
140
+ used = ARGV.select { |a| a.include?('--') } # so we can remove used options
141
+
142
+ method_options = @command_class.all_commands[@current_command].options.keys
143
+ class_options = @command_class.class_options.keys
144
+
145
+ all_options = method_options + class_options + ['help']
146
+
147
+ all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
148
+ filtered_options = all_options - used
149
+ filtered_options.uniq
150
+ end
151
+
152
+ # Useful for debugging. Using puts messes up completion.
153
+ def log(msg)
154
+ File.open("/tmp/complete.log", "a") do |file|
155
+ file.puts(msg)
156
+ end
157
+ end
158
+ end
159
+ end
@@ -0,0 +1,11 @@
1
+ module S3Antivirus
2
+ module Conf
3
+ extend Memoist
4
+
5
+ def conf
6
+ conf = Config.new
7
+ conf.data
8
+ end
9
+ memoize :conf
10
+ end
11
+ end
@@ -0,0 +1,29 @@
1
+ module S3Antivirus
2
+ class Config
3
+ include Logger
4
+
5
+ attr_reader :data
6
+ def initialize(path=nil)
7
+ @data = load(path)
8
+ end
9
+
10
+ def load(path)
11
+ YAML.load_file(lookup_path(path))
12
+ end
13
+
14
+ def lookup_path(path=nil)
15
+ paths = [
16
+ path,
17
+ "./s3-antivirus.conf",
18
+ "#{ENV['HOME']}/.s3-antivirus.conf",
19
+ "/etc/s3-antivirus.conf"
20
+ ].compact
21
+ found = paths.find { |p| File.exist?(p) }
22
+ unless found
23
+ logger.fatal("FATAL: unable to find the s3-antivirus.conf file. Paths considered: #{paths}")
24
+ exit 1
25
+ end
26
+ found
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,20 @@
1
+ ## Examples
2
+
3
+ s3-antivirus completion
4
+
5
+ Prints words for TAB auto-completion.
6
+
7
+ s3-antivirus completion
8
+ s3-antivirus completion hello
9
+ s3-antivirus completion hello name
10
+
11
+ To enable, TAB auto-completion add the following to your profile:
12
+
13
+ eval $(s3-antivirus completion_script)
14
+
15
+ Auto-completion example usage:
16
+
17
+ s3-antivirus [TAB]
18
+ s3-antivirus hello [TAB]
19
+ s3-antivirus hello name [TAB]
20
+ s3-antivirus hello name --[TAB]
@@ -0,0 +1,3 @@
1
+ To use, add the following to your `~/.bashrc` or `~/.profile`
2
+
3
+ eval $(s3-antivirus completion_script)
@@ -0,0 +1,24 @@
1
+ ## Example
2
+
3
+ $ s3-antivirus scan
4
+ Polling SQS queue for S3 antivirus findings. Started 2019-12-15 04:07:09 +0000...
5
+ Checking s3://test-bucket/eicar.txt
6
+ Downloading s3://test-bucket/eicar.txt to /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b...
7
+ Scanning s3://test-bucket/eicar.txt...
8
+ => clamdscan /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b
9
+ /tmp/b5e986d1-4356-454a-87fe-bc01e5747a7b: Eicar-Test-Signature FOUND
10
+
11
+ ----------- SCAN SUMMARY -----------
12
+ Infected files: 1
13
+ Time: 0.001 sec (0 m 0 s)
14
+ s3://test-bucket/eicar.txt is infected (deleting)
15
+ Checking s3://test-bucket/a.txt
16
+ Downloading s3://test-bucket/a.txt to /tmp/56da4b41-a672-42d4-a766-1727f5dc256d...
17
+ Scanning s3://test-bucket/a.txt...
18
+ => clamdscan /tmp/56da4b41-a672-42d4-a766-1727f5dc256d
19
+ /tmp/56da4b41-a672-42d4-a766-1727f5dc256d: OK
20
+
21
+ ----------- SCAN SUMMARY -----------
22
+ Infected files: 0
23
+ Time: 0.001 sec (0 m 0 s)
24
+ s3://test-bucket/a.txt is clean (tagging)
@@ -0,0 +1,9 @@
1
+ module S3Antivirus::Help
2
+ class << self
3
+ def text(namespaced_command)
4
+ path = namespaced_command.to_s.gsub(':','/')
5
+ path = File.expand_path("../help/#{path}.md", __FILE__)
6
+ IO.read(path) if File.exist?(path)
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,7 @@
1
+ module S3Antivirus
2
+ module Logger
3
+ def logger
4
+ $logger ||= Tee.new("s3-antivirus")
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,35 @@
1
+ module S3Antivirus
2
+ class Notifier
3
+ include AwsServices
4
+ include Conf
5
+
6
+ def initialize(s3_record)
7
+ @s3_record = s3_record
8
+ @bucket, @key, @version = s3_record.bucket, s3_record.key, s3_record.version
9
+ end
10
+
11
+ def notify(status:, action:)
12
+ data = {
13
+ action: action,
14
+ bucket: @bucket,
15
+ key: @key,
16
+ status: status,
17
+ }
18
+ data[:version] = @version if @version
19
+ message_attributes = data.inject({}) do |result, (k,v)|
20
+ result.merge(
21
+ k => {
22
+ data_type: "String",
23
+ string_value: v
24
+ }
25
+ )
26
+ end
27
+ sns.publish(
28
+ topic_arn: conf['topic'],
29
+ message: "#{@s3_record.human_key} is #{status}, #{action} action executed",
30
+ subject: "s3-antivirus s3://#{@bucket}",
31
+ message_attributes: message_attributes
32
+ )
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,39 @@
1
+ module S3Antivirus
2
+ class S3Record
3
+ include Conf
4
+
5
+ def initialize(record)
6
+ @record = record # record data from SQS event structure
7
+ end
8
+
9
+ def human_key
10
+ text = "s3://#{bucket}/#{key}"
11
+ text += " (version: #{version})" if version
12
+ text
13
+ end
14
+
15
+ def bucket
16
+ @record['s3']['bucket']['name']
17
+ end
18
+
19
+ def key
20
+ URI.decode(@record['s3']['object']['key']).gsub('+', ' ')
21
+ end
22
+
23
+ def version
24
+ @record['s3']['object']['versionId']
25
+ end
26
+
27
+ def oversized?
28
+ size > max_size
29
+ end
30
+
31
+ def size
32
+ @record['s3']['object']['size']
33
+ end
34
+
35
+ def max_size
36
+ conf['volume_size'] * 1073741824 / 2 # in bytes
37
+ end
38
+ end
39
+ end