s3-secure 0.2.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +16 -0
  4. data/LICENSE.txt +201 -22
  5. data/README.md +134 -16
  6. data/lib/s3_secure.rb +3 -2
  7. data/lib/s3_secure/access_logs.rb +30 -0
  8. data/lib/s3_secure/access_logs/base.rb +4 -0
  9. data/lib/s3_secure/access_logs/disable.rb +37 -0
  10. data/lib/s3_secure/access_logs/enable.rb +41 -0
  11. data/lib/s3_secure/access_logs/list.rb +25 -0
  12. data/lib/s3_secure/access_logs/show.rb +89 -0
  13. data/lib/s3_secure/aws_services.rb +1 -30
  14. data/lib/s3_secure/aws_services/s3.rb +54 -0
  15. data/lib/s3_secure/cli.rb +26 -0
  16. data/lib/s3_secure/command.rb +7 -0
  17. data/lib/s3_secure/encryption.rb +2 -0
  18. data/lib/s3_secure/encryption/disable.rb +4 -8
  19. data/lib/s3_secure/encryption/enable.rb +4 -8
  20. data/lib/s3_secure/encryption/list.rb +12 -16
  21. data/lib/s3_secure/encryption/show.rb +11 -6
  22. data/lib/s3_secure/help/batch.md +14 -0
  23. data/lib/s3_secure/help/encryption/disable.md +5 -0
  24. data/lib/s3_secure/help/encryption/enable.md +6 -0
  25. data/lib/s3_secure/help/encryption/list.md +5 -0
  26. data/lib/s3_secure/help/lifecycle/add.md +13 -0
  27. data/lib/s3_secure/help/lifecycle/list.md +22 -0
  28. data/lib/s3_secure/help/lifecycle/remove.md +5 -0
  29. data/lib/s3_secure/help/lifecycle/show.md +13 -0
  30. data/lib/s3_secure/help/policy/enforce_ssl.md +34 -0
  31. data/lib/s3_secure/help/policy/list.md +5 -0
  32. data/lib/s3_secure/help/policy/unforce_ssl.md +61 -0
  33. data/lib/s3_secure/help/summary.md +22 -0
  34. data/lib/s3_secure/lifecycle.rb +31 -0
  35. data/lib/s3_secure/lifecycle/add.rb +33 -0
  36. data/lib/s3_secure/lifecycle/base.rb +5 -0
  37. data/lib/s3_secure/lifecycle/builder.rb +47 -0
  38. data/lib/s3_secure/lifecycle/list.rb +24 -0
  39. data/lib/s3_secure/lifecycle/remove.rb +28 -0
  40. data/lib/s3_secure/lifecycle/show.rb +40 -0
  41. data/lib/s3_secure/policy.rb +2 -0
  42. data/lib/s3_secure/policy/document.rb +1 -1
  43. data/lib/s3_secure/policy/enforce.rb +3 -6
  44. data/lib/s3_secure/policy/list.rb +13 -17
  45. data/lib/s3_secure/policy/show.rb +8 -6
  46. data/lib/s3_secure/policy/unforce.rb +5 -8
  47. data/lib/s3_secure/remediate_all.rb +11 -0
  48. data/lib/s3_secure/summary.rb +13 -0
  49. data/lib/s3_secure/summary/item.rb +16 -0
  50. data/lib/s3_secure/summary/items.rb +65 -0
  51. data/lib/s3_secure/table.rb +18 -0
  52. data/lib/s3_secure/version.rb +1 -1
  53. data/lib/s3_secure/versioning.rb +29 -0
  54. data/lib/s3_secure/versioning/base.rb +4 -0
  55. data/lib/s3_secure/versioning/disable.rb +19 -0
  56. data/lib/s3_secure/versioning/enable.rb +19 -0
  57. data/lib/s3_secure/versioning/list.rb +24 -0
  58. data/lib/s3_secure/versioning/show.rb +27 -0
  59. data/s3-secure.gemspec +5 -2
  60. data/spec/lib/lifecycle/builder_spec.rb +85 -0
  61. metadata +71 -6
  62. data/Gemfile.lock +0 -89
  63. data/lib/s3_secure/help/hello.md +0 -5
@@ -0,0 +1,30 @@
1
+ module S3Secure
2
+ class AccessLogs < Command
3
+ desc "list", "List bucket access_logs setting"
4
+ long_desc Help.text("access_logs/list")
5
+ option :format, desc: "Format options: #{CliFormat.formats.join(', ')}"
6
+ option :access_logs, type: :boolean, desc: "Filter for access_logs: all, true, false"
7
+ def list
8
+ List.new(options).run
9
+ end
10
+
11
+ desc "show BUCKET", "show bucket access_logs"
12
+ long_desc Help.text("access_logs/show")
13
+ def show(bucket)
14
+ Show.new(options.merge(bucket: bucket)).run
15
+ end
16
+
17
+ desc "enable BUCKET", "enable bucket access_logs"
18
+ long_desc Help.text("access_logs/enable")
19
+ option :target_bucket, desc: "Target s3 bucket"
20
+ def enable(bucket)
21
+ Enable.new(options.merge(bucket: bucket)).run
22
+ end
23
+
24
+ desc "disable BUCKET", "disable bucket access_logs"
25
+ long_desc Help.text("access_logs/disable")
26
+ def disable(bucket)
27
+ Disable.new(options.merge(bucket: bucket)).run
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,4 @@
1
+ class S3Secure::AccessLogs
2
+ class Base < S3Secure::AbstractBase
3
+ end
4
+ end
@@ -0,0 +1,37 @@
1
+ class S3Secure::AccessLogs
2
+ class Disable < Base
3
+ def run
4
+ @show = Show.new(bucket: @bucket)
5
+
6
+ remove_access_logging
7
+ remove_bucket_acl
8
+ end
9
+
10
+ def remove_access_logging
11
+ unless @show.logging_enabled?
12
+ puts "Bucket #{@bucket} is not configured with access logging. So nothing to remove."
13
+ return
14
+ end
15
+
16
+ s3.put_bucket_logging(
17
+ bucket: @bucket, # source
18
+ bucket_logging_status: {}, # empty hash to remove
19
+ )
20
+ puts "Bucket #{@bucket} access logging removed"
21
+ end
22
+
23
+ def remove_bucket_acl
24
+ unless @show.acl_enabled?
25
+ puts "Bucket #{@bucket} is not configured the log delivery ACL. So nothing to remove."
26
+ return
27
+ end
28
+
29
+ access_control_policy = @show.access_control_policy_without_log_delivery_permissions
30
+ s3.put_bucket_acl(
31
+ bucket: @bucket,
32
+ access_control_policy: access_control_policy,
33
+ )
34
+ puts "Bucket #{@bucket} ACL Log Delivery removed"
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,41 @@
1
+ class S3Secure::AccessLogs
2
+ class Enable < Base
3
+ def run
4
+ @show = Show.new(bucket: @bucket)
5
+ add_bucket_acl
6
+ enable_access_logging
7
+ end
8
+
9
+ # Bucket ACL applies on the target bucket only
10
+ def add_bucket_acl
11
+ if @show.acl_enabled?
12
+ puts "Bucket acl already has log delivery ACL"
13
+ return
14
+ end
15
+
16
+ s3.put_bucket_acl(
17
+ bucket: @bucket,
18
+ access_control_policy: @show.access_control_policy_with_log_delivery_permissions,
19
+ )
20
+ puts "Added to bucket acl that grants log delivery"
21
+ end
22
+
23
+ def enable_access_logging
24
+ if @show.logging_enabled?
25
+ puts "Bucket access logging already enabled"
26
+ return
27
+ end
28
+
29
+ s3.put_bucket_logging(
30
+ bucket: @bucket, # source
31
+ bucket_logging_status: {
32
+ logging_enabled: {
33
+ target_bucket: @show.target_bucket,
34
+ target_prefix: @show.target_prefix,
35
+ },
36
+ },
37
+ )
38
+ puts "Enabled access logging on the source bucket #{@bucket} to be delivered to the target bucket #{@show.target_bucket}"
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,25 @@
1
+ class S3Secure::AccessLogs
2
+ class List < Base
3
+ def run
4
+ presenter = CliFormat::Presenter.new(@options)
5
+ presenter.header = ["Bucket", "Access Logs?"]
6
+
7
+ buckets.each do |bucket|
8
+ $stderr.puts "Getting access log setting for bucket #{bucket.color(:green)}"
9
+ show = Show.new(bucket: bucket)
10
+
11
+ enabled = show.logging_enabled?
12
+ row = [bucket, enabled]
13
+ if @options[:enabled].nil?
14
+ presenter.rows << row # always show policy
15
+ elsif @options[:enabled]
16
+ presenter.rows << row if enabled # only show if bucket has some encryption rules
17
+ else
18
+ presenter.rows << row unless enabled # only show if bucket doesnt have any encryption rules
19
+ end
20
+ end
21
+
22
+ presenter.show
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,89 @@
1
+ class S3Secure::AccessLogs
2
+ class Show < Base
3
+ def run
4
+ puts "Bucket ACL:"
5
+ pp bucket_acl_grants
6
+ puts "Bucket Logging:"
7
+ pp bucket_logging
8
+ end
9
+
10
+ def bucket_logging
11
+ # Tricky here, need to swtich the s3 client in case target_bucket is in another region
12
+ with_regional_s3(target_bucket) do
13
+ s3.get_bucket_logging(bucket: target_bucket).to_h
14
+ end
15
+ end
16
+ memoize :bucket_logging
17
+
18
+ def bucket_acl
19
+ # Tricky here, need to swtich the s3 client in case target_bucket is in another region
20
+ with_regional_s3(target_bucket) do
21
+ s3.get_bucket_acl(bucket: target_bucket)
22
+ end
23
+ end
24
+ memoize :bucket_acl
25
+
26
+ def bucket_acl_grants
27
+ bucket_acl.grants.map(&:to_h)
28
+ end
29
+
30
+ def enabled?
31
+ acl_enabled? && logging_enabled?
32
+ end
33
+
34
+ def acl_enabled?
35
+ grants = bucket_acl_grants & log_delivery_access_grants
36
+ !grants.empty?
37
+ end
38
+
39
+ def logging_enabled?
40
+ !bucket_logging.empty?
41
+ end
42
+
43
+ def log_delivery_access_grants
44
+ [
45
+ {
46
+ grantee: {type: "Group", uri: "http://acs.amazonaws.com/groups/s3/LogDelivery"},
47
+ permission: "WRITE"
48
+ },{
49
+ grantee: {type: "Group", uri: "http://acs.amazonaws.com/groups/s3/LogDelivery"},
50
+ permission: "READ_ACP"
51
+ }
52
+ ]
53
+ end
54
+
55
+ def access_control_policy_with_log_delivery_permissions
56
+ grants = bucket_acl_grants + log_delivery_access_grants
57
+ { grants: grants, owner: owner }
58
+ end
59
+
60
+ def access_control_policy_without_log_delivery_permissions
61
+ grants = bucket_acl_grants - log_delivery_access_grants
62
+ { grants: grants, owner: owner }
63
+ end
64
+
65
+ def owner
66
+ {
67
+ display_name: bucket_acl.owner.display_name,
68
+ id: bucket_acl.owner.id,
69
+ }
70
+ end
71
+
72
+ def target_bucket
73
+ @options[:target_bucket] || @bucket
74
+ end
75
+
76
+ def target_prefix
77
+ prefix = @options[:target_prefix] || "access-logs"
78
+ prefix += "/" unless prefix.ends_with?("/")
79
+ prefix
80
+ end
81
+
82
+ def with_regional_s3(bucket)
83
+ current_bucket, @bucket = @bucket, bucket
84
+ result = yield
85
+ @bucket = current_bucket
86
+ result
87
+ end
88
+ end
89
+ end
@@ -2,35 +2,6 @@ require "aws-sdk-s3"
2
2
 
3
3
  module S3Secure
4
4
  module AwsServices
5
- extend Memoist
6
-
7
- @@buckets = {} # holds bucket => region map
8
- def s3_regional_client(bucket)
9
- region = @@buckets[bucket]
10
-
11
- unless region
12
- resp = s3_client.get_bucket_location(bucket: bucket)
13
- region = resp.location_constraint
14
- region = 'us-east-1' if region.empty? # "" means us-east-1
15
- end
16
-
17
- new_s3_regional_client(region)
18
- end
19
-
20
- def new_s3_regional_client(region=nil)
21
- options = {}
22
- options[:endpoint] = "https://s3.#{region}.amazonaws.com" if region
23
- options[:region] = region if region
24
- Aws::S3::Client.new(options)
25
- end
26
- memoize :new_s3_regional_client
27
-
28
- # Generic s3 client. Will be configured to whatever region user has locally configured in ~/.aws/config
29
- # Used to call get_bucket_location to get each specific bucket's location.
30
- # Generally use the s3_regional_client instead of this.
31
- def s3_client
32
- Aws::S3::Client.new
33
- end
34
- memoize :s3_client
5
+ include S3
35
6
  end
36
7
  end
@@ -0,0 +1,54 @@
1
+ module S3Secure::AwsServices
2
+ module S3
3
+ extend Memoist
4
+
5
+ @@s3_clients = {} # holds cached s3 regional clients cache
6
+ def s3
7
+ check_bucket!
8
+ @@s3_clients[@bucket] ||= new_s3_regional_client
9
+ end
10
+
11
+ def new_s3_regional_client
12
+ options = {}
13
+ options[:endpoint] = "https://s3.#{region}.amazonaws.com"
14
+ options[:region] = region
15
+ Aws::S3::Client.new(options)
16
+ rescue Aws::STS::Errors::RegionDisabledException
17
+ puts "ERROR: Fail to establish client connection to region #{region}".color(:red)
18
+ raise
19
+ end
20
+
21
+ # Generic s3 client. Will be configured to whatever region user has locally configured in ~/.aws/config
22
+ # Used to call get_bucket_location to get each specific bucket's location.
23
+ # Generally use the s3_regional_client instead of this.
24
+ def s3_client
25
+ Aws::S3::Client.new
26
+ end
27
+ memoize :s3_client
28
+
29
+ def check_bucket!
30
+ # IMPORANT: The class that includes this module must set @bucket before using the s3 method.
31
+ unless @bucket
32
+ raise "@bucket #{@bucket.inspect} is not set. The class must set @bucket before using the any client method."
33
+ end
34
+ region_map # triggers building region map for specific @bucket
35
+ end
36
+
37
+ @@region_map = {} # bucket to region map cache
38
+ def region_map
39
+ region = @@region_map[@bucket]
40
+ return @@region_map if region # return cache
41
+
42
+ # build cache
43
+ resp = s3_client.get_bucket_location(bucket: @bucket)
44
+ region = resp.location_constraint
45
+ region = 'us-east-1' if region.empty? # "" means us-east-1
46
+ @@region_map[@bucket] = region
47
+ @@region_map
48
+ end
49
+
50
+ def region
51
+ region_map[@bucket]
52
+ end
53
+ end
54
+ end
@@ -3,6 +3,10 @@ module S3Secure
3
3
  class_option :verbose, type: :boolean
4
4
  class_option :noop, type: :boolean
5
5
 
6
+ desc "access_logs SUBCOMMAND", "access_logs subcommands"
7
+ long_desc Help.text(:access_logs)
8
+ subcommand "access_logs", AccessLogs
9
+
6
10
  desc "encryption SUBCOMMAND", "encryption subcommands"
7
11
  long_desc Help.text(:encryption)
8
12
  subcommand "encryption", Encryption
@@ -11,6 +15,28 @@ module S3Secure
11
15
  long_desc Help.text(:policy)
12
16
  subcommand "policy", Policy
13
17
 
18
+ desc "versioning SUBCOMMAND", "versioning subcommands"
19
+ long_desc Help.text(:versioning)
20
+ subcommand "versioning", Versioning
21
+
22
+ desc "lifecycle SUBCOMMAND", "lifecycle subcommands"
23
+ long_desc Help.text(:lifecycle)
24
+ subcommand "lifecycle", Lifecycle
25
+
26
+ desc "remediate_all", "Remediate all. For more fine-grain control use each of the commands directly."
27
+ long_desc Help.text("remediate_all")
28
+ def remediate_all(bucket)
29
+ RemediateAll.new(options.merge(bucket: bucket)).run
30
+ end
31
+
32
+ desc "summary", "Summarize buckets"
33
+ long_desc Help.text("summary")
34
+ option :encrypted, default: "any", desc: "filter for encryption enabled. Examples: any, yes, no"
35
+ option :ssl, default: "any", desc: "filter for ssl enforcement. Examples: any, yes, no"
36
+ def summary
37
+ Summary.new(options).run
38
+ end
39
+
14
40
  desc "batch *PARAMS", "Batch wrapper method"
15
41
  long_desc Help.text(:batch)
16
42
  def batch(*params)
@@ -77,6 +77,13 @@ module S3Secure
77
77
  def website
78
78
  ""
79
79
  end
80
+
81
+ # https://github.com/erikhuda/thor/issues/244
82
+ # Deprecation warning: Thor exit with status 0 on errors. To keep this behavior, you must define `exit_on_failure?` in `Lono::CLI`
83
+ # You can silence deprecations warning by setting the environment variable THOR_SILENCE_DEPRECATION.
84
+ def exit_on_failure?
85
+ true
86
+ end
80
87
  end
81
88
  end
82
89
  end
@@ -2,6 +2,8 @@ module S3Secure
2
2
  class Encryption < Command
3
3
  desc "list", "List bucket encryptions"
4
4
  long_desc Help.text("encryption/list")
5
+ option :format, desc: "Format options: #{CliFormat.formats.join(', ')}"
6
+ option :encryption, type: :boolean, desc: "Filter for encryption: all, true, false"
5
7
  def list
6
8
  List.new(options).run
7
9
  end
@@ -1,17 +1,13 @@
1
1
  class S3Secure::Encryption
2
2
  class Disable < Base
3
3
  def run
4
- @s3 = s3_regional_client(@bucket)
4
+ show = Show.new(@options)
5
5
 
6
- list = S3Secure::Encryption::List.new(@options)
7
- list.set_s3(@s3)
8
-
9
- rules = list.get_encryption_rules(@bucket)
10
- if rules
11
- @s3.delete_bucket_encryption(bucket: @bucket) # returns resp = #<struct Aws::EmptyStructure>
6
+ if show.enabled?
7
+ s3.delete_bucket_encryption(bucket: @bucket) # returns resp = #<struct Aws::EmptyStructure>
12
8
  puts "Bucket #{@bucket} encryption has been removed"
13
9
  else
14
- puts "WARN: Bucket #{@bucket} is not configured with encryption at the bucket level".color(:yellow)
10
+ puts "Bucket #{@bucket} is not configured with encryption at the bucket level"
15
11
  end
16
12
  end
17
13
  end
@@ -1,16 +1,12 @@
1
1
  class S3Secure::Encryption
2
2
  class Enable < Base
3
3
  def run
4
- @s3 = s3_regional_client(@bucket)
4
+ show = Show.new(@options)
5
5
 
6
- list = S3Secure::Encryption::List.new(@options)
7
- list.set_s3(@s3)
8
-
9
- rules = list.get_encryption_rules(@bucket)
10
- if rules
6
+ if show.enabled?
11
7
  # check rules to see if encryption is already set of some sort
12
8
  puts "Bucket #{@bucket} already has encryption rules:"
13
- puts rules.map(&:to_h)
9
+ puts show.rules.map(&:to_h)
14
10
  else
15
11
  # Set encryption rules
16
12
  # Ruby docs: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#put_bucket_encryption-instance_method
@@ -18,7 +14,7 @@ class S3Secure::Encryption
18
14
  #
19
15
  # put_bucket_encryption returns #<struct Aws::EmptyStructure>
20
16
  #
21
- @s3.put_bucket_encryption(
17
+ s3.put_bucket_encryption(
22
18
  bucket: @bucket,
23
19
  server_side_encryption_configuration: {
24
20
  rules: [rule]})
@@ -1,28 +1,24 @@
1
1
  class S3Secure::Encryption
2
2
  class List < Base
3
3
  def run
4
+ presenter = CliFormat::Presenter.new(@options)
5
+ presenter.header = ["Bucket", "Has Encryption?"]
6
+
4
7
  buckets.each do |bucket|
5
- @s3 = s3_regional_client(bucket)
6
- puts "Policy for bucket #{bucket.color(:green)}"
7
- encryption_rules = get_encryption_rules(bucket)
8
+ $stderr.puts "Getting encryption for bucket #{bucket.color(:green)}"
9
+ show = Show.new(bucket: bucket)
8
10
 
9
- if encryption_rules
10
- puts encryption_rules
11
+ row = [bucket, show.enabled?]
12
+ if @options[:encryption].nil?
13
+ presenter.rows << row # always show policy
14
+ elsif @options[:encryption]
15
+ presenter.rows << row if show.enabled? # only show if bucket has some encryption rules
11
16
  else
12
- puts "Bucket does not have bucket encryption enabled"
17
+ presenter.rows << row unless show.enabled? # only show if bucket doesnt have any encryption rules
13
18
  end
14
19
  end
15
- end
16
-
17
- def get_encryption_rules(bucket)
18
- resp = @s3.get_bucket_encryption(bucket: bucket)
19
- resp.server_side_encryption_configuration.rules # Aws::Xml::DefaultList object
20
- rescue Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError
21
- end
22
20
 
23
- # Useful when calling List outside of the list CLI
24
- def set_s3(client)
25
- @s3 = client
21
+ presenter.show
26
22
  end
27
23
  end
28
24
  end