porky_lib 0.9.4 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3fdbc4311ca319836e44898d3ad018416d03decf224941ae3f24a37ea1de9bad
-  data.tar.gz: bf572e33f77188a0a5b057c77bc91034e98044ba1b8e5f54d0a59d1561557948
+  metadata.gz: cbd7245d269cc4567f5d51dac830da9baeb088df1b4fb26a47c57115b6b29292
+  data.tar.gz: b1c55f42e2b30acfbe3ca9d88bd55ac9ef5d4b2e080c44563ac7a065eaeb1ff4
 SHA512:
-  metadata.gz: f25cc53a69235ee25eca951ac3fe6fcce8317baf99e7188a7a83d880d13545eeb73d6f68572999361d6e595087013b322e90b3a513815a448083b9090aab9d1f
-  data.tar.gz: cbf5794b8098c9a0fbb6562fa6c9e3d1cceed6e24798f1133eab5cee5f776797f82676a1bd6c8356a37046b642d8d7907c3e7cf2657d9c75018a45dff58b9a0c
+  metadata.gz: 44a202622d1b47fe683e051ef790664b18fe0314cff2f1dfed90e630e4f76582cae2ba826349c01b91275413c4df142f6ed92c86d4c309ff19fe74796dbbe236
+  data.tar.gz: d10b7f7b4ac97ef7c3fff635087445e9f92f69d72d5b5c985188afbaa02e77015b54afa4c478eae125fbd11f4a39bb53c67dfabf23f2f5cfb29f201a5772e6f4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    porky_lib (0.9.4)
+    porky_lib (0.9.5)
       aws-sdk-kms
       aws-sdk-s3
       msgpack
@@ -13,17 +13,17 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.243.0)
-    aws-sdk-core (3.80.0)
+    aws-partitions (1.259.0)
+    aws-sdk-core (3.86.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.25.0)
+    aws-sdk-kms (1.27.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.57.0)
-      aws-sdk-core (~> 3, >= 3.77.0)
+    aws-sdk-s3 (1.60.1)
+      aws-sdk-core (~> 3, >= 3.83.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
@@ -32,6 +32,8 @@ GEM
       bundler (>= 1.2.0, < 3)
       thor (~> 0.18)
     byebug (11.0.1)
+    codacy-coverage (2.1.5)
+      simplecov
     codecov (0.1.16)
       json
       simplecov
@@ -69,7 +71,7 @@ GEM
     rspec-support (3.9.0)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.76.0)
+    rubocop (0.78.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.6)
@@ -78,7 +80,7 @@ GEM
       unicode-display_width (>= 1.4.0, < 1.7)
     rubocop-performance (1.5.1)
       rubocop (>= 0.71.0)
-    rubocop-rspec (1.37.0)
+    rubocop-rspec (1.37.1)
       rubocop (>= 0.68.1)
     rubocop_runner (2.2.0)
     ruby-progressbar (1.10.1)
@@ -101,6 +103,7 @@ DEPENDENCIES
   bundler
   bundler-audit
   byebug
+  codacy-coverage
   codecov
   msgpack
   porky_lib!

data/README.md

@@ -160,7 +160,7 @@ file_data = PorkyLib::Unencrypted::FileService.read(bucket_name, file_key)
 # Where file is the data to encrypt and upload to S3 (can be a path or raw data or ruby file object)
 # bucket_name is the name of the S3 bucket to write to
 # key_id is the ID of the CMK to use to generate a data encryption key to encrypt the file data
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::FileService.write(file, bucket_name, key_id, options)
 ```
 
@@ -169,7 +169,7 @@ file_key = PorkyLib::FileService.write(file, bucket_name, key_id, options)
 # Where file is the data to encrypt and upload to S3 (can be a path or ruby file object)
 # bucket_name is the name of the S3 bucket to write to
 # key_id is the ID of the CMK to use to generate a data encryption key to encrypt the file data
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::FileService.write_file(file, bucket_name, key_id, options)
 ```
 
@@ -178,7 +178,7 @@ file_key = PorkyLib::FileService.write_file(file, bucket_name, key_id, options)
 # Where data is the raw data to encrypt and upload to S3
 # bucket_name is the name of the S3 bucket to write to
 # key_id is the ID of the CMK to use to generate a data encryption key to encrypt the file data
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::FileService.write_data(data, bucket_name, key_id, options)
 ```
 
@@ -187,7 +187,7 @@ file_key = PorkyLib::FileService.write_data(data, bucket_name, key_id, options)
 # --- DEPRECATED --- Please use write_data or write_file instead of write
 # Where file is the data to upload to S3 (can be a path or raw data or ruby file object)
 # bucket_name is the name of the S3 bucket to write to
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::Unencrypted::FileService.write(file, bucket_name, options)
 ```
 
@@ -195,7 +195,7 @@ file_key = PorkyLib::Unencrypted::FileService.write(file, bucket_name, options)
 ```ruby
 # Where file is the data to encrypt and upload to S3 (can be a path or ruby file object)
 # bucket_name is the name of the S3 bucket to write to
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::Unencrypted::FileService.write_file(file, bucket_name, options)
 ```
 
@@ -203,7 +203,7 @@ file_key = PorkyLib::Unencrypted::FileService.write_file(file, bucket_name, opti
 ```ruby
 # Where data is the raw data to encrypt and upload to S3
 # bucket_name is the name of the S3 bucket to write to
-# options is an optional parameter for specifying optional metadata about the file
+# options is an optional parameter for specifying optional metadata about the file and the storage_class of the object
 file_key = PorkyLib::Unencrypted::FileService.write_data(data, bucket_name, options)
 ```
 
@@ -224,6 +224,45 @@ To generate a new presigned GET url (used to download files directly from AWS S3
 url = PorkyLib::Symmetric.instance.presigned_get_url(bucket_name, file_key)
 ```
 
+## Rake task
+If you want to write or read an encrypted file from the command line, there is a Rake write and read task.
+
+> Note: the environment variables can be set globally or by prepending them to the rake task command
+
+### Write file
+
+Rake task name: `file:write`
+
+Environment variables:
+* Required
+  * `FILE_PATH` - Absolute or relative file path 
+  * `CMK_KEY_ID` - Alias of the CMK key
+  * `AWS_S3_BUCKET` - AWS S3 bucket name
+  * `AWS_REGION` - AWS region name
+  * `AWS_ACCESS_KEY_ID` - AWS access key ID (credentials)
+  * `AWS_ACCESS_KEY` - AWS secret access key (credentials)
+* Optional
+  * `AWS_S3_MOCK_CLIENT` - PorkyLib's AWS KMS mock client (defaults to `true`)
+  * `AWS_S3_MAX_FILE_SIZE` - Max file size (defaults to `1MB`)
+  * `AWS_S3_STORAGE_CLASS` - One of STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE (defaults to `STANDARD`)
+  * `AWS_S3_KEEP_FILE_NAME` - Saves the file in AWS S3 with the original file name (defaults to `true`)
+  
+### Read file
+
+Rake task name: `file:read`
+
+Environment variables:
+* Required
+  * `FILE_KEY` - AWS S3 object file key
+  * `AWS_S3_BUCKET` - AWS S3 bucket name
+  * `AWS_REGION` - AWS region name
+  * `AWS_ACCESS_KEY_ID` - AWS access key ID (credentials)
+  * `AWS_ACCESS_KEY` - AWS secret access key (credentials)
+* Optional
+  * `AWS_S3_MOCK_CLIENT` - PorkyLib's AWS KMS mock client (defaults to `true`)
+  * `AWS_S3_MAX_FILE_SIZE` - Max file size (defaults to `1MB`)
+  * `DESTINATION` - Location to save the file (defaults to `FILE_KEY`)
+
 ## Development
 
 Development on this project should occur on separate feature branches and pull requests should be submitted. When submitting a

data/Rakefile

@@ -2,6 +2,7 @@
 
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+import 'lib/tasks/file.rake'
 
 RSpec::Core::RakeTask.new(:spec)
 

data/lib/porky_lib/file_service.rb

@@ -77,7 +77,11 @@ class PorkyLib::FileService
     raise FileServiceError, 'Invalid input. One or more input values is nil' if input_invalid?(data, bucket_name, key_id)
     raise FileSizeTooLargeError, "Data size is larger than maximum allowed size of #{max_file_size}" if data_size_invalid?(data)
 
-    file_key = generate_file_key(options)
+    file_key = if options.key?(:file_name)
+                 options[:file_name]
+               else
+                 generate_file_key(options)
+               end
     tempfile = encrypt_file_contents(data, key_id, file_key, options)
 
     begin

data/lib/porky_lib/file_service_helper.rb

@@ -36,11 +36,13 @@ module PorkyLib::FileServiceHelper
 
   def perform_upload(bucket_name, file_key, tempfile, options)
     obj = s3.bucket(bucket_name).object(file_key)
-    if options.key?(:metadata)
-      obj.upload_file(tempfile.path, metadata: options[:metadata])
-    else
-      obj.upload_file(tempfile.path)
-    end
+
+    upload_options = {
+      metadata: (options[:metadata] if options.key?(:metadata)),
+      storage_class: (options[:storage_class] if options.key?(:storage_class))
+    }.compact
+
+    obj.upload_file(tempfile.path, upload_options)
   end
 
   def s3

data/lib/porky_lib/unencrypted/file_service.rb

@@ -47,7 +47,11 @@ class PorkyLib::Unencrypted::FileService
     raise FileServiceError, 'Invalid input. One or more input values is nil' if input_invalid?(data, bucket_name)
     raise FileSizeTooLargeError, "Data size is larger than maximum allowed size of #{max_file_size}" if data_size_invalid?(data)
 
-    file_key = generate_file_key(options)
+    file_key = if options.key?(:file_name)
+                 options[:file_name]
+               else
+                 generate_file_key(options)
+               end
     tempfile = write_tempfile(data, file_key)
 
     begin

data/lib/porky_lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PorkyLib
-  VERSION = "0.9.4"
+  VERSION = "0.9.5"
 end

data/lib/tasks/file.rake

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'porky_lib'
+
+namespace :file do
+  desc "Read a file from AWS S3"
+  task :read do
+    # Optional arguments
+    use_mock_client = ENV.fetch('AWS_S3_MOCK_CLIENT', 'true') == 'true'
+    max_file_size = ENV.fetch('AWS_S3_MAX_FILE_SIZE', 1_048_576).to_i
+    destination = ENV.fetch('DESTINATION', ENV['FILE_KEY'])
+
+    # Required arguments
+    arguments = {
+      file_key: ENV['FILE_KEY'],
+      aws_s3_bucket: ENV['AWS_S3_BUCKET'],
+      aws_region: ENV['AWS_REGION'],
+      aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+      aws_access_key: ENV['AWS_ACCESS_KEY']
+    }
+
+    # Checks presence of required arguments and configures porky_lib
+    check_arguments(arguments)
+    setup_porky_lib(arguments, use_mock_client, max_file_size)
+
+    # Reads and writes the file
+    message, = PorkyLib::FileService.instance.read(arguments[:aws_s3_bucket], arguments[:file_key])
+    file = File.open(destination, 'w')
+    file.puts(message)
+    file.close
+
+    puts "SUCCESS - Saved file: '#{destination}' with content of the bucket: '#{arguments[:aws_s3_bucket]}' with file_key: '#{arguments[:file_key]}'"
+  end
+
+  desc "Write a file to AWS S3"
+  task :write do
+    # Optional arguments
+    use_mock_client = ENV.fetch('AWS_S3_MOCK_CLIENT', 'true') == 'true'
+    max_file_size = ENV.fetch('AWS_S3_MAX_FILE_SIZE', 1_048_576).to_i
+    storage_class = ENV.fetch('AWS_S3_STORAGE_CLASS', 'STANDARD')
+    keep_file_name = ENV.fetch('AWS_S3_KEEP_FILE_NAME', 'true') == 'true'
+
+    # Required arguments
+    arguments = {
+      file_path: ENV['FILE_PATH'],
+      cmk_key_id: ENV['CMK_KEY_ID'],
+      aws_s3_bucket: ENV['AWS_S3_BUCKET'],
+      aws_region: ENV['AWS_REGION'],
+      aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+      aws_access_key: ENV['AWS_ACCESS_KEY']
+    }
+
+    # Checks presence of required arguments and configures porky_lib
+    check_arguments(arguments)
+    setup_porky_lib(arguments, use_mock_client, max_file_size)
+
+    write_options = {
+      storage_class: storage_class,
+      file_name: (File.basename(arguments[:file_path]) if keep_file_name)
+    }.compact
+
+    # Creates CMK key with empty tags and stores file
+    PorkyLib::Symmetric.instance.create_key([{}], arguments[:cmk_key_id]) unless PorkyLib::Symmetric.instance.cmk_alias_exists?(arguments[:cmk_key_id])
+    file_key = PorkyLib::FileService.instance.write_file(arguments[:file_path], arguments[:aws_s3_bucket], arguments[:cmk_key_id], write_options)
+
+    puts "SUCCESS - Created file: '#{arguments[:file_path]}' bucket: '#{arguments[:aws_s3_bucket]}' file_key: '#{file_key}'"
+  end
+end
+
+private
+
+def check_arguments(arguments)
+  nil_arguments = []
+  arguments.map { |key, value| nil_arguments.push(key.to_s.upcase) if value.nil? && !key.nil? }
+  abort "ERROR - Need to provide as environment variables: #{nil_arguments.join(', ')}" unless nil_arguments.empty?
+end
+
+def setup_porky_lib(arguments, use_mock_client, max_file_size)
+  PorkyLib::Config.configure(aws_region: arguments[:aws_region],
+                             aws_key_id: arguments[:aws_access_key_id],
+                             aws_key_secret: arguments[:aws_access_key],
+                             aws_client_mock: use_mock_client,
+                             max_file_size: max_file_size)
+  PorkyLib::Config.initialize_aws
+end

data/porky_lib.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'bundler-audit'
   spec.add_development_dependency 'byebug'
+  spec.add_development_dependency 'codacy-coverage'
   spec.add_development_dependency 'codecov'
   spec.add_development_dependency 'msgpack'
   spec.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: porky_lib
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.5
 platform: ruby
 authors:
 - Greg Fletcher
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-26 00:00:00.000000000 Z
+date: 2019-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-kms
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: codacy-coverage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -390,6 +404,7 @@ files:
 - lib/porky_lib/unencrypted.rb
 - lib/porky_lib/unencrypted/file_service.rb
 - lib/porky_lib/version.rb
+- lib/tasks/file.rake
 - porky_lib.gemspec
 homepage: https://github.com/Zetatango/porky_lib
 licenses: []