data_migrater 0.7.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad45bfd5bedd2d2b8a8bddd76afc4d10ce030c99d1507e5a57226c2d9be4a7bc
-  data.tar.gz: cce9a45c1224c2b9b69e0238ec7f39881b98ea96632f4a98aae11591b90c5158
+  metadata.gz: d23aa11964d870b88a42f582ada1573f58a781d08a359cc25eea5e2dbf7dab62
+  data.tar.gz: cd3bbe075f0cc3ff091284ef1a00c23c27a8f725989f269cb33eebd2f7c07ebb
 SHA512:
-  metadata.gz: 450dd08d38c0122a9b50ebc5d620d497e548a6f15e683887dab95a763a04006b4246ab5b697d512c3dbcbd8f694b7bd61d4a80efb54950aec3e2b99489e9e785
-  data.tar.gz: ce1181a6c82f55ac2748072d566793e3fb8f0f22df57c6942719fe991bfa346f07bb4f37cf9c27a83d0b96c71a933ac80d973d0a9a803fcbdc5c4d75fc422c05
+  metadata.gz: 8fadf53e7bd9f4eaeb709847d5f73b6a53369eca3a93ee766d0192e56195a940122cc330b687ff93a346f8ca550f7eeb150f2b71d0dd757818f4631330ed0d29
+  data.tar.gz: e6beaa0daecc1926844ee50fe15684a673e5710b7bb3e68ae7eefcc31b508e634079a4fc78649953b1ed1e07257a741bada5640e5327b792d556beb5cc5cccff

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## v1.0.0
+
+### Break
+
+- Removes the `tmp_dir` options, now the file will be download at `path` place.
+
+### Fixes
+
+- Do not raise error when S3 file does not exist;
+
+### News
+
+- Uses `us-east-1` as default region for S3 module;
+- Adds `csv_delete` method to be able to delete the S3 file.
+
+### Updates
+
+- Uses new syntax of SDK S3.
+
 ## v0.7.0
 
 ### News

data/README.md

@@ -110,7 +110,7 @@ class MyDataMigration
   def execute
     # [
     #   { first_name: 'Washington', last_name: 'Botelho' },
-    #   { first_name: 'Lucas'     , last_name: 'Souza' }
+    #   { first_name: 'Vanessa'   , last_name: 'Queiroz' }
     # ]
     csv.each { |line| Object.create line }
   end
@@ -134,14 +134,14 @@ end
 ```
 
 #### Options
-dum
-- `dir`: Directory where CSV is located;
-- `file`: File name;
-- `path`: Composition of `dir/file` when you want give a fully qualified path. Default: `db/data_migrate/support/csv/class_name.csv`.
+
+- `dir`: Directory where CSV is located, by default `db/data_migrate/support/csv`;
+- `file`: File name, by default is the class name underscored: `my_data_migration.csv`;
+- `path`: Composition of `dir` + `/` + `file` when you want give a fully qualified path.
 
 ---
 
-##### CSV Options:
+##### CSV Options
 
 - `chunk_size`: Batch parse size;
 - `key_mapping`: Key name alias.
@@ -150,14 +150,13 @@ For more CSV options, check the project [Smarter CSV](https://github.com/tilo/sm
 
 ## S3
 
-You can download your CSV directly from [Amazon S3](https://aws.amazon.com/s3) using the module `DataMigrater::CSV` with some configs.
-You *must* keep the path as `:s3` to activate S3 feature.
+You can download your CSV directly from [Amazon S3](https://aws.amazon.com/s3) using the module `DataMigrater::CSV` with some configs. You *must* set `provider` as `:s3` to activate S3 feature.
 
 ```ruby
 class MyDataMigration
   include DataMigrater::CSV
 
-  data_csv path: :s3
+  data_csv bucket: 'my-bucket', provider: :s3
 
   def execute
     csv.each { |line| Object.create line }
@@ -165,41 +164,20 @@ class MyDataMigration
 end
 ```
 
-By default, the class name is used as the file name in `underscore` style: `my_data_migration.csv`. You can change it:
-
-```ruby
-class MyDataMigration
-  include DataMigrater::CSV
-
-  data_csv path: :s3, file: 'custom-name.csv'
-
-  def execute
-    csv.each { |line| Object.create line }
-  end
-end
-```
-
-By default, the bucket name is `data-migrater`, to change it, just declare the `bucket` options:
-
-```ruby
-class MyDataMigration
-  include DataMigrater::CSV
-
-  data_csv path: :s3, bucket: 'custom-bucket'
+### Credentials
 
-  def execute
-    csv.each { |line| Object.create line }
-  end
-end
-```
-
-When file is downloaded, it is keeped in a temporary (`/tmp`) folder waiting to be parsed, using the options `tmp_dir` you change it:
+By default, when you use the S3 feature, the envs `ACCESS_KEY_ID`, `REGION` (default `us-east-1`) and `SECRET_ACCESS_KEY` will be used.
+If you do not want export it globally and need to pass it inside you class, just declare de `credentials` options:
 
 ```ruby
 class MyDataMigration
   include DataMigrater::CSV
 
-  data_csv path: :s3, tmp_dir: '/Users/wbotelhos'
+  data_csv provider: :s3, credentials: {
+    access_key_id:     'foo',
+    region:            'us-east-1',
+    secret_access_key: 'bar'
+  }
 
   def execute
     csv.each { |line| Object.create line }
@@ -207,34 +185,29 @@ class MyDataMigration
 end
 ```
 
-#### Credentials
+### CSV Delete
 
-By default, when you use the S3 feature, the envs `ACCESS_KEY_ID`, `REGION` and `SECRET_ACCESS_KEY` will be used.
-If you do not want export it globally and need to pass it inside you class, just declare de `credentials` options:
+You can delete the S3 file from your migration after process the CSV.
 
 ```ruby
 class MyDataMigration
   include DataMigrater::CSV
 
-  data_csv path: :s3, credentials: {
-    access_key_id:     'foo',
-    region:            'us-east-1',
-    secret_access_key: 'bar'
-  }
+  data_csv provider: :s3
 
   def execute
     csv.each { |line| Object.create line }
+
+    csv_delete
   end
 end
 ```
 
-#### Options
+#### S3 Options
 
-- `bucket`: Bucket name;
+- `bucket`: The bucket name. By default `data-migrater`.
 - `credentials`: AWS credentials: `access_key_id`, `region` and `secret_access_key`;
-- `file`: File name;
-- `path`: `:s3` to indicate the S3 support;
-- `tmp_dir`: Directory where CSV will be keeped after download.
+- `provider`: `:s3` to indicate the S3 provider;
 
 #### Skip Run
 

data/lib/data_migrater/csv.rb

@@ -8,7 +8,7 @@ module DataMigrater
 
     included do
       def csv(processor: ::SmarterCSV)
-        return s3.download(processor: processor) if csv_s3?
+        s3.download if csv_options.delete(:provider) == :s3
 
         processor.process csv_path, csv_options
       end
@@ -19,6 +19,10 @@ module DataMigrater
         @csv_path ||= [csv_dir, csv_file].join('/')
       end
 
+      def csv_delete
+        s3.delete
+      end
+
       private
 
       def csv_bucket
@@ -37,25 +41,12 @@ module DataMigrater
         self.class.csv_options
       end
 
-      def csv_s3?
-        csv_path.to_s == 's3'
-      end
-
-      def csv_tmp_dir
-        csv_options.delete(:tmp_dir) || '/tmp'
-      end
-
       def s3_credentials
         csv_options.delete(:credentials) || {}
       end
 
       def s3
-        DataMigrater::S3.new(
-          credentials: s3_credentials,
-          bucket:      csv_bucket,
-          file:        csv_file,
-          tmp_dir:     csv_tmp_dir
-        )
+        @s3 ||= DataMigrater::S3.new(csv_bucket, s3_credentials, csv_path)
       end
     end
 

data/lib/data_migrater/s3.rb

@@ -4,43 +4,42 @@ module DataMigrater
   class S3
     require 'aws-sdk-s3'
 
-    def initialize(bucket:, credentials: {}, file:, tmp_dir:)
+    def initialize(bucket, credentials, csv_path)
       @bucket      = bucket
-      @credentials = credentials.reverse_merge(default_credentials)
-      @file        = file
-      @tmp_dir     = tmp_dir
+      @credentials = default_credentials.merge(credentials)
+      @csv_path    = csv_path
 
       ::Aws.config.update @credentials
     end
 
-    def download(processor:)
-      File.open(file_path, 'w+') do |file|
-        client.get_object options, target: file
+    def delete
+      client.delete_object options
+    end
+
+    def download
+      client.head_object options
 
-        processor.process file
-      end
+      client.get_object options.merge(response_target: @csv_path)
+    rescue Aws::S3::Errors::NotFound
+      []
     end
 
     private
 
     def client
-      Aws::S3::Client.new
+      @client ||= Aws::S3::Client.new
     end
 
     def default_credentials
       {
         access_key_id:     ENV['AWS_ACCESS_KEY_ID'],
-        region:            ENV['AWS_REGION'],
+        region:            ENV.fetch('AWS_REGION', 'us-east-1'),
         secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
       }
     end
 
-    def file_path
-      [@tmp_dir, @file].join('/')
-    end
-
     def options
-      { bucket: @bucket, key: @file }
+      { bucket: @bucket, key: @csv_path.split('/').last }
     end
   end
 end

data/lib/data_migrater/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataMigrater
-  VERSION = '0.7.0'
+  VERSION = '1.0.0'
 end

data/spec/csv/csv_delete_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe '#csv_delete' do
+  let!(:s3) { instance_double 'DataMigrater::S3' }
+
+  before do
+    stub_const 'Dummy', Class.new
+
+    Dummy.class_eval { include DataMigrater::CSV }
+    Dummy.class_eval { data_csv provider: :s3 }
+
+    allow(DataMigrater::S3).to receive(:new)
+      .with('data-migrater', {}, 'db/data_migrate/support/csv/dummy.csv').and_return s3
+  end
+
+  it 'delegates delete to s3 object' do
+    expect(s3).to receive(:delete)
+
+    Dummy.new.csv_delete
+  end
+end

data/spec/csv/csv_spec.rb

@@ -30,23 +30,21 @@ RSpec.describe '#data_csv' do
     end
   end
 
-  context 'when path is about s3' do
+  context 'when provider is s3' do
     before do
       stub_const 'Dummy', Class.new
 
       Dummy.class_eval { include DataMigrater::CSV }
-      Dummy.class_eval { data_csv path: :s3 }
-
-      allow(DataMigrater::S3).to receive(:new).with(
-        credentials: {},
-        bucket:      'data-migrater',
-        file:        'dummy.csv',
-        tmp_dir:     '/tmp'
-      ) { double download: :result }
+      Dummy.class_eval { data_csv provider: :s3 }
+
+      allow(DataMigrater::S3).to receive(:new)
+        .with('data-migrater', {}, 'db/data_migrate/support/csv/dummy.csv').and_return double(download: true)
     end
 
     it 'reads csv from s3' do
-      expect(Dummy.new.csv).to eq :result
+      expect(::SmarterCSV).to receive(:process).with('db/data_migrate/support/csv/dummy.csv', {})
+
+      Dummy.new.csv
     end
   end
 end

data/spec/s3/delete_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe DataMigrater::S3, '.delete' do
+  subject(:s3) { described_class.new 'bucket', {}, 'path/dummy.csv' }
+
+  let!(:client) { instance_double('Aws::S3::Client').as_null_object }
+
+  before { allow(Aws::S3::Client).to receive(:new) { client } }
+
+  context 'when file is found' do
+    it 'deletes the file' do
+      expect(client).to receive(:delete_object).with(bucket: 'bucket', key: 'dummy.csv')
+
+      s3.delete
+    end
+  end
+end

data/spec/s3/download_spec.rb

@@ -3,27 +3,34 @@
 require 'spec_helper'
 
 RSpec.describe DataMigrater::S3, '.download' do
-  subject { described_class.new bucket: 'data-migrater', file: 'dummy.csv', tmp_dir: '/tmp' }
+  subject(:s3) { described_class.new options[:bucket], {}, 'csv_path' }
 
-  let!(:client)    { double(Aws::S3).as_null_object }
-  let!(:file_open) { double(File).as_null_object }
-  let!(:processor) { double.as_null_object }
-  let!(:temp_file) { double(File).as_null_object }
+  let!(:client)  { double('Aws::S3::Client').as_null_object }
+  let!(:options) { { bucket: 'data-migrater', key: 'csv_path' } }
 
-  before do
-    allow(Aws::S3::Client).to receive(:new) { client }
-    allow(File).to receive(:open).with('/tmp/dummy.csv', 'w+').and_yield temp_file
-  end
+  before { allow(Aws::S3::Client).to receive(:new) { client } }
+
+  context 'when file is found' do
+    it 'downloads the file' do
+      expect(client).to receive(:get_object).with(options.merge(response_target: 'csv_path'))
 
-  it 'downloads the csv file' do
-    expect(client).to receive(:get_object).with({ bucket: 'data-migrater', key: 'dummy.csv' }, target: temp_file)
+      s3.download
+    end
 
-    subject.download processor: processor
+    it 'returns the value of get object' do
+      expect(client).to receive(:get_object).with(options.merge(response_target: 'csv_path')).and_return :success
+
+      expect(s3.download).to eq :success
+    end
   end
 
-  it 'process the csv content with given processor' do
-    expect(processor).to receive(:process).with temp_file
+  context 'when file is not found' do
+    let!(:error) { Aws::S3::Errors::NotFound.new 'error', 'message' }
+
+    before { allow(client).to receive(:head_object).with(options).and_raise error }
 
-    subject.download processor: processor
+    it 'returns an empty array' do
+      expect(s3.download).to eq []
+    end
   end
 end

data/spec/s3/initialize_spec.rb

@@ -3,59 +3,39 @@
 require 'spec_helper'
 
 RSpec.describe DataMigrater::S3, 'initialize' do
+  subject(:s3) { described_class.new 'data-migrater', credentials, 'dummy.csv' }
+
   before do
     allow(ENV).to receive(:[]).with('AWS_ACCESS_KEY_ID') { 'AWS_ACCESS_KEY_ID' }
-    allow(ENV).to receive(:[]).with('AWS_REGION') { 'AWS_REGION' }
+    allow(ENV).to receive(:fetch).with('AWS_REGION', 'us-east-1') { 'AWS_REGION' }
     allow(ENV).to receive(:[]).with('AWS_SECRET_ACCESS_KEY') { 'AWS_SECRET_ACCESS_KEY' }
   end
 
-  context 'when only mandatory params is given' do
-    subject { described_class.new bucket: 'data-migrater', file: 'dummy.csv', tmp_dir: '/tmp' }
+  context 'when credentials is not given' do
+    let!(:credentials) { {} }
+
+    let!(:credentials_env) do
+      { access_key_id: 'AWS_ACCESS_KEY_ID', region: 'AWS_REGION', secret_access_key: 'AWS_SECRET_ACCESS_KEY' }
+    end
 
     it 'caches default values and uses exported envs' do
-      expect(subject.instance_variable_get(:@bucket)).to  eq 'data-migrater'
-      expect(subject.instance_variable_get(:@file)).to    eq 'dummy.csv'
-      expect(subject.instance_variable_get(:@tmp_dir)).to eq '/tmp'
-
-      expect(subject.instance_variable_get(:@credentials)).to eq(
-        access_key_id:     'AWS_ACCESS_KEY_ID',
-        region:            'AWS_REGION',
-        secret_access_key: 'AWS_SECRET_ACCESS_KEY'
-      )
+      expect(s3.instance_variable_get(:@bucket)).to      eq 'data-migrater'
+      expect(s3.instance_variable_get(:@csv_path)).to    eq 'dummy.csv'
+      expect(s3.instance_variable_get(:@credentials)).to eq credentials_env
     end
 
     it 'updates the aws config' do
-      expect(::Aws.config).to receive(:update).with(
-        access_key_id:     'AWS_ACCESS_KEY_ID',
-        region:            'AWS_REGION',
-        secret_access_key: 'AWS_SECRET_ACCESS_KEY'
-      )
+      expect(::Aws.config).to receive(:update).with credentials_env
 
       subject
     end
   end
 
-  context 'when some credential is given' do
-    subject do
-      described_class.new(
-        bucket:  'data-migrater',
-        file:    'dummy.csv',
-        tmp_dir: '/tmp',
-
-        credentials: {
-          access_key_id:     'access_key_id',
-          region:            'region',
-          secret_access_key: 'secret_access_key'
-        }
-      )
-    end
+  context 'when credentials is given' do
+    let!(:credentials) { { access_key_id: 'access_key_id', region: 'region', secret_access_key: 'secret_access_key' } }
 
     it 'is used' do
-      expect(::Aws.config).to receive(:update).with(
-        access_key_id:     'access_key_id',
-        region:            'region',
-        secret_access_key: 'secret_access_key'
-      )
+      expect(::Aws.config).to receive(:update).with credentials
 
       subject
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_migrater
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Washington Botelho
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-06 00:00:00.000000000 Z
+date: 2019-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -162,6 +162,7 @@ files:
 - lib/generators/data_migrater/templates/db/migrate/create_data_migrations.rb
 - spec/collection_spec.rb
 - spec/csv/converters_spec.rb
+- spec/csv/csv_delete_spec.rb
 - spec/csv/csv_path_spec.rb
 - spec/csv/csv_spec.rb
 - spec/csv/options_spec.rb
@@ -170,6 +171,7 @@ files:
 - spec/logger/logger_path_spec.rb
 - spec/migrater_spec.rb
 - spec/migration_spec.rb
+- spec/s3/delete_spec.rb
 - spec/s3/download_spec.rb
 - spec/s3/initialize_spec.rb
 - spec/spec_helper.rb
@@ -201,20 +203,22 @@ signing_key:
 specification_version: 4
 summary: A Data Migrator gem
 test_files:
+- spec/spec_helper.rb
+- spec/collection_spec.rb
+- spec/s3/initialize_spec.rb
+- spec/s3/delete_spec.rb
+- spec/s3/download_spec.rb
 - spec/logger/data_logger_spec.rb
 - spec/logger/logger_path_spec.rb
-- spec/collection_spec.rb
-- spec/support/database_cleaner.rb
-- spec/support/common.rb
+- spec/migration_spec.rb
 - spec/support/migrate.rb
+- spec/support/common.rb
 - spec/support/csv/dummy.csv
-- spec/spec_helper.rb
+- spec/support/database_cleaner.rb
 - spec/migrater_spec.rb
-- spec/s3/download_spec.rb
-- spec/s3/initialize_spec.rb
-- spec/data_migration_spec.rb
-- spec/migration_spec.rb
 - spec/csv/csv_spec.rb
 - spec/csv/options_spec.rb
 - spec/csv/csv_path_spec.rb
+- spec/csv/csv_delete_spec.rb
 - spec/csv/converters_spec.rb
+- spec/data_migration_spec.rb