data_migrater 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d18ad057ff627de12fb89e183f32f399fc5982bf
-  data.tar.gz: 71d9bc74af6bedeb81e810512584b20649d26aa0
+  metadata.gz: 2e763846467f7bf1af5607a89750f416e4addb70
+  data.tar.gz: 666f66ff8908a468b27f4da84fb486c0e0919ffb
 SHA512:
-  metadata.gz: acd8775f066a903ed373309b4c06c934301aaa76ec1a41bcfe769271d3d2c33c69d8f0559bc7d35bdfb91921978e17bf909bfaa7c0de831acd35391759443988
-  data.tar.gz: 7b8c18ec30448b76280a87bca602a335133d11f11b09d2aa4344ae98a0eda6824a0d49620b4883df1574721fc9bebee4f6ecc9560a08f10b3c62cc28543dc0a4
+  metadata.gz: c3c6a9c17e22316dc01aa1385d62134063ab3801db8528615577dcd8394bc1dc334e6b288912120fac6fdc504f22e0099e3a216d3a778fc03fc2eaccd55a6a76
+  data.tar.gz: c98bf19f3c91c7bde8f9a38a4472504eef266d6e149150d864f984f3fd7ef8faa18a4ed6d02c7d9e387d26065c4c0d00fe83df101eda1d0064f1b0e91aa6b13a

data/CHANGELOG.md

@@ -1,6 +1,13 @@
+# 0.4.0
+
+- Add Rails 5 support;
+- Bump Ruby until 2.4.1.
+
 # 0.3.0
 
-- Added support for CSV.
+- Added support for CSV;
+- Added options `dir` to specify the directory on Logger;
+- Added options `file` to specify the Logger file name.
 
 # 0.2.0
 

data/README.md

@@ -1,5 +1,8 @@
 # Data Migrater
 
+[![Build Status](https://travis-ci.org/getninjas/data_migrater.svg)](https://travis-ci.org/getninjas/data_migrater)
+[![Gem Version](https://badge.fury.io/rb/data_migrater.svg)](https://badge.fury.io/rb/data_migrater)
+
 Data Migration is like Migrate, but it will changes the data of your tables,
 not your schema. This gems creates a file where you can write code to make
 change along you schema changes.
@@ -73,7 +76,6 @@ class MyDataMigration
   include DataMigrater::CSV
 
   def execute
-    # parsed from `db/data_migrate/support/csv/my_data_migration.csv`
     csv.each do |line|
       Object.create! line
     end
@@ -87,10 +89,9 @@ By default, the class name is used and the file is parsed from `db/data_migrate/
 class MyDataMigration
   include DataMigrater::CSV
 
-  data_csv path: "/tmp/objects.csv"
+  data_csv path: '/tmp/objects.csv'
 
   def execute
-    # parsed from `/tmp/objects.csv`
     csv.each do |line|
       Object.create! line
     end
@@ -107,16 +108,11 @@ class MyDataMigration
   data_csv chunk_size: 2
 
   def execute
-    csv.each do |line|
-      # line:
-      #
-      # [
-      #   { first_name: 'Washington', last_name: 'Botelho' },
-      #   { first_name: 'Lucas'     , last_name: 'Souza' }
-      # ]
-
-      Object.create! line
-    end
+    # [
+    #   { first_name: 'Washington', last_name: 'Botelho' },
+    #   { first_name: 'Lucas'     , last_name: 'Souza' }
+    # ]
+    csv.each { |line| Object.create line }
   end
 end
 ```
@@ -138,7 +134,7 @@ end
 ```
 
 #### Options
-
+dum
 - `dir`: Directory where CSV is located;
 - `file`: File name;
 - `path`: Composition of `dir/file` when you want give a fully qualified path. Default: `db/data_migrate/support/csv/class_name.csv`.
@@ -152,18 +148,90 @@ end
 
 For more CSV options, check the project [Smarter CSV](https://github.com/tilo/smarter_csv):
 
-## Test
+## S3
 
-Before send pull request, check if specs is passing.
+You can download your CSV directly from [Amazon S3](https://aws.amazon.com/s3) using the module `DataMigrater::CSV` with some configs.
+You *must* keep the path as `:s3` to activate S3 feature.
 
-```bash
-rspec spec
 ```
+class MyDataMigration
+  include DataMigrater::CSV
+
+  data_csv path: :s3
 
-## Code Style
+  def execute
+    csv.each { |line| Object.create line }
+  end
+end
+```
 
-And check if the code style is good.
+By default, the class name is used as the file name in `underscore` style: `my_data_migration.csv`. You can change it:
 
-```bash
-rubocop --debug --display-cop-names
 ```
+class MyDataMigration
+  include DataMigrater::CSV
+
+  data_csv path: :s3, file: 'custom-name.csv'
+
+  def execute
+    csv.each { |line| Object.create line }
+  end
+end
+```
+
+By default, the bucket name is `data-migrater`, to change it, just declare the `bucket` options:
+
+```
+class MyDataMigration
+  include DataMigrater::CSV
+
+  data_csv path: :s3, bucket: 'custom-bucket'
+
+  def execute
+    csv.each { |line| Object.create line }
+  end
+end
+```
+
+When file is downloaded, it is keeped in a temporary (`/tmp`) folder waiting to be parsed, using the options `tmp_dir` you change it:
+
+```
+class MyDataMigration
+  include DataMigrater::CSV
+
+  data_csv path: :s3, tmp_dir: '/Users/wbotelhos'
+
+  def execute
+    csv.each { |line| Object.create line }
+  end
+end
+```
+
+#### Credentials
+
+By default, when you use the S3 feature, the envs `ACCESS_KEY_ID`, `REGION` and `SECRET_ACCESS_KEY` will be used.
+If you do not want export it globally and need to pass it inside you class, just declare de `credentials` options:
+
+```
+class MyDataMigration
+  include DataMigrater::CSV
+
+  data_csv path: :s3, credentials: {
+    access_key_id:     'foo',
+    region:            'us-east-1',
+    secret_access_key: 'bar'
+  }
+
+  def execute
+    csv.each { |line| Object.create line }
+  end
+end
+```
+
+#### Options
+
+- `bucket`: Bucket name;
+- `credentials`: AWS credentials: `access_key_id`, `region` and `secret_access_key`;
+- `file`: File name;
+- `path`: `:s3` to indicate the S3 support;
+- `tmp_dir`: Directory where CSV will be keeped after download.

data/lib/data_migrater.rb

@@ -10,4 +10,5 @@ require 'data_migrater/csv'
 require 'data_migrater/logger'
 require 'data_migrater/migration'
 require 'data_migrater/migrator'
+require 'data_migrater/s3'
 require 'data_migrater/version'

data/lib/data_migrater/csv.rb

@@ -7,20 +7,24 @@ module DataMigrater
     require 'smarter_csv'
 
     included do
-      def csv
-        result = ::SmarterCSV.process(csv_path, csv_options)
+      def csv(processor: ::SmarterCSV)
+        return s3.download(processor: processor) if csv_s3?
 
-        result
+        processor.process csv_path, csv_options
       end
 
       def csv_path
         return csv_options[:path] if csv_options[:path].present?
 
-        [csv_dir, csv_file].join '/'
+        @csv_path ||= [csv_dir, csv_file].join('/')
       end
 
       private
 
+      def csv_bucket
+        csv_options.delete(:bucket) || 'data-migrater'
+      end
+
       def csv_dir
         csv_options.delete(:dir) || 'db/data_migrate/support/csv'
       end
@@ -32,15 +36,36 @@ module DataMigrater
       def csv_options
         self.class.csv_options
       end
+
+      def csv_s3?
+        csv_path.to_s == 's3'
+      end
+
+      def csv_tmp_dir
+        csv_options.delete(:tmp_dir) || '/tmp'
+      end
+
+      def s3_credentials
+        csv_options.delete(:credentials) || {}
+      end
+
+      def s3
+        DataMigrater::S3.new(
+          credentials: s3_credentials,
+          bucket:      csv_bucket,
+          file:        csv_file,
+          tmp_dir:     csv_tmp_dir
+        )
+      end
     end
 
     module ClassMethods
       def data_csv(options = {})
-        @options = options
+        @csv_options = options
       end
 
       def csv_options
-        @options || {}
+        @csv_options || {}
       end
     end
   end

data/lib/data_migrater/logger.rb

@@ -44,11 +44,11 @@ module DataMigrater
 
     module ClassMethods
       def data_logger(options = {})
-        @options = options
+        @logger_options = options
       end
 
       def logger_options
-        @options || {}
+        @logger_options || {}
       end
     end
   end

data/lib/data_migrater/s3.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DataMigrater
+  class S3
+    require 'aws-sdk'
+
+    def initialize(bucket:, credentials: {}, file:, tmp_dir:)
+      @bucket      = bucket
+      @credentials = credentials.reverse_merge(default_credentials)
+      @file        = file
+      @tmp_dir     = tmp_dir
+
+      ::Aws.config.update @credentials
+    end
+
+    def download(processor:)
+      File.open(file_path, 'w+') do |file|
+        client.get_object options, target: file
+
+        processor.process file
+      end
+    end
+
+    private
+
+    def client
+      Aws::S3::Client.new
+    end
+
+    def default_credentials
+      {
+        access_key_id:     ENV['AWS_ACCESS_KEY_ID'],
+        region:            ENV['AWS_REGION'],
+        secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
+      }
+    end
+
+    def file_path
+      [@tmp_dir, @file].join('/')
+    end
+
+    def options
+      { bucket: @bucket, key: @file }
+    end
+  end
+end

data/lib/data_migrater/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataMigrater
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.5.0'
 end

data/spec/csv/csv_spec.rb

@@ -3,28 +3,50 @@
 require 'spec_helper'
 
 RSpec.describe '#data_csv' do
-  before do
-    stub_const 'Dummy', Class.new
+  context 'when path is not about s3' do
+    before do
+      stub_const 'Dummy', Class.new
 
-    Dummy.class_eval { include DataMigrater::CSV }
-    Dummy.class_eval { data_csv dir: 'spec/support/csv' }
+      Dummy.class_eval { include DataMigrater::CSV }
+      Dummy.class_eval { data_csv dir: 'spec/support/csv' }
+    end
+
+    it 'returns an array of hash' do
+      expect(Dummy.new.csv).to eq [
+        {
+          first_name: 'Washington',
+          last_name:  'Botelho',
+          username:   'wbotelhos',
+          age:        32,
+          birthday:   '23/10/1984'
+        },
+
+        {
+          first_name: 'Lucas',
+          last_name:  'Souza',
+          username:   'lucasas'
+        }
+      ]
+    end
   end
 
-  it 'returns an array of hash' do
-    expect(Dummy.new.csv).to eq [
-      {
-        first_name: 'Washington',
-        last_name:  'Botelho',
-        username:   'wbotelhos',
-        age:        32,
-        birthday:   '23/10/1984'
-      },
-
-      {
-        first_name: 'Lucas',
-        last_name:  'Souza',
-        username:   'lucasas'
-      }
-    ]
+  context 'when path is about s3' do
+    before do
+      stub_const 'Dummy', Class.new
+
+      Dummy.class_eval { include DataMigrater::CSV }
+      Dummy.class_eval { data_csv path: :s3 }
+
+      allow(DataMigrater::S3).to receive(:new).with(
+        credentials: {},
+        bucket:      'data-migrater',
+        file:        'dummy.csv',
+        tmp_dir:     '/tmp'
+      ) { double download: :result }
+    end
+
+    it 'reads csv from s3' do
+      expect(Dummy.new.csv).to eq :result
+    end
   end
 end

data/spec/logger/data_logger_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe '#data_logger' do
     it 'logs on the given path file with right content' do
       subject.logger.info 'done!'
 
-      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} -\d{4}\] INFO Dummy: done!\n/
+      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (-|\+)\d{4}\] INFO Dummy: done!\n/
 
       expect(File.readlines('custom.log').last).to match result
     end
@@ -35,7 +35,7 @@ RSpec.describe '#data_logger' do
     it 'logs on log folder with class name with right content' do
       subject.logger.info 'done!'
 
-      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} -\d{4}\] INFO Dummy: done!\n/
+      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (-|\+)\d{4}\] INFO Dummy: done!\n/
 
       expect(File.readlines('log/dummy.log').last).to match result
     end
@@ -53,7 +53,7 @@ RSpec.describe '#data_logger' do
     it 'logs on log folder with class name with right content' do
       subject.logger.info 'done!'
 
-      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} -\d{4}\] INFO Dummy: done!\n/
+      result = /\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (-|\+)\d{4}\] INFO Dummy: done!\n/
 
       expect(File.readlines('log/dummy.log').last).to match result
     end

data/spec/migration_spec.rb

@@ -34,7 +34,7 @@ describe DataMigrater::Migration do
         DataMigration.create version: version
       end
 
-      specify { expect(subject.execute).to be_falsy }
+      specify { expect(subject.execute).to eq false }
 
       it 'does not executes' do
         expect(migration).not_to receive :execute
@@ -53,7 +53,7 @@ describe DataMigrater::Migration do
       it 'creates a new data_migrations' do
         subject.execute
 
-        expect(DataMigration.exists?(version: version)).to be_truthy
+        expect(DataMigration.exists?(version: version)).to eq true
       end
 
       context 'and some error is raised' do
@@ -67,7 +67,7 @@ describe DataMigrater::Migration do
           begin
             subject.execute
           rescue
-            expect(DataMigration.exists?(version: version)).to be_falsy
+            expect(DataMigration.exists?(version: version)).to eq false
           end
         end
       end

data/spec/s3/download_spec.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe DataMigrater::S3, '.download' do
+  subject { described_class.new bucket: 'data-migrater', file: 'dummy.csv', tmp_dir: '/tmp' }
+
+  let!(:client)    { double(Aws::S3).as_null_object }
+  let!(:file_open) { double(File).as_null_object }
+  let!(:processor) { double.as_null_object }
+  let!(:temp_file) { double(File).as_null_object }
+
+  before do
+    allow(Aws::S3::Client).to receive(:new) { client }
+    allow(File).to receive(:open).with('/tmp/dummy.csv', 'w+').and_yield temp_file
+  end
+
+  it 'downloads the csv file' do
+    expect(client).to receive(:get_object).with({ bucket: 'data-migrater', key: 'dummy.csv' }, target: temp_file)
+
+    subject.download processor: processor
+  end
+
+  it 'process the csv content with given processor' do
+    expect(processor).to receive(:process).with temp_file
+
+    subject.download processor: processor
+  end
+end

data/spec/s3/initialize_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe DataMigrater::S3, 'initialize' do
+  before do
+    allow(ENV).to receive(:[]).with('AWS_ACCESS_KEY_ID') { 'AWS_ACCESS_KEY_ID' }
+    allow(ENV).to receive(:[]).with('AWS_REGION') { 'AWS_REGION' }
+    allow(ENV).to receive(:[]).with('AWS_SECRET_ACCESS_KEY') { 'AWS_SECRET_ACCESS_KEY' }
+  end
+
+  context 'when only mandatory params is given' do
+    subject { described_class.new bucket: 'data-migrater', file: 'dummy.csv', tmp_dir: '/tmp' }
+
+    it 'caches default values and uses exported envs' do
+      expect(subject.instance_variable_get(:@bucket)).to  eq 'data-migrater'
+      expect(subject.instance_variable_get(:@file)).to    eq 'dummy.csv'
+      expect(subject.instance_variable_get(:@tmp_dir)).to eq '/tmp'
+
+      expect(subject.instance_variable_get(:@credentials)).to eq(
+        access_key_id:     'AWS_ACCESS_KEY_ID',
+        region:            'AWS_REGION',
+        secret_access_key: 'AWS_SECRET_ACCESS_KEY'
+      )
+    end
+
+    it 'updates the aws config' do
+      expect(::Aws.config).to receive(:update).with(
+        access_key_id:     'AWS_ACCESS_KEY_ID',
+        region:            'AWS_REGION',
+        secret_access_key: 'AWS_SECRET_ACCESS_KEY'
+      )
+
+      subject
+    end
+  end
+
+  context 'when some credential is given' do
+    subject do
+      described_class.new(
+        bucket:  'data-migrater',
+        file:    'dummy.csv',
+        tmp_dir: '/tmp',
+
+        credentials: {
+          access_key_id:     'access_key_id',
+          region:            'region',
+          secret_access_key: 'secret_access_key'
+        }
+      )
+    end
+
+    it 'is used' do
+      expect(::Aws.config).to receive(:update).with(
+        access_key_id:     'access_key_id',
+        region:            'region',
+        secret_access_key: 'secret_access_key'
+      )
+
+      subject
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -9,3 +9,7 @@ require 'pry-byebug'
 ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
 
 Dir[File.expand_path('support/**/*.rb', __dir__)].each { |file| require file }
+
+def local_path
+  "#{File.expand_path(__dir__)}/support/local"
+end

data/spec/support/local/dummy.csv

@@ -1,3 +0,0 @@
-first name,last name,username,age,birthday
-Washington,Botelho,wbotelhos,32,23/10/1984
-Lucas,Souza,lucasas,,

metadata

@@ -1,43 +1,56 @@
 --- !ruby/object:Gem::Specification
 name: data_migrater
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
+- Washington Botelho
 - GetNinjas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-31 00:00:00.000000000 Z
+date: 2017-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 4.1.14
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.14
+        version: '4.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 4.1.14
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.14
+        version: '4.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: smarter_csv
   requirement: !ruby/object:Gem::Requirement
@@ -53,19 +66,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2.9'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.9'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -108,20 +121,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -165,7 +164,9 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Generates Data Migrations on Migrate style.
-email: tech@getninjas.com.br
+email:
+- wbotelhos@gmail.com
+- tech@getninjas.com.br
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -179,6 +180,7 @@ files:
 - lib/data_migrater/logger.rb
 - lib/data_migrater/migration.rb
 - lib/data_migrater/migrator.rb
+- lib/data_migrater/s3.rb
 - lib/data_migrater/version.rb
 - lib/data_migration.rb
 - lib/generators/data_migrater/create_generator.rb
@@ -196,6 +198,8 @@ files:
 - spec/logger/logger_path_spec.rb
 - spec/migrater_spec.rb
 - spec/migration_spec.rb
+- spec/s3/download_spec.rb
+- spec/s3/initialize_spec.rb
 - spec/spec_helper.rb
 - spec/support/common.rb
 - spec/support/csv/dummy.csv
@@ -222,7 +226,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A Data Migrator gem
@@ -237,6 +241,8 @@ test_files:
 - spec/logger/logger_path_spec.rb
 - spec/migrater_spec.rb
 - spec/migration_spec.rb
+- spec/s3/download_spec.rb
+- spec/s3/initialize_spec.rb
 - spec/spec_helper.rb
 - spec/support/common.rb
 - spec/support/csv/dummy.csv