impex 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
 - data/.gitignore +10 -0
 - data/.travis.yml +5 -0
 - data/Gemfile +4 -0
 - data/LICENSE.txt +21 -0
 - data/README.md +113 -0
 - data/Rakefile +14 -0
 - data/bin/console +14 -0
 - data/bin/setup +8 -0
 - data/impex.gemspec +31 -0
 - data/lib/generators/impex/install_generator.rb +17 -0
 - data/lib/generators/impex/templates/create_impex_history.rb +13 -0
 - data/lib/generators/impex/templates/impex.rb +12 -0
 - data/lib/impex.rb +14 -0
 - data/lib/impex/configuration.rb +31 -0
 - data/lib/impex/engine.rb +50 -0
 - data/lib/impex/file.rb +17 -0
 - data/lib/impex/file_formatter.rb +28 -0
 - data/lib/impex/file_loader/base.rb +15 -0
 - data/lib/impex/file_loader/errors.rb +5 -0
 - data/lib/impex/file_loaders/file_system.rb +22 -0
 - data/lib/impex/history_manager/base.rb +14 -0
 - data/lib/impex/history_manager/errors.rb +5 -0
 - data/lib/impex/history_managers/active_record.rb +86 -0
 - data/lib/impex/lookup.rb +69 -0
 - data/lib/impex/row.rb +10 -0
 - data/lib/impex/version.rb +3 -0
 - data/lib/tasks/all.rake +6 -0
 - metadata +195 -0
 
    
        checksums.yaml
    ADDED
    
    | 
         @@ -0,0 +1,7 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            ---
         
     | 
| 
      
 2 
     | 
    
         
            +
            SHA1:
         
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: a15777de5afa825929a201162fdfcd88fd136195
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: e8d135c401ff7dca41b200d82ef4162b2b4ec98c
         
     | 
| 
      
 5 
     | 
    
         
            +
            SHA512:
         
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 4fcaeea5fe105570eea52bd7a74ab7aa891ec0189ae43b63457dd81329682167a1f57e2676031c9d1a0c35405d16edaf84ddfb4812ee4ff63b2ecdf3052d6660
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 1d8f4d4b0c4b44e32c8e8c856f473eb4cb8b37f91e4d4f5be469520a586c3430680da71142d08794c374f29472f33266b944fa13f9e8938ae758871afcca097f
         
     | 
    
        data/.gitignore
    ADDED
    
    
    
        data/.travis.yml
    ADDED
    
    
    
        data/Gemfile
    ADDED
    
    
    
        data/LICENSE.txt
    ADDED
    
    | 
         @@ -0,0 +1,21 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            The MIT License (MIT)
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            Copyright (c) 2016 Mehdi FARSI
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            Permission is hereby granted, free of charge, to any person obtaining a copy
         
     | 
| 
      
 6 
     | 
    
         
            +
            of this software and associated documentation files (the "Software"), to deal
         
     | 
| 
      
 7 
     | 
    
         
            +
            in the Software without restriction, including without limitation the rights
         
     | 
| 
      
 8 
     | 
    
         
            +
            to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
         
     | 
| 
      
 9 
     | 
    
         
            +
            copies of the Software, and to permit persons to whom the Software is
         
     | 
| 
      
 10 
     | 
    
         
            +
            furnished to do so, subject to the following conditions:
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
            The above copyright notice and this permission notice shall be included in
         
     | 
| 
      
 13 
     | 
    
         
            +
            all copies or substantial portions of the Software.
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
            THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
         
     | 
| 
      
 16 
     | 
    
         
            +
            IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
         
     | 
| 
      
 17 
     | 
    
         
            +
            FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
         
     | 
| 
      
 18 
     | 
    
         
            +
            AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
         
     | 
| 
      
 19 
     | 
    
         
            +
            LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
         
     | 
| 
      
 20 
     | 
    
         
            +
            OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
         
     | 
| 
      
 21 
     | 
    
         
            +
            THE SOFTWARE.
         
     | 
    
        data/README.md
    ADDED
    
    | 
         @@ -0,0 +1,113 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            # CsvImporter
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            ## Installation
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            Add this line to your application's Gemfile:
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 8 
     | 
    
         
            +
            gem 'impex'
         
     | 
| 
      
 9 
     | 
    
         
            +
            ```
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
            And then execute:
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
                $ bundle
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
            ## Usage
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 18 
     | 
    
         
            +
            ?> rails g impex:install
         
     | 
| 
      
 19 
     | 
    
         
            +
            ?> rake db:migrate
         
     | 
| 
      
 20 
     | 
    
         
            +
            ?> rake impex:all
         
     | 
| 
      
 21 
     | 
    
         
            +
            ```
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
            ## How it Works
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
            By default, `impex` looks for CSV files in `public/csv_import/TABLE_NAME/*.csv`. `TABLE_NAME` stands for the name of table (e.g: `users`, ...). Each file may require a header that contains the exact name of the field. If it's not the case, a [hook]() is provided to re-organize the mapping of the file before check. 
         
     | 
| 
      
 26 
     | 
    
         
            +
             
     | 
| 
      
 27 
     | 
    
         
            +
            Before the update of a field, `impex` will check if csv column value has never been a value of the table field (e.g: Update the email only if has never been an email of the person).
         
     | 
| 
      
 28 
     | 
    
         
            +
             
     | 
| 
      
 29 
     | 
    
         
            +
            However, it's possible to custom the list of field to look up via a `whitelisting` [config](https://github.com/mehdi-farsi/impex/blob/master/lib/generators/impex/templates/impex.rb). 
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
            By default, the reference field (a unique field that's used as identifier to search in history) corresponds to a field named `reference`.
         
     | 
| 
      
 32 
     | 
    
         
            +
            You can customize the reference for each tables in [config](https://github.com/mehdi-farsi/impex/blob/master/lib/generators/impex/templates/impex.rb).
         
     | 
| 
      
 33 
     | 
    
         
            +
             
     | 
| 
      
 34 
     | 
    
         
            +
            # Tools
         
     | 
| 
      
 35 
     | 
    
         
            +
             
     | 
| 
      
 36 
     | 
    
         
            +
            The gem provides 2 tools: a rake tasks and a rails generator.
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
            The rails generator `impex:install` provides 2 files:
         
     | 
| 
      
 39 
     | 
    
         
            +
             
     | 
| 
      
 40 
     | 
    
         
            +
            ```shell
         
     | 
| 
      
 41 
     | 
    
         
            +
            $ rails g impex:install
         
     | 
| 
      
 42 
     | 
    
         
            +
              create  config/initializers/impex.rb
         
     | 
| 
      
 43 
     | 
    
         
            +
              create  db/migrate/20160831080017_create_impex_histories.rb
         
     | 
| 
      
 44 
     | 
    
         
            +
            ```
         
     | 
| 
      
 45 
     | 
    
         
            +
             
     | 
| 
      
 46 
     | 
    
         
            +
            The `impex.rb` initializer permits to manage the path of the CSV files and the whitelisting for the history.
         
     | 
| 
      
 47 
     | 
    
         
            +
             
     | 
| 
      
 48 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 49 
     | 
    
         
            +
            Impex.configure({
         
     | 
| 
      
 50 
     | 
    
         
            +
              file_loader: { loader: :file_system, path: "#{Rails.root}public/" },
         
     | 
| 
      
 51 
     | 
    
         
            +
              history_whitelisting: {
         
     | 
| 
      
 52 
     | 
    
         
            +
                buildings: [:manager_name],
         
     | 
| 
      
 53 
     | 
    
         
            +
                people:    [:email, :phone_number, :cell_phone, :address]
         
     | 
| 
      
 54 
     | 
    
         
            +
              }
         
     | 
| 
      
 55 
     | 
    
         
            +
            })
         
     | 
| 
      
 56 
     | 
    
         
            +
            ```
         
     | 
| 
      
 57 
     | 
    
         
            +
             
     | 
| 
      
 58 
     | 
    
         
            +
            The rake task `impex:all` iterates through each file available in the `path` defined in config.
         
     | 
| 
      
 59 
     | 
    
         
            +
             
     | 
| 
      
 60 
     | 
    
         
            +
            ## Extend functionality
         
     | 
| 
      
 61 
     | 
    
         
            +
             
     | 
| 
      
 62 
     | 
    
         
            +
            ### Internals
         
     | 
| 
      
 63 
     | 
    
         
            +
             
     | 
| 
      
 64 
     | 
    
         
            +
            The project is based on the principle of "composition over inheritance". There is 3 main entities in the project: the core, the `FileLoader` system and the `HistoryManager` system.
         
     | 
| 
      
 65 
     | 
    
         
            +
             
     | 
| 
      
 66 
     | 
    
         
            +
            #### FileLoader
         
     | 
| 
      
 67 
     | 
    
         
            +
             
     | 
| 
      
 68 
     | 
    
         
            +
            The `FileLoader` is in charge of fetching and converting CSV files into a format that the `impex` can consume.
         
     | 
| 
      
 69 
     | 
    
         
            +
            The project provides one file loader by default: `Impex::FileLoader::FileSystem`.
         
     | 
| 
      
 70 
     | 
    
         
            +
             
     | 
| 
      
 71 
     | 
    
         
            +
            If you want to add a new `FileLoader` (e.g `FileLoader::FileLoader::S3`), you can create a new class that inherits from `Impex::FileLoader::Base` and implements the instance method `#load` that returns an array of `Impex::File`.
         
     | 
| 
      
 72 
     | 
    
         
            +
             
     | 
| 
      
 73 
     | 
    
         
            +
            In order to ease the convertion of the CSV file to a format that is consumable by the `impex`, the project provides a `Impex::FileFormatter.build(INSTANCE_OF_FILE_CLASS)` method that takes an instance of the ruby `File` class anf returns a `Impex::File`.
         
     | 
| 
      
 74 
     | 
    
         
            +
             
     | 
| 
      
 75 
     | 
    
         
            +
            #### HistoryManager
         
     | 
| 
      
 76 
     | 
    
         
            +
             
     | 
| 
      
 77 
     | 
    
         
            +
            The `HistoryManager` is in charge of filter csv columns by maintaining an history of the value used in past for each column.
         
     | 
| 
      
 78 
     | 
    
         
            +
            The project provides one history manager by default: `Impex::HistoryManager::ActiveRecord`.
         
     | 
| 
      
 79 
     | 
    
         
            +
             
     | 
| 
      
 80 
     | 
    
         
            +
            If you want to add a new `HistoryManager` (e.g `FileLoader::HistoryManager::Redis`), you can create a new class that inherits from `Impex::HistoryManager::Base` and implements the instance methods `#filter_data_with_history(row)` and `#update_history(row)`.
         
     | 
| 
      
 81 
     | 
    
         
            +
             
     | 
| 
      
 82 
     | 
    
         
            +
            Once a new `FileLoader` or `HistoryManager` is added to the project then you can choose it by providing a config (configs name have to be in `underscore` syntax)
         
     | 
| 
      
 83 
     | 
    
         
            +
             
     | 
| 
      
 84 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 85 
     | 
    
         
            +
            file_loader: { loader: :s3, path: "#{Rails.root}public/" },
         
     | 
| 
      
 86 
     | 
    
         
            +
            history_manager: { manager: :redis, table: "impex_histories" }
         
     | 
| 
      
 87 
     | 
    
         
            +
            ```
         
     | 
| 
      
 88 
     | 
    
         
            +
             
     | 
| 
      
 89 
     | 
    
         
            +
            ### Hooks
         
     | 
| 
      
 90 
     | 
    
         
            +
             
     | 
| 
      
 91 
     | 
    
         
            +
            The entry point of the project is the method `Impex::Engine.run`. This method accepts a block. The block (that takes a `Impex::Row` instance as parameter) is called just before history checks and insertion of each row.
         
     | 
| 
      
 92 
     | 
    
         
            +
             
     | 
| 
      
 93 
     | 
    
         
            +
            So you can re-arrange the mapping of the row (recall: each header value has to correspond to a table column name)
         
     | 
| 
      
 94 
     | 
    
         
            +
             
     | 
| 
      
 95 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 96 
     | 
    
         
            +
            Impex::Engine.run do |row|
         
     | 
| 
      
 97 
     | 
    
         
            +
              if row.table == "users"
         
     | 
| 
      
 98 
     | 
    
         
            +
                row.columns[:email] = row.columns.delete('Company email')
         
     | 
| 
      
 99 
     | 
    
         
            +
                # ...
         
     | 
| 
      
 100 
     | 
    
         
            +
              end
         
     | 
| 
      
 101 
     | 
    
         
            +
            end
         
     | 
| 
      
 102 
     | 
    
         
            +
            ```
         
     | 
| 
      
 103 
     | 
    
         
            +
             
     | 
| 
      
 104 
     | 
    
         
            +
            by overriding the method `Impex::Engine#insert_row(record, row)` you can manage the way to update the record according to the filtered row. 
         
     | 
| 
      
 105 
     | 
    
         
            +
             
     | 
| 
      
 106 
     | 
    
         
            +
            ### Contact
         
     | 
| 
      
 107 
     | 
    
         
            +
             
     | 
| 
      
 108 
     | 
    
         
            +
            Thanks for reading.
         
     | 
| 
      
 109 
     | 
    
         
            +
             
     | 
| 
      
 110 
     | 
    
         
            +
            [Twitter](https://twitter.com/farsi_mehdi)<br/>
         
     | 
| 
      
 111 
     | 
    
         
            +
            [GitHub](https://github.com/mehdi-farsi/)<br/>
         
     | 
| 
      
 112 
     | 
    
         
            +
            [LinkedIn](https://fr.linkedin.com/in/mehdifarsi)<br/>
         
     | 
| 
      
 113 
     | 
    
         
            +
             
     | 
    
        data/Rakefile
    ADDED
    
    | 
         @@ -0,0 +1,14 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require "bundler/gem_tasks"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "rake/testtask"
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            Rake::TestTask.new(:test) do |t|
         
     | 
| 
      
 5 
     | 
    
         
            +
              t.libs << "test"
         
     | 
| 
      
 6 
     | 
    
         
            +
              t.libs << "lib"
         
     | 
| 
      
 7 
     | 
    
         
            +
              t.test_files = %w(
         
     | 
| 
      
 8 
     | 
    
         
            +
                test/engine_test.rb
         
     | 
| 
      
 9 
     | 
    
         
            +
                test/lookup_test.rb
         
     | 
| 
      
 10 
     | 
    
         
            +
                test/version_test.rb
         
     | 
| 
      
 11 
     | 
    
         
            +
              )
         
     | 
| 
      
 12 
     | 
    
         
            +
            end
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
            task :default => :test
         
     | 
    
        data/bin/console
    ADDED
    
    | 
         @@ -0,0 +1,14 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            #!/usr/bin/env ruby
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            require "bundler/setup"
         
     | 
| 
      
 4 
     | 
    
         
            +
            require "impex"
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            # You can add fixtures and/or initialization code here to make experimenting
         
     | 
| 
      
 7 
     | 
    
         
            +
            # with your gem easier. You can also use a different console, if you like.
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
            # (If you use this, don't forget to add pry to your Gemfile!)
         
     | 
| 
      
 10 
     | 
    
         
            +
            # require "pry"
         
     | 
| 
      
 11 
     | 
    
         
            +
            # Pry.start
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
            require "irb"
         
     | 
| 
      
 14 
     | 
    
         
            +
            IRB.start
         
     | 
    
        data/bin/setup
    ADDED
    
    
    
        data/impex.gemspec
    ADDED
    
    | 
         @@ -0,0 +1,31 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            # coding: utf-8
         
     | 
| 
      
 2 
     | 
    
         
            +
            lib = File.expand_path('../lib', __FILE__)
         
     | 
| 
      
 3 
     | 
    
         
            +
            $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
         
     | 
| 
      
 4 
     | 
    
         
            +
            require 'impex/version'
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            Gem::Specification.new do |spec|
         
     | 
| 
      
 7 
     | 
    
         
            +
              spec.name          = "impex"
         
     | 
| 
      
 8 
     | 
    
         
            +
              spec.version       = Impex::VERSION
         
     | 
| 
      
 9 
     | 
    
         
            +
              spec.date          = Time.now.strftime("%F")
         
     | 
| 
      
 10 
     | 
    
         
            +
              spec.authors       = ["mehdi-farsi"]
         
     | 
| 
      
 11 
     | 
    
         
            +
              spec.email         = ["mehdifarsi.pro@gmail.com"]
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
              spec.summary       = "An idempotent CSV import system"
         
     | 
| 
      
 14 
     | 
    
         
            +
              spec.homepage      = "https://github.com/mehdi-farsi/impex"
         
     | 
| 
      
 15 
     | 
    
         
            +
              spec.license       = "MIT"
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
              spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
         
     | 
| 
      
 18 
     | 
    
         
            +
              spec.bindir        = "exe"
         
     | 
| 
      
 19 
     | 
    
         
            +
              spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
         
     | 
| 
      
 20 
     | 
    
         
            +
              spec.require_paths = ["lib"]
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
              spec.add_development_dependency "rails", "4.2.6"
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
              spec.add_development_dependency "bundler", "~> 1.12"
         
     | 
| 
      
 25 
     | 
    
         
            +
              spec.add_development_dependency "rake", "~> 10.0"
         
     | 
| 
      
 26 
     | 
    
         
            +
              spec.add_development_dependency "minitest", "~> 5.0"
         
     | 
| 
      
 27 
     | 
    
         
            +
              spec.add_development_dependency  "minitest-reporters", "~> 1.1", ">= 1.1.11"
         
     | 
| 
      
 28 
     | 
    
         
            +
              spec.add_development_dependency "mocha", "~> 1.1"
         
     | 
| 
      
 29 
     | 
    
         
            +
              spec.add_development_dependency "sqlite3", "~> 1.3", ">= 1.3.11"
         
     | 
| 
      
 30 
     | 
    
         
            +
              spec.add_development_dependency "rubocop", "~> 0.42.0"
         
     | 
| 
      
 31 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,17 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 2 
     | 
    
         
            +
              class InstallGenerator < Rails::Generators::Base
         
     | 
| 
      
 3 
     | 
    
         
            +
                source_root ::File.expand_path('../templates', __FILE__)
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
                argument :history_manager, :type => :string, default: "activerecord"
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                def generate_export
         
     | 
| 
      
 8 
     | 
    
         
            +
                  initializer_path = "#{Rails.root}/config/initializers/impex.rb"
         
     | 
| 
      
 9 
     | 
    
         
            +
                  copy_file "impex.rb", initializer_path
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                  if history_manager == "activerecord"
         
     | 
| 
      
 12 
     | 
    
         
            +
                    migration_path = "#{Rails.root}/db/migrate/#{Time.now.strftime("%Y%m%d%H%M%S")}_create_impex_histories.rb"
         
     | 
| 
      
 13 
     | 
    
         
            +
                    copy_file "create_impex_history.rb", migration_path
         
     | 
| 
      
 14 
     | 
    
         
            +
                  end
         
     | 
| 
      
 15 
     | 
    
         
            +
                end
         
     | 
| 
      
 16 
     | 
    
         
            +
              end
         
     | 
| 
      
 17 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,13 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            class CreateImpexHistories < ActiveRecord::Migration
         
     | 
| 
      
 2 
     | 
    
         
            +
              def change
         
     | 
| 
      
 3 
     | 
    
         
            +
                create_table :impex_histories do |t|
         
     | 
| 
      
 4 
     | 
    
         
            +
                  t.column :reference, :string
         
     | 
| 
      
 5 
     | 
    
         
            +
                  t.column :table, :string
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                  t.column :key, :string
         
     | 
| 
      
 8 
     | 
    
         
            +
                  t.column :value, :string
         
     | 
| 
      
 9 
     | 
    
         
            +
                end
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                add_index(:impex_histories, [:reference, :table], order: {reference: :desc, table: :desc})
         
     | 
| 
      
 12 
     | 
    
         
            +
              end
         
     | 
| 
      
 13 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/impex.rb
    ADDED
    
    | 
         @@ -0,0 +1,14 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require "impex/configuration"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "impex/engine"
         
     | 
| 
      
 3 
     | 
    
         
            +
            require "impex/lookup"
         
     | 
| 
      
 4 
     | 
    
         
            +
            require "impex/version"
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 7 
     | 
    
         
            +
              if defined?(Rails)
         
     | 
| 
      
 8 
     | 
    
         
            +
                class Railtie < Rails::Railtie
         
     | 
| 
      
 9 
     | 
    
         
            +
                  rake_tasks do
         
     | 
| 
      
 10 
     | 
    
         
            +
                    ::Dir[::File.join(::File.dirname(__FILE__),'tasks/*.rake')].each { |f| load f }
         
     | 
| 
      
 11 
     | 
    
         
            +
                  end
         
     | 
| 
      
 12 
     | 
    
         
            +
                end
         
     | 
| 
      
 13 
     | 
    
         
            +
              end
         
     | 
| 
      
 14 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,31 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require "yaml"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "rails"
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 5 
     | 
    
         
            +
              @config = {
         
     | 
| 
      
 6 
     | 
    
         
            +
                file_loader: { loader: :file_system, relative_path: "public/" },
         
     | 
| 
      
 7 
     | 
    
         
            +
                history_manager: { manager: :active_record, table: "impex_histories" },
         
     | 
| 
      
 8 
     | 
    
         
            +
                history_whitelisting: {},
         
     | 
| 
      
 9 
     | 
    
         
            +
                history_references: {}
         
     | 
| 
      
 10 
     | 
    
         
            +
              }
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
              @valid_config_keys = %I[
         
     | 
| 
      
 13 
     | 
    
         
            +
                file_loader
         
     | 
| 
      
 14 
     | 
    
         
            +
                history_manager
         
     | 
| 
      
 15 
     | 
    
         
            +
                history_whitelisting
         
     | 
| 
      
 16 
     | 
    
         
            +
                history_references
         
     | 
| 
      
 17 
     | 
    
         
            +
              ]
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
              # Configure through hash
         
     | 
| 
      
 20 
     | 
    
         
            +
              def self.configure(options = {})
         
     | 
| 
      
 21 
     | 
    
         
            +
                options.each do |k,v|
         
     | 
| 
      
 22 
     | 
    
         
            +
                  @config[k.to_sym] = v if @valid_config_keys.include? k.to_sym
         
     | 
| 
      
 23 
     | 
    
         
            +
                end
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
                @config
         
     | 
| 
      
 26 
     | 
    
         
            +
              end
         
     | 
| 
      
 27 
     | 
    
         
            +
             
     | 
| 
      
 28 
     | 
    
         
            +
              def self.config
         
     | 
| 
      
 29 
     | 
    
         
            +
                @config
         
     | 
| 
      
 30 
     | 
    
         
            +
              end
         
     | 
| 
      
 31 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/impex/engine.rb
    ADDED
    
    | 
         @@ -0,0 +1,50 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 2 
     | 
    
         
            +
              class Engine
         
     | 
| 
      
 3 
     | 
    
         
            +
                def initialize(options = {})
         
     | 
| 
      
 4 
     | 
    
         
            +
                end
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
                def run(&block)
         
     | 
| 
      
 7 
     | 
    
         
            +
                  @config = Impex.config
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
                  @lookup = Impex::Lookup.new(@config)
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                  @files_loader    = @lookup.file_loader
         
     | 
| 
      
 12 
     | 
    
         
            +
                  @history_manager = @lookup.history_manager
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
                  @files = @files_loader.load
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
                  @files.each do |file|
         
     | 
| 
      
 17 
     | 
    
         
            +
                    model = file.table
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
                    file.each do |row|
         
     | 
| 
      
 20 
     | 
    
         
            +
                      # the user can re-organize each row before saving.
         
     | 
| 
      
 21 
     | 
    
         
            +
                      row = block.call(row) if block
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
                      row = @history_manager.filter_data_with_history(row)
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
                      reference_column = @config[:history_references][row.table.to_sym] || "reference"
         
     | 
| 
      
 26 
     | 
    
         
            +
                      reference = row.columns[reference_column.to_s]
         
     | 
| 
      
 27 
     | 
    
         
            +
             
     | 
| 
      
 28 
     | 
    
         
            +
                      record = model.find_or_initialize_by(reference_column => reference)
         
     | 
| 
      
 29 
     | 
    
         
            +
             
     | 
| 
      
 30 
     | 
    
         
            +
                      insert_row(record, row.columns.except([reference_column]))
         
     | 
| 
      
 31 
     | 
    
         
            +
             
     | 
| 
      
 32 
     | 
    
         
            +
                      @history_manager.update_history(row)
         
     | 
| 
      
 33 
     | 
    
         
            +
                    end
         
     | 
| 
      
 34 
     | 
    
         
            +
                  end
         
     | 
| 
      
 35 
     | 
    
         
            +
                end
         
     | 
| 
      
 36 
     | 
    
         
            +
             
     | 
| 
      
 37 
     | 
    
         
            +
                private
         
     | 
| 
      
 38 
     | 
    
         
            +
                # override this method to change the insertion behavior.
         
     | 
| 
      
 39 
     | 
    
         
            +
                # For example. you can skip validations for some specific models, etc..
         
     | 
| 
      
 40 
     | 
    
         
            +
                def insert_row(record, columns)
         
     | 
| 
      
 41 
     | 
    
         
            +
                  record.update!(columns)
         
     | 
| 
      
 42 
     | 
    
         
            +
                end
         
     | 
| 
      
 43 
     | 
    
         
            +
             
     | 
| 
      
 44 
     | 
    
         
            +
                class << self
         
     | 
| 
      
 45 
     | 
    
         
            +
                  def run(options = {}, &block)
         
     | 
| 
      
 46 
     | 
    
         
            +
                    Impex::Engine.new(options).run(&block)
         
     | 
| 
      
 47 
     | 
    
         
            +
                  end
         
     | 
| 
      
 48 
     | 
    
         
            +
                end
         
     | 
| 
      
 49 
     | 
    
         
            +
              end
         
     | 
| 
      
 50 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/impex/file.rb
    ADDED
    
    | 
         @@ -0,0 +1,17 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require_relative "row.rb"
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 4 
     | 
    
         
            +
              class File
         
     | 
| 
      
 5 
     | 
    
         
            +
                extend  Forwardable
         
     | 
| 
      
 6 
     | 
    
         
            +
                def_delegators :@rows, :each, :[], :<<
         
     | 
| 
      
 7 
     | 
    
         
            +
                alias each_row each
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
                attr_reader :file, :table
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                def initialize(file, config = {})
         
     | 
| 
      
 12 
     | 
    
         
            +
                  @file = file
         
     | 
| 
      
 13 
     | 
    
         
            +
                  @rows = []
         
     | 
| 
      
 14 
     | 
    
         
            +
                  @table = config[:table].classify.constantize
         
     | 
| 
      
 15 
     | 
    
         
            +
                end
         
     | 
| 
      
 16 
     | 
    
         
            +
              end
         
     | 
| 
      
 17 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,28 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require_relative "file.rb"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require_relative "row.rb"
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            require "csv"
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 7 
     | 
    
         
            +
              class FileFormatter
         
     | 
| 
      
 8 
     | 
    
         
            +
                class << self
         
     | 
| 
      
 9 
     | 
    
         
            +
                  def build(csv_file)
         
     | 
| 
      
 10 
     | 
    
         
            +
                    table = find_table_name(csv_file.path)
         
     | 
| 
      
 11 
     | 
    
         
            +
                    file_config = {
         
     | 
| 
      
 12 
     | 
    
         
            +
                      table: table
         
     | 
| 
      
 13 
     | 
    
         
            +
                    }
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
                    file = Impex::File.new(csv_file, file_config)
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
                    ::CSV.read(csv_file, headers: true).each do |row|
         
     | 
| 
      
 18 
     | 
    
         
            +
                      file << Impex::Row.new(row.to_h, file_config)
         
     | 
| 
      
 19 
     | 
    
         
            +
                    end
         
     | 
| 
      
 20 
     | 
    
         
            +
                    file
         
     | 
| 
      
 21 
     | 
    
         
            +
                  end
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
                  def find_table_name(filename)
         
     | 
| 
      
 24 
     | 
    
         
            +
                    filename[/\/(\w+)\/\w+.csv$/, 1]
         
     | 
| 
      
 25 
     | 
    
         
            +
                  end
         
     | 
| 
      
 26 
     | 
    
         
            +
                end
         
     | 
| 
      
 27 
     | 
    
         
            +
              end
         
     | 
| 
      
 28 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,22 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require_relative "../file_loader/base.rb"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require_relative "../file_formatter.rb"
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 5 
     | 
    
         
            +
              module FileLoader
         
     | 
| 
      
 6 
     | 
    
         
            +
                class FileSystem < Base
         
     | 
| 
      
 7 
     | 
    
         
            +
                  # The method #load is responsible to
         
     | 
| 
      
 8 
     | 
    
         
            +
                  # fetch all files (instances of Impex::File).
         
     | 
| 
      
 9 
     | 
    
         
            +
                  # Each rows are stored as an array of Impex::Row.
         
     | 
| 
      
 10 
     | 
    
         
            +
                  # The routine CSVImport::FileFormatter.build takes an instance of
         
     | 
| 
      
 11 
     | 
    
         
            +
                  # File as parameter and returns an instance of Impex::File
         
     | 
| 
      
 12 
     | 
    
         
            +
                  # which contains a set of Impex::Row accessible via :each method
         
     | 
| 
      
 13 
     | 
    
         
            +
                  def load
         
     | 
| 
      
 14 
     | 
    
         
            +
                    files = []
         
     | 
| 
      
 15 
     | 
    
         
            +
                    ::Dir.glob("#{::Rails.root}/#{@options[:relative_path]}/csv_import/**/*.csv").each do |f|
         
     | 
| 
      
 16 
     | 
    
         
            +
                      files << Impex::FileFormatter.build(::File.open(f))
         
     | 
| 
      
 17 
     | 
    
         
            +
                    end
         
     | 
| 
      
 18 
     | 
    
         
            +
                    files
         
     | 
| 
      
 19 
     | 
    
         
            +
                  end
         
     | 
| 
      
 20 
     | 
    
         
            +
                end
         
     | 
| 
      
 21 
     | 
    
         
            +
              end
         
     | 
| 
      
 22 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,86 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require_relative "../history_manager/base.rb"
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 4 
     | 
    
         
            +
              module HistoryManager
         
     | 
| 
      
 5 
     | 
    
         
            +
                class ActiveRecord < Base
         
     | 
| 
      
 6 
     | 
    
         
            +
                  def filter_data_with_history(row)
         
     | 
| 
      
 7 
     | 
    
         
            +
                    reference = row.columns[@options[:history_references][row.table.to_sym].to_s || "reference"]
         
     | 
| 
      
 8 
     | 
    
         
            +
                    select_column = row.columns.keys.join(", ")
         
     | 
| 
      
 9 
     | 
    
         
            +
                    whitelist = @whitelist[row.table]
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                    query = <<-SQL.gsub(/[\n\t\s]+/, ' ')
         
     | 
| 
      
 12 
     | 
    
         
            +
                      SELECT *
         
     | 
| 
      
 13 
     | 
    
         
            +
                      FROM #{history_table} AS cih
         
     | 
| 
      
 14 
     | 
    
         
            +
                      WHERE
         
     | 
| 
      
 15 
     | 
    
         
            +
                        cih.`reference`=#{quote(reference)}
         
     | 
| 
      
 16 
     | 
    
         
            +
                        AND cih.`table`=#{quote(row.table)}
         
     | 
| 
      
 17 
     | 
    
         
            +
                    SQL
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
                    query << "AND cih.`key` IN (#{quote(whitelist).join(',')})" unless whitelist.nil? || whitelist.empty?
         
     | 
| 
      
 20 
     | 
    
         
            +
             
     | 
| 
      
 21 
     | 
    
         
            +
                    records = connection.exec_query(query).to_hash
         
     | 
| 
      
 22 
     | 
    
         
            +
                    return row if records.empty?
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
                    history = Hash.new { |h, k| h[k] = [] }
         
     | 
| 
      
 25 
     | 
    
         
            +
             
     | 
| 
      
 26 
     | 
    
         
            +
                    records.each { |record| history[record["key"]] << record["value"] }
         
     | 
| 
      
 27 
     | 
    
         
            +
             
     | 
| 
      
 28 
     | 
    
         
            +
                    row.columns.each do |column_name, column_value|
         
     | 
| 
      
 29 
     | 
    
         
            +
                      row.columns.delete(column_name) if history[column_name].include?(column_value)
         
     | 
| 
      
 30 
     | 
    
         
            +
                    end
         
     | 
| 
      
 31 
     | 
    
         
            +
                    row
         
     | 
| 
      
 32 
     | 
    
         
            +
                  end
         
     | 
| 
      
 33 
     | 
    
         
            +
             
     | 
| 
      
 34 
     | 
    
         
            +
                  def update_history(row)
         
     | 
| 
      
 35 
     | 
    
         
            +
                    reference = row.columns.delete(@options[:history_references][row.table.to_sym].to_s || "reference")
         
     | 
| 
      
 36 
     | 
    
         
            +
                    return if row.columns.empty?
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
                    query = <<-SQL.gsub(/[\n\t\s]+/, ' ')
         
     | 
| 
      
 39 
     | 
    
         
            +
                      INSERT INTO
         
     | 
| 
      
 40 
     | 
    
         
            +
                      #{history_table}
         
     | 
| 
      
 41 
     | 
    
         
            +
                      (`reference`, `table`, `key`, `value`)
         
     | 
| 
      
 42 
     | 
    
         
            +
                      VALUES
         
     | 
| 
      
 43 
     | 
    
         
            +
                    SQL
         
     | 
| 
      
 44 
     | 
    
         
            +
             
     | 
| 
      
 45 
     | 
    
         
            +
                    records = []
         
     | 
| 
      
 46 
     | 
    
         
            +
                    row.columns.each do |column_name, column_value|
         
     | 
| 
      
 47 
     | 
    
         
            +
                      next unless whitelist_include?(row.table, column_name)
         
     | 
| 
      
 48 
     | 
    
         
            +
                      values = [
         
     | 
| 
      
 49 
     | 
    
         
            +
                        reference,
         
     | 
| 
      
 50 
     | 
    
         
            +
                        row.table,
         
     | 
| 
      
 51 
     | 
    
         
            +
                        column_name, column_value
         
     | 
| 
      
 52 
     | 
    
         
            +
                      ].map { |value| quote(value) }.join(',')
         
     | 
| 
      
 53 
     | 
    
         
            +
             
     | 
| 
      
 54 
     | 
    
         
            +
                      records << "(#{values})"
         
     | 
| 
      
 55 
     | 
    
         
            +
                    end
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
                    query << "#{records.join(',')};"
         
     | 
| 
      
 58 
     | 
    
         
            +
                    connection.execute(query) unless records.empty?
         
     | 
| 
      
 59 
     | 
    
         
            +
                  end
         
     | 
| 
      
 60 
     | 
    
         
            +
             
     | 
| 
      
 61 
     | 
    
         
            +
                  private
         
     | 
| 
      
 62 
     | 
    
         
            +
                  def whitelist_include?(table, key)
         
     | 
| 
      
 63 
     | 
    
         
            +
                    # if list not provided then accept any fields
         
     | 
| 
      
 64 
     | 
    
         
            +
                    return true if @whitelist[table].nil? || @whitelist[table].empty?
         
     | 
| 
      
 65 
     | 
    
         
            +
                    return true if @whitelist[table].map(&:to_s).include?(key.to_s)
         
     | 
| 
      
 66 
     | 
    
         
            +
                    false
         
     | 
| 
      
 67 
     | 
    
         
            +
                  end
         
     | 
| 
      
 68 
     | 
    
         
            +
             
     | 
| 
      
 69 
     | 
    
         
            +
                  def history_table
         
     | 
| 
      
 70 
     | 
    
         
            +
                    @history_table ||= @options[:history_manager][:table] || "impex_histories"
         
     | 
| 
      
 71 
     | 
    
         
            +
                  end
         
     | 
| 
      
 72 
     | 
    
         
            +
             
     | 
| 
      
 73 
     | 
    
         
            +
                  def connection
         
     | 
| 
      
 74 
     | 
    
         
            +
                    @ar_connection ||= ::ActiveRecord::Base.connection
         
     | 
| 
      
 75 
     | 
    
         
            +
                  end
         
     | 
| 
      
 76 
     | 
    
         
            +
             
     | 
| 
      
 77 
     | 
    
         
            +
                  def quote(value)
         
     | 
| 
      
 78 
     | 
    
         
            +
                    if value.is_a? Array
         
     | 
| 
      
 79 
     | 
    
         
            +
                      value.map { |v| connection.quote(v) }
         
     | 
| 
      
 80 
     | 
    
         
            +
                    else
         
     | 
| 
      
 81 
     | 
    
         
            +
                      connection.quote(value)
         
     | 
| 
      
 82 
     | 
    
         
            +
                    end
         
     | 
| 
      
 83 
     | 
    
         
            +
                  end
         
     | 
| 
      
 84 
     | 
    
         
            +
                end
         
     | 
| 
      
 85 
     | 
    
         
            +
              end
         
     | 
| 
      
 86 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/impex/lookup.rb
    ADDED
    
    | 
         @@ -0,0 +1,69 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require_relative "file_loader/base.rb"
         
     | 
| 
      
 2 
     | 
    
         
            +
            Dir.glob(File.join(File.dirname(__FILE__), 'file_loaders/*.rb')) do |f|
         
     | 
| 
      
 3 
     | 
    
         
            +
              require f
         
     | 
| 
      
 4 
     | 
    
         
            +
            end
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            Dir.glob(File.join(File.dirname(__FILE__), 'history_managers/*.rb')) do |f|
         
     | 
| 
      
 7 
     | 
    
         
            +
              require f
         
     | 
| 
      
 8 
     | 
    
         
            +
            end
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
            require "active_support/hash_with_indifferent_access"
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
            module Impex
         
     | 
| 
      
 13 
     | 
    
         
            +
              class Lookup
         
     | 
| 
      
 14 
     | 
    
         
            +
                attr_reader :config, :file_loaders, :history_managers
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
                def initialize(config = {})
         
     | 
| 
      
 17 
     | 
    
         
            +
                  @config = config
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
                  @file_loaders     = ::ActiveSupport::HashWithIndifferentAccess.new
         
     | 
| 
      
 20 
     | 
    
         
            +
                  @history_managers = ::ActiveSupport::HashWithIndifferentAccess.new
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
                  setup_file_loaders
         
     | 
| 
      
 23 
     | 
    
         
            +
                  setup_history_managers
         
     | 
| 
      
 24 
     | 
    
         
            +
                end
         
     | 
| 
      
 25 
     | 
    
         
            +
             
     | 
| 
      
 26 
     | 
    
         
            +
                def file_loader
         
     | 
| 
      
 27 
     | 
    
         
            +
                  lookup_for_file_loader
         
     | 
| 
      
 28 
     | 
    
         
            +
                end
         
     | 
| 
      
 29 
     | 
    
         
            +
             
     | 
| 
      
 30 
     | 
    
         
            +
                def history_manager
         
     | 
| 
      
 31 
     | 
    
         
            +
                  lookup_for_history_manager
         
     | 
| 
      
 32 
     | 
    
         
            +
                end
         
     | 
| 
      
 33 
     | 
    
         
            +
             
     | 
| 
      
 34 
     | 
    
         
            +
                private
         
     | 
| 
      
 35 
     | 
    
         
            +
                def lookup_for_file_loader
         
     | 
| 
      
 36 
     | 
    
         
            +
                  klass = @file_loaders[@config[:file_loader][:loader]]
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
                  raise Impex::FileLoader::UnknownFileLoaderError,
         
     | 
| 
      
 39 
     | 
    
         
            +
                        "undefined class #{@config[:file_loader][:loader].to_s.camelize}" if klass.nil?
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
                  klass.new(@config[:file_loader])
         
     | 
| 
      
 42 
     | 
    
         
            +
                end
         
     | 
| 
      
 43 
     | 
    
         
            +
             
     | 
| 
      
 44 
     | 
    
         
            +
                def lookup_for_history_manager
         
     | 
| 
      
 45 
     | 
    
         
            +
                  klass = @history_managers[@config[:history_manager][:manager]]
         
     | 
| 
      
 46 
     | 
    
         
            +
             
     | 
| 
      
 47 
     | 
    
         
            +
                  raise Impex::HistoryManager::UnknownHistoryManagerError,
         
     | 
| 
      
 48 
     | 
    
         
            +
                        "undefined class #{@config[:history_manager][:manager].to_s.camelize}" if klass.nil?
         
     | 
| 
      
 49 
     | 
    
         
            +
             
     | 
| 
      
 50 
     | 
    
         
            +
                  klass.new(@config)
         
     | 
| 
      
 51 
     | 
    
         
            +
                end
         
     | 
| 
      
 52 
     | 
    
         
            +
             
     | 
| 
      
 53 
     | 
    
         
            +
                def setup_file_loaders
         
     | 
| 
      
 54 
     | 
    
         
            +
                  ::Dir.glob(::File.join(::File.dirname(__FILE__), 'file_loaders/*.rb')).each do |file|
         
     | 
| 
      
 55 
     | 
    
         
            +
                    /(?<klass>\w+)\.rb/ =~ file
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
                    @file_loaders[klass] = "::Impex::FileLoader::#{klass.camelize}".constantize
         
     | 
| 
      
 58 
     | 
    
         
            +
                  end
         
     | 
| 
      
 59 
     | 
    
         
            +
                end
         
     | 
| 
      
 60 
     | 
    
         
            +
             
     | 
| 
      
 61 
     | 
    
         
            +
                def setup_history_managers
         
     | 
| 
      
 62 
     | 
    
         
            +
                  ::Dir.glob(::File.join(::File.dirname(__FILE__), 'history_managers/*.rb')).each do |file|
         
     | 
| 
      
 63 
     | 
    
         
            +
                    /(?<klass>\w+)\.rb/ =~ file
         
     | 
| 
      
 64 
     | 
    
         
            +
             
     | 
| 
      
 65 
     | 
    
         
            +
                    @history_managers[klass] = "::Impex::HistoryManager::#{klass.camelize}".constantize
         
     | 
| 
      
 66 
     | 
    
         
            +
                  end
         
     | 
| 
      
 67 
     | 
    
         
            +
                end
         
     | 
| 
      
 68 
     | 
    
         
            +
              end
         
     | 
| 
      
 69 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/impex/row.rb
    ADDED
    
    
    
        data/lib/tasks/all.rake
    ADDED
    
    
    
        metadata
    ADDED
    
    | 
         @@ -0,0 +1,195 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            --- !ruby/object:Gem::Specification
         
     | 
| 
      
 2 
     | 
    
         
            +
            name: impex
         
     | 
| 
      
 3 
     | 
    
         
            +
            version: !ruby/object:Gem::Version
         
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.1.0
         
     | 
| 
      
 5 
     | 
    
         
            +
            platform: ruby
         
     | 
| 
      
 6 
     | 
    
         
            +
            authors:
         
     | 
| 
      
 7 
     | 
    
         
            +
            - mehdi-farsi
         
     | 
| 
      
 8 
     | 
    
         
            +
            autorequire: 
         
     | 
| 
      
 9 
     | 
    
         
            +
            bindir: exe
         
     | 
| 
      
 10 
     | 
    
         
            +
            cert_chain: []
         
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2016-09-24 00:00:00.000000000 Z
         
     | 
| 
      
 12 
     | 
    
         
            +
            dependencies:
         
     | 
| 
      
 13 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 14 
     | 
    
         
            +
              name: rails
         
     | 
| 
      
 15 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 16 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 17 
     | 
    
         
            +
                - - '='
         
     | 
| 
      
 18 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 19 
     | 
    
         
            +
                    version: 4.2.6
         
     | 
| 
      
 20 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 21 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 22 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 23 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 24 
     | 
    
         
            +
                - - '='
         
     | 
| 
      
 25 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 26 
     | 
    
         
            +
                    version: 4.2.6
         
     | 
| 
      
 27 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 28 
     | 
    
         
            +
              name: bundler
         
     | 
| 
      
 29 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 30 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 31 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 32 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 33 
     | 
    
         
            +
                    version: '1.12'
         
     | 
| 
      
 34 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 35 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 36 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 37 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 38 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 39 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 40 
     | 
    
         
            +
                    version: '1.12'
         
     | 
| 
      
 41 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 42 
     | 
    
         
            +
              name: rake
         
     | 
| 
      
 43 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 44 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 45 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 46 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 47 
     | 
    
         
            +
                    version: '10.0'
         
     | 
| 
      
 48 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 49 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 50 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 51 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 52 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 53 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 54 
     | 
    
         
            +
                    version: '10.0'
         
     | 
| 
      
 55 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 56 
     | 
    
         
            +
              name: minitest
         
     | 
| 
      
 57 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 58 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 59 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 60 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 61 
     | 
    
         
            +
                    version: '5.0'
         
     | 
| 
      
 62 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 63 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 64 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 65 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 66 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 67 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 68 
     | 
    
         
            +
                    version: '5.0'
         
     | 
| 
      
 69 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 70 
     | 
    
         
            +
              name: minitest-reporters
         
     | 
| 
      
 71 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 72 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 73 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 74 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 75 
     | 
    
         
            +
                    version: '1.1'
         
     | 
| 
      
 76 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 77 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 78 
     | 
    
         
            +
                    version: 1.1.11
         
     | 
| 
      
 79 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 80 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 81 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 82 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 83 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 84 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 85 
     | 
    
         
            +
                    version: '1.1'
         
     | 
| 
      
 86 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 87 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 88 
     | 
    
         
            +
                    version: 1.1.11
         
     | 
| 
      
 89 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 90 
     | 
    
         
            +
              name: mocha
         
     | 
| 
      
 91 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 92 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 93 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 94 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 95 
     | 
    
         
            +
                    version: '1.1'
         
     | 
| 
      
 96 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 97 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 98 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 99 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 100 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 101 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 102 
     | 
    
         
            +
                    version: '1.1'
         
     | 
| 
      
 103 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 104 
     | 
    
         
            +
              name: sqlite3
         
     | 
| 
      
 105 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 106 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 107 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 108 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 109 
     | 
    
         
            +
                    version: '1.3'
         
     | 
| 
      
 110 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 111 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 112 
     | 
    
         
            +
                    version: 1.3.11
         
     | 
| 
      
 113 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 114 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 115 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 116 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 117 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 118 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 119 
     | 
    
         
            +
                    version: '1.3'
         
     | 
| 
      
 120 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 121 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 122 
     | 
    
         
            +
                    version: 1.3.11
         
     | 
| 
      
 123 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 124 
     | 
    
         
            +
              name: rubocop
         
     | 
| 
      
 125 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 126 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 127 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 128 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 129 
     | 
    
         
            +
                    version: 0.42.0
         
     | 
| 
      
 130 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 131 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 132 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 133 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 134 
     | 
    
         
            +
                - - "~>"
         
     | 
| 
      
 135 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 136 
     | 
    
         
            +
                    version: 0.42.0
         
     | 
| 
      
 137 
     | 
    
         
            +
            description: 
         
     | 
| 
      
 138 
     | 
    
         
            +
            email:
         
     | 
| 
      
 139 
     | 
    
         
            +
            - mehdifarsi.pro@gmail.com
         
     | 
| 
      
 140 
     | 
    
         
            +
            executables: []
         
     | 
| 
      
 141 
     | 
    
         
            +
            extensions: []
         
     | 
| 
      
 142 
     | 
    
         
            +
            extra_rdoc_files: []
         
     | 
| 
      
 143 
     | 
    
         
            +
            files:
         
     | 
| 
      
 144 
     | 
    
         
            +
            - ".gitignore"
         
     | 
| 
      
 145 
     | 
    
         
            +
            - ".travis.yml"
         
     | 
| 
      
 146 
     | 
    
         
            +
            - Gemfile
         
     | 
| 
      
 147 
     | 
    
         
            +
            - LICENSE.txt
         
     | 
| 
      
 148 
     | 
    
         
            +
            - README.md
         
     | 
| 
      
 149 
     | 
    
         
            +
            - Rakefile
         
     | 
| 
      
 150 
     | 
    
         
            +
            - bin/console
         
     | 
| 
      
 151 
     | 
    
         
            +
            - bin/setup
         
     | 
| 
      
 152 
     | 
    
         
            +
            - impex.gemspec
         
     | 
| 
      
 153 
     | 
    
         
            +
            - lib/generators/impex/install_generator.rb
         
     | 
| 
      
 154 
     | 
    
         
            +
            - lib/generators/impex/templates/create_impex_history.rb
         
     | 
| 
      
 155 
     | 
    
         
            +
            - lib/generators/impex/templates/impex.rb
         
     | 
| 
      
 156 
     | 
    
         
            +
            - lib/impex.rb
         
     | 
| 
      
 157 
     | 
    
         
            +
            - lib/impex/configuration.rb
         
     | 
| 
      
 158 
     | 
    
         
            +
            - lib/impex/engine.rb
         
     | 
| 
      
 159 
     | 
    
         
            +
            - lib/impex/file.rb
         
     | 
| 
      
 160 
     | 
    
         
            +
            - lib/impex/file_formatter.rb
         
     | 
| 
      
 161 
     | 
    
         
            +
            - lib/impex/file_loader/base.rb
         
     | 
| 
      
 162 
     | 
    
         
            +
            - lib/impex/file_loader/errors.rb
         
     | 
| 
      
 163 
     | 
    
         
            +
            - lib/impex/file_loaders/file_system.rb
         
     | 
| 
      
 164 
     | 
    
         
            +
            - lib/impex/history_manager/base.rb
         
     | 
| 
      
 165 
     | 
    
         
            +
            - lib/impex/history_manager/errors.rb
         
     | 
| 
      
 166 
     | 
    
         
            +
            - lib/impex/history_managers/active_record.rb
         
     | 
| 
      
 167 
     | 
    
         
            +
            - lib/impex/lookup.rb
         
     | 
| 
      
 168 
     | 
    
         
            +
            - lib/impex/row.rb
         
     | 
| 
      
 169 
     | 
    
         
            +
            - lib/impex/version.rb
         
     | 
| 
      
 170 
     | 
    
         
            +
            - lib/tasks/all.rake
         
     | 
| 
      
 171 
     | 
    
         
            +
            homepage: https://github.com/mehdi-farsi/impex
         
     | 
| 
      
 172 
     | 
    
         
            +
            licenses:
         
     | 
| 
      
 173 
     | 
    
         
            +
            - MIT
         
     | 
| 
      
 174 
     | 
    
         
            +
            metadata: {}
         
     | 
| 
      
 175 
     | 
    
         
            +
            post_install_message: 
         
     | 
| 
      
 176 
     | 
    
         
            +
            rdoc_options: []
         
     | 
| 
      
 177 
     | 
    
         
            +
            require_paths:
         
     | 
| 
      
 178 
     | 
    
         
            +
            - lib
         
     | 
| 
      
 179 
     | 
    
         
            +
            required_ruby_version: !ruby/object:Gem::Requirement
         
     | 
| 
      
 180 
     | 
    
         
            +
              requirements:
         
     | 
| 
      
 181 
     | 
    
         
            +
              - - ">="
         
     | 
| 
      
 182 
     | 
    
         
            +
                - !ruby/object:Gem::Version
         
     | 
| 
      
 183 
     | 
    
         
            +
                  version: '0'
         
     | 
| 
      
 184 
     | 
    
         
            +
            required_rubygems_version: !ruby/object:Gem::Requirement
         
     | 
| 
      
 185 
     | 
    
         
            +
              requirements:
         
     | 
| 
      
 186 
     | 
    
         
            +
              - - ">="
         
     | 
| 
      
 187 
     | 
    
         
            +
                - !ruby/object:Gem::Version
         
     | 
| 
      
 188 
     | 
    
         
            +
                  version: '0'
         
     | 
| 
      
 189 
     | 
    
         
            +
            requirements: []
         
     | 
| 
      
 190 
     | 
    
         
            +
            rubyforge_project: 
         
     | 
| 
      
 191 
     | 
    
         
            +
            rubygems_version: 2.5.1
         
     | 
| 
      
 192 
     | 
    
         
            +
            signing_key: 
         
     | 
| 
      
 193 
     | 
    
         
            +
            specification_version: 4
         
     | 
| 
      
 194 
     | 
    
         
            +
            summary: An idempotent CSV import system
         
     | 
| 
      
 195 
     | 
    
         
            +
            test_files: []
         
     |