csv_fast_importer 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1c6850b0ee75facc689b0992b21b3a3f94918bc9
+  data.tar.gz: 19d416bfb8f7c61f56bfab8906e25adecf329ba8
+SHA512:
+  metadata.gz: f5c35806af277b60c034b04f938dcf2faca4f9433a9194c985d098d5866230a58c07e9de678e69219d90097c0cbaac5fd9eb89d271936e8f35adf7c1121ac8e0
+  data.tar.gz: cd058154e47b07196e6ad1448eab4e40ae9e452cba095f383bf4589615139f290decf223f966b19e7de90635dc7c4d8cf87b5836b4ff62f9a04d16a139d170ca

data/.gitignore

@@ -0,0 +1,36 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.ruby-version

@@ -0,0 +1 @@
+2.2.1

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+script:
+  - bundle exec rake test:db:create
+  - bundle exec rspec spec
+  - if [ "$RUN_BENCHMARK" = "true" ]; then DATASET_SIZE=100 bundle exec rake benchmark; fi
+rvm:
+  2.2.0
+matrix:
+  include:
+    - env:
+      - DB_TYPE=mysql
+      - DB_USERNAME=travis
+    - env:
+      - DB_TYPE=postgres
+      - RUN_BENCHMARK=true

data/CONTRIBUTING.md

@@ -0,0 +1,24 @@
+# Contributing to CSV Fast Importer
+
+## Report a bug
+
+You just found a bug and you want to report it? It's very simple, open a ticket
+on [our bug tracker](https://github.com/sogilis/csv_fast_importer/issues). Use
+the research bar to find similar tickets and to avoid duplicate tickets.
+
+When you report a bug, please make sure to include useful information such as:
+
+- version of Ruby
+- which database was used and which version
+- steps to reproduce the bug
+- what you expected and what happened instead
+- if you can, include the related CSV file
+
+## Contribute to development
+
+- Fork the project
+- Create a new branch (e.g. `git checkout -b feature/my-awesome-feature master`)
+- Make your changes and create a pull request against our master branch
+- Check your commits are self-explanatory and are clear
+- Check your code does not fail code linting nor tests
+- Make sure to describe your pull request

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,128 @@
+PATH
+  remote: .
+  specs:
+    csv_fast_importer (1.0.0)
+      activerecord (>= 3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    active_importer (0.2.6)
+      roo
+    activemodel (4.2.5)
+      activesupport (= 4.2.5)
+      builder (~> 3.1)
+    activerecord (4.2.5)
+      activemodel (= 4.2.5)
+      activesupport (= 4.2.5)
+      arel (~> 6.0)
+    activerecord-import (0.10.0)
+      activerecord (>= 3.0)
+    activesupport (4.2.5)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.3)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    builder (3.2.2)
+    bulk_insert (1.5.0)
+      activerecord (>= 4.1.0)
+    codacy-coverage (0.2.3)
+      rest-client (~> 1.8)
+      simplecov (~> 0.10.0)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    csv-importer (0.3.2)
+      virtus
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.20160216)
+      unf (>= 0.0.5, < 1.0.0)
+    equalizer (0.0.11)
+    ferry (2.0.0)
+      highline (~> 1.6.21)
+      progressbar (~> 0.21.0)
+    highline (1.6.21)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    i18n (0.7.0)
+    ice_nine (0.11.2)
+    json (1.8.3)
+    mime-types (2.99.1)
+    mini_portile2 (2.2.0)
+    minitest (5.8.3)
+    mysql2 (0.3.20)
+    netrc (0.11.0)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    pg (0.18.4)
+    progressbar (0.21.0)
+    rake (10.4.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    roo (2.7.1)
+      nokogiri (~> 1)
+      rubyzip (~> 1.1, < 2.0.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rubyzip (1.2.1)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    smarter_csv (1.1.4)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.2)
+    upsert (2.2.1)
+    virtus (1.0.5)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  active_importer
+  activerecord-import
+  bulk_insert
+  bundler (~> 1.10)
+  codacy-coverage
+  csv-importer
+  csv_fast_importer!
+  ferry
+  mysql2 (>= 0.3.10)
+  pg (>= 0.18.4)
+  rake (~> 10.0)
+  rspec
+  smarter_csv
+  upsert
+
+BUNDLED WITH
+   1.13.2

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Jean-Baptiste Mille
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,186 @@
+[![Build Status](https://travis-ci.org/sogilis/csv_fast_importer.svg?branch=master)](https://travis-ci.org/sogilis/csv_fast_importer) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/3747d356ba004b7da2d0aec6bf1160f0)](https://www.codacy.com/app/Jibidus/csv_fast_importer?utm_source=github.com&utm_medium=referral&utm_content=sogilis/csv_fast_importer&utm_campaign=Badge_Grade)
+
+# CSV Fast Importer
+
+A gem to import CSV files' content into a PostgreSQL or MySQL database. It is respectively based on [PostgreSQL `COPY`](https://wiki.postgresql.org/wiki/COPY) and [MySQL `LOAD DATA INFILE`](https://dev.mysql.com/doc/refman/5.7/en/load-data.html) which are designed to be as fast as possible.
+
+## Why?
+
+CSV importation is a common task which can be done by more than 6 different gems, but none of them is able to import **1 million of lines in a few seconds** (see benchmark below), hence the creation of this gem.
+
+Here is an indicative benchmark to compare available solutions. It represents the **duration (ms)** to import a **10 000 lines** csv file into a local PostgreSQL instance on a laptop running OSX (lower is better):
+
+![Benchmark](benchmark/results.png?raw=true "Benchmark")
+
+Like all benchmarks, some tuning can produce different results, yet this chart gives a big picture. See [benchmark details](benchmark/README.md).
+
+## Requirements
+
+- Rails (ActiveRecord in fact)
+- PostgreSQL or MySQL
+
+## Limitations
+
+- Usual ActiveRecord process (validations, callbacks, computed fields like `created_at`...) is bypassed. This is the price for performance
+- Custom enclosing field (ex: `"`) is not supported yet
+- Custom line serparator (ex: `\r\n` for windows file) is not supported yet
+- MySQL: encoding is not supported yet
+- MySQL: transaction is not supported yet
+- MySQL: row_index is not supported yet
+
+## Installation
+
+Add the dependency to your Gemfile:
+
+```gemfile
+gem 'csv_fast_importer`
+```
+
+Run `bundle install`.
+
+You can install the gem by yourself too:
+
+```sh
+$ gem install csv_fast_importer
+```
+
+**For MySQL** :warning: enable `local_infile` for both [client](https://dev.mysql.com/doc/refman/5.7/en/source-configuration-options.html#option_cmake_enabled_local_infile) and [server](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_local_infile). In Rails application, juste add `local_infile: true` to your database config file `databse.yml` to configure the database client. See [Security Issues with LOAD DATA LOCAL](https://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) for more details.
+
+
+## Usage
+
+Actually, CSV Fast Importer needs `active_record` to work. Setup your database
+configuration as in a usual Rails project. Then, use the `CsvFastImporter`
+class:
+
+```ruby
+require 'csv_fast_importer'
+
+file = File.new '/path/to/knights.csv'
+imported_lines_count = CsvFastImporter.import(file)
+
+puts imported_lines_count
+```
+
+Under the hood, CSV Fast Importer deletes data from the `knights` table and
+imports those from `knights.csv` by mapping columns' names to table's fields.
+Note: mapping is case insensitive so **database fields' names must be lowercase**.
+For instance, a `FIRSTNAME` CSV column will be mapped to the `firstname` field.
+
+### Options
+
+| Option key | Purpose | Default value |
+| ------------ | ------------- | ------------- |
+| *encoding* | File encoding. *PostgreSQL only*| `'UTF-8'` |
+| *col_sep* | Column separator in file | `';'` |
+| *destination* | Destination table | given base filename (without extension) |
+| *mapping* | Column mapping | `{}` |
+| *row_index_column* | Column name where inserting file row index (not used when `nil`). *PostgreSQL only* | `nil` |
+| *transaction* | Execute DELETE and INSERT in same transaction. *PostgreSQL only* | `:enabled` |
+| *deletion* | Row deletion method (`:delete` for SQL DELETE, `:truncate` for SQL TRUNCATE or `:none` for no deletion before import) | `:delete` |
+
+Your CSV file should be encoding in UTF-8 but you can specify another encoding
+with the `encoding` option (*PostgreSQL only*).
+
+```ruby
+CsvFastImporter.import file, encoding: 'ISO-8859-1'
+```
+
+You can specify a different separator column with the `col_sep` option (`;` by
+default):
+
+```ruby
+CsvFastImporter.import file, col_sep: '|'
+```
+
+By default, CSV Fast Importer computes the database table's name by taking the
+`basename` of the imported file. For instance, considering the imported file
+`/path/to/knights.csv`, the table's name will be `knights`. To bypass
+this default behaviour, specify the `destination` option:
+
+```ruby
+file = File.new '/path/to/clients.csv'
+CsvFastImporter.import file, destination: 'knights'
+```
+
+Finally, you can precise a custom mapping between CSV file's columns and
+database fields with the `mapping` option.
+
+Considering the following `knights.csv` file:
+
+```csv
+NAME;KNIGHT_EMAIL
+Perceval;perceval@logre.cel
+Lancelot;lancelot@logre.cel
+```
+
+To map the `KNIGHT_EMAIL` column to the `email` database field:
+
+```ruby
+CsvFastImporter.import file, mapping: { email: :knight_email }
+```
+
+## How to contribute?
+
+You can fork and submit new pull request (with tests and explanations).
+First of all, you need to initialize your environment :
+
+```sh
+$ bundle install
+```
+
+Then, start your PostgreSQL database (ex: [Postgres.app](http://postgresapp.com) for the Mac) and setup database environment:
+
+```sh
+$ bundle exec rake test:db:create
+```
+This will connect to `localhost` PostgreSQL database without user (see `config/database.postgres.yml`) and create a new database dedicated to tests.
+
+Finally, you can run all tests with RSpec like this:
+
+```sh
+$ bundle exec rspec
+```
+
+By default, PostgreSQL is used. You can set another database with environment variables like this for MySQL:
+```sh
+$ DB_TYPE=mysql DB_ROOT_PASSWORD=password DB_USERNAME=username bundle exec rake test:db:create
+$ DB_TYPE=mysql DB_USERNAME=username bundle exec rspec
+```
+This will connect to mysql with `root` user (with `password` as password) and create database for user `username`.
+Use `DB_TYPE=mysql DB_USERNAME=` (with empty username) for anonymous account.
+
+*Warning*: Mysql tests require your local database permits LOCAL works. Check your Mysql instance with following command: `SHOW GLOBAL VARIABLES LIKE 'local_infile'` (should be `ON`).
+
+## Versioning
+
+`master` is the development branch and releases are published as tags.
+
+**We're not ready for the production yet (version < 1.0) so use this gem with
+precaution.**
+
+We follow the [Semantic Versioning 2.0.0](http://semver.org/) for our gem
+releases.
+
+In few words:
+
+> Given a version number MAJOR.MINOR.PATCH, increment the:
+>
+> 1. MAJOR version when you make incompatible API changes,
+> 2. MINOR version when you add functionality in a backwards-compatible manner,
+>    and
+> 3. PATCH version when you make backwards-compatible bug fixes.
+
+## Backlog (unordered)
+
+- [ ] Support any column and table case
+- [ ] Support custom enclosing field (ex: `"`)
+- [ ] Support custom line serparator (ex: \r\n for windows file)
+- [ ] Support custom type convertion
+- [ ] MySQL: support encoding parameter. See https://dev.mysql.com/doc/refman/5.5/en/charset-charsets.html
+- [ ] MySQL: support transaction parameter
+- [ ] MySQL: support row_index_column parameter
+- [ ] MySQL: run multiple SQL queries in single statement
+- [ ] Refactor tests (with should-> must / should -> expect / subject...)
+- [ ] Reduce technical debt on db connection (test & benchmark)
+- [ ] SQLite support

data/Rakefile

@@ -0,0 +1,44 @@
+require 'bundler/gem_tasks'
+
+namespace :test do
+  namespace :db do
+
+    desc "Create test database"
+    task :create do
+
+      require_relative './spec/config/test_database.rb'
+
+      db = TestDatabase.new
+      case db.type
+        when :postgres
+          require 'pg'
+          db.connect 'postgres'
+          ActiveRecord::Base.connection.execute "CREATE DATABASE #{db.name}"
+
+        when :mysql
+          require 'mysql2'
+          client_config = db.configuration
+                            .merge(database: nil,
+                                   username: 'root',
+                                   password: ENV['DB_ROOT_PASSWORD'],
+                                   flags: Mysql2::Client::MULTI_STATEMENTS)
+          client = Mysql2::Client.new(client_config)
+          client.query <<-SQL
+            CREATE DATABASE #{db.name};
+            GRANT ALL ON #{db.name}.* TO '#{db.configuration['username']}'@'#{db.configuration['host']}';
+            FLUSH PRIVILEGES;
+          SQL
+        else
+          raise "Unknown database type: #{db.type}"
+      end
+
+      puts "Test database \"#{db.name}\" created."
+    end
+
+  end
+end
+
+desc "Benchmark all available solutions to import CSV in Postgres database"
+task :benchmark do
+  require_relative 'benchmark/benchmark'
+end