csv_fast_importer 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/tests.yml +91 -0
- data/.gitignore +1 -1
- data/.ruby-version +1 -1
- data/Gemfile.lock +25 -39
- data/README.md +31 -15
- data/Rakefile +1 -1
- data/benchmark/README.md +1 -1
- data/csv_fast_importer.gemspec +3 -2
- data/doc/faq.md +64 -0
- data/lib/csv_fast_importer/version.rb +1 -1
- data/rakelib/release.rake +6 -3
- data/sample-app/Gemfile.lock +2 -2
- data/sample-app/README.md +1 -1
- metadata +27 -13
- data/.travis.yml +0 -21
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: e64fb873391ad1bac7f05bf0bcefc26f0482d0917b71244e7388a19cd866b6cb
|
|
4
|
+
data.tar.gz: 6db4589951a6256be1b4334c0791cde38b10f4e87974bfbf85835ed39c01c061
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1d2ecf83bebef80e492e9799b4c2345c9b0a5e8c6a645c219a9acb2ed0012263ebf1322baa8ede53c9dc6390e6fb3846c9737979259957f1d2cc779ab654282e
|
|
7
|
+
data.tar.gz: 33ad1e337b5f4c649d86d96bb8a6f803168009719c47484dc76681e7dc70f6acb538c6f1a694e7f8939bc2ff20a2705bf5a0c08c2cdd44f4a8eed4b6faea8cf6
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on: push
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
postgres:
|
|
7
|
+
name: Test against PostgreSQL
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
env:
|
|
10
|
+
DB_TYPE: postgres
|
|
11
|
+
DB_PASSWORD: password
|
|
12
|
+
DB_USERNAME: postgres
|
|
13
|
+
services:
|
|
14
|
+
postgres:
|
|
15
|
+
image: postgres
|
|
16
|
+
env:
|
|
17
|
+
POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
|
|
18
|
+
ports:
|
|
19
|
+
- 5432:5432
|
|
20
|
+
options: >-
|
|
21
|
+
--health-cmd pg_isready
|
|
22
|
+
--health-interval 10s
|
|
23
|
+
--health-timeout 5s
|
|
24
|
+
--health-retries 5
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
- uses: ruby/setup-ruby@v1
|
|
28
|
+
with:
|
|
29
|
+
bundler-cache: true
|
|
30
|
+
- run: bundle exec rake test:db:create
|
|
31
|
+
- run: bundle exec rspec spec
|
|
32
|
+
- uses: actions/upload-artifact@v3
|
|
33
|
+
with:
|
|
34
|
+
name: postgres-test-coverage
|
|
35
|
+
path: coverage/lcov/csv_fast_importer.lcov
|
|
36
|
+
|
|
37
|
+
mysql:
|
|
38
|
+
name: Test against MySQL
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
env:
|
|
41
|
+
DB_TYPE: mysql
|
|
42
|
+
DB_HOST: 127.0.0.1
|
|
43
|
+
DB_PORT: 3306
|
|
44
|
+
DB_ROOT_PASSWORD: password
|
|
45
|
+
DB_USERNAME: mysql
|
|
46
|
+
DB_PASSWORD: password
|
|
47
|
+
DB_DATABASE: csv_fast_importer_test
|
|
48
|
+
services:
|
|
49
|
+
mysql:
|
|
50
|
+
image: mysql
|
|
51
|
+
env:
|
|
52
|
+
MYSQL_DATABASE: ${{ env.DB_DATABASE }}
|
|
53
|
+
MYSQL_USER: ${{ env.DB_USERNAME }}
|
|
54
|
+
MYSQL_PASSWORD: ${{ env.DB_PASSWORD }}
|
|
55
|
+
MYSQL_ROOT_PASSWORD: ${{ env.DB_ROOT_PASSWORD }}
|
|
56
|
+
ports:
|
|
57
|
+
- 3306:3306
|
|
58
|
+
options: >-
|
|
59
|
+
--health-cmd="mysqladmin ping"
|
|
60
|
+
--health-interval=10s
|
|
61
|
+
--health-timeout=5s
|
|
62
|
+
--health-retries=3
|
|
63
|
+
steps:
|
|
64
|
+
- uses: actions/checkout@v4
|
|
65
|
+
- uses: ruby/setup-ruby@v1
|
|
66
|
+
with:
|
|
67
|
+
bundler-cache: true
|
|
68
|
+
- name: Enable local data loading
|
|
69
|
+
run: |
|
|
70
|
+
mysql --host=${{ env.DB_HOST }} --user=root --password=${{ env.DB_ROOT_PASSWORD }} ${{ env.DB_DATABASE }} <<MY_QUERY
|
|
71
|
+
set global local_infile=true;
|
|
72
|
+
MY_QUERY
|
|
73
|
+
- run: bundle exec rspec spec
|
|
74
|
+
- uses: actions/upload-artifact@v3
|
|
75
|
+
with:
|
|
76
|
+
name: mysql-test-coverage
|
|
77
|
+
path: coverage/lcov/csv_fast_importer.lcov
|
|
78
|
+
|
|
79
|
+
test-coverage:
|
|
80
|
+
name: Publish test coverage
|
|
81
|
+
needs: [postgres, mysql]
|
|
82
|
+
runs-on: ubuntu-latest
|
|
83
|
+
steps:
|
|
84
|
+
- name: Recover all test coverages
|
|
85
|
+
uses: actions/download-artifact@v3
|
|
86
|
+
- run: ls -R
|
|
87
|
+
- name: Upload coverage to Codacy
|
|
88
|
+
uses: codacy/codacy-coverage-reporter-action@v1
|
|
89
|
+
with:
|
|
90
|
+
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
|
91
|
+
coverage-reports: postgres-test-coverage/csv_fast_importer.lcov, mysql-test-coverage/csv_fast_importer.lcov
|
data/.gitignore
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
/test/tmp/
|
|
10
10
|
/test/version_tmp/
|
|
11
11
|
/tmp/
|
|
12
|
+
/.idea/
|
|
12
13
|
|
|
13
14
|
## Specific to RubyMotion:
|
|
14
15
|
.dat*
|
|
@@ -18,7 +19,6 @@ build/
|
|
|
18
19
|
## Documentation cache and generated files:
|
|
19
20
|
/.yardoc/
|
|
20
21
|
/_yardoc/
|
|
21
|
-
/doc/
|
|
22
22
|
/rdoc/
|
|
23
23
|
|
|
24
24
|
## Environment normalization:
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.
|
|
1
|
+
2.6.10
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
csv_fast_importer (1.
|
|
4
|
+
csv_fast_importer (1.2.0)
|
|
5
5
|
activerecord (~> 4.2)
|
|
6
6
|
|
|
7
7
|
GEM
|
|
@@ -9,16 +9,16 @@ GEM
|
|
|
9
9
|
specs:
|
|
10
10
|
active_importer (0.2.6)
|
|
11
11
|
roo
|
|
12
|
-
activemodel (4.2.
|
|
13
|
-
activesupport (= 4.2.
|
|
12
|
+
activemodel (4.2.11.3)
|
|
13
|
+
activesupport (= 4.2.11.3)
|
|
14
14
|
builder (~> 3.1)
|
|
15
|
-
activerecord (4.2.
|
|
16
|
-
activemodel (= 4.2.
|
|
17
|
-
activesupport (= 4.2.
|
|
15
|
+
activerecord (4.2.11.3)
|
|
16
|
+
activemodel (= 4.2.11.3)
|
|
17
|
+
activesupport (= 4.2.11.3)
|
|
18
18
|
arel (~> 6.0)
|
|
19
19
|
activerecord-import (0.10.0)
|
|
20
20
|
activerecord (>= 3.0)
|
|
21
|
-
activesupport (4.2.
|
|
21
|
+
activesupport (4.2.11.3)
|
|
22
22
|
i18n (~> 0.7)
|
|
23
23
|
minitest (~> 5.1)
|
|
24
24
|
thread_safe (~> 0.3, >= 0.3.4)
|
|
@@ -28,48 +28,34 @@ GEM
|
|
|
28
28
|
descendants_tracker (~> 0.0.4)
|
|
29
29
|
ice_nine (~> 0.11.0)
|
|
30
30
|
thread_safe (~> 0.3, >= 0.3.1)
|
|
31
|
-
builder (3.2.
|
|
31
|
+
builder (3.2.4)
|
|
32
32
|
bulk_insert (1.5.0)
|
|
33
33
|
activerecord (>= 4.1.0)
|
|
34
|
-
codacy-coverage (0.2.3)
|
|
35
|
-
rest-client (~> 1.8)
|
|
36
|
-
simplecov (~> 0.10.0)
|
|
37
34
|
coercible (1.0.0)
|
|
38
35
|
descendants_tracker (~> 0.0.1)
|
|
39
|
-
concurrent-ruby (1.
|
|
36
|
+
concurrent-ruby (1.2.2)
|
|
40
37
|
csv-importer (0.3.2)
|
|
41
38
|
virtus
|
|
42
39
|
descendants_tracker (0.0.4)
|
|
43
40
|
thread_safe (~> 0.3, >= 0.3.1)
|
|
44
41
|
diff-lcs (1.2.5)
|
|
45
|
-
docile (1.
|
|
46
|
-
domain_name (0.5.20160216)
|
|
47
|
-
unf (>= 0.0.5, < 1.0.0)
|
|
42
|
+
docile (1.4.0)
|
|
48
43
|
equalizer (0.0.11)
|
|
49
44
|
ferry (2.0.0)
|
|
50
45
|
highline (~> 1.6.21)
|
|
51
46
|
progressbar (~> 0.21.0)
|
|
52
47
|
highline (1.6.21)
|
|
53
|
-
|
|
54
|
-
domain_name (~> 0.5)
|
|
55
|
-
i18n (0.9.3)
|
|
48
|
+
i18n (0.9.5)
|
|
56
49
|
concurrent-ruby (~> 1.0)
|
|
57
50
|
ice_nine (0.11.2)
|
|
58
|
-
json (1.8.6)
|
|
59
|
-
mime-types (2.99.1)
|
|
60
51
|
mini_portile2 (2.3.0)
|
|
61
|
-
minitest (5.
|
|
52
|
+
minitest (5.20.0)
|
|
62
53
|
mysql2 (0.4.10)
|
|
63
|
-
netrc (0.11.0)
|
|
64
54
|
nokogiri (1.8.2)
|
|
65
55
|
mini_portile2 (~> 2.3.0)
|
|
66
|
-
pg (0.
|
|
56
|
+
pg (0.21.0)
|
|
67
57
|
progressbar (0.21.0)
|
|
68
58
|
rake (10.4.2)
|
|
69
|
-
rest-client (1.8.0)
|
|
70
|
-
http-cookie (>= 1.0.2, < 2.0)
|
|
71
|
-
mime-types (>= 1.16, < 3.0)
|
|
72
|
-
netrc (~> 0.7)
|
|
73
59
|
roo (2.7.1)
|
|
74
60
|
nokogiri (~> 1)
|
|
75
61
|
rubyzip (~> 1.1, < 2.0.0)
|
|
@@ -86,19 +72,18 @@ GEM
|
|
|
86
72
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
87
73
|
rspec-support (~> 3.4.0)
|
|
88
74
|
rspec-support (3.4.1)
|
|
89
|
-
rubyzip (1.
|
|
90
|
-
simplecov (0.
|
|
91
|
-
docile (~> 1.1
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
simplecov-html (0.
|
|
75
|
+
rubyzip (1.3.0)
|
|
76
|
+
simplecov (0.22.0)
|
|
77
|
+
docile (~> 1.1)
|
|
78
|
+
simplecov-html (~> 0.11)
|
|
79
|
+
simplecov_json_formatter (~> 0.1)
|
|
80
|
+
simplecov-html (0.12.3)
|
|
81
|
+
simplecov-lcov (0.8.0)
|
|
82
|
+
simplecov_json_formatter (0.1.4)
|
|
95
83
|
smarter_csv (1.1.4)
|
|
96
84
|
thread_safe (0.3.6)
|
|
97
|
-
tzinfo (1.2.
|
|
85
|
+
tzinfo (1.2.11)
|
|
98
86
|
thread_safe (~> 0.1)
|
|
99
|
-
unf (0.1.4)
|
|
100
|
-
unf_ext
|
|
101
|
-
unf_ext (0.0.7.2)
|
|
102
87
|
upsert (2.2.1)
|
|
103
88
|
virtus (1.0.5)
|
|
104
89
|
axiom-types (~> 0.1)
|
|
@@ -114,14 +99,15 @@ DEPENDENCIES
|
|
|
114
99
|
activerecord-import
|
|
115
100
|
bulk_insert
|
|
116
101
|
bundler (~> 1.10)
|
|
117
|
-
codacy-coverage
|
|
118
102
|
csv-importer
|
|
119
103
|
csv_fast_importer!
|
|
120
104
|
ferry
|
|
121
105
|
mysql2 (>= 0.4.0)
|
|
122
|
-
pg (
|
|
106
|
+
pg (~> 0.21.0)
|
|
123
107
|
rake (~> 10.0)
|
|
124
108
|
rspec
|
|
109
|
+
simplecov
|
|
110
|
+
simplecov-lcov
|
|
125
111
|
smarter_csv
|
|
126
112
|
upsert
|
|
127
113
|
|
data/README.md
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
[](https://badge.fury.io/rb/csv_fast_importer)
|
|
1
|
+
[](https://badge.fury.io/rb/csv_fast_importer)  [](https://www.codacy.com/gh/sogilis/csv_fast_importer/dashboard?utm_source=github.com&utm_medium=referral&utm_content=sogilis/csv_fast_importer&utm_campaign=Badge_Grade)
|
|
2
|
+
|
|
2
3
|
|
|
3
4
|
# CSV Fast Importer
|
|
4
5
|
|
|
@@ -23,17 +24,19 @@ Like all benchmarks, some tuning can produce different results, yet this chart g
|
|
|
23
24
|
|
|
24
25
|
- Usual ActiveRecord process (validations, callbacks, computed fields like `created_at`...) is bypassed. This is the price for performance
|
|
25
26
|
- Custom enclosing field (ex: `"`) is not supported yet
|
|
26
|
-
- Custom line
|
|
27
|
+
- Custom line separator (ex: `\r\n` for windows file) is not supported yet
|
|
27
28
|
- MySQL: encoding is not supported yet
|
|
28
29
|
- MySQL: transaction is not supported yet
|
|
29
30
|
- MySQL: row_index is not supported yet
|
|
30
31
|
|
|
32
|
+
Note about custom line separator: it might work by opening the file with the `universal_newline` argument (e.g. `file = File.new(path, universal_newline: true)`). Unfortunately, we weren't able to reproduce and test it so we don't support it "officialy". You can find more information in [this ticket](https://github.com/sogilis/csv_fast_importer/pull/45#issuecomment-326578839) (in French).
|
|
33
|
+
|
|
31
34
|
## Installation
|
|
32
35
|
|
|
33
36
|
Add the dependency to your Gemfile:
|
|
34
37
|
|
|
35
|
-
```
|
|
36
|
-
gem 'csv_fast_importer
|
|
38
|
+
```ruby
|
|
39
|
+
gem 'csv_fast_importer'
|
|
37
40
|
```
|
|
38
41
|
|
|
39
42
|
Run `bundle install`.
|
|
@@ -71,7 +74,7 @@ For instance, a `FIRSTNAME` CSV column will be mapped to the `firstname` field.
|
|
|
71
74
|
|
|
72
75
|
| Option key | Purpose | Default value |
|
|
73
76
|
| ------------ | ------------- | ------------- |
|
|
74
|
-
| *encoding* | File encoding. *PostgreSQL only
|
|
77
|
+
| *encoding* | File encoding. *PostgreSQL only* (see [FAQ](doc/faq.md) for more details)| `'UTF-8'` |
|
|
75
78
|
| *col_sep* | Column separator in file | `';'` |
|
|
76
79
|
| *destination* | Destination table | given base filename (without extension) |
|
|
77
80
|
| *mapping* | Column mapping | `{}` |
|
|
@@ -79,11 +82,10 @@ For instance, a `FIRSTNAME` CSV column will be mapped to the `firstname` field.
|
|
|
79
82
|
| *transaction* | Execute DELETE and INSERT in same transaction. *PostgreSQL only* | `:enabled` |
|
|
80
83
|
| *deletion* | Row deletion method (`:delete` for SQL DELETE, `:truncate` for SQL TRUNCATE or `:none` for no deletion before import) | `:delete` |
|
|
81
84
|
|
|
82
|
-
|
|
83
|
-
with the `encoding` option (*PostgreSQL only*).
|
|
85
|
+
If your CSV file is not encoded with same table than your database, you can specify encoding at the file opening (see [FAQ](doc/faq.md) for more details):
|
|
84
86
|
|
|
85
87
|
```ruby
|
|
86
|
-
|
|
88
|
+
file = File.new '/path/to/knights.csv', encoding: 'ISO-8859-1'
|
|
87
89
|
```
|
|
88
90
|
|
|
89
91
|
You can specify a different separator column with the `col_sep` option (`;` by
|
|
@@ -117,15 +119,21 @@ Lancelot;lancelot@logre.cel
|
|
|
117
119
|
To map the `KNIGHT_EMAIL` column to the `email` database field:
|
|
118
120
|
|
|
119
121
|
```ruby
|
|
120
|
-
CsvFastImporter.import file, mapping: {
|
|
122
|
+
CsvFastImporter.import file, mapping: { knight_email: :email }
|
|
121
123
|
```
|
|
122
124
|
|
|
125
|
+
## Need help?
|
|
126
|
+
|
|
127
|
+
See [FAQ](doc/faq.md).
|
|
128
|
+
|
|
123
129
|
## How to contribute?
|
|
124
130
|
|
|
125
131
|
You can fork and submit new pull request (with tests and explanations).
|
|
126
132
|
First of all, you need to initialize your environment :
|
|
127
133
|
|
|
128
134
|
```sh
|
|
135
|
+
$ brew install postgresql # in macOS
|
|
136
|
+
$ apt-get install libpq-dev # in Linux
|
|
129
137
|
$ bundle install
|
|
130
138
|
```
|
|
131
139
|
|
|
@@ -136,6 +144,8 @@ $ bundle exec rake test:db:create
|
|
|
136
144
|
```
|
|
137
145
|
This will connect to `localhost` PostgreSQL database without user (see `config/database.postgres.yml`) and create a new database dedicated to tests.
|
|
138
146
|
|
|
147
|
+
*Warning:* database instance have to allow database creation with `UTF-8` encoding.
|
|
148
|
+
|
|
139
149
|
Finally, you can run all tests with RSpec like this:
|
|
140
150
|
|
|
141
151
|
```sh
|
|
@@ -156,9 +166,6 @@ Use `DB_TYPE=mysql DB_USERNAME=` (with empty username) for anonymous account.
|
|
|
156
166
|
|
|
157
167
|
`master` is the development branch and releases are published as tags.
|
|
158
168
|
|
|
159
|
-
**We're not ready for the production yet (version < 1.0) so use this gem with
|
|
160
|
-
precaution.**
|
|
161
|
-
|
|
162
169
|
We follow the [Semantic Versioning 2.0.0](http://semver.org/) for our gem
|
|
163
170
|
releases.
|
|
164
171
|
|
|
@@ -187,9 +194,18 @@ In few words:
|
|
|
187
194
|
|
|
188
195
|
## How to release new version?
|
|
189
196
|
|
|
190
|
-
|
|
197
|
+
Setup rubygems.org account:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
curl -u {your_gem_account_name} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
|
201
|
+
chmod 0600 ~/.gem/credentials
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Make sure you are in `master` branch and run:
|
|
205
|
+
|
|
191
206
|
```bash
|
|
192
|
-
bundle exec rake release:make[major|minor|patch|x.y.z]
|
|
207
|
+
bundle exec rake "release:make[major|minor|patch|x.y.z]"
|
|
193
208
|
```
|
|
209
|
+
Example: `bundle exec rake "release:make[minor]"`
|
|
194
210
|
|
|
195
|
-
|
|
211
|
+
Then, follow instructions.
|
data/Rakefile
CHANGED
|
@@ -13,7 +13,7 @@ namespace :test do
|
|
|
13
13
|
when :postgres
|
|
14
14
|
require 'pg'
|
|
15
15
|
db.connect 'postgres'
|
|
16
|
-
ActiveRecord::Base.connection.execute "CREATE DATABASE #{db.name}"
|
|
16
|
+
ActiveRecord::Base.connection.execute "CREATE DATABASE #{db.name} ENCODING='UTF-8'"
|
|
17
17
|
|
|
18
18
|
when :mysql
|
|
19
19
|
require 'mysql2'
|
data/benchmark/README.md
CHANGED
data/csv_fast_importer.gemspec
CHANGED
|
@@ -24,9 +24,10 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
|
|
25
25
|
spec.add_development_dependency "bundler", "~> 1.10"
|
|
26
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
|
27
|
-
spec.add_development_dependency "pg", "
|
|
27
|
+
spec.add_development_dependency "pg", "~> 0.21.0"
|
|
28
28
|
spec.add_development_dependency "mysql2", ">= 0.4.0"
|
|
29
|
-
spec.add_development_dependency "
|
|
29
|
+
spec.add_development_dependency "simplecov"
|
|
30
|
+
spec.add_development_dependency "simplecov-lcov"
|
|
30
31
|
spec.add_development_dependency "rspec"
|
|
31
32
|
|
|
32
33
|
# Only for benchmark
|
data/doc/faq.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Frequently Asked Questions
|
|
2
|
+
|
|
3
|
+
## How to specify encoding?
|
|
4
|
+
|
|
5
|
+
Multiple components are involved when `CSV Fast Importer` is executed:
|
|
6
|
+
|
|
7
|
+
- file
|
|
8
|
+
- ruby `File` wrapper
|
|
9
|
+
- database client (managed by `ActiveRecord` connection)
|
|
10
|
+
- SQL command (`COPY` for PostgreSQL)
|
|
11
|
+
- database server
|
|
12
|
+
|
|
13
|
+
Encoding must be consistent accross all these components. Here is how to specify or check each component encoding.
|
|
14
|
+
|
|
15
|
+
### File
|
|
16
|
+
|
|
17
|
+
You can get current file encoding with `file -i [file_path]` (`-I` on macOS) command.
|
|
18
|
+
Some tools like [iconv](http://www.gnu.org/savannah-checkouts/gnu/libiconv/documentation/libiconv-1.15/iconv.1.html) can modify file encoding.
|
|
19
|
+
|
|
20
|
+
### Ruby `File` wrapper
|
|
21
|
+
|
|
22
|
+
`File` uses default Ruby encoding (given by `Encoding.default_external`. See [External / Internal Encoding](https://ruby-doc.org/core-2.4.1/Encoding.html#class-Encoding-label-External+encoding) which might be different from file enoding!
|
|
23
|
+
|
|
24
|
+
```ruby
|
|
25
|
+
File.new 'path/to/file.csv'
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
But, you can specify encoding with `encoding` parameter:
|
|
29
|
+
|
|
30
|
+
```ruby
|
|
31
|
+
File.new 'path/to/file.csv', encoding: 'ISO-8859-1'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Ruby `File` can also handle internal and external encoding (see [File::new](https://ruby-doc.org/core-2.4.1/File.html#method-c-new) which can be useful to manage automatic conversion:
|
|
35
|
+
|
|
36
|
+
```ruby
|
|
37
|
+
File.new 'path/to/file.csv', external_encoding: 'ISO-8859-1', internal_encoding: 'UTF-8'
|
|
38
|
+
# or
|
|
39
|
+
File.new 'path/to/file.csv', encoding: 'ISO-8859-1:UTF-8'
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Database client
|
|
43
|
+
|
|
44
|
+
Database is accessed through a dedicated client.
|
|
45
|
+
This client is managed by `ActiveRecord` with some configuration (`database.yml` in Rails application) where `encoding` parameter can be defined.
|
|
46
|
+
|
|
47
|
+
### SQL Command
|
|
48
|
+
|
|
49
|
+
By default, `COPY` and `LOAD DATA INFILE` commands follow database client encoding configuration. But you can override this with dedicated parameter.
|
|
50
|
+
This is the purpose of `CSV FAST Importer`'s `encoding` parameter.
|
|
51
|
+
|
|
52
|
+
### Database server
|
|
53
|
+
|
|
54
|
+
Each Postgres server instance is encoded with a specific table. You can show this with following command:
|
|
55
|
+
|
|
56
|
+
```shell
|
|
57
|
+
psql -l
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Or, from `psql` client:
|
|
61
|
+
|
|
62
|
+
```sql
|
|
63
|
+
\l
|
|
64
|
+
```
|
data/rakelib/release.rake
CHANGED
|
@@ -12,9 +12,11 @@ namespace :release do
|
|
|
12
12
|
release.apply!
|
|
13
13
|
|
|
14
14
|
puts
|
|
15
|
-
puts "Changelog from #{release.last_version} to #{release.version}
|
|
15
|
+
puts "Changelog from #{release.last_version} to #{release.version}"
|
|
16
|
+
puts "---------------------------------------------------------------------"
|
|
16
17
|
puts release.changelog
|
|
17
18
|
puts "(Chores ignored)"
|
|
19
|
+
puts "---------------------------------------------------------------------"
|
|
18
20
|
|
|
19
21
|
next if args[:dry_run]
|
|
20
22
|
|
|
@@ -25,9 +27,10 @@ namespace :release do
|
|
|
25
27
|
puts
|
|
26
28
|
puts "Run following commands to publish version #{release.version}:"
|
|
27
29
|
puts "$ git push && git push --tags"
|
|
28
|
-
puts "$
|
|
30
|
+
puts "$ bundle exec rake release"
|
|
29
31
|
puts
|
|
30
|
-
puts "After that, do not forget to report changelog in Github
|
|
32
|
+
puts "After that, do not forget to report changelog in Github Releases page:"
|
|
33
|
+
puts "https://github.com/sogilis/csv_fast_importer/releases"
|
|
31
34
|
end
|
|
32
35
|
end
|
|
33
36
|
|
data/sample-app/Gemfile.lock
CHANGED
|
@@ -54,7 +54,7 @@ GEM
|
|
|
54
54
|
debug_inspector (0.0.3)
|
|
55
55
|
erubis (2.7.0)
|
|
56
56
|
execjs (2.7.0)
|
|
57
|
-
ffi (1.
|
|
57
|
+
ffi (1.16.1)
|
|
58
58
|
globalid (0.4.1)
|
|
59
59
|
activesupport (>= 4.2.0)
|
|
60
60
|
i18n (0.9.5)
|
|
@@ -169,4 +169,4 @@ DEPENDENCIES
|
|
|
169
169
|
web-console (~> 2.0)
|
|
170
170
|
|
|
171
171
|
BUNDLED WITH
|
|
172
|
-
1.
|
|
172
|
+
1.17.2
|
data/sample-app/README.md
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: csv_fast_importer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sogilis
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-09-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -42,16 +42,16 @@ dependencies:
|
|
|
42
42
|
name: pg
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
|
-
- - "
|
|
45
|
+
- - "~>"
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: 0.
|
|
47
|
+
version: 0.21.0
|
|
48
48
|
type: :development
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
|
-
- - "
|
|
52
|
+
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 0.
|
|
54
|
+
version: 0.21.0
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: mysql2
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -67,7 +67,21 @@ dependencies:
|
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
68
|
version: 0.4.0
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
70
|
+
name: simplecov
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - ">="
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '0'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - ">="
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '0'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: simplecov-lcov
|
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
|
72
86
|
requirements:
|
|
73
87
|
- - ">="
|
|
@@ -215,9 +229,9 @@ executables: []
|
|
|
215
229
|
extensions: []
|
|
216
230
|
extra_rdoc_files: []
|
|
217
231
|
files:
|
|
232
|
+
- ".github/workflows/tests.yml"
|
|
218
233
|
- ".gitignore"
|
|
219
234
|
- ".ruby-version"
|
|
220
|
-
- ".travis.yml"
|
|
221
235
|
- CONTRIBUTING.md
|
|
222
236
|
- Gemfile
|
|
223
237
|
- Gemfile.lock
|
|
@@ -232,6 +246,7 @@ files:
|
|
|
232
246
|
- benchmark/strategies.rb
|
|
233
247
|
- benchmark/tools.rb
|
|
234
248
|
- csv_fast_importer.gemspec
|
|
249
|
+
- doc/faq.md
|
|
235
250
|
- lib/csv_fast_importer.rb
|
|
236
251
|
- lib/csv_fast_importer/configuration.rb
|
|
237
252
|
- lib/csv_fast_importer/database/mysql.rb
|
|
@@ -312,7 +327,7 @@ homepage: https://github.com/sogilis/csv_fast_importer
|
|
|
312
327
|
licenses:
|
|
313
328
|
- MIT
|
|
314
329
|
metadata: {}
|
|
315
|
-
post_install_message:
|
|
330
|
+
post_install_message:
|
|
316
331
|
rdoc_options: []
|
|
317
332
|
require_paths:
|
|
318
333
|
- lib
|
|
@@ -327,9 +342,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
327
342
|
- !ruby/object:Gem::Version
|
|
328
343
|
version: '0'
|
|
329
344
|
requirements: []
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
signing_key:
|
|
345
|
+
rubygems_version: 3.0.3.1
|
|
346
|
+
signing_key:
|
|
333
347
|
specification_version: 4
|
|
334
348
|
summary: Fast CSV Importer
|
|
335
349
|
test_files: []
|
data/.travis.yml
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
language: ruby
|
|
2
|
-
before_install:
|
|
3
|
-
- gem install bundler
|
|
4
|
-
script:
|
|
5
|
-
- bundle exec rake test:db:create
|
|
6
|
-
- bundle exec rspec spec
|
|
7
|
-
- if [ "$RUN_BENCHMARK" = "true" ]; then DATASET_SIZE=100 bundle exec rake benchmark; fi
|
|
8
|
-
rvm:
|
|
9
|
-
2.3.0
|
|
10
|
-
jobs:
|
|
11
|
-
include:
|
|
12
|
-
- env:
|
|
13
|
-
- DB_TYPE=mysql
|
|
14
|
-
- DB_USERNAME=travis
|
|
15
|
-
services:
|
|
16
|
-
- mysql
|
|
17
|
-
- env:
|
|
18
|
-
- DB_TYPE=postgres
|
|
19
|
-
- RUN_BENCHMARK=true
|
|
20
|
-
services:
|
|
21
|
-
- postgresql
|