csv_fast_importer 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/tests.yml +91 -0
- data/.gitignore +1 -1
- data/.ruby-version +1 -1
- data/Gemfile.lock +25 -39
- data/README.md +31 -15
- data/Rakefile +1 -1
- data/benchmark/README.md +1 -1
- data/csv_fast_importer.gemspec +3 -2
- data/doc/faq.md +64 -0
- data/lib/csv_fast_importer/version.rb +1 -1
- data/rakelib/release.rake +6 -3
- data/sample-app/Gemfile.lock +2 -2
- data/sample-app/README.md +1 -1
- metadata +27 -13
- data/.travis.yml +0 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e64fb873391ad1bac7f05bf0bcefc26f0482d0917b71244e7388a19cd866b6cb
|
4
|
+
data.tar.gz: 6db4589951a6256be1b4334c0791cde38b10f4e87974bfbf85835ed39c01c061
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d2ecf83bebef80e492e9799b4c2345c9b0a5e8c6a645c219a9acb2ed0012263ebf1322baa8ede53c9dc6390e6fb3846c9737979259957f1d2cc779ab654282e
|
7
|
+
data.tar.gz: 33ad1e337b5f4c649d86d96bb8a6f803168009719c47484dc76681e7dc70f6acb538c6f1a694e7f8939bc2ff20a2705bf5a0c08c2cdd44f4a8eed4b6faea8cf6
|
@@ -0,0 +1,91 @@
|
|
1
|
+
name: Tests
|
2
|
+
|
3
|
+
on: push
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
postgres:
|
7
|
+
name: Test against PostgreSQL
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
env:
|
10
|
+
DB_TYPE: postgres
|
11
|
+
DB_PASSWORD: password
|
12
|
+
DB_USERNAME: postgres
|
13
|
+
services:
|
14
|
+
postgres:
|
15
|
+
image: postgres
|
16
|
+
env:
|
17
|
+
POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
|
18
|
+
ports:
|
19
|
+
- 5432:5432
|
20
|
+
options: >-
|
21
|
+
--health-cmd pg_isready
|
22
|
+
--health-interval 10s
|
23
|
+
--health-timeout 5s
|
24
|
+
--health-retries 5
|
25
|
+
steps:
|
26
|
+
- uses: actions/checkout@v4
|
27
|
+
- uses: ruby/setup-ruby@v1
|
28
|
+
with:
|
29
|
+
bundler-cache: true
|
30
|
+
- run: bundle exec rake test:db:create
|
31
|
+
- run: bundle exec rspec spec
|
32
|
+
- uses: actions/upload-artifact@v3
|
33
|
+
with:
|
34
|
+
name: postgres-test-coverage
|
35
|
+
path: coverage/lcov/csv_fast_importer.lcov
|
36
|
+
|
37
|
+
mysql:
|
38
|
+
name: Test against MySQL
|
39
|
+
runs-on: ubuntu-latest
|
40
|
+
env:
|
41
|
+
DB_TYPE: mysql
|
42
|
+
DB_HOST: 127.0.0.1
|
43
|
+
DB_PORT: 3306
|
44
|
+
DB_ROOT_PASSWORD: password
|
45
|
+
DB_USERNAME: mysql
|
46
|
+
DB_PASSWORD: password
|
47
|
+
DB_DATABASE: csv_fast_importer_test
|
48
|
+
services:
|
49
|
+
mysql:
|
50
|
+
image: mysql
|
51
|
+
env:
|
52
|
+
MYSQL_DATABASE: ${{ env.DB_DATABASE }}
|
53
|
+
MYSQL_USER: ${{ env.DB_USERNAME }}
|
54
|
+
MYSQL_PASSWORD: ${{ env.DB_PASSWORD }}
|
55
|
+
MYSQL_ROOT_PASSWORD: ${{ env.DB_ROOT_PASSWORD }}
|
56
|
+
ports:
|
57
|
+
- 3306:3306
|
58
|
+
options: >-
|
59
|
+
--health-cmd="mysqladmin ping"
|
60
|
+
--health-interval=10s
|
61
|
+
--health-timeout=5s
|
62
|
+
--health-retries=3
|
63
|
+
steps:
|
64
|
+
- uses: actions/checkout@v4
|
65
|
+
- uses: ruby/setup-ruby@v1
|
66
|
+
with:
|
67
|
+
bundler-cache: true
|
68
|
+
- name: Enable local data loading
|
69
|
+
run: |
|
70
|
+
mysql --host=${{ env.DB_HOST }} --user=root --password=${{ env.DB_ROOT_PASSWORD }} ${{ env.DB_DATABASE }} <<MY_QUERY
|
71
|
+
set global local_infile=true;
|
72
|
+
MY_QUERY
|
73
|
+
- run: bundle exec rspec spec
|
74
|
+
- uses: actions/upload-artifact@v3
|
75
|
+
with:
|
76
|
+
name: mysql-test-coverage
|
77
|
+
path: coverage/lcov/csv_fast_importer.lcov
|
78
|
+
|
79
|
+
test-coverage:
|
80
|
+
name: Publish test coverage
|
81
|
+
needs: [postgres, mysql]
|
82
|
+
runs-on: ubuntu-latest
|
83
|
+
steps:
|
84
|
+
- name: Recover all test coverages
|
85
|
+
uses: actions/download-artifact@v3
|
86
|
+
- run: ls -R
|
87
|
+
- name: Upload coverage to Codacy
|
88
|
+
uses: codacy/codacy-coverage-reporter-action@v1
|
89
|
+
with:
|
90
|
+
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
91
|
+
coverage-reports: postgres-test-coverage/csv_fast_importer.lcov, mysql-test-coverage/csv_fast_importer.lcov
|
data/.gitignore
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
/test/tmp/
|
10
10
|
/test/version_tmp/
|
11
11
|
/tmp/
|
12
|
+
/.idea/
|
12
13
|
|
13
14
|
## Specific to RubyMotion:
|
14
15
|
.dat*
|
@@ -18,7 +19,6 @@ build/
|
|
18
19
|
## Documentation cache and generated files:
|
19
20
|
/.yardoc/
|
20
21
|
/_yardoc/
|
21
|
-
/doc/
|
22
22
|
/rdoc/
|
23
23
|
|
24
24
|
## Environment normalization:
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.6.10
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
csv_fast_importer (1.
|
4
|
+
csv_fast_importer (1.2.0)
|
5
5
|
activerecord (~> 4.2)
|
6
6
|
|
7
7
|
GEM
|
@@ -9,16 +9,16 @@ GEM
|
|
9
9
|
specs:
|
10
10
|
active_importer (0.2.6)
|
11
11
|
roo
|
12
|
-
activemodel (4.2.
|
13
|
-
activesupport (= 4.2.
|
12
|
+
activemodel (4.2.11.3)
|
13
|
+
activesupport (= 4.2.11.3)
|
14
14
|
builder (~> 3.1)
|
15
|
-
activerecord (4.2.
|
16
|
-
activemodel (= 4.2.
|
17
|
-
activesupport (= 4.2.
|
15
|
+
activerecord (4.2.11.3)
|
16
|
+
activemodel (= 4.2.11.3)
|
17
|
+
activesupport (= 4.2.11.3)
|
18
18
|
arel (~> 6.0)
|
19
19
|
activerecord-import (0.10.0)
|
20
20
|
activerecord (>= 3.0)
|
21
|
-
activesupport (4.2.
|
21
|
+
activesupport (4.2.11.3)
|
22
22
|
i18n (~> 0.7)
|
23
23
|
minitest (~> 5.1)
|
24
24
|
thread_safe (~> 0.3, >= 0.3.4)
|
@@ -28,48 +28,34 @@ GEM
|
|
28
28
|
descendants_tracker (~> 0.0.4)
|
29
29
|
ice_nine (~> 0.11.0)
|
30
30
|
thread_safe (~> 0.3, >= 0.3.1)
|
31
|
-
builder (3.2.
|
31
|
+
builder (3.2.4)
|
32
32
|
bulk_insert (1.5.0)
|
33
33
|
activerecord (>= 4.1.0)
|
34
|
-
codacy-coverage (0.2.3)
|
35
|
-
rest-client (~> 1.8)
|
36
|
-
simplecov (~> 0.10.0)
|
37
34
|
coercible (1.0.0)
|
38
35
|
descendants_tracker (~> 0.0.1)
|
39
|
-
concurrent-ruby (1.
|
36
|
+
concurrent-ruby (1.2.2)
|
40
37
|
csv-importer (0.3.2)
|
41
38
|
virtus
|
42
39
|
descendants_tracker (0.0.4)
|
43
40
|
thread_safe (~> 0.3, >= 0.3.1)
|
44
41
|
diff-lcs (1.2.5)
|
45
|
-
docile (1.
|
46
|
-
domain_name (0.5.20160216)
|
47
|
-
unf (>= 0.0.5, < 1.0.0)
|
42
|
+
docile (1.4.0)
|
48
43
|
equalizer (0.0.11)
|
49
44
|
ferry (2.0.0)
|
50
45
|
highline (~> 1.6.21)
|
51
46
|
progressbar (~> 0.21.0)
|
52
47
|
highline (1.6.21)
|
53
|
-
|
54
|
-
domain_name (~> 0.5)
|
55
|
-
i18n (0.9.3)
|
48
|
+
i18n (0.9.5)
|
56
49
|
concurrent-ruby (~> 1.0)
|
57
50
|
ice_nine (0.11.2)
|
58
|
-
json (1.8.6)
|
59
|
-
mime-types (2.99.1)
|
60
51
|
mini_portile2 (2.3.0)
|
61
|
-
minitest (5.
|
52
|
+
minitest (5.20.0)
|
62
53
|
mysql2 (0.4.10)
|
63
|
-
netrc (0.11.0)
|
64
54
|
nokogiri (1.8.2)
|
65
55
|
mini_portile2 (~> 2.3.0)
|
66
|
-
pg (0.
|
56
|
+
pg (0.21.0)
|
67
57
|
progressbar (0.21.0)
|
68
58
|
rake (10.4.2)
|
69
|
-
rest-client (1.8.0)
|
70
|
-
http-cookie (>= 1.0.2, < 2.0)
|
71
|
-
mime-types (>= 1.16, < 3.0)
|
72
|
-
netrc (~> 0.7)
|
73
59
|
roo (2.7.1)
|
74
60
|
nokogiri (~> 1)
|
75
61
|
rubyzip (~> 1.1, < 2.0.0)
|
@@ -86,19 +72,18 @@ GEM
|
|
86
72
|
diff-lcs (>= 1.2.0, < 2.0)
|
87
73
|
rspec-support (~> 3.4.0)
|
88
74
|
rspec-support (3.4.1)
|
89
|
-
rubyzip (1.
|
90
|
-
simplecov (0.
|
91
|
-
docile (~> 1.1
|
92
|
-
|
93
|
-
|
94
|
-
simplecov-html (0.
|
75
|
+
rubyzip (1.3.0)
|
76
|
+
simplecov (0.22.0)
|
77
|
+
docile (~> 1.1)
|
78
|
+
simplecov-html (~> 0.11)
|
79
|
+
simplecov_json_formatter (~> 0.1)
|
80
|
+
simplecov-html (0.12.3)
|
81
|
+
simplecov-lcov (0.8.0)
|
82
|
+
simplecov_json_formatter (0.1.4)
|
95
83
|
smarter_csv (1.1.4)
|
96
84
|
thread_safe (0.3.6)
|
97
|
-
tzinfo (1.2.
|
85
|
+
tzinfo (1.2.11)
|
98
86
|
thread_safe (~> 0.1)
|
99
|
-
unf (0.1.4)
|
100
|
-
unf_ext
|
101
|
-
unf_ext (0.0.7.2)
|
102
87
|
upsert (2.2.1)
|
103
88
|
virtus (1.0.5)
|
104
89
|
axiom-types (~> 0.1)
|
@@ -114,14 +99,15 @@ DEPENDENCIES
|
|
114
99
|
activerecord-import
|
115
100
|
bulk_insert
|
116
101
|
bundler (~> 1.10)
|
117
|
-
codacy-coverage
|
118
102
|
csv-importer
|
119
103
|
csv_fast_importer!
|
120
104
|
ferry
|
121
105
|
mysql2 (>= 0.4.0)
|
122
|
-
pg (
|
106
|
+
pg (~> 0.21.0)
|
123
107
|
rake (~> 10.0)
|
124
108
|
rspec
|
109
|
+
simplecov
|
110
|
+
simplecov-lcov
|
125
111
|
smarter_csv
|
126
112
|
upsert
|
127
113
|
|
data/README.md
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
[![Gem Version](https://badge.fury.io/rb/csv_fast_importer.svg)](https://badge.fury.io/rb/csv_fast_importer)
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/csv_fast_importer.svg)](https://badge.fury.io/rb/csv_fast_importer) ![Tests status](https://github.com/sogilis/csv_fast_importer/actions/workflows/tests.yml/badge.svg) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/1ecd555b2ff3414d92bc8674b29c68ea)](https://www.codacy.com/gh/sogilis/csv_fast_importer/dashboard?utm_source=github.com&utm_medium=referral&utm_content=sogilis/csv_fast_importer&utm_campaign=Badge_Grade)
|
2
|
+
|
2
3
|
|
3
4
|
# CSV Fast Importer
|
4
5
|
|
@@ -23,17 +24,19 @@ Like all benchmarks, some tuning can produce different results, yet this chart g
|
|
23
24
|
|
24
25
|
- Usual ActiveRecord process (validations, callbacks, computed fields like `created_at`...) is bypassed. This is the price for performance
|
25
26
|
- Custom enclosing field (ex: `"`) is not supported yet
|
26
|
-
- Custom line
|
27
|
+
- Custom line separator (ex: `\r\n` for windows file) is not supported yet
|
27
28
|
- MySQL: encoding is not supported yet
|
28
29
|
- MySQL: transaction is not supported yet
|
29
30
|
- MySQL: row_index is not supported yet
|
30
31
|
|
32
|
+
Note about custom line separator: it might work by opening the file with the `universal_newline` argument (e.g. `file = File.new(path, universal_newline: true)`). Unfortunately, we weren't able to reproduce and test it so we don't support it "officialy". You can find more information in [this ticket](https://github.com/sogilis/csv_fast_importer/pull/45#issuecomment-326578839) (in French).
|
33
|
+
|
31
34
|
## Installation
|
32
35
|
|
33
36
|
Add the dependency to your Gemfile:
|
34
37
|
|
35
|
-
```
|
36
|
-
gem 'csv_fast_importer
|
38
|
+
```ruby
|
39
|
+
gem 'csv_fast_importer'
|
37
40
|
```
|
38
41
|
|
39
42
|
Run `bundle install`.
|
@@ -71,7 +74,7 @@ For instance, a `FIRSTNAME` CSV column will be mapped to the `firstname` field.
|
|
71
74
|
|
72
75
|
| Option key | Purpose | Default value |
|
73
76
|
| ------------ | ------------- | ------------- |
|
74
|
-
| *encoding* | File encoding. *PostgreSQL only
|
77
|
+
| *encoding* | File encoding. *PostgreSQL only* (see [FAQ](doc/faq.md) for more details)| `'UTF-8'` |
|
75
78
|
| *col_sep* | Column separator in file | `';'` |
|
76
79
|
| *destination* | Destination table | given base filename (without extension) |
|
77
80
|
| *mapping* | Column mapping | `{}` |
|
@@ -79,11 +82,10 @@ For instance, a `FIRSTNAME` CSV column will be mapped to the `firstname` field.
|
|
79
82
|
| *transaction* | Execute DELETE and INSERT in same transaction. *PostgreSQL only* | `:enabled` |
|
80
83
|
| *deletion* | Row deletion method (`:delete` for SQL DELETE, `:truncate` for SQL TRUNCATE or `:none` for no deletion before import) | `:delete` |
|
81
84
|
|
82
|
-
|
83
|
-
with the `encoding` option (*PostgreSQL only*).
|
85
|
+
If your CSV file is not encoded with same table than your database, you can specify encoding at the file opening (see [FAQ](doc/faq.md) for more details):
|
84
86
|
|
85
87
|
```ruby
|
86
|
-
|
88
|
+
file = File.new '/path/to/knights.csv', encoding: 'ISO-8859-1'
|
87
89
|
```
|
88
90
|
|
89
91
|
You can specify a different separator column with the `col_sep` option (`;` by
|
@@ -117,15 +119,21 @@ Lancelot;lancelot@logre.cel
|
|
117
119
|
To map the `KNIGHT_EMAIL` column to the `email` database field:
|
118
120
|
|
119
121
|
```ruby
|
120
|
-
CsvFastImporter.import file, mapping: {
|
122
|
+
CsvFastImporter.import file, mapping: { knight_email: :email }
|
121
123
|
```
|
122
124
|
|
125
|
+
## Need help?
|
126
|
+
|
127
|
+
See [FAQ](doc/faq.md).
|
128
|
+
|
123
129
|
## How to contribute?
|
124
130
|
|
125
131
|
You can fork and submit new pull request (with tests and explanations).
|
126
132
|
First of all, you need to initialize your environment :
|
127
133
|
|
128
134
|
```sh
|
135
|
+
$ brew install postgresql # in macOS
|
136
|
+
$ apt-get install libpq-dev # in Linux
|
129
137
|
$ bundle install
|
130
138
|
```
|
131
139
|
|
@@ -136,6 +144,8 @@ $ bundle exec rake test:db:create
|
|
136
144
|
```
|
137
145
|
This will connect to `localhost` PostgreSQL database without user (see `config/database.postgres.yml`) and create a new database dedicated to tests.
|
138
146
|
|
147
|
+
*Warning:* database instance have to allow database creation with `UTF-8` encoding.
|
148
|
+
|
139
149
|
Finally, you can run all tests with RSpec like this:
|
140
150
|
|
141
151
|
```sh
|
@@ -156,9 +166,6 @@ Use `DB_TYPE=mysql DB_USERNAME=` (with empty username) for anonymous account.
|
|
156
166
|
|
157
167
|
`master` is the development branch and releases are published as tags.
|
158
168
|
|
159
|
-
**We're not ready for the production yet (version < 1.0) so use this gem with
|
160
|
-
precaution.**
|
161
|
-
|
162
169
|
We follow the [Semantic Versioning 2.0.0](http://semver.org/) for our gem
|
163
170
|
releases.
|
164
171
|
|
@@ -187,9 +194,18 @@ In few words:
|
|
187
194
|
|
188
195
|
## How to release new version?
|
189
196
|
|
190
|
-
|
197
|
+
Setup rubygems.org account:
|
198
|
+
|
199
|
+
```bash
|
200
|
+
curl -u {your_gem_account_name} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
201
|
+
chmod 0600 ~/.gem/credentials
|
202
|
+
```
|
203
|
+
|
204
|
+
Make sure you are in `master` branch and run:
|
205
|
+
|
191
206
|
```bash
|
192
|
-
bundle exec rake release:make[major|minor|patch|x.y.z]
|
207
|
+
bundle exec rake "release:make[major|minor|patch|x.y.z]"
|
193
208
|
```
|
209
|
+
Example: `bundle exec rake "release:make[minor]"`
|
194
210
|
|
195
|
-
|
211
|
+
Then, follow instructions.
|
data/Rakefile
CHANGED
@@ -13,7 +13,7 @@ namespace :test do
|
|
13
13
|
when :postgres
|
14
14
|
require 'pg'
|
15
15
|
db.connect 'postgres'
|
16
|
-
ActiveRecord::Base.connection.execute "CREATE DATABASE #{db.name}"
|
16
|
+
ActiveRecord::Base.connection.execute "CREATE DATABASE #{db.name} ENCODING='UTF-8'"
|
17
17
|
|
18
18
|
when :mysql
|
19
19
|
require 'mysql2'
|
data/benchmark/README.md
CHANGED
data/csv_fast_importer.gemspec
CHANGED
@@ -24,9 +24,10 @@ Gem::Specification.new do |spec|
|
|
24
24
|
|
25
25
|
spec.add_development_dependency "bundler", "~> 1.10"
|
26
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
-
spec.add_development_dependency "pg", "
|
27
|
+
spec.add_development_dependency "pg", "~> 0.21.0"
|
28
28
|
spec.add_development_dependency "mysql2", ">= 0.4.0"
|
29
|
-
spec.add_development_dependency "
|
29
|
+
spec.add_development_dependency "simplecov"
|
30
|
+
spec.add_development_dependency "simplecov-lcov"
|
30
31
|
spec.add_development_dependency "rspec"
|
31
32
|
|
32
33
|
# Only for benchmark
|
data/doc/faq.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Frequently Asked Questions
|
2
|
+
|
3
|
+
## How to specify encoding?
|
4
|
+
|
5
|
+
Multiple components are involved when `CSV Fast Importer` is executed:
|
6
|
+
|
7
|
+
- file
|
8
|
+
- ruby `File` wrapper
|
9
|
+
- database client (managed by `ActiveRecord` connection)
|
10
|
+
- SQL command (`COPY` for PostgreSQL)
|
11
|
+
- database server
|
12
|
+
|
13
|
+
Encoding must be consistent accross all these components. Here is how to specify or check each component encoding.
|
14
|
+
|
15
|
+
### File
|
16
|
+
|
17
|
+
You can get current file encoding with `file -i [file_path]` (`-I` on macOS) command.
|
18
|
+
Some tools like [iconv](http://www.gnu.org/savannah-checkouts/gnu/libiconv/documentation/libiconv-1.15/iconv.1.html) can modify file encoding.
|
19
|
+
|
20
|
+
### Ruby `File` wrapper
|
21
|
+
|
22
|
+
`File` uses default Ruby encoding (given by `Encoding.default_external`. See [External / Internal Encoding](https://ruby-doc.org/core-2.4.1/Encoding.html#class-Encoding-label-External+encoding) which might be different from file enoding!
|
23
|
+
|
24
|
+
```ruby
|
25
|
+
File.new 'path/to/file.csv'
|
26
|
+
```
|
27
|
+
|
28
|
+
But, you can specify encoding with `encoding` parameter:
|
29
|
+
|
30
|
+
```ruby
|
31
|
+
File.new 'path/to/file.csv', encoding: 'ISO-8859-1'
|
32
|
+
```
|
33
|
+
|
34
|
+
Ruby `File` can also handle internal and external encoding (see [File::new](https://ruby-doc.org/core-2.4.1/File.html#method-c-new) which can be useful to manage automatic conversion:
|
35
|
+
|
36
|
+
```ruby
|
37
|
+
File.new 'path/to/file.csv', external_encoding: 'ISO-8859-1', internal_encoding: 'UTF-8'
|
38
|
+
# or
|
39
|
+
File.new 'path/to/file.csv', encoding: 'ISO-8859-1:UTF-8'
|
40
|
+
```
|
41
|
+
|
42
|
+
### Database client
|
43
|
+
|
44
|
+
Database is accessed through a dedicated client.
|
45
|
+
This client is managed by `ActiveRecord` with some configuration (`database.yml` in Rails application) where `encoding` parameter can be defined.
|
46
|
+
|
47
|
+
### SQL Command
|
48
|
+
|
49
|
+
By default, `COPY` and `LOAD DATA INFILE` commands follow database client encoding configuration. But you can override this with dedicated parameter.
|
50
|
+
This is the purpose of `CSV FAST Importer`'s `encoding` parameter.
|
51
|
+
|
52
|
+
### Database server
|
53
|
+
|
54
|
+
Each Postgres server instance is encoded with a specific table. You can show this with following command:
|
55
|
+
|
56
|
+
```shell
|
57
|
+
psql -l
|
58
|
+
```
|
59
|
+
|
60
|
+
Or, from `psql` client:
|
61
|
+
|
62
|
+
```sql
|
63
|
+
\l
|
64
|
+
```
|
data/rakelib/release.rake
CHANGED
@@ -12,9 +12,11 @@ namespace :release do
|
|
12
12
|
release.apply!
|
13
13
|
|
14
14
|
puts
|
15
|
-
puts "Changelog from #{release.last_version} to #{release.version}
|
15
|
+
puts "Changelog from #{release.last_version} to #{release.version}"
|
16
|
+
puts "---------------------------------------------------------------------"
|
16
17
|
puts release.changelog
|
17
18
|
puts "(Chores ignored)"
|
19
|
+
puts "---------------------------------------------------------------------"
|
18
20
|
|
19
21
|
next if args[:dry_run]
|
20
22
|
|
@@ -25,9 +27,10 @@ namespace :release do
|
|
25
27
|
puts
|
26
28
|
puts "Run following commands to publish version #{release.version}:"
|
27
29
|
puts "$ git push && git push --tags"
|
28
|
-
puts "$
|
30
|
+
puts "$ bundle exec rake release"
|
29
31
|
puts
|
30
|
-
puts "After that, do not forget to report changelog in Github
|
32
|
+
puts "After that, do not forget to report changelog in Github Releases page:"
|
33
|
+
puts "https://github.com/sogilis/csv_fast_importer/releases"
|
31
34
|
end
|
32
35
|
end
|
33
36
|
|
data/sample-app/Gemfile.lock
CHANGED
@@ -54,7 +54,7 @@ GEM
|
|
54
54
|
debug_inspector (0.0.3)
|
55
55
|
erubis (2.7.0)
|
56
56
|
execjs (2.7.0)
|
57
|
-
ffi (1.
|
57
|
+
ffi (1.16.1)
|
58
58
|
globalid (0.4.1)
|
59
59
|
activesupport (>= 4.2.0)
|
60
60
|
i18n (0.9.5)
|
@@ -169,4 +169,4 @@ DEPENDENCIES
|
|
169
169
|
web-console (~> 2.0)
|
170
170
|
|
171
171
|
BUNDLED WITH
|
172
|
-
1.
|
172
|
+
1.17.2
|
data/sample-app/README.md
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv_fast_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sogilis
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-09-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -42,16 +42,16 @@ dependencies:
|
|
42
42
|
name: pg
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.21.0
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.21.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: mysql2
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,7 +67,21 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.4.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: simplecov
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: simplecov-lcov
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
87
|
- - ">="
|
@@ -215,9 +229,9 @@ executables: []
|
|
215
229
|
extensions: []
|
216
230
|
extra_rdoc_files: []
|
217
231
|
files:
|
232
|
+
- ".github/workflows/tests.yml"
|
218
233
|
- ".gitignore"
|
219
234
|
- ".ruby-version"
|
220
|
-
- ".travis.yml"
|
221
235
|
- CONTRIBUTING.md
|
222
236
|
- Gemfile
|
223
237
|
- Gemfile.lock
|
@@ -232,6 +246,7 @@ files:
|
|
232
246
|
- benchmark/strategies.rb
|
233
247
|
- benchmark/tools.rb
|
234
248
|
- csv_fast_importer.gemspec
|
249
|
+
- doc/faq.md
|
235
250
|
- lib/csv_fast_importer.rb
|
236
251
|
- lib/csv_fast_importer/configuration.rb
|
237
252
|
- lib/csv_fast_importer/database/mysql.rb
|
@@ -312,7 +327,7 @@ homepage: https://github.com/sogilis/csv_fast_importer
|
|
312
327
|
licenses:
|
313
328
|
- MIT
|
314
329
|
metadata: {}
|
315
|
-
post_install_message:
|
330
|
+
post_install_message:
|
316
331
|
rdoc_options: []
|
317
332
|
require_paths:
|
318
333
|
- lib
|
@@ -327,9 +342,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
327
342
|
- !ruby/object:Gem::Version
|
328
343
|
version: '0'
|
329
344
|
requirements: []
|
330
|
-
|
331
|
-
|
332
|
-
signing_key:
|
345
|
+
rubygems_version: 3.0.3.1
|
346
|
+
signing_key:
|
333
347
|
specification_version: 4
|
334
348
|
summary: Fast CSV Importer
|
335
349
|
test_files: []
|
data/.travis.yml
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
before_install:
|
3
|
-
- gem install bundler
|
4
|
-
script:
|
5
|
-
- bundle exec rake test:db:create
|
6
|
-
- bundle exec rspec spec
|
7
|
-
- if [ "$RUN_BENCHMARK" = "true" ]; then DATASET_SIZE=100 bundle exec rake benchmark; fi
|
8
|
-
rvm:
|
9
|
-
2.3.0
|
10
|
-
jobs:
|
11
|
-
include:
|
12
|
-
- env:
|
13
|
-
- DB_TYPE=mysql
|
14
|
-
- DB_USERNAME=travis
|
15
|
-
services:
|
16
|
-
- mysql
|
17
|
-
- env:
|
18
|
-
- DB_TYPE=postgres
|
19
|
-
- RUN_BENCHMARK=true
|
20
|
-
services:
|
21
|
-
- postgresql
|