samidare 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +15 -15
- data/README.md +179 -117
- data/Rakefile +7 -7
- data/lib/samidare.rb +42 -42
- data/lib/samidare/bigquery_utility.rb +85 -85
- data/lib/samidare/embulk.rb +60 -60
- data/lib/samidare/embulk_utility.rb +42 -42
- data/lib/samidare/mysql.rb +117 -117
- data/lib/samidare/version.rb +3 -3
- data/samidare.gemspec +3 -3
- data/spec/samidare/bigquery_utility_spec.rb +85 -85
- data/spec/samidare/embulk_spec.rb +23 -23
- data/spec/samidare/embulk_utility_spec.rb +119 -119
- data/spec/samidare/mysql_spec.rb +135 -135
- data/spec/samidare_spec.rb +7 -7
- data/spec/support/databe.yml +13 -13
- data/spec/support/table.yml +11 -11
- metadata +9 -9
data/spec/samidare/mysql_spec.rb
CHANGED
@@ -1,135 +1,135 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Samidare::MySQL::TableConfig do
|
4
|
-
|
5
|
-
describe '.generate_table_configs' do
|
6
|
-
subject { Samidare::MySQL::TableConfig.generate_table_configs('spec/support/table.yml') }
|
7
|
-
let(:db01_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge', 'daily_snapshot' => true }) }
|
8
|
-
let(:db01_simple) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
|
9
|
-
let(:db02_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
10
|
-
let(:db02_with_condition) { Samidare::MySQL::TableConfig.new({ 'name' => 'with_condition', 'condition' => 'created_at < CURRENT_DATE()' }) }
|
11
|
-
|
12
|
-
it { expect(subject['db01'][0]).to eq db01_hoge }
|
13
|
-
it { expect(subject['db01'][1]).to eq db01_simple }
|
14
|
-
it { expect(subject['db02'][0]).to eq db02_fuga }
|
15
|
-
it { expect(subject['db02'][1]).to eq db02_with_condition }
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
describe Samidare::MySQL::Column do
|
20
|
-
let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
|
21
|
-
let(:column_name) { 'id' }
|
22
|
-
let(:data_type) { 'int' }
|
23
|
-
|
24
|
-
it { expect(column.column_name).to eq 'id' }
|
25
|
-
it { expect(column.data_type).to eq 'int' }
|
26
|
-
|
27
|
-
describe '#bigquery_data_type' do
|
28
|
-
subject { column.bigquery_data_type }
|
29
|
-
|
30
|
-
context 'int' do
|
31
|
-
let(:data_type) { 'int' }
|
32
|
-
it { expect(subject).to eq 'integer' }
|
33
|
-
end
|
34
|
-
|
35
|
-
context 'tinyint' do
|
36
|
-
let(:data_type) { 'tinyint' }
|
37
|
-
it { expect(subject).to eq 'integer' }
|
38
|
-
end
|
39
|
-
|
40
|
-
context 'smallint' do
|
41
|
-
let(:data_type) { 'smallint' }
|
42
|
-
it { expect(subject).to eq 'integer' }
|
43
|
-
end
|
44
|
-
|
45
|
-
context 'mediumint' do
|
46
|
-
let(:data_type) { 'mediumint' }
|
47
|
-
it { expect(subject).to eq 'integer' }
|
48
|
-
end
|
49
|
-
|
50
|
-
context 'bigint' do
|
51
|
-
let(:data_type) { 'bigint' }
|
52
|
-
it { expect(subject).to eq 'integer' }
|
53
|
-
end
|
54
|
-
|
55
|
-
context 'float' do
|
56
|
-
let(:data_type) { 'float' }
|
57
|
-
it { expect(subject).to eq 'float' }
|
58
|
-
end
|
59
|
-
|
60
|
-
context 'double' do
|
61
|
-
let(:data_type) { 'double' }
|
62
|
-
it { expect(subject).to eq 'float' }
|
63
|
-
end
|
64
|
-
|
65
|
-
context 'decimal' do
|
66
|
-
let(:data_type) { 'decimal' }
|
67
|
-
it { expect(subject).to eq 'float' }
|
68
|
-
end
|
69
|
-
|
70
|
-
context 'char' do
|
71
|
-
let(:data_type) { 'char' }
|
72
|
-
it { expect(subject).to eq 'string' }
|
73
|
-
end
|
74
|
-
|
75
|
-
context 'varchar' do
|
76
|
-
let(:data_type) { 'varchar' }
|
77
|
-
it { expect(subject).to eq 'string' }
|
78
|
-
end
|
79
|
-
|
80
|
-
context 'tinytext' do
|
81
|
-
let(:data_type) { 'tinytext' }
|
82
|
-
it { expect(subject).to eq 'string' }
|
83
|
-
end
|
84
|
-
|
85
|
-
context 'text' do
|
86
|
-
let(:data_type) { 'text' }
|
87
|
-
it { expect(subject).to eq 'string' }
|
88
|
-
end
|
89
|
-
|
90
|
-
context 'date' do
|
91
|
-
let(:data_type) { 'date' }
|
92
|
-
it { expect(subject).to eq 'timestamp' }
|
93
|
-
end
|
94
|
-
|
95
|
-
context 'datetime' do
|
96
|
-
let(:data_type) { 'datetime' }
|
97
|
-
it { expect(subject).to eq 'timestamp' }
|
98
|
-
end
|
99
|
-
|
100
|
-
context 'timestamp' do
|
101
|
-
let(:data_type) { 'timestamp' }
|
102
|
-
it { expect(subject).to eq 'timestamp' }
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
describe '#converted_value' do
|
107
|
-
subject { column.converted_value }
|
108
|
-
|
109
|
-
context 'datetime' do
|
110
|
-
let(:column_name) { 'create_at' }
|
111
|
-
let(:data_type) { 'datetime' }
|
112
|
-
it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
|
113
|
-
end
|
114
|
-
|
115
|
-
context 'int' do
|
116
|
-
let(:column_name) { 'id' }
|
117
|
-
let(:data_type) { 'int' }
|
118
|
-
it { expect(subject).to eq '`id`' }
|
119
|
-
end
|
120
|
-
|
121
|
-
context 'varchar' do
|
122
|
-
let(:column_name) { 'explanation' }
|
123
|
-
let(:data_type) { 'varchar' }
|
124
|
-
it { expect(subject).to eq '`explanation`' }
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
describe '#to_json' do
|
129
|
-
subject { column.to_json }
|
130
|
-
|
131
|
-
let(:column_name) { 'id' }
|
132
|
-
let(:data_type) { 'int' }
|
133
|
-
it { expect(subject).to eq '{"name":"id","type":"integer"}' }
|
134
|
-
end
|
135
|
-
end
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Samidare::MySQL::TableConfig do
|
4
|
+
|
5
|
+
describe '.generate_table_configs' do
|
6
|
+
subject { Samidare::MySQL::TableConfig.generate_table_configs('spec/support/table.yml') }
|
7
|
+
let(:db01_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge', 'daily_snapshot' => true }) }
|
8
|
+
let(:db01_simple) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
|
9
|
+
let(:db02_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
10
|
+
let(:db02_with_condition) { Samidare::MySQL::TableConfig.new({ 'name' => 'with_condition', 'condition' => 'created_at < CURRENT_DATE()' }) }
|
11
|
+
|
12
|
+
it { expect(subject['db01'][0]).to eq db01_hoge }
|
13
|
+
it { expect(subject['db01'][1]).to eq db01_simple }
|
14
|
+
it { expect(subject['db02'][0]).to eq db02_fuga }
|
15
|
+
it { expect(subject['db02'][1]).to eq db02_with_condition }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe Samidare::MySQL::Column do
|
20
|
+
let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
|
21
|
+
let(:column_name) { 'id' }
|
22
|
+
let(:data_type) { 'int' }
|
23
|
+
|
24
|
+
it { expect(column.column_name).to eq 'id' }
|
25
|
+
it { expect(column.data_type).to eq 'int' }
|
26
|
+
|
27
|
+
describe '#bigquery_data_type' do
|
28
|
+
subject { column.bigquery_data_type }
|
29
|
+
|
30
|
+
context 'int' do
|
31
|
+
let(:data_type) { 'int' }
|
32
|
+
it { expect(subject).to eq 'integer' }
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'tinyint' do
|
36
|
+
let(:data_type) { 'tinyint' }
|
37
|
+
it { expect(subject).to eq 'integer' }
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'smallint' do
|
41
|
+
let(:data_type) { 'smallint' }
|
42
|
+
it { expect(subject).to eq 'integer' }
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'mediumint' do
|
46
|
+
let(:data_type) { 'mediumint' }
|
47
|
+
it { expect(subject).to eq 'integer' }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'bigint' do
|
51
|
+
let(:data_type) { 'bigint' }
|
52
|
+
it { expect(subject).to eq 'integer' }
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'float' do
|
56
|
+
let(:data_type) { 'float' }
|
57
|
+
it { expect(subject).to eq 'float' }
|
58
|
+
end
|
59
|
+
|
60
|
+
context 'double' do
|
61
|
+
let(:data_type) { 'double' }
|
62
|
+
it { expect(subject).to eq 'float' }
|
63
|
+
end
|
64
|
+
|
65
|
+
context 'decimal' do
|
66
|
+
let(:data_type) { 'decimal' }
|
67
|
+
it { expect(subject).to eq 'float' }
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'char' do
|
71
|
+
let(:data_type) { 'char' }
|
72
|
+
it { expect(subject).to eq 'string' }
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'varchar' do
|
76
|
+
let(:data_type) { 'varchar' }
|
77
|
+
it { expect(subject).to eq 'string' }
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'tinytext' do
|
81
|
+
let(:data_type) { 'tinytext' }
|
82
|
+
it { expect(subject).to eq 'string' }
|
83
|
+
end
|
84
|
+
|
85
|
+
context 'text' do
|
86
|
+
let(:data_type) { 'text' }
|
87
|
+
it { expect(subject).to eq 'string' }
|
88
|
+
end
|
89
|
+
|
90
|
+
context 'date' do
|
91
|
+
let(:data_type) { 'date' }
|
92
|
+
it { expect(subject).to eq 'timestamp' }
|
93
|
+
end
|
94
|
+
|
95
|
+
context 'datetime' do
|
96
|
+
let(:data_type) { 'datetime' }
|
97
|
+
it { expect(subject).to eq 'timestamp' }
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'timestamp' do
|
101
|
+
let(:data_type) { 'timestamp' }
|
102
|
+
it { expect(subject).to eq 'timestamp' }
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
describe '#converted_value' do
|
107
|
+
subject { column.converted_value }
|
108
|
+
|
109
|
+
context 'datetime' do
|
110
|
+
let(:column_name) { 'create_at' }
|
111
|
+
let(:data_type) { 'datetime' }
|
112
|
+
it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
|
113
|
+
end
|
114
|
+
|
115
|
+
context 'int' do
|
116
|
+
let(:column_name) { 'id' }
|
117
|
+
let(:data_type) { 'int' }
|
118
|
+
it { expect(subject).to eq '`id`' }
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'varchar' do
|
122
|
+
let(:column_name) { 'explanation' }
|
123
|
+
let(:data_type) { 'varchar' }
|
124
|
+
it { expect(subject).to eq '`explanation`' }
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
describe '#to_json' do
|
129
|
+
subject { column.to_json }
|
130
|
+
|
131
|
+
let(:column_name) { 'id' }
|
132
|
+
let(:data_type) { 'int' }
|
133
|
+
it { expect(subject).to eq '{"name":"id","type":"integer"}' }
|
134
|
+
end
|
135
|
+
end
|
data/spec/samidare_spec.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Samidare do
|
4
|
-
it 'has a version number' do
|
5
|
-
expect(Samidare::VERSION).not_to be nil
|
6
|
-
end
|
7
|
-
end
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Samidare do
|
4
|
+
it 'has a version number' do
|
5
|
+
expect(Samidare::VERSION).not_to be nil
|
6
|
+
end
|
7
|
+
end
|
data/spec/support/databe.yml
CHANGED
@@ -1,13 +1,13 @@
|
|
1
|
-
db01:
|
2
|
-
host: localhost
|
3
|
-
username: root
|
4
|
-
password:
|
5
|
-
database: embulk
|
6
|
-
bq_dataset: mysql
|
7
|
-
|
8
|
-
db02:
|
9
|
-
host: localhost
|
10
|
-
username: root
|
11
|
-
password:
|
12
|
-
database: embulk2
|
13
|
-
bq_dataset: mysql2
|
1
|
+
db01:
|
2
|
+
host: localhost
|
3
|
+
username: root
|
4
|
+
password:
|
5
|
+
database: embulk
|
6
|
+
bq_dataset: mysql
|
7
|
+
|
8
|
+
db02:
|
9
|
+
host: localhost
|
10
|
+
username: root
|
11
|
+
password:
|
12
|
+
database: embulk2
|
13
|
+
bq_dataset: mysql2
|
data/spec/support/table.yml
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
db01:
|
2
|
-
tables:
|
3
|
-
- name: hoge
|
4
|
-
daily_snapshot: true
|
5
|
-
- name: simple
|
6
|
-
|
7
|
-
db02:
|
8
|
-
tables:
|
9
|
-
- name: fuga
|
10
|
-
- name: with_condition
|
11
|
-
condition: created_at < CURRENT_DATE()
|
1
|
+
db01:
|
2
|
+
tables:
|
3
|
+
- name: hoge
|
4
|
+
daily_snapshot: true
|
5
|
+
- name: simple
|
6
|
+
|
7
|
+
db02:
|
8
|
+
tables:
|
9
|
+
- name: fuga
|
10
|
+
- name: with_condition
|
11
|
+
condition: created_at < CURRENT_DATE()
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: samidare
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryoji Kobori
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -100,42 +100,42 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - '='
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.
|
103
|
+
version: 0.4.3
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - '='
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.
|
110
|
+
version: 0.4.3
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: embulk-input-mysql
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - '='
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
117
|
+
version: 0.8.2
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - '='
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
124
|
+
version: 0.8.2
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: embulk-parser-jsonl
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - '='
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 0.0
|
131
|
+
version: 0.2.0
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - '='
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: 0.0
|
138
|
+
version: 0.2.0
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: embulk-formatter-jsonl
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -212,7 +212,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
212
212
|
version: '0'
|
213
213
|
requirements: []
|
214
214
|
rubyforge_project:
|
215
|
-
rubygems_version: 2.
|
215
|
+
rubygems_version: 2.6.8
|
216
216
|
signing_key:
|
217
217
|
specification_version: 4
|
218
218
|
summary: Embulk utility for MySQL to BigQuery
|