samidare 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +15 -15
- data/README.md +179 -117
- data/Rakefile +7 -7
- data/lib/samidare.rb +42 -42
- data/lib/samidare/bigquery_utility.rb +85 -85
- data/lib/samidare/embulk.rb +60 -60
- data/lib/samidare/embulk_utility.rb +42 -42
- data/lib/samidare/mysql.rb +117 -117
- data/lib/samidare/version.rb +3 -3
- data/samidare.gemspec +3 -3
- data/spec/samidare/bigquery_utility_spec.rb +85 -85
- data/spec/samidare/embulk_spec.rb +23 -23
- data/spec/samidare/embulk_utility_spec.rb +119 -119
- data/spec/samidare/mysql_spec.rb +135 -135
- data/spec/samidare_spec.rb +7 -7
- data/spec/support/databe.yml +13 -13
- data/spec/support/table.yml +11 -11
- metadata +9 -9
@@ -1,85 +1,85 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'unindent'
|
3
|
-
require 'timecop'
|
4
|
-
|
5
|
-
describe Samidare::BigQueryUtility do
|
6
|
-
describe '.generate_schema' do
|
7
|
-
subject { Samidare::BigQueryUtility.generate_schema(columns) }
|
8
|
-
|
9
|
-
let(:columns) { [
|
10
|
-
Samidare::MySQL::Column.new('id', 'int'),
|
11
|
-
Samidare::MySQL::Column.new('name', 'varchar'),
|
12
|
-
Samidare::MySQL::Column.new('created_at', 'datetime')
|
13
|
-
] }
|
14
|
-
let(:schema_json) {
|
15
|
-
<<-JSON.unindent
|
16
|
-
[
|
17
|
-
{"name":"id","type":"integer"},
|
18
|
-
{"name":"name","type":"string"},
|
19
|
-
{"name":"created_at","type":"timestamp"}
|
20
|
-
]
|
21
|
-
JSON
|
22
|
-
}
|
23
|
-
it { expect(subject).to eq schema_json }
|
24
|
-
end
|
25
|
-
|
26
|
-
describe '.generate_sql' do
|
27
|
-
subject { Samidare::BigQueryUtility.generate_sql(table_config, columns) }
|
28
|
-
|
29
|
-
let(:columns) { [
|
30
|
-
Samidare::MySQL::Column.new('id', 'int'),
|
31
|
-
Samidare::MySQL::Column.new('name', 'varchar'),
|
32
|
-
Samidare::MySQL::Column.new('created_at', 'datetime')
|
33
|
-
] }
|
34
|
-
|
35
|
-
context 'no condition' do
|
36
|
-
let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
|
37
|
-
let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple\n" }
|
38
|
-
it { expect(subject).to eq sql }
|
39
|
-
end
|
40
|
-
|
41
|
-
context 'has condition' do
|
42
|
-
let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple', 'condition' => 'created_at >= CURRENT_DATE() - INTERVAL 3 MONTH' }) }
|
43
|
-
let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple WHERE created_at >= CURRENT_DATE() - INTERVAL 3 MONTH\n" }
|
44
|
-
it { expect(subject).to eq sql }
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
describe '#actual_table_name' do
|
49
|
-
before { Timecop.freeze(Time.now) }
|
50
|
-
|
51
|
-
after { Timecop.return }
|
52
|
-
|
53
|
-
subject { Samidare::BigQueryUtility.new({}).actual_table_name(table_name, daily_snapshot) }
|
54
|
-
let(:table_name) { 'users' }
|
55
|
-
let(:daily_snapshot) { false }
|
56
|
-
|
57
|
-
context 'do not use daily snapshot' do
|
58
|
-
it { expect(subject).to eq table_name }
|
59
|
-
end
|
60
|
-
|
61
|
-
context 'use daily snapshot' do
|
62
|
-
let(:daily_snapshot) { true }
|
63
|
-
it { expect(subject).to eq table_name + Time.now.strftime('%Y%m%d') }
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
describe '#actual_table_name' do
|
68
|
-
before { Timecop.freeze(Time.now) }
|
69
|
-
|
70
|
-
after { Timecop.return }
|
71
|
-
|
72
|
-
subject { Samidare::BigQueryUtility.new({}).actual_table_name(table_name, daily_snapshot) }
|
73
|
-
let(:table_name) { 'users' }
|
74
|
-
let(:daily_snapshot) { false }
|
75
|
-
|
76
|
-
context 'do not use daily snapshot' do
|
77
|
-
it { expect(subject).to eq table_name }
|
78
|
-
end
|
79
|
-
|
80
|
-
context 'use daily snapshot' do
|
81
|
-
let(:daily_snapshot) { true }
|
82
|
-
it { expect(subject).to eq table_name + Time.now.strftime('%Y%m%d') }
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'unindent'
|
3
|
+
require 'timecop'
|
4
|
+
|
5
|
+
describe Samidare::BigQueryUtility do
|
6
|
+
describe '.generate_schema' do
|
7
|
+
subject { Samidare::BigQueryUtility.generate_schema(columns) }
|
8
|
+
|
9
|
+
let(:columns) { [
|
10
|
+
Samidare::MySQL::Column.new('id', 'int'),
|
11
|
+
Samidare::MySQL::Column.new('name', 'varchar'),
|
12
|
+
Samidare::MySQL::Column.new('created_at', 'datetime')
|
13
|
+
] }
|
14
|
+
let(:schema_json) {
|
15
|
+
<<-JSON.unindent
|
16
|
+
[
|
17
|
+
{"name":"id","type":"integer"},
|
18
|
+
{"name":"name","type":"string"},
|
19
|
+
{"name":"created_at","type":"timestamp"}
|
20
|
+
]
|
21
|
+
JSON
|
22
|
+
}
|
23
|
+
it { expect(subject).to eq schema_json }
|
24
|
+
end
|
25
|
+
|
26
|
+
describe '.generate_sql' do
|
27
|
+
subject { Samidare::BigQueryUtility.generate_sql(table_config, columns) }
|
28
|
+
|
29
|
+
let(:columns) { [
|
30
|
+
Samidare::MySQL::Column.new('id', 'int'),
|
31
|
+
Samidare::MySQL::Column.new('name', 'varchar'),
|
32
|
+
Samidare::MySQL::Column.new('created_at', 'datetime')
|
33
|
+
] }
|
34
|
+
|
35
|
+
context 'no condition' do
|
36
|
+
let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
|
37
|
+
let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple\n" }
|
38
|
+
it { expect(subject).to eq sql }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'has condition' do
|
42
|
+
let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple', 'condition' => 'created_at >= CURRENT_DATE() - INTERVAL 3 MONTH' }) }
|
43
|
+
let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple WHERE created_at >= CURRENT_DATE() - INTERVAL 3 MONTH\n" }
|
44
|
+
it { expect(subject).to eq sql }
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe '#actual_table_name' do
|
49
|
+
before { Timecop.freeze(Time.now) }
|
50
|
+
|
51
|
+
after { Timecop.return }
|
52
|
+
|
53
|
+
subject { Samidare::BigQueryUtility.new({}).actual_table_name(table_name, daily_snapshot) }
|
54
|
+
let(:table_name) { 'users' }
|
55
|
+
let(:daily_snapshot) { false }
|
56
|
+
|
57
|
+
context 'do not use daily snapshot' do
|
58
|
+
it { expect(subject).to eq table_name }
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'use daily snapshot' do
|
62
|
+
let(:daily_snapshot) { true }
|
63
|
+
it { expect(subject).to eq table_name + Time.now.strftime('%Y%m%d') }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe '#actual_table_name' do
|
68
|
+
before { Timecop.freeze(Time.now) }
|
69
|
+
|
70
|
+
after { Timecop.return }
|
71
|
+
|
72
|
+
subject { Samidare::BigQueryUtility.new({}).actual_table_name(table_name, daily_snapshot) }
|
73
|
+
let(:table_name) { 'users' }
|
74
|
+
let(:daily_snapshot) { false }
|
75
|
+
|
76
|
+
context 'do not use daily snapshot' do
|
77
|
+
it { expect(subject).to eq table_name }
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'use daily snapshot' do
|
81
|
+
let(:daily_snapshot) { true }
|
82
|
+
it { expect(subject).to eq table_name + Time.now.strftime('%Y%m%d') }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -1,23 +1,23 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Samidare::Embulk do
|
4
|
-
describe '#target_table_configs' do
|
5
|
-
subject { Samidare::Embulk.new(nil, nil).target_table_configs(table_configs, target_table_names) }
|
6
|
-
|
7
|
-
context 'all tables' do
|
8
|
-
let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
|
9
|
-
let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
10
|
-
let(:table_configs) { [table_hoge, table_fuga] }
|
11
|
-
let(:target_table_names) { [] }
|
12
|
-
it { expect(subject).to match(table_configs) }
|
13
|
-
end
|
14
|
-
|
15
|
-
context 'target table selected' do
|
16
|
-
let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
|
17
|
-
let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
18
|
-
let(:table_configs) { [table_hoge, table_fuga] }
|
19
|
-
let(:target_table_names) { ['hoge'] }
|
20
|
-
it { expect(subject).to match([table_hoge]) }
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Samidare::Embulk do
|
4
|
+
describe '#target_table_configs' do
|
5
|
+
subject { Samidare::Embulk.new(nil, nil).target_table_configs(table_configs, target_table_names) }
|
6
|
+
|
7
|
+
context 'all tables' do
|
8
|
+
let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
|
9
|
+
let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
10
|
+
let(:table_configs) { [table_hoge, table_fuga] }
|
11
|
+
let(:target_table_names) { [] }
|
12
|
+
it { expect(subject).to match(table_configs) }
|
13
|
+
end
|
14
|
+
|
15
|
+
context 'target table selected' do
|
16
|
+
let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
|
17
|
+
let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
|
18
|
+
let(:table_configs) { [table_hoge, table_fuga] }
|
19
|
+
let(:target_table_names) { ['hoge'] }
|
20
|
+
it { expect(subject).to match([table_hoge]) }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,119 +1,119 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Samidare::MySQL::Column do
|
4
|
-
let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
|
5
|
-
let(:column_name) { 'id' }
|
6
|
-
let(:data_type) { 'int' }
|
7
|
-
|
8
|
-
it { expect(column.column_name).to eq 'id' }
|
9
|
-
it { expect(column.data_type).to eq 'int' }
|
10
|
-
|
11
|
-
describe '#bigquery_data_type' do
|
12
|
-
subject { column.bigquery_data_type }
|
13
|
-
|
14
|
-
context 'int' do
|
15
|
-
let(:data_type) { 'int' }
|
16
|
-
it { expect(subject).to eq 'integer' }
|
17
|
-
end
|
18
|
-
|
19
|
-
context 'tinyint' do
|
20
|
-
let(:data_type) { 'tinyint' }
|
21
|
-
it { expect(subject).to eq 'integer' }
|
22
|
-
end
|
23
|
-
|
24
|
-
context 'smallint' do
|
25
|
-
let(:data_type) { 'smallint' }
|
26
|
-
it { expect(subject).to eq 'integer' }
|
27
|
-
end
|
28
|
-
|
29
|
-
context 'mediumint' do
|
30
|
-
let(:data_type) { 'mediumint' }
|
31
|
-
it { expect(subject).to eq 'integer' }
|
32
|
-
end
|
33
|
-
|
34
|
-
context 'bigint' do
|
35
|
-
let(:data_type) { 'bigint' }
|
36
|
-
it { expect(subject).to eq 'integer' }
|
37
|
-
end
|
38
|
-
|
39
|
-
context 'float' do
|
40
|
-
let(:data_type) { 'float' }
|
41
|
-
it { expect(subject).to eq 'float' }
|
42
|
-
end
|
43
|
-
|
44
|
-
context 'double' do
|
45
|
-
let(:data_type) { 'double' }
|
46
|
-
it { expect(subject).to eq 'float' }
|
47
|
-
end
|
48
|
-
|
49
|
-
context 'decimal' do
|
50
|
-
let(:data_type) { 'decimal' }
|
51
|
-
it { expect(subject).to eq 'float' }
|
52
|
-
end
|
53
|
-
|
54
|
-
context 'char' do
|
55
|
-
let(:data_type) { 'char' }
|
56
|
-
it { expect(subject).to eq 'string' }
|
57
|
-
end
|
58
|
-
|
59
|
-
context 'varchar' do
|
60
|
-
let(:data_type) { 'varchar' }
|
61
|
-
it { expect(subject).to eq 'string' }
|
62
|
-
end
|
63
|
-
|
64
|
-
context 'tinytext' do
|
65
|
-
let(:data_type) { 'tinytext' }
|
66
|
-
it { expect(subject).to eq 'string' }
|
67
|
-
end
|
68
|
-
|
69
|
-
context 'text' do
|
70
|
-
let(:data_type) { 'text' }
|
71
|
-
it { expect(subject).to eq 'string' }
|
72
|
-
end
|
73
|
-
|
74
|
-
context 'date' do
|
75
|
-
let(:data_type) { 'date' }
|
76
|
-
it { expect(subject).to eq 'timestamp' }
|
77
|
-
end
|
78
|
-
|
79
|
-
context 'datetime' do
|
80
|
-
let(:data_type) { 'datetime' }
|
81
|
-
it { expect(subject).to eq 'timestamp' }
|
82
|
-
end
|
83
|
-
|
84
|
-
context 'timestamp' do
|
85
|
-
let(:data_type) { 'timestamp' }
|
86
|
-
it { expect(subject).to eq 'timestamp' }
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
describe '#converted_value' do
|
91
|
-
subject { column.converted_value }
|
92
|
-
|
93
|
-
context 'datetime' do
|
94
|
-
let(:column_name) { 'create_at' }
|
95
|
-
let(:data_type) { 'datetime' }
|
96
|
-
it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
|
97
|
-
end
|
98
|
-
|
99
|
-
context 'int' do
|
100
|
-
let(:column_name) { 'id' }
|
101
|
-
let(:data_type) { 'int' }
|
102
|
-
it { expect(subject).to eq '`id`' }
|
103
|
-
end
|
104
|
-
|
105
|
-
context 'varchar' do
|
106
|
-
let(:column_name) { 'explanation' }
|
107
|
-
let(:data_type) { 'varchar' }
|
108
|
-
it { expect(subject).to eq '`explanation`' }
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
describe '#to_json' do
|
113
|
-
subject { column.to_json }
|
114
|
-
|
115
|
-
let(:column_name) { 'id' }
|
116
|
-
let(:data_type) { 'int' }
|
117
|
-
it { expect(subject).to eq '{"name":"id","type":"integer"}' }
|
118
|
-
end
|
119
|
-
end
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Samidare::MySQL::Column do
|
4
|
+
let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
|
5
|
+
let(:column_name) { 'id' }
|
6
|
+
let(:data_type) { 'int' }
|
7
|
+
|
8
|
+
it { expect(column.column_name).to eq 'id' }
|
9
|
+
it { expect(column.data_type).to eq 'int' }
|
10
|
+
|
11
|
+
describe '#bigquery_data_type' do
|
12
|
+
subject { column.bigquery_data_type }
|
13
|
+
|
14
|
+
context 'int' do
|
15
|
+
let(:data_type) { 'int' }
|
16
|
+
it { expect(subject).to eq 'integer' }
|
17
|
+
end
|
18
|
+
|
19
|
+
context 'tinyint' do
|
20
|
+
let(:data_type) { 'tinyint' }
|
21
|
+
it { expect(subject).to eq 'integer' }
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'smallint' do
|
25
|
+
let(:data_type) { 'smallint' }
|
26
|
+
it { expect(subject).to eq 'integer' }
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'mediumint' do
|
30
|
+
let(:data_type) { 'mediumint' }
|
31
|
+
it { expect(subject).to eq 'integer' }
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'bigint' do
|
35
|
+
let(:data_type) { 'bigint' }
|
36
|
+
it { expect(subject).to eq 'integer' }
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'float' do
|
40
|
+
let(:data_type) { 'float' }
|
41
|
+
it { expect(subject).to eq 'float' }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'double' do
|
45
|
+
let(:data_type) { 'double' }
|
46
|
+
it { expect(subject).to eq 'float' }
|
47
|
+
end
|
48
|
+
|
49
|
+
context 'decimal' do
|
50
|
+
let(:data_type) { 'decimal' }
|
51
|
+
it { expect(subject).to eq 'float' }
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'char' do
|
55
|
+
let(:data_type) { 'char' }
|
56
|
+
it { expect(subject).to eq 'string' }
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'varchar' do
|
60
|
+
let(:data_type) { 'varchar' }
|
61
|
+
it { expect(subject).to eq 'string' }
|
62
|
+
end
|
63
|
+
|
64
|
+
context 'tinytext' do
|
65
|
+
let(:data_type) { 'tinytext' }
|
66
|
+
it { expect(subject).to eq 'string' }
|
67
|
+
end
|
68
|
+
|
69
|
+
context 'text' do
|
70
|
+
let(:data_type) { 'text' }
|
71
|
+
it { expect(subject).to eq 'string' }
|
72
|
+
end
|
73
|
+
|
74
|
+
context 'date' do
|
75
|
+
let(:data_type) { 'date' }
|
76
|
+
it { expect(subject).to eq 'timestamp' }
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'datetime' do
|
80
|
+
let(:data_type) { 'datetime' }
|
81
|
+
it { expect(subject).to eq 'timestamp' }
|
82
|
+
end
|
83
|
+
|
84
|
+
context 'timestamp' do
|
85
|
+
let(:data_type) { 'timestamp' }
|
86
|
+
it { expect(subject).to eq 'timestamp' }
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
describe '#converted_value' do
|
91
|
+
subject { column.converted_value }
|
92
|
+
|
93
|
+
context 'datetime' do
|
94
|
+
let(:column_name) { 'create_at' }
|
95
|
+
let(:data_type) { 'datetime' }
|
96
|
+
it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
|
97
|
+
end
|
98
|
+
|
99
|
+
context 'int' do
|
100
|
+
let(:column_name) { 'id' }
|
101
|
+
let(:data_type) { 'int' }
|
102
|
+
it { expect(subject).to eq '`id`' }
|
103
|
+
end
|
104
|
+
|
105
|
+
context 'varchar' do
|
106
|
+
let(:column_name) { 'explanation' }
|
107
|
+
let(:data_type) { 'varchar' }
|
108
|
+
it { expect(subject).to eq '`explanation`' }
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
describe '#to_json' do
|
113
|
+
subject { column.to_json }
|
114
|
+
|
115
|
+
let(:column_name) { 'id' }
|
116
|
+
let(:data_type) { 'int' }
|
117
|
+
it { expect(subject).to eq '{"name":"id","type":"integer"}' }
|
118
|
+
end
|
119
|
+
end
|