samidare 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/samidare/bigquery_utility.rb +0 -1
- data/lib/samidare/version.rb +3 -3
- data/lib/samidare.rb +51 -51
- metadata +2 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e9297d4a4da6cde678e001213c21b9ad3483b6eb
|
4
|
+
data.tar.gz: 1aa36de501c88dc84e0af7890bc365f028066ab5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c3fe570abcc9ea01af76be93257eb04e29b8cfbf98af94d567687012f044430ff8c0dbf607707899f26700f293eb73748b12cc4a73432fe6acaee6f627a3910
|
7
|
+
data.tar.gz: e5ab011f63fb3ffcd95e904239d24c45efd860a85007b3a84d2c701d0885b5b41aa5056cae49b02392f607aff64d97b4e838ee5134726fc2b1d5f66799562107
|
@@ -62,7 +62,6 @@ module Samidare
|
|
62
62
|
schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_info.name}.json"
|
63
63
|
path_prefix = "/var/tmp/embulk_#{db_name}_#{table_info.name}"
|
64
64
|
|
65
|
-
#File.open('lib/samidare/embulk_config.erb') { |f| ERB.new(f.read).result(binding) }
|
66
65
|
ERB.new(CONTENTS).result(binding)
|
67
66
|
end
|
68
67
|
|
data/lib/samidare/version.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
module Samidare
|
2
|
-
VERSION = "0.0.
|
3
|
-
end
|
1
|
+
module Samidare
|
2
|
+
VERSION = "0.0.5"
|
3
|
+
end
|
data/lib/samidare.rb
CHANGED
@@ -1,51 +1,51 @@
|
|
1
|
-
require 'samidare/version'
|
2
|
-
require 'samidare/embulk_utility'
|
3
|
-
|
4
|
-
module Samidare
|
5
|
-
class EmbulkClient
|
6
|
-
def generate_config(config)
|
7
|
-
Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
|
8
|
-
end
|
9
|
-
|
10
|
-
def run(config)
|
11
|
-
db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
|
12
|
-
table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
|
13
|
-
db_infos.keys.each do |db_name|
|
14
|
-
embulk_run(db_name, db_infos[db_name]['bq_dataset'], table_infos[db_name], config)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
private
|
19
|
-
def embulk_run(db_name, bq_dataset, tables, config)
|
20
|
-
process_times = []
|
21
|
-
big_query = Samidare::BigQueryUtility.new(config)
|
22
|
-
tables.each do |table|
|
23
|
-
start_time = Time.now
|
24
|
-
log "table: #{table.name} - start"
|
25
|
-
|
26
|
-
begin
|
27
|
-
big_query.delete_table(bq_dataset, table.name)
|
28
|
-
log "table: #{table.name} - deleted"
|
29
|
-
rescue
|
30
|
-
log "table: #{table.name} - not
|
31
|
-
end
|
32
|
-
|
33
|
-
cmd = "embulk run #{ENV['EMBULK_CONFIG_DIR']}/#{db_name}/#{table.name}.yml"
|
34
|
-
log "cmd: #{cmd}"
|
35
|
-
result = system(cmd) ? 'success' : 'error'
|
36
|
-
|
37
|
-
process_time = Time.now - start_time
|
38
|
-
message = "table: #{table.name} - result: #{result} #{sprintf('%10.1f', process_time)}sec"
|
39
|
-
log message
|
40
|
-
process_times << message
|
41
|
-
end
|
42
|
-
log '------------------------------------'
|
43
|
-
log "db_name: #{db_name}"
|
44
|
-
process_times.each { |process_time| log process_time }
|
45
|
-
end
|
46
|
-
|
47
|
-
def log(message)
|
48
|
-
puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
1
|
+
require 'samidare/version'
|
2
|
+
require 'samidare/embulk_utility'
|
3
|
+
|
4
|
+
module Samidare
|
5
|
+
class EmbulkClient
|
6
|
+
def generate_config(config)
|
7
|
+
Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
|
8
|
+
end
|
9
|
+
|
10
|
+
def run(config)
|
11
|
+
db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
|
12
|
+
table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
|
13
|
+
db_infos.keys.each do |db_name|
|
14
|
+
embulk_run(db_name, db_infos[db_name]['bq_dataset'], table_infos[db_name], config)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def embulk_run(db_name, bq_dataset, tables, config)
|
20
|
+
process_times = []
|
21
|
+
big_query = Samidare::BigQueryUtility.new(config)
|
22
|
+
tables.each do |table|
|
23
|
+
start_time = Time.now
|
24
|
+
log "table: #{table.name} - start"
|
25
|
+
|
26
|
+
begin
|
27
|
+
big_query.delete_table(bq_dataset, table.name)
|
28
|
+
log "table: #{table.name} - deleted"
|
29
|
+
rescue
|
30
|
+
log "table: #{table.name} - does not exist"
|
31
|
+
end
|
32
|
+
|
33
|
+
cmd = "embulk run #{ENV['EMBULK_CONFIG_DIR']}/#{db_name}/#{table.name}.yml"
|
34
|
+
log "cmd: #{cmd}"
|
35
|
+
result = system(cmd) ? 'success' : 'error'
|
36
|
+
|
37
|
+
process_time = Time.now - start_time
|
38
|
+
message = "table: #{table.name} - result: #{result} #{sprintf('%10.1f', process_time)}sec"
|
39
|
+
log message
|
40
|
+
process_times << message
|
41
|
+
end
|
42
|
+
log '------------------------------------'
|
43
|
+
log "db_name: #{db_name}"
|
44
|
+
process_times.each { |process_time| log process_time }
|
45
|
+
end
|
46
|
+
|
47
|
+
def log(message)
|
48
|
+
puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: samidare
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryoji Kobori
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -215,4 +215,3 @@ test_files:
|
|
215
215
|
- spec/samidare/embulk_utility_spec.rb
|
216
216
|
- spec/samidare_spec.rb
|
217
217
|
- spec/spec_helper.rb
|
218
|
-
has_rdoc:
|