samidare 0.0.4 → 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cebd0e6fe168080a1c3e36dc9299564b0fbeee72
4
- data.tar.gz: b2bacde505cd91182e92592d3e8835dea8055fb8
3
+ metadata.gz: e9297d4a4da6cde678e001213c21b9ad3483b6eb
4
+ data.tar.gz: 1aa36de501c88dc84e0af7890bc365f028066ab5
5
5
  SHA512:
6
- metadata.gz: 9d2564ddc962b6ce9fcbcbd1a75cb6530eff6f77cca2d3a326ff337680a26be43f55595b10d125e7d88c6aeef8123d1ace470afa3883af98083483732a34c93b
7
- data.tar.gz: 1b4174505b57d13c5a484af60015e81d8913edfa700147e9068d8c77aab7e15bd5e2280d5dca6cd0e9e9a34cab20c18bb365e58b2abe7c592e3433a32d7e6cac
6
+ metadata.gz: 3c3fe570abcc9ea01af76be93257eb04e29b8cfbf98af94d567687012f044430ff8c0dbf607707899f26700f293eb73748b12cc4a73432fe6acaee6f627a3910
7
+ data.tar.gz: e5ab011f63fb3ffcd95e904239d24c45efd860a85007b3a84d2c701d0885b5b41aa5056cae49b02392f607aff64d97b4e838ee5134726fc2b1d5f66799562107
@@ -62,7 +62,6 @@ module Samidare
62
62
  schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_info.name}.json"
63
63
  path_prefix = "/var/tmp/embulk_#{db_name}_#{table_info.name}"
64
64
 
65
- #File.open('lib/samidare/embulk_config.erb') { |f| ERB.new(f.read).result(binding) }
66
65
  ERB.new(CONTENTS).result(binding)
67
66
  end
68
67
 
@@ -1,3 +1,3 @@
1
- module Samidare
2
- VERSION = "0.0.4"
3
- end
1
+ module Samidare
2
+ VERSION = "0.0.5"
3
+ end
data/lib/samidare.rb CHANGED
@@ -1,51 +1,51 @@
1
- require 'samidare/version'
2
- require 'samidare/embulk_utility'
3
-
4
- module Samidare
5
- class EmbulkClient
6
- def generate_config(config)
7
- Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
8
- end
9
-
10
- def run(config)
11
- db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
12
- table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
13
- db_infos.keys.each do |db_name|
14
- embulk_run(db_name, db_infos[db_name]['bq_dataset'], table_infos[db_name], config)
15
- end
16
- end
17
-
18
- private
19
- def embulk_run(db_name, bq_dataset, tables, config)
20
- process_times = []
21
- big_query = Samidare::BigQueryUtility.new(config)
22
- tables.each do |table|
23
- start_time = Time.now
24
- log "table: #{table.name} - start"
25
-
26
- begin
27
- big_query.delete_table(bq_dataset, table.name)
28
- log "table: #{table.name} - deleted"
29
- rescue
30
- log "table: #{table.name} - not exists"
31
- end
32
-
33
- cmd = "embulk run #{ENV['EMBULK_CONFIG_DIR']}/#{db_name}/#{table.name}.yml"
34
- log "cmd: #{cmd}"
35
- result = system(cmd) ? 'success' : 'error'
36
-
37
- process_time = Time.now - start_time
38
- message = "table: #{table.name} - result: #{result} #{sprintf('%10.1f', process_time)}sec"
39
- log message
40
- process_times << message
41
- end
42
- log '------------------------------------'
43
- log "db_name: #{db_name}"
44
- process_times.each { |process_time| log process_time }
45
- end
46
-
47
- def log(message)
48
- puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
49
- end
50
- end
51
- end
1
+ require 'samidare/version'
2
+ require 'samidare/embulk_utility'
3
+
4
+ module Samidare
5
+ class EmbulkClient
6
+ def generate_config(config)
7
+ Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
8
+ end
9
+
10
+ def run(config)
11
+ db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
12
+ table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
13
+ db_infos.keys.each do |db_name|
14
+ embulk_run(db_name, db_infos[db_name]['bq_dataset'], table_infos[db_name], config)
15
+ end
16
+ end
17
+
18
+ private
19
+ def embulk_run(db_name, bq_dataset, tables, config)
20
+ process_times = []
21
+ big_query = Samidare::BigQueryUtility.new(config)
22
+ tables.each do |table|
23
+ start_time = Time.now
24
+ log "table: #{table.name} - start"
25
+
26
+ begin
27
+ big_query.delete_table(bq_dataset, table.name)
28
+ log "table: #{table.name} - deleted"
29
+ rescue
30
+ log "table: #{table.name} - does not exist"
31
+ end
32
+
33
+ cmd = "embulk run #{ENV['EMBULK_CONFIG_DIR']}/#{db_name}/#{table.name}.yml"
34
+ log "cmd: #{cmd}"
35
+ result = system(cmd) ? 'success' : 'error'
36
+
37
+ process_time = Time.now - start_time
38
+ message = "table: #{table.name} - result: #{result} #{sprintf('%10.1f', process_time)}sec"
39
+ log message
40
+ process_times << message
41
+ end
42
+ log '------------------------------------'
43
+ log "db_name: #{db_name}"
44
+ process_times.each { |process_time| log process_time }
45
+ end
46
+
47
+ def log(message)
48
+ puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
49
+ end
50
+ end
51
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: samidare
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryoji Kobori
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-15 00:00:00.000000000 Z
11
+ date: 2015-07-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -215,4 +215,3 @@ test_files:
215
215
  - spec/samidare/embulk_utility_spec.rb
216
216
  - spec/samidare_spec.rb
217
217
  - spec/spec_helper.rb
218
- has_rdoc: