rseed 1.0.7 → 1.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.rdoc +7 -0
- data/lib/rseed/helpers.rb +2 -1
- data/lib/rseed/processor.rb +47 -31
- data/lib/rseed/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MGU2YjBiM2ExYWZhZjhiMGVmMjNkNjdkODFjZjMwODliZjA3NWUwYw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YzRiNjZiOGNkMWI3YTg2NTM1NDFkYmU4M2Y3MmVkOWMwZjliMGQyMw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NmVhNjQ2MDNhYWY3ODliNDgyMzRiMDc3YmFiZWI4YWQxZDQ3MTcwMjg5MmIw
|
10
|
+
Y2Q4MWU2MTI2ZGE1MmE3YWU3NzBmMzY2MjM1N2Q0MTIwOTNkYjQ4ODI1MTAx
|
11
|
+
N2EzNTAwYmFmMTNmMWRmYzFkNGUzZDhhMDRlZDdhZWFmYWVmZjc=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZjcyNjYxZDJmNjE5ZDQyNWZjZGRiMTM2ZmVkNzRmM2U4MDE3ZTA3MTM3NGI3
|
14
|
+
MjdhNTIzNDA2YzI2NWEzZjNkOTUyMDMzMzZkNWQxNDc4MjRkMzE1ZjYxMTlk
|
15
|
+
NjRiNWY5OTBjNjFkY2JhNDFlNDc5ZWU4MGNkMWM0ZDBmMWIyYzE=
|
data/README.rdoc
CHANGED
@@ -124,6 +124,13 @@ These rake tasks allow you to run seeds manually:
|
|
124
124
|
|
125
125
|
In this case the file in db/rseed/users.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case options["give_admin_access"] will evaluate to "true".
|
126
126
|
|
127
|
+
== Processor Options
|
128
|
+
|
129
|
+
* :within_transaction
|
130
|
+
|
131
|
+
Setting this to true will wrap the entire deserialize in an transaction, avoiding calling a commit after each line.
|
132
|
+
For large data sets, this should speed up insertion.
|
133
|
+
|
127
134
|
== Seeding
|
128
135
|
|
129
136
|
If you want to seed your Rails application using Rseed. The best method is to add lines like the following to db/seeds.rb
|
data/lib/rseed/helpers.rb
CHANGED
@@ -39,8 +39,9 @@ module Rseed
|
|
39
39
|
progress_bar.format "#{"Complete".green} %t <%B> %C (%a)"
|
40
40
|
progress_bar.finish if progress_bar.total
|
41
41
|
when :error
|
42
|
-
processor.logger.error "Error processing
|
42
|
+
processor.logger.error "Error during processing"
|
43
43
|
processor.logger.error result[:message].to_s.red
|
44
|
+
processor.logger.error meta.to_s.cyan if meta
|
44
45
|
processor.logger.error result[:error]
|
45
46
|
processor.logger.error result[:backtrace].join('\n') if result[:backtrace]
|
46
47
|
end
|
data/lib/rseed/processor.rb
CHANGED
@@ -22,6 +22,7 @@ module Rseed
|
|
22
22
|
end
|
23
23
|
converter.options = HashWithIndifferentAccess.new(converter_options)
|
24
24
|
end
|
25
|
+
@within_transaction = options[:within_transaction]
|
25
26
|
@adapter = adapter
|
26
27
|
@converter = converter
|
27
28
|
end
|
@@ -46,42 +47,45 @@ module Rseed
|
|
46
47
|
if @converter.before_deserialize
|
47
48
|
yield :processing
|
48
49
|
start_time = Time.now
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
50
|
+
wrap_inserts do
|
51
|
+
adapter.process do |values, meta|
|
52
|
+
result = {values: values}
|
53
|
+
meta ||= {}
|
54
|
+
begin
|
55
|
+
if @converter.deserialize_raw(values)
|
56
|
+
result[:success] = true
|
57
|
+
else
|
58
|
+
result[:success] = false
|
59
|
+
result[:message] = "Failed to convert"
|
60
|
+
result[:error] = @converter.error
|
61
|
+
end
|
62
|
+
rescue Exception => e
|
56
63
|
result[:success] = false
|
57
|
-
result[:message] = "
|
58
|
-
result[:error] =
|
64
|
+
result[:message] = "Exception during deserialize"
|
65
|
+
result[:error] = e.message
|
66
|
+
result[:backtrace] = e.backtrace
|
59
67
|
end
|
60
|
-
rescue Exception => e
|
61
|
-
result[:success] = false
|
62
|
-
result[:message] = "Exception during deserialize"
|
63
|
-
result[:error] = e.message
|
64
|
-
result[:backtrace] = e.backtrace
|
65
|
-
end
|
66
68
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
69
|
+
total_records = meta[:total_records] unless meta[:total_records].nil?
|
70
|
+
record_count = meta[:record_count] unless meta[:record_count].nil?
|
71
|
+
# Calculate the ETA
|
72
|
+
if record_count and total_records
|
73
|
+
remaining = total_records - record_count
|
74
|
+
tpr = (Time.now - start_time)/record_count
|
75
|
+
meta[:eta] = remaining * tpr
|
76
|
+
end
|
75
77
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
78
|
+
# Log any errors
|
79
|
+
unless result[:success]
|
80
|
+
logger.error result[:message].to_s.red
|
81
|
+
logger.error result[:error].to_s.red
|
82
|
+
logger.error meta.to_s.cyan
|
83
|
+
logger.error result[:backtrace].to_s unless result[:backtrace].to_s.blank?
|
84
|
+
end
|
85
|
+
yield :processing, result, meta
|
81
86
|
end
|
82
|
-
|
87
|
+
@converter.after_deserialize
|
83
88
|
end
|
84
|
-
@converter.after_deserialize
|
85
89
|
else
|
86
90
|
yield :error, {success: false, message: 'Before deserialize failed', error: @converter.error}
|
87
91
|
end
|
@@ -91,7 +95,19 @@ module Rseed
|
|
91
95
|
rescue Exception => e
|
92
96
|
yield :error, {success: false, message: 'Exception during Processing', error: e.message, backtrace: e.backtrace}
|
93
97
|
end
|
94
|
-
yield :complete, {success: true
|
98
|
+
yield :complete, {success: true}, {total_records: total_records, record_count: record_count}
|
99
|
+
end
|
100
|
+
|
101
|
+
protected
|
102
|
+
|
103
|
+
def wrap_inserts &block
|
104
|
+
if @within_transaction
|
105
|
+
ActiveRecord::Base.transaction do
|
106
|
+
yield
|
107
|
+
end
|
108
|
+
else
|
109
|
+
yield
|
110
|
+
end
|
95
111
|
end
|
96
112
|
end
|
97
113
|
end
|
data/lib/rseed/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rseed
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Monagle
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-11-
|
11
|
+
date: 2013-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: colorize
|