relational_exporter 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/relational_exporter.rb +26 -114
- data/lib/relational_exporter/csv_builder.rb +71 -0
- data/lib/relational_exporter/record_worker.rb +130 -0
- data/lib/relational_exporter/version.rb +1 -1
- data/relational_exporter.gemspec +1 -0
- metadata +32 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9fef27ab7b4fa6f13f35191995f9c8590ac6bd59
|
4
|
+
data.tar.gz: e6bfdaae73b812140fe7ded8747d03384d8a9f4d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13c2c195e7c783b734b93d51e6aba5edac1bebf48fad6f3b149e55a28525a2f276eae232ac4aa2d3676702ba04b869d3d70d3fcb09f8a20134cda1204e744419
|
7
|
+
data.tar.gz: 0413a341cfed6cad14169aadfb296a2cc841cf7aadc7909b63fc992efb67bdc48b50d9d607bb676bf40f829ddd8e4e0dd470ff3b9b9c842bf549ac3011dd4fe6
|
data/lib/relational_exporter.rb
CHANGED
@@ -3,6 +3,9 @@ require 'csv'
|
|
3
3
|
require 'hashie'
|
4
4
|
require 'relational_exporter/version'
|
5
5
|
require 'relational_exporter/active_record_extension'
|
6
|
+
require 'relational_exporter/csv_builder'
|
7
|
+
require 'relational_exporter/record_worker'
|
8
|
+
require 'celluloid'
|
6
9
|
|
7
10
|
module RelationalExporter
|
8
11
|
class Runner
|
@@ -26,6 +29,7 @@ module RelationalExporter
|
|
26
29
|
|
27
30
|
def export(output_config, &block)
|
28
31
|
ActiveRecord::Base.logger = @logger
|
32
|
+
Celluloid.logger = @logger
|
29
33
|
|
30
34
|
output_config = Hashie::Mash.new output_config
|
31
35
|
|
@@ -33,128 +37,36 @@ module RelationalExporter
|
|
33
37
|
|
34
38
|
main_klass.set_scope_from_hash output_config.output.scope.as_json
|
35
39
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
return unless single
|
55
|
-
end
|
56
|
-
|
57
|
-
row = []
|
58
|
-
|
59
|
-
# Add main record headers
|
60
|
-
serialized_attributes(single).each do |field, value|
|
61
|
-
header_row << csv_header_prefix_for_key(main_klass, field) if csv.header_row?
|
62
|
-
row << value
|
63
|
-
end
|
64
|
-
|
65
|
-
output_config.output.associations.each do |association_accessor, association_options|
|
66
|
-
association_accessor = association_accessor.to_s.to_sym
|
67
|
-
association_klass = association_accessor.to_s.classify.constantize
|
68
|
-
scope = symbolize_options association_options.scope
|
69
|
-
|
70
|
-
associated = single.send association_accessor
|
71
|
-
# TODO - this might suck for single associations (has_one) because they don't return an ar::associations::collectionproxy
|
72
|
-
associated = associated.find_all_by_scope(scope) unless scope.blank? || !associated.respond_to?(:find_all_by_scope)
|
73
|
-
|
74
|
-
if associated.is_a? Hash
|
75
|
-
associated = [ associated ]
|
76
|
-
elsif associated.blank?
|
77
|
-
associated = []
|
78
|
-
end
|
79
|
-
|
80
|
-
foreign_key = main_klass.reflections[association_accessor].foreign_key rescue nil
|
81
|
-
|
82
|
-
fields = serialized_attributes(association_klass).keys
|
83
|
-
|
84
|
-
fields.reject! {|v| v == foreign_key } if foreign_key
|
85
|
-
|
86
|
-
if csv.header_row?
|
87
|
-
case main_klass.reflections[association_accessor].macro
|
88
|
-
when :has_many
|
89
|
-
max_associated = association_klass.find_all_by_scope(scope)
|
90
|
-
.joins(main_klass.table_name.to_sym)
|
91
|
-
.order('count_all desc')
|
92
|
-
.group(foreign_key)
|
93
|
-
.limit(1).count.flatten[1]
|
94
|
-
when :has_one
|
95
|
-
max_associated = 1
|
96
|
-
end
|
97
|
-
|
98
|
-
max_associated = 0 if max_associated.nil?
|
99
|
-
|
100
|
-
max_associations[association_accessor] = max_associated
|
101
|
-
|
102
|
-
max_associated.times do |i|
|
103
|
-
fields.each do |field|
|
104
|
-
header_row << csv_header_prefix_for_key(association_klass, field, i+1)
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
get_row_arr(associated, fields, max_associations[association_accessor]) {|field| row << field}
|
110
|
-
end
|
111
|
-
|
112
|
-
csv << header_row if csv.header_row?
|
113
|
-
if row.count != header_row.count
|
114
|
-
@logger.error "Encountered invalid row, skipping."
|
115
|
-
end
|
116
|
-
csv << row
|
40
|
+
csv_builder = RelationalExporter::CsvBuilder.new output_config.file_path
|
41
|
+
Celluloid::Actor[:csv_builder] = csv_builder
|
42
|
+
result = csv_builder.future.start
|
43
|
+
pool = RelationalExporter::RecordWorker.pool size: 8
|
44
|
+
get_headers = true
|
45
|
+
|
46
|
+
record_sequence = -1
|
47
|
+
main_klass.find_all_by_scope(output_config.output.scope.as_json).find_in_batches(batch_size: 100) do |records|
|
48
|
+
records.each do |record|
|
49
|
+
record_sequence += 1
|
50
|
+
|
51
|
+
args = [record_sequence, record, output_config.output.associations, get_headers]
|
52
|
+
if get_headers
|
53
|
+
pool.get_csv_row(*args)
|
54
|
+
get_headers = false
|
55
|
+
else
|
56
|
+
pool.async.get_csv_row(*args)
|
117
57
|
end
|
118
58
|
end
|
119
59
|
end
|
120
|
-
end
|
121
|
-
|
122
|
-
private
|
123
|
-
|
124
|
-
def csv_header_prefix_for_key(klass, key, index=nil)
|
125
|
-
if klass.respond_to?(:active_model_serializer) && !klass.active_model_serializer.nil? && klass.active_model_serializer.respond_to?(:csv_header_prefix_for_key)
|
126
|
-
header_prefix = klass.active_model_serializer.csv_header_prefix_for_key key.to_sym
|
127
|
-
else
|
128
|
-
header_prefix = klass.to_s
|
129
|
-
end
|
130
60
|
|
131
|
-
|
132
|
-
end
|
133
|
-
|
134
|
-
def serialized_attributes(object)
|
135
|
-
return {} if object.nil?
|
136
|
-
|
137
|
-
klass, model = object.is_a?(Class) ? [object, object.first] : [object.class, object]
|
61
|
+
csv_builder.end_index = record_sequence
|
138
62
|
|
139
|
-
|
140
|
-
|
141
|
-
if model.respond_to?(:active_model_serializer) && !model.active_model_serializer.nil?
|
142
|
-
serialized = model.active_model_serializer.new(model).as_json(root: false)
|
143
|
-
end
|
63
|
+
@logger.info "CSV export complete" if result.value === true
|
144
64
|
|
145
|
-
|
146
|
-
|
65
|
+
pool.terminate
|
66
|
+
csv_builder.terminate
|
147
67
|
end
|
148
68
|
|
149
|
-
|
150
|
-
max_count.times do |i|
|
151
|
-
record = records[i].nil? ? {} : serialized_attributes(records[i])
|
152
|
-
fields.each do |field|
|
153
|
-
val = record[field]
|
154
|
-
yield val
|
155
|
-
end
|
156
|
-
end
|
157
|
-
end
|
69
|
+
private
|
158
70
|
|
159
71
|
def symbolize_options(options)
|
160
72
|
options = options.as_json
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
|
3
|
+
module RelationalExporter
|
4
|
+
class CsvBuilder
|
5
|
+
include Celluloid
|
6
|
+
include Celluloid::Logger
|
7
|
+
|
8
|
+
attr_accessor :queue, :end_index
|
9
|
+
|
10
|
+
trap_exit :actor_died
|
11
|
+
def actor_died(actor, reason)
|
12
|
+
warn "Oh no! #{actor.inspect} has died because of a #{reason.class}" unless reason.nil?
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(file_path=nil)
|
16
|
+
@header_row = []
|
17
|
+
@index = 0
|
18
|
+
@end_index = nil
|
19
|
+
@queue = {}
|
20
|
+
@file_path = file_path
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
csv_args = @file_path.blank? ? STDOUT : @file_path
|
25
|
+
|
26
|
+
csv_options = {headers: true}
|
27
|
+
if @file_path.blank?
|
28
|
+
csv_method = :instance
|
29
|
+
csv_args = [STDOUT, csv_options]
|
30
|
+
else
|
31
|
+
csv_method = :open
|
32
|
+
csv_args = [@file_path, 'wb', csv_options]
|
33
|
+
end
|
34
|
+
|
35
|
+
::CSV.send(csv_method, *csv_args) do |csv|
|
36
|
+
until @index == @end_index
|
37
|
+
if row = @queue.delete(@index)
|
38
|
+
write_row(row, csv)
|
39
|
+
@index += 1
|
40
|
+
else
|
41
|
+
sleep 1
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
true
|
47
|
+
end
|
48
|
+
|
49
|
+
def remaining
|
50
|
+
@end_index - @index if @end_index
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def write_row(row, csv)
|
56
|
+
headers, values = row.is_a?(Celluloid::Future) ? row.value : row
|
57
|
+
if csv.header_row?
|
58
|
+
@header_row = headers
|
59
|
+
info "Writing headers to file (#{headers.count})"
|
60
|
+
csv << @header_row
|
61
|
+
end
|
62
|
+
if values.count == @header_row.count
|
63
|
+
info "Writing row to file (#{@index})"
|
64
|
+
csv << values
|
65
|
+
else
|
66
|
+
# @logger.error "Encountered invalid row, skipping."
|
67
|
+
error "Bad row! #{values.count} vs #{@header_row.count}", @header_row.join(','), values.join(',')
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,130 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
|
3
|
+
module RelationalExporter
|
4
|
+
class RecordWorker
|
5
|
+
include Celluloid
|
6
|
+
include Celluloid::Logger
|
7
|
+
|
8
|
+
trap_exit :actor_died
|
9
|
+
def actor_died(actor, reason)
|
10
|
+
puts "Oh no! #{actor.inspect} has died because of a #{reason.class}" unless reason.nil?
|
11
|
+
end
|
12
|
+
|
13
|
+
@@MAX_ASSOCIATED = {}
|
14
|
+
|
15
|
+
def get_csv_row(record_sequence, record, associations, with_headers=false)
|
16
|
+
@record = record
|
17
|
+
@associations = associations
|
18
|
+
|
19
|
+
get_rows with_headers
|
20
|
+
|
21
|
+
Celluloid::Actor[:csv_builder].queue[record_sequence] = [@header_row, @value_row]
|
22
|
+
end
|
23
|
+
|
24
|
+
def get_rows(get_headers=false)
|
25
|
+
@header_row = []
|
26
|
+
@value_row = []
|
27
|
+
main_klass = @record.class
|
28
|
+
|
29
|
+
RecordWorker.serialized_attributes_for_object_or_class(@record).each do |field, value|
|
30
|
+
@header_row << RecordWorker.csv_header_prefix_for_key(main_klass, field) if get_headers
|
31
|
+
@value_row << value
|
32
|
+
end
|
33
|
+
|
34
|
+
@associations.each do |association_accessor, association_options|
|
35
|
+
association_accessor = association_accessor.to_s.to_sym
|
36
|
+
association_klass = association_accessor.to_s.classify.constantize
|
37
|
+
scope = RecordWorker.symbolize_options association_options.scope
|
38
|
+
|
39
|
+
associated = @record.send association_accessor
|
40
|
+
# TODO - this might suck for single associations (has_one) because they don't return an ar::associations::collectionproxy
|
41
|
+
associated = associated.find_all_by_scope(scope) unless scope.blank? || !associated.respond_to?(:find_all_by_scope)
|
42
|
+
|
43
|
+
if associated.is_a? Hash
|
44
|
+
associated = [ associated ]
|
45
|
+
elsif associated.blank?
|
46
|
+
associated = []
|
47
|
+
end
|
48
|
+
|
49
|
+
foreign_key = main_klass.reflections[association_accessor].foreign_key rescue nil
|
50
|
+
|
51
|
+
fields = RecordWorker.serialized_attributes_for_object_or_class(association_klass).keys
|
52
|
+
|
53
|
+
fields.reject! {|v| v == foreign_key } if foreign_key
|
54
|
+
|
55
|
+
if get_headers
|
56
|
+
@@MAX_ASSOCIATED[association_accessor] ||= begin
|
57
|
+
case main_klass.reflections[association_accessor].macro
|
58
|
+
when :has_many
|
59
|
+
max_associated = association_klass.find_all_by_scope(scope)
|
60
|
+
.joins(main_klass.table_name.to_sym)
|
61
|
+
.order('count_all desc')
|
62
|
+
.group(foreign_key)
|
63
|
+
.limit(1).count.flatten[1]
|
64
|
+
when :has_one
|
65
|
+
max_associated = 1
|
66
|
+
end
|
67
|
+
|
68
|
+
max_associated = 0 if max_associated.nil?
|
69
|
+
max_associated
|
70
|
+
end
|
71
|
+
|
72
|
+
@@MAX_ASSOCIATED[association_accessor].times do |i|
|
73
|
+
fields.each do |field|
|
74
|
+
@header_row << RecordWorker.csv_header_prefix_for_key(association_klass, field, i+1)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
RecordWorker.get_maxed_row_arr(associated, fields, @@MAX_ASSOCIATED[association_accessor]) do |field|
|
80
|
+
@value_row << field
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.get_maxed_row_arr(records, fields, max_count=0, &block)
|
86
|
+
return if max_count.nil?
|
87
|
+
max_count.times do |i|
|
88
|
+
record = records[i].nil? ? {} : RecordWorker.serialized_attributes_for_object_or_class(records[i])
|
89
|
+
fields.each do |field|
|
90
|
+
val = record[field]
|
91
|
+
yield val
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.csv_header_prefix_for_key(klass, key, index=nil)
|
97
|
+
if klass.respond_to?(:active_model_serializer) && !klass.active_model_serializer.nil? && klass.active_model_serializer.respond_to?(:csv_header_prefix_for_key)
|
98
|
+
header_prefix = klass.active_model_serializer.csv_header_prefix_for_key key.to_sym
|
99
|
+
else
|
100
|
+
header_prefix = klass.to_s
|
101
|
+
end
|
102
|
+
|
103
|
+
header_prefix + index.to_s + key.to_s.classify
|
104
|
+
end
|
105
|
+
|
106
|
+
def self.serialized_attributes_for_object_or_class(object)
|
107
|
+
return {} if object.nil?
|
108
|
+
|
109
|
+
klass, model = object.is_a?(Class) ? [object, object.first] : [object.class, object]
|
110
|
+
|
111
|
+
return {} if model.nil?
|
112
|
+
|
113
|
+
if model.respond_to?(:active_model_serializer) && !model.active_model_serializer.nil?
|
114
|
+
serialized = model.active_model_serializer.new(model).as_json(root: false)
|
115
|
+
end
|
116
|
+
|
117
|
+
serialized = model.attributes if serialized.nil?
|
118
|
+
serialized
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.symbolize_options(options)
|
122
|
+
options = options.as_json
|
123
|
+
if options.is_a? Hash
|
124
|
+
options.deep_symbolize_keys!
|
125
|
+
elsif options.is_a? Array
|
126
|
+
options.map { |val| RecordWorker.symbolize_options val }
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/relational_exporter.gemspec
CHANGED
metadata
CHANGED
@@ -1,83 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: relational_exporter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Hammond
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-02-
|
11
|
+
date: 2014-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: hashie
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: celluloid
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
39
53
|
- !ruby/object:Gem::Version
|
40
54
|
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bundler
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
45
|
-
- - ~>
|
59
|
+
- - "~>"
|
46
60
|
- !ruby/object:Gem::Version
|
47
61
|
version: '1.3'
|
48
62
|
type: :development
|
49
63
|
prerelease: false
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
|
-
- - ~>
|
66
|
+
- - "~>"
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '1.3'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: rake
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
|
-
- -
|
73
|
+
- - ">="
|
60
74
|
- !ruby/object:Gem::Version
|
61
75
|
version: '0'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
|
-
- -
|
80
|
+
- - ">="
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: byebug
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- -
|
87
|
+
- - ">="
|
74
88
|
- !ruby/object:Gem::Version
|
75
89
|
version: '0'
|
76
90
|
type: :development
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- -
|
94
|
+
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
83
97
|
description: Export relational databases as flat files
|
@@ -87,13 +101,15 @@ executables: []
|
|
87
101
|
extensions: []
|
88
102
|
extra_rdoc_files: []
|
89
103
|
files:
|
90
|
-
- .gitignore
|
104
|
+
- ".gitignore"
|
91
105
|
- Gemfile
|
92
106
|
- LICENSE.txt
|
93
107
|
- README.md
|
94
108
|
- Rakefile
|
95
109
|
- lib/relational_exporter.rb
|
96
110
|
- lib/relational_exporter/active_record_extension.rb
|
111
|
+
- lib/relational_exporter/csv_builder.rb
|
112
|
+
- lib/relational_exporter/record_worker.rb
|
97
113
|
- lib/relational_exporter/version.rb
|
98
114
|
- relational_exporter.gemspec
|
99
115
|
homepage: http://github.com/andrhamm
|
@@ -106,19 +122,18 @@ require_paths:
|
|
106
122
|
- lib
|
107
123
|
required_ruby_version: !ruby/object:Gem::Requirement
|
108
124
|
requirements:
|
109
|
-
- -
|
125
|
+
- - ">="
|
110
126
|
- !ruby/object:Gem::Version
|
111
127
|
version: '0'
|
112
128
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
129
|
requirements:
|
114
|
-
- -
|
130
|
+
- - ">="
|
115
131
|
- !ruby/object:Gem::Version
|
116
132
|
version: '0'
|
117
133
|
requirements: []
|
118
134
|
rubyforge_project:
|
119
|
-
rubygems_version: 2.
|
135
|
+
rubygems_version: 2.2.0
|
120
136
|
signing_key:
|
121
137
|
specification_version: 4
|
122
138
|
summary: Export relational databases as flat files
|
123
139
|
test_files: []
|
124
|
-
has_rdoc:
|