db_facet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +147 -0
- data/Rakefile +10 -0
- data/_config.yml +1 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/db_facet.gemspec +29 -0
- data/lib/db_facet.rb +8 -0
- data/lib/db_facet/db_spider.rb +130 -0
- data/lib/db_facet/db_spider_node_set.rb +21 -0
- data/lib/db_facet/db_spider_reader_node.rb +69 -0
- data/lib/db_facet/db_spider_root_merger.rb +40 -0
- data/lib/db_facet/db_spider_weaver.rb +125 -0
- data/lib/db_facet/db_spider_writer_node.rb +69 -0
- data/lib/db_facet/version.rb +3 -0
- metadata +120 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8bf8ace9b9fd87f1fbe6c32f337cd64977e05d07
|
4
|
+
data.tar.gz: f98e7e12bf892f9623484fc4a030c928ad363913
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ec8b6cd661df12f93d6eec00658adc154bc6059bd0e4e7dd8328ac1d98c38ec237069ba54537d42fda383b39bb78c96df3127047d0cba690324004cfe69e7410
|
7
|
+
data.tar.gz: 37f21f47c59665da12bafb38abd46206f3ab292550671e6969cdba7b2c6597fff6f1ad61d58c21a20588f845189fed1cac3da83cbc846b3a74289e615e423973
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Tom Lobato
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
# db_facet
|
2
|
+
|
3
|
+
db_facet extracts and inserts subsets of a database content, like a full user account with all its photos, invoices and history.
|
4
|
+
|
5
|
+
DbSpider recursively fetch records from a database and generates a Hash structure representing the data entities and its relations.
|
6
|
+
|
7
|
+
The Hash structure can be read by DbSpiderWeaver to insert the data into the database again.
|
8
|
+
|
9
|
+
Common usages would be to export and import an account, or build a fresh account by cloning an existing one.
|
10
|
+
|
11
|
+
It\`s designed to ensure a blazing fast database write (DbSpiderWeaver, that relies on [activerecord-import](https://github.com/zdennis/activerecord-import)) and supports rails [globalize](https://github.com/globalize/globalize).
|
12
|
+
|
13
|
+
db_facet is written to work on RubyOnRails, but can be used in any system just by writing the activerecord models and its relations representing your database.
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
Add this line to your application's Gemfile:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
gem 'db_facet'
|
21
|
+
```
|
22
|
+
|
23
|
+
And then execute:
|
24
|
+
|
25
|
+
$ bundle
|
26
|
+
|
27
|
+
Or install it yourself as:
|
28
|
+
|
29
|
+
$ gem install db_facet
|
30
|
+
|
31
|
+
## Usage
|
32
|
+
|
33
|
+
Here is a sample class using db_facet.
|
34
|
+
It "clones" an user account with all its dependencies to a fresh new user.
|
35
|
+
|
36
|
+
```ruby
|
37
|
+
|
38
|
+
# Usage:
|
39
|
+
# new_attrs = {name: 'Demo account', email: 'demo@example.com'}
|
40
|
+
# new_fresh_user = CloneAccount.new(template_user.id, new_attrs).build
|
41
|
+
#
|
42
|
+
|
43
|
+
class CloneAccount
|
44
|
+
|
45
|
+
INCLUDE_TEMPLATE_MODELS = %w(
|
46
|
+
User
|
47
|
+
Album
|
48
|
+
Photo
|
49
|
+
Video
|
50
|
+
Invoice
|
51
|
+
)
|
52
|
+
|
53
|
+
def initialize template_user_id, new_attrs
|
54
|
+
@template_user_id = template_user_id
|
55
|
+
@new_attrs = new_attrs.deep_dup
|
56
|
+
end
|
57
|
+
|
58
|
+
def build
|
59
|
+
seed = fetch_seed @template_user_id
|
60
|
+
tmp_user = build_user
|
61
|
+
|
62
|
+
override_seed! seed, overrides(tmp_user)
|
63
|
+
new_user_id = save! seed
|
64
|
+
|
65
|
+
User.find new_user_id
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def template_overrides user
|
71
|
+
@new_attrs.merge!(
|
72
|
+
profile_theme: 34
|
73
|
+
)
|
74
|
+
|
75
|
+
# children overrides
|
76
|
+
@new_attrs.merge!(
|
77
|
+
lang_config: {locale: 'fr'},
|
78
|
+
invoices: lambda {|data| data[:cc_end] = nil }
|
79
|
+
)
|
80
|
+
|
81
|
+
@new_attrs
|
82
|
+
end
|
83
|
+
|
84
|
+
def build_user
|
85
|
+
user.new
|
86
|
+
end
|
87
|
+
|
88
|
+
# db_facet interface
|
89
|
+
|
90
|
+
def fetch_seed template_user_id
|
91
|
+
# You could cache the generated data structure if using
|
92
|
+
# it often and it is viable to clear when changed.
|
93
|
+
# Rails.cache.fetch "export-import-seed-#{template_user_id}" do
|
94
|
+
DbSpider.new(User.find(template_user_id), INCLUDE_MODELS).spide
|
95
|
+
#end
|
96
|
+
end
|
97
|
+
|
98
|
+
def override_seed! seed, overrides
|
99
|
+
DbSpiderRootMerger.new(seed).merge! overrides
|
100
|
+
end
|
101
|
+
|
102
|
+
def save! seed
|
103
|
+
DbSpiderWeaver.new(seed, timer: true).weave!
|
104
|
+
end
|
105
|
+
end
|
106
|
+
```
|
107
|
+
|
108
|
+
## Hash structure
|
109
|
+
|
110
|
+
```yml
|
111
|
+
{
|
112
|
+
class_name: 'User',
|
113
|
+
data: {name: 'Chuck Norris!'},
|
114
|
+
reflections: {
|
115
|
+
albuns: [
|
116
|
+
{
|
117
|
+
class_name: 'Albuns',
|
118
|
+
data: {name: 'Day off 2017/02'},
|
119
|
+
reflections: {
|
120
|
+
photos: ...
|
121
|
+
|
122
|
+
```
|
123
|
+
|
124
|
+
## Classes descriptions
|
125
|
+
|
126
|
+
- DbSpider - Crawls db and generates the Hash structure.
|
127
|
+
- DbSpiderReaderNode - Wrapper for an AR model record.
|
128
|
+
- DbSpiderNodeSet - Proxy class to instantiate and reuse DbSpiderReaderNode`s.
|
129
|
+
- DbSpiderWeaver - Reads the Hash structure generated by DbSpider and INSERTS`s into the database.
|
130
|
+
- DbSpiderWriterNode - Wrapper for a node generated by DbSpiderReaderNode.
|
131
|
+
- DbSpiderRootMerger - Apply a diff to the Hash structure. Accepts a simplified data structure as parameter.
|
132
|
+
|
133
|
+
## Development
|
134
|
+
|
135
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
136
|
+
|
137
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
138
|
+
|
139
|
+
## Contributing
|
140
|
+
|
141
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/tomlobato/db_facet.
|
142
|
+
|
143
|
+
|
144
|
+
## License
|
145
|
+
|
146
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
147
|
+
|
data/Rakefile
ADDED
data/_config.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
theme: jekyll-theme-cayman
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "db_facet"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/db_facet.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'db_facet/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "db_facet"
|
8
|
+
spec.version = DbFacet::VERSION
|
9
|
+
spec.authors = ["Tom Lobato"]
|
10
|
+
spec.email = ["tomlobato@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{db_facet extracts and inserts subsets of a database content, like a full user account with all its photos, invoices and history.}
|
13
|
+
spec.description = %q{db_facet extracts and inserts subsets of a database content, like a full user account with all its photos, invoices and history..}
|
14
|
+
spec.homepage = "https://tomlobato.github.io/db_facet/"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
25
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
spec.add_development_dependency "minitest", "~> 5.0"
|
27
|
+
|
28
|
+
spec.add_runtime_dependency "activerecord-import", "~> 0.19.1"
|
29
|
+
end
|
data/lib/db_facet.rb
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
|
2
|
+
# TODO: Support for :has_and_belongs_to_many and :through.
|
3
|
+
|
4
|
+
class DbSpider
|
5
|
+
|
6
|
+
DEFAULT_IGNORE_COLUMNS = [
|
7
|
+
%w(id created_at updated_at),
|
8
|
+
/password|passwd|pass|token|senha/i,
|
9
|
+
]
|
10
|
+
|
11
|
+
def initialize root_rec,
|
12
|
+
allow_models = nil, # nil allows all
|
13
|
+
keep_columns: {},
|
14
|
+
ignore_columns: DEFAULT_IGNORE_COLUMNS, # unflattened list of regex, strings or symbols
|
15
|
+
allow_root_class_as_child: false
|
16
|
+
|
17
|
+
@root_rec = root_rec
|
18
|
+
@allow_models = allow_models
|
19
|
+
@keep_columns = keep_columns
|
20
|
+
@ignore_columns = ignore_columns
|
21
|
+
|
22
|
+
@node_set = DbSpiderNodeSet.new
|
23
|
+
@cache = {data_column: {}}
|
24
|
+
@deny_models = []
|
25
|
+
|
26
|
+
unless allow_root_class_as_child
|
27
|
+
@deny_models << @root_rec.class.name
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def spide print: false
|
32
|
+
root_node = traverse @root_rec, nil
|
33
|
+
print_tree root_node if print
|
34
|
+
root_node.data_tree
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def traverse rec, parent_rec, parent_node = nil, excl_foreign_key = nil
|
40
|
+
data_columns = get_data_columns(rec.class) - [excl_foreign_key]
|
41
|
+
|
42
|
+
node = @node_set.find_or_create rec, data_columns
|
43
|
+
|
44
|
+
unless node.traversed?
|
45
|
+
node.reflections.each do |reflection|
|
46
|
+
|
47
|
+
if follow_reflection? reflection, parent_rec
|
48
|
+
|
49
|
+
excl_src, excl_dst = exclude_foreign_key reflection
|
50
|
+
node.excl_data_cols excl_src
|
51
|
+
|
52
|
+
node.reflection_records(reflection).each do |reflection_rec|
|
53
|
+
reflection_node = traverse reflection_rec, rec, node, excl_dst
|
54
|
+
node.add_reflection_node reflection_node, reflection
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
node.traversed!
|
60
|
+
end
|
61
|
+
|
62
|
+
node
|
63
|
+
end
|
64
|
+
|
65
|
+
def exclude_foreign_key ref
|
66
|
+
pk = ref.foreign_key
|
67
|
+
case ref.macro
|
68
|
+
when :belongs_to
|
69
|
+
[pk, nil]
|
70
|
+
when :has_many, :has_one
|
71
|
+
[nil, pk]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_data_columns model
|
76
|
+
@cache[:data_column].fetch! model.name do
|
77
|
+
cols = model.column_names
|
78
|
+
.select{|col|
|
79
|
+
allow_column? model, col
|
80
|
+
}
|
81
|
+
if model.translates?
|
82
|
+
cols += model.translated_attribute_names.map(&:to_s)
|
83
|
+
end
|
84
|
+
cols
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def allow_column? model, col
|
89
|
+
if @keep_columns[model.name].to_a.include? col
|
90
|
+
return true
|
91
|
+
end
|
92
|
+
|
93
|
+
@ignore_columns.to_a.flatten.each do |ignorer|
|
94
|
+
if ignorer.is_a? Regexp
|
95
|
+
if col =~ ignorer
|
96
|
+
return false
|
97
|
+
end
|
98
|
+
elsif col == ignorer.to_s
|
99
|
+
return false
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
true
|
104
|
+
end
|
105
|
+
|
106
|
+
def follow_reflection? ref, parent_rec
|
107
|
+
return false if !ref.macro.in? [:belongs_to, :has_many, :has_one]
|
108
|
+
return false if ref.options[:through] # Skip :through assossiations
|
109
|
+
return false if !ref.active_record.name == ref.class_name # Skip self joins
|
110
|
+
return false if !ref.class_name.in? @allow_models.to_a
|
111
|
+
return false if @deny_models.include? ref.class_name
|
112
|
+
return false if parent_rec and parent_rec.class == ref.klass # Deny model go back up to the parent class
|
113
|
+
true
|
114
|
+
end
|
115
|
+
|
116
|
+
def print_tree node, level = 0
|
117
|
+
def tab l; "\t" * l; end
|
118
|
+
puts "#{ tab level }#{node.rec.class} #{node.rec.id}"
|
119
|
+
last_class = nil
|
120
|
+
node.children.each do |child_node|
|
121
|
+
if last_class and child_node.rec.class != last_class
|
122
|
+
puts "#{ tab (level+1) }--"
|
123
|
+
end
|
124
|
+
last_class = child_node.rec.class
|
125
|
+
print_tree child_node, (level+1)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
@@ -0,0 +1,21 @@
|
|
1
|
+
class DbSpiderNodeSet
|
2
|
+
attr_reader :count, :uniq_count
|
3
|
+
|
4
|
+
def initialize
|
5
|
+
@set = {}
|
6
|
+
@count = 0
|
7
|
+
@uniq_count = 0
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_or_create *args
|
11
|
+
@count += 1
|
12
|
+
obj = args[0]
|
13
|
+
if node = @set[obj]
|
14
|
+
node
|
15
|
+
else
|
16
|
+
@uniq_count += 1
|
17
|
+
@set[obj] = DbSpiderReaderNode.new *args
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
class DbSpiderReaderNode
|
2
|
+
attr_reader :rec
|
3
|
+
|
4
|
+
def initialize rec, data_columns = nil
|
5
|
+
@rec = rec
|
6
|
+
@data_columns = data_columns || @rec.class.column_names
|
7
|
+
@ref_nodes = {}
|
8
|
+
@traversed = false
|
9
|
+
end
|
10
|
+
|
11
|
+
def excl_data_cols excl
|
12
|
+
@data_columns -= [excl]
|
13
|
+
end
|
14
|
+
|
15
|
+
def traversed?
|
16
|
+
@traversed
|
17
|
+
end
|
18
|
+
|
19
|
+
def traversed!
|
20
|
+
@traversed = true
|
21
|
+
end
|
22
|
+
|
23
|
+
def data_tree
|
24
|
+
{
|
25
|
+
data: data,
|
26
|
+
class_name: @rec.class.name,
|
27
|
+
original_id: @rec.id,
|
28
|
+
reflections: reflections_data
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
def reflections_data
|
33
|
+
rd = {}
|
34
|
+
@ref_nodes.each_pair do |ref_name, nodes|
|
35
|
+
rd[ref_name] = nodes.map &:data_tree
|
36
|
+
end
|
37
|
+
rd
|
38
|
+
end
|
39
|
+
|
40
|
+
def reflections
|
41
|
+
@rec.class.reflections.values
|
42
|
+
end
|
43
|
+
|
44
|
+
def reflection_records ref
|
45
|
+
recs = @rec.send ref.name
|
46
|
+
[recs].flatten.compact
|
47
|
+
end
|
48
|
+
|
49
|
+
def add_reflection_node node, ref
|
50
|
+
@ref_nodes[ref.name] ||= []
|
51
|
+
@ref_nodes[ref.name] << node
|
52
|
+
end
|
53
|
+
|
54
|
+
def eql? other # used by Hash to compare keys
|
55
|
+
@rec.eql? other
|
56
|
+
end
|
57
|
+
|
58
|
+
def data
|
59
|
+
@rec.attributes.slice *@data_columns
|
60
|
+
end
|
61
|
+
|
62
|
+
def children
|
63
|
+
reflections
|
64
|
+
.map{|ref| @ref_nodes[ref.name].to_a}
|
65
|
+
.flatten
|
66
|
+
.compact
|
67
|
+
.sort_by{|n| n.rec.class.name}
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
class DbSpiderRootMerger
|
3
|
+
|
4
|
+
def initialize root_node
|
5
|
+
@root_node = root_node
|
6
|
+
end
|
7
|
+
|
8
|
+
def merge! data
|
9
|
+
root_model = @root_node[:class_name].constantize
|
10
|
+
root_model.reflections.each do |ref_name, reflection|
|
11
|
+
next unless data[ref_name]
|
12
|
+
|
13
|
+
case data[ref_name]
|
14
|
+
when Hash, Array, ActiveRecord::Base
|
15
|
+
[data[ref_name]].flatten.each do |ref_data|
|
16
|
+
rec = if ref_data.is_a? ActiveRecord::Base
|
17
|
+
ref_data
|
18
|
+
else
|
19
|
+
reflection.klass.new ref_data
|
20
|
+
end
|
21
|
+
@root_node[:reflections][ref_name] ||= []
|
22
|
+
@root_node[:reflections][ref_name] << DbSpiderReaderNode.new(rec).data_tree
|
23
|
+
end
|
24
|
+
|
25
|
+
when Proc
|
26
|
+
@root_node[:reflections][ref_name].each do |ref_node|
|
27
|
+
data[ref_name].call ref_node[:data]
|
28
|
+
end
|
29
|
+
|
30
|
+
else
|
31
|
+
raise "Invalid value. data[ref_name] must be a Hash, Array of Hash`es or Lambda. Found #{data[ref_name].class} for ref_name #{ref_name}."
|
32
|
+
end
|
33
|
+
|
34
|
+
data.delete ref_name
|
35
|
+
end
|
36
|
+
|
37
|
+
@root_node[:data].merge! data.stringify_keys
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
|
2
|
+
##################
|
3
|
+
# DbSpiderWeaver #
|
4
|
+
##################
|
5
|
+
|
6
|
+
class DbSpiderWeaver
|
7
|
+
VALIDATE_INSERTS = false
|
8
|
+
IMPORT_OPTS = {timestamps: true, validate: false, recursive: false}
|
9
|
+
|
10
|
+
def initialize data_tree, timer: false
|
11
|
+
@root_node = DbSpiderWriterNode.new data_tree
|
12
|
+
@translation_buffer = []
|
13
|
+
@timer = if timer
|
14
|
+
ProcTimer.new "DbSpiderWeaver", own_logfile: false
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def weave!
|
19
|
+
@timer.try :start
|
20
|
+
ActiveRecord::Base.transaction do
|
21
|
+
traverse [@root_node]
|
22
|
+
insert_translations
|
23
|
+
end
|
24
|
+
@timer.try :finish
|
25
|
+
@root_node.id
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def traverse nodes, parent = nil, parent_reflection: nil
|
31
|
+
return if nodes.blank?
|
32
|
+
set_foreign_keys_on_children nodes, parent_reflection, parent
|
33
|
+
insert nodes
|
34
|
+
update_foreign_key_on_parent parent, parent_reflection, nodes
|
35
|
+
nodes.each do |node|
|
36
|
+
node.reflection_nodes.each do |ref, ref_nodes|
|
37
|
+
traverse ref_nodes, node, parent_reflection: ref
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def set_foreign_keys_on_children nodes, reflection, parent
|
43
|
+
return unless parent and
|
44
|
+
reflection and
|
45
|
+
reflection.macro.in? [:has_many, :has_one]
|
46
|
+
|
47
|
+
if parent.id.blank?
|
48
|
+
raise "no fk val for #{parent.model} #{parent.data.to_json}"
|
49
|
+
end
|
50
|
+
|
51
|
+
fk_data = {reflection.foreign_key.to_s => parent.id}
|
52
|
+
|
53
|
+
if reflection.options[:as] # polymorphic
|
54
|
+
fk_data[reflection.type] = nodes.first.model.name
|
55
|
+
end
|
56
|
+
|
57
|
+
nodes.each {|n| n.data.merge! fk_data}
|
58
|
+
|
59
|
+
nodes
|
60
|
+
end
|
61
|
+
|
62
|
+
def update_foreign_key_on_parent parent, reflection, nodes
|
63
|
+
return unless parent and
|
64
|
+
reflection and
|
65
|
+
reflection.macro == :belongs_to
|
66
|
+
child_node = nodes.first
|
67
|
+
atts = {}
|
68
|
+
atts[reflection.foreign_key] = child_node[:data][child_node[:class_name].constantize.primary_key]
|
69
|
+
if reflection.options[:polymorphic]
|
70
|
+
atts[reflection.foreign_type] = child_node[:class_name]
|
71
|
+
end
|
72
|
+
parent.update_columns atts
|
73
|
+
end
|
74
|
+
|
75
|
+
def insert nodes
|
76
|
+
raise 'blank nodes' if nodes.blank?
|
77
|
+
|
78
|
+
model = nodes.first.model
|
79
|
+
|
80
|
+
atts_list = nodes.map &:insert_data
|
81
|
+
|
82
|
+
validate model, atts_list if VALIDATE_INSERTS
|
83
|
+
|
84
|
+
result = model.import atts_list, IMPORT_OPTS
|
85
|
+
check_insert result, model, nodes.length
|
86
|
+
|
87
|
+
nodes.each_with_index {|node, idx| node.id = result.ids[idx].to_i}
|
88
|
+
@translation_buffer << nodes if model.translates?
|
89
|
+
end
|
90
|
+
|
91
|
+
def check_insert result, model, length
|
92
|
+
if result.failed_instances.any?
|
93
|
+
raise "Insert failed for model #{model}. failed_instances: #{ result.failed_instances.map{|int| int.errors.messages } }"
|
94
|
+
end
|
95
|
+
if result.num_inserts == 0
|
96
|
+
raise "Insert failed for model #{model}: num_inserts = #{result.num_inserts}"
|
97
|
+
end
|
98
|
+
if result.ids.length < length
|
99
|
+
raise "Insert failed for model #{model}: ids.length < nodes.length #{result.ids.length}/#{length}"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def validate model, atts_list
|
104
|
+
atts_list.each do |atts|
|
105
|
+
inst = model.new atts
|
106
|
+
unless inst.valid?
|
107
|
+
puts "INVALID OBJ: #{model.name} #{atts} #{inst.errors.messages}"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def insert_translations
|
113
|
+
n = @translation_buffer.flatten.group_by{|node| node.t_model}
|
114
|
+
n.each_pair do |t_model, nodes|
|
115
|
+
t_model.import nodes.map(&:t_data), IMPORT_OPTS
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
|
121
|
+
######################
|
122
|
+
# DbSpiderWriterNode #
|
123
|
+
######################
|
124
|
+
|
125
|
+
|
@@ -0,0 +1,69 @@
|
|
1
|
+
class DbSpiderWriterNode
|
2
|
+
|
3
|
+
def initialize reader_node
|
4
|
+
@r_node = reader_node
|
5
|
+
end
|
6
|
+
|
7
|
+
def id=(v)
|
8
|
+
data[pk] = v
|
9
|
+
end
|
10
|
+
|
11
|
+
def id
|
12
|
+
data[pk]
|
13
|
+
end
|
14
|
+
|
15
|
+
def model
|
16
|
+
@model ||= @r_node[:class_name].constantize
|
17
|
+
end
|
18
|
+
|
19
|
+
def pk
|
20
|
+
model.primary_key
|
21
|
+
end
|
22
|
+
|
23
|
+
def reflections
|
24
|
+
@r_node[:reflections]
|
25
|
+
end
|
26
|
+
|
27
|
+
def reflection_nodes
|
28
|
+
rn = {}
|
29
|
+
reflections.each_pair do |ref_name, reader_nodes|
|
30
|
+
rn[model.reflections[ref_name]] = reader_nodes.map{|reader_node|
|
31
|
+
DbSpiderWriterNode.new reader_node
|
32
|
+
}
|
33
|
+
end
|
34
|
+
rn
|
35
|
+
end
|
36
|
+
|
37
|
+
def data
|
38
|
+
@r_node[:data]
|
39
|
+
end
|
40
|
+
|
41
|
+
def insert_data
|
42
|
+
data.slice *model.column_names
|
43
|
+
end
|
44
|
+
|
45
|
+
def t_data
|
46
|
+
data.slice(*t_cols).merge(
|
47
|
+
locale: 'pt-BR',
|
48
|
+
t_fk => id
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
52
|
+
def t_fk
|
53
|
+
@t_fk ||= t_model.reflections[:globalized_model].foreign_key
|
54
|
+
end
|
55
|
+
|
56
|
+
def t_model
|
57
|
+
@t_model ||= model.translation_class
|
58
|
+
end
|
59
|
+
|
60
|
+
def t_cols
|
61
|
+
@t_cols ||= model.translated_attribute_names.map &:to_s
|
62
|
+
end
|
63
|
+
|
64
|
+
def update_columns atts
|
65
|
+
raise "Trying to update record without primary key: #{model} #{atts}" if id.blank?
|
66
|
+
model.where(id: id).update_columns atts
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
metadata
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: db_facet
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tom Lobato
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.14'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '5.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '5.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activerecord-import
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.19.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.19.1
|
69
|
+
description: db_facet extracts and inserts subsets of a database content, like a full
|
70
|
+
user account with all its photos, invoices and history..
|
71
|
+
email:
|
72
|
+
- tomlobato@gmail.com
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- ".gitignore"
|
78
|
+
- ".travis.yml"
|
79
|
+
- Gemfile
|
80
|
+
- LICENSE.txt
|
81
|
+
- README.md
|
82
|
+
- Rakefile
|
83
|
+
- _config.yml
|
84
|
+
- bin/console
|
85
|
+
- bin/setup
|
86
|
+
- db_facet.gemspec
|
87
|
+
- lib/db_facet.rb
|
88
|
+
- lib/db_facet/db_spider.rb
|
89
|
+
- lib/db_facet/db_spider_node_set.rb
|
90
|
+
- lib/db_facet/db_spider_reader_node.rb
|
91
|
+
- lib/db_facet/db_spider_root_merger.rb
|
92
|
+
- lib/db_facet/db_spider_weaver.rb
|
93
|
+
- lib/db_facet/db_spider_writer_node.rb
|
94
|
+
- lib/db_facet/version.rb
|
95
|
+
homepage: https://tomlobato.github.io/db_facet/
|
96
|
+
licenses:
|
97
|
+
- MIT
|
98
|
+
metadata: {}
|
99
|
+
post_install_message:
|
100
|
+
rdoc_options: []
|
101
|
+
require_paths:
|
102
|
+
- lib
|
103
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0'
|
108
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - ">="
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project:
|
115
|
+
rubygems_version: 2.5.1
|
116
|
+
signing_key:
|
117
|
+
specification_version: 4
|
118
|
+
summary: db_facet extracts and inserts subsets of a database content, like a full
|
119
|
+
user account with all its photos, invoices and history.
|
120
|
+
test_files: []
|