seiso-import_master 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +14 -0
- data/Rakefile +3 -1
- data/lib/seiso/import_master.rb +61 -75
- data/lib/seiso/import_master/errors/invalid_document_error.rb +11 -0
- data/lib/seiso/import_master/importers/base_importer.rb +26 -0
- data/lib/seiso/import_master/importers/node_importer.rb +79 -0
- data/lib/seiso/import_master/importers/service_importer.rb +135 -0
- data/lib/seiso/import_master/importers/service_instance_importer.rb +40 -0
- data/lib/seiso/import_master/importers/simple_importer.rb +28 -0
- data/lib/seiso/import_master/mappers/master_item_mapper.rb +189 -0
- data/lib/seiso/import_master/mappers/node_mapper.rb +33 -0
- data/lib/seiso/import_master/mappers/service_instance_mapper.rb +62 -0
- data/lib/seiso/import_master/mappers/service_mapper.rb +35 -0
- data/lib/seiso/import_master/util/rest_connector.rb +102 -0
- data/lib/seiso/import_master/util/uri_factory_v1.rb +61 -0
- data/lib/seiso/import_master/util/uri_factory_v2.rb +34 -0
- data/lib/seiso/import_master/validators/base_validator.rb +19 -0
- data/lib/seiso/import_master/validators/node_validator.rb +46 -0
- data/lib/seiso/import_master/validators/service_validator.rb +45 -0
- data/seiso-import_master.gemspec +4 -1
- data/test/test_master_item_mapper.rb +5 -30
- data/test/test_node_mapper.rb +40 -0
- metadata +36 -6
- data/lib/seiso/import_master/master_item_mapper.rb +0 -295
- data/refresh +0 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZTRjZDQ3YzYzOWUyOTdjYWY2MTc0ZGIxM2E0YTI4OGQ3ZTAzOWRjNw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NzNhOWRlNTNjMGQxZGI1ODhhMzExMzY0ZTdkOWQ4OGM4ZDdlM2RmNg==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NTY3N2EzOWY2NGY0OGI0N2VmYmMwNTcyMjIwZDI1NDk2Y2MwMDI4NmExZTA4
|
10
|
+
NWQ5NjJiYjMyZjFmMzk1Y2ZmNmMyMTU1MTk3ZjA0N2YyZGIyNTY1Yzg0ZWRl
|
11
|
+
Y2M3ZTZlODJmZmRmMzBlNjYyNjkxNjAwZGFhZWE4NWQ5MDM1ZTE=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NTg4MDMxMWMyMjM1NmEwMjUyZWZjZTQ2MDEwYTc0YjM5NjkzMTc4M2VlYWFl
|
14
|
+
ZWI5ZWYxNGZlYTU2ZTMxNjNiYWE3YjdhNTZmMGNjMGEyMDE2ZTBhMzU0NGYx
|
15
|
+
NjI1NjJiYjc4NWY0ZDFlYjkzN2ExMDVkNzhjYjA5MDg4MTlkMmU=
|
data/README.md
CHANGED
@@ -39,3 +39,17 @@ Or install it yourself as:
|
|
39
39
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
40
40
|
4. Push to the branch (`git push origin my-new-feature`)
|
41
41
|
5. Create a new pull request.
|
42
|
+
|
43
|
+
## Development
|
44
|
+
|
45
|
+
Rake tasks available via
|
46
|
+
|
47
|
+
$ rake tasks
|
48
|
+
|
49
|
+
In particular, run the unit tests using
|
50
|
+
|
51
|
+
$ rake test
|
52
|
+
|
53
|
+
Build and install the gem using Rake:
|
54
|
+
|
55
|
+
$ rake install
|
data/Rakefile
CHANGED
@@ -5,11 +5,13 @@ task :default => [ :test ]
|
|
5
5
|
|
6
6
|
task :tasks do
|
7
7
|
puts "Tasks:"
|
8
|
-
puts "
|
8
|
+
puts "install : Build and install seiso-import_master gem"
|
9
|
+
puts "test : Run unit tests"
|
9
10
|
end
|
10
11
|
|
11
12
|
Rake::TestTask.new do |t|
|
12
13
|
t.libs << 'test'
|
13
14
|
t.test_files = FileList[ 'test/test_*.rb' ]
|
14
15
|
t.verbose = true
|
16
|
+
t.warning = true
|
15
17
|
end
|
data/lib/seiso/import_master.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
require "json"
|
2
|
+
require "require_all"
|
2
3
|
require "seiso/connector"
|
3
4
|
require "yaml"
|
4
|
-
require_relative "import_master/master_item_mapper"
|
5
5
|
|
6
|
-
|
6
|
+
require_rel "import_master"
|
7
|
+
|
7
8
|
module Seiso
|
8
9
|
|
9
10
|
# Imports Seiso data master files into Seiso.
|
@@ -15,31 +16,47 @@ module Seiso
|
|
15
16
|
|
16
17
|
# Initializes the importer with a Seiso connector.
|
17
18
|
def initialize(seiso_settings)
|
18
|
-
@seiso = Seiso::Connector.new(seiso_settings)
|
19
|
-
@mapper = Seiso::ImportMaster::MasterItemMapper.new
|
20
19
|
@loaders = {
|
21
20
|
'json' => ->(file) { JSON.parse(IO.read(file)) },
|
22
21
|
'yaml' => ->(file) { YAML.load_file file }
|
23
22
|
}
|
24
|
-
end
|
25
23
|
|
26
|
-
|
27
|
-
@
|
28
|
-
|
24
|
+
# TODO Add other validators
|
25
|
+
@validators = {
|
26
|
+
'nodes' => Validators::NodeValidator.new,
|
27
|
+
'services' => Validators::ServiceValidator.new
|
28
|
+
}
|
29
29
|
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
basic_mapper = Mappers::MasterItemMapper.new
|
31
|
+
node_mapper = Mappers::NodeMapper.new
|
32
|
+
service_mapper = Mappers::ServiceMapper.new
|
33
|
+
service_instance_mapper = Mappers::ServiceInstanceMapper.new
|
34
|
+
|
35
|
+
seiso = Seiso::Connector.new seiso_settings
|
36
|
+
uri_factory_v1 = uri_factory_v1 seiso_settings
|
37
|
+
uri_factory_v2 = uri_factory_v2 seiso_settings
|
38
|
+
rest_connector_v1 = rest_connector(seiso_settings, "application/json")
|
39
|
+
rest_connector_v2 = rest_connector(seiso_settings, "application/hal+json")
|
33
40
|
|
34
|
-
|
35
|
-
@
|
41
|
+
@simple_importer = Importers::SimpleImporter.new(basic_mapper, seiso, uri_factory_v1, rest_connector_v1)
|
42
|
+
@importers = {
|
43
|
+
'nodes' => Importers::NodeImporter.new(node_mapper, uri_factory_v1, rest_connector_v1),
|
44
|
+
'services' => Importers::ServiceImporter.new(
|
45
|
+
service_mapper,
|
46
|
+
basic_mapper,
|
47
|
+
uri_factory_v1,
|
48
|
+
uri_factory_v2,
|
49
|
+
rest_connector_v1,
|
50
|
+
rest_connector_v2),
|
51
|
+
'service-instances' => Importers::ServiceInstanceImporter.new(service_instance_mapper, seiso, uri_factory_v1, rest_connector_v1)
|
52
|
+
}
|
36
53
|
end
|
37
|
-
|
54
|
+
|
38
55
|
# Imports a list of master files in order. Legal formats are 'json' (default) and 'yaml'.
|
39
56
|
def import_files(files, format = 'json')
|
40
57
|
loop do
|
41
58
|
file = files.pop
|
42
|
-
puts "Processing #{file}"
|
59
|
+
puts "Processing: #{file}"
|
43
60
|
import_file(file, format)
|
44
61
|
break if files.empty?
|
45
62
|
end
|
@@ -47,74 +64,43 @@ module Seiso
|
|
47
64
|
|
48
65
|
# Imports a data master file. Legal formats are 'json' (default) and 'yaml'.
|
49
66
|
def import_file(file, format = 'json')
|
50
|
-
loader = loaders[format]
|
67
|
+
loader = @loaders[format]
|
51
68
|
raise ArgumentError, "Illegal format: #{format}" if loader.nil?
|
52
69
|
doc = loader.call(file)
|
53
|
-
import_doc doc
|
54
|
-
end
|
55
|
-
|
56
|
-
# Imports a data master document.
|
57
|
-
def import_doc(doc)
|
58
70
|
type = doc['type']
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
if type == 'nodes'
|
63
|
-
si_key = doc['serviceInstance']
|
64
|
-
do_import_nodes(si_key, master_items)
|
65
|
-
elsif type == 'services'
|
66
|
-
do_import_services master_items
|
67
|
-
elsif type == 'service-instances'
|
68
|
-
do_import_service_instances master_items
|
69
|
-
else
|
70
|
-
do_import_items(type, master_items)
|
71
|
-
end
|
71
|
+
validator = @validators[type]
|
72
|
+
validator.validate(doc) unless validator.nil?
|
73
|
+
(@importers[type] || @simple_importer).import doc
|
72
74
|
end
|
73
|
-
|
75
|
+
|
74
76
|
private
|
75
|
-
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
nips = detach_children(nodes, 'node', 'name', 'ipAddresses')
|
84
|
-
nodes.each do |n|
|
85
|
-
n['serviceInstance'] = si_key
|
86
|
-
end
|
87
|
-
do_import_items('nodes', nodes)
|
88
|
-
do_import_items('node-ip-addresses', nips)
|
89
|
-
end
|
90
|
-
|
91
|
-
def do_import_services(services)
|
92
|
-
doc_links = detach_children(services, 'service', 'key', 'docLinks')
|
93
|
-
do_import_items('services', services)
|
94
|
-
# do_import_items('doc-links', docLinks)
|
95
|
-
doc_links.each { |dl| puts dl.to_json }
|
77
|
+
|
78
|
+
def uri_factory_v1(settings)
|
79
|
+
host = settings['host']
|
80
|
+
port = settings['port']
|
81
|
+
use_ssl = settings['use_ssl'] || false
|
82
|
+
scheme = use_ssl ? "https" : "http"
|
83
|
+
base_uri = "#{scheme}://#{host}:#{port}/v1"
|
84
|
+
Util::UriFactoryV1.new base_uri
|
96
85
|
end
|
97
86
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
87
|
+
def uri_factory_v2(settings)
|
88
|
+
host = settings['host']
|
89
|
+
port = settings['port']
|
90
|
+
use_ssl = settings['use_ssl'] || false
|
91
|
+
scheme = use_ssl ? "https" : "http"
|
92
|
+
base_uri = "#{scheme}://#{host}:#{port}/v2"
|
93
|
+
Util::UriFactoryV2.new base_uri
|
105
94
|
end
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
p.delete child_prop
|
116
|
-
end
|
117
|
-
all_children
|
95
|
+
|
96
|
+
def rest_connector(settings, media_type)
|
97
|
+
host = settings['host']
|
98
|
+
port = settings['port']
|
99
|
+
use_ssl = settings['use_ssl'] || false
|
100
|
+
ignore_cert = settings['ignore_cert'] || false
|
101
|
+
username = settings['username']
|
102
|
+
password = settings['password']
|
103
|
+
Util::RestConnector.new(host, port, use_ssl, ignore_cert, username, password, media_type)
|
118
104
|
end
|
119
105
|
end
|
120
106
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Seiso
|
2
|
+
class ImportMaster
|
3
|
+
module Importers
|
4
|
+
|
5
|
+
# Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
|
6
|
+
# Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
|
7
|
+
# License:: Apache 2.0
|
8
|
+
class BaseImporter
|
9
|
+
|
10
|
+
# Enriches children with a link to the parent, detaches them from the parent, and returns
|
11
|
+
# all detached children.
|
12
|
+
def detach_children(parents, parent_prop, parent_key, child_prop)
|
13
|
+
all_children = []
|
14
|
+
parents.each do |p|
|
15
|
+
children = p[child_prop]
|
16
|
+
next if children.nil?
|
17
|
+
children.each { |c| c[parent_prop] = p[parent_key] }
|
18
|
+
all_children.push(*children)
|
19
|
+
p.delete child_prop
|
20
|
+
end
|
21
|
+
all_children
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require_relative "base_importer"
|
2
|
+
|
3
|
+
module Seiso
|
4
|
+
class ImportMaster
|
5
|
+
module Importers
|
6
|
+
|
7
|
+
# Imports a node document into Seiso.
|
8
|
+
#
|
9
|
+
# Each node doc corresponds to a service instance. This importer compares the nodes in Seiso with the
|
10
|
+
# nodes in the node doc, and removes anything from Seiso that isn't in the node doc. Then it imports
|
11
|
+
# the nodes from the node doc into Seiso.
|
12
|
+
#
|
13
|
+
# Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
|
14
|
+
# Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
|
15
|
+
# License:: Apache 2.0
|
16
|
+
class NodeImporter < BaseImporter
|
17
|
+
|
18
|
+
def initialize(mapper, uri_factory_v1, rest_connector_v1)
|
19
|
+
@mapper = mapper
|
20
|
+
@uri_factory_v1 = uri_factory_v1
|
21
|
+
@rest_connector_v1 = rest_connector_v1
|
22
|
+
end
|
23
|
+
|
24
|
+
# Imports the node document.
|
25
|
+
def import(doc)
|
26
|
+
si_key = doc['serviceInstance']
|
27
|
+
nodes = doc['items']
|
28
|
+
puts "Importing node document for service instance #{si_key}"
|
29
|
+
|
30
|
+
delete_stale_nodes_from_seiso
|
31
|
+
|
32
|
+
# Node masters contain the service instance at the top-level, so enrich the nodes themselves
|
33
|
+
# with the service instance.
|
34
|
+
nodes.each do |n|
|
35
|
+
n['serviceInstance'] = si_key
|
36
|
+
end
|
37
|
+
|
38
|
+
nips = detach_children(nodes, 'node', 'name', 'ipAddresses')
|
39
|
+
|
40
|
+
seiso_nodes = nodes.map { |n| @mapper.seiso_node n }
|
41
|
+
nodes_uri = @uri_factory_v1.nodes_uri true
|
42
|
+
@rest_connector_v1.post(nodes_uri, seiso_nodes)
|
43
|
+
|
44
|
+
# import_nips_slowly_without_keepalive nips
|
45
|
+
import_nips_quickly_with_keepalive nips
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def delete_stale_nodes_from_seiso
|
51
|
+
# TODO
|
52
|
+
# puts "TODO Delete stale nodes from seiso"
|
53
|
+
end
|
54
|
+
|
55
|
+
def import_nips_slowly_without_keepalive(nips)
|
56
|
+
seiso_nips = nips.map { |nip| @mapper.seiso_nip nip }
|
57
|
+
seiso_nips.each do |nip|
|
58
|
+
node_name = nip['node']['name']
|
59
|
+
ip_address = nip['ipAddress']
|
60
|
+
nip_uri = @uri_factory_v1.node_ip_address_uri(node_name, ip_address)
|
61
|
+
@rest_connector_v1.put(nip_uri, nip)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def import_nips_quickly_with_keepalive(nips)
|
66
|
+
seiso_nip_pairs = nips.map do |nip|
|
67
|
+
node_name = nip['node']
|
68
|
+
ip_address = nip['ipAddress']
|
69
|
+
{
|
70
|
+
"uri" => @uri_factory_v1.node_ip_address_uri(node_name, ip_address),
|
71
|
+
"resource" => @mapper.seiso_nip(nip)
|
72
|
+
}
|
73
|
+
end
|
74
|
+
@rest_connector_v1.put_all seiso_nip_pairs
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require_relative "base_importer"
|
2
|
+
|
3
|
+
module Seiso
|
4
|
+
class ImportMaster
|
5
|
+
module Importers
|
6
|
+
|
7
|
+
# Imports a service document into Seiso.
|
8
|
+
#
|
9
|
+
# Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
|
10
|
+
# Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
|
11
|
+
# License:: Apache 2.0
|
12
|
+
class ServiceImporter < BaseImporter
|
13
|
+
|
14
|
+
def initialize(
|
15
|
+
service_mapper,
|
16
|
+
basic_mapper,
|
17
|
+
uri_factory_v1,
|
18
|
+
uri_factory_v2,
|
19
|
+
rest_connector_v1,
|
20
|
+
rest_connector_v2)
|
21
|
+
|
22
|
+
@service_mapper = service_mapper
|
23
|
+
@basic_mapper = basic_mapper
|
24
|
+
@uri_factory_v1 = uri_factory_v1
|
25
|
+
@uri_factory_v2 = uri_factory_v2
|
26
|
+
@rest_connector_v1 = rest_connector_v1
|
27
|
+
@rest_connector_v2 = rest_connector_v2
|
28
|
+
end
|
29
|
+
|
30
|
+
# Imports the service document
|
31
|
+
def import(doc)
|
32
|
+
services = doc['items']
|
33
|
+
puts "Importing service document"
|
34
|
+
|
35
|
+
doc_links = detach_children(services, 'service', 'key', 'docLinks')
|
36
|
+
|
37
|
+
seiso_services = services.map { |s| @service_mapper.seiso_service s }
|
38
|
+
services_uri = @uri_factory_v1.services_uri true
|
39
|
+
@rest_connector_v1.post(services_uri, seiso_services)
|
40
|
+
|
41
|
+
import_doc_links doc_links
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def import_doc_links(doc_links)
|
47
|
+
doc_links_by_service = group_doc_links_by_service doc_links
|
48
|
+
|
49
|
+
doc_links_by_service.each do |service_key, doc_links|
|
50
|
+
doc_links_uri = @uri_factory_v2.doc_links_uri service_key
|
51
|
+
|
52
|
+
# Get service's doc links so we can
|
53
|
+
# 1) Delete stale doc links
|
54
|
+
# 2) Avoid duplicate imports
|
55
|
+
seiso_doc_links_response = @rest_connector_v2.get(doc_links_uri)
|
56
|
+
seiso_doc_links = JSON.parse(seiso_doc_links_response.body)
|
57
|
+
|
58
|
+
to_delete = seiso_doc_links_to_delete(doc_links, seiso_doc_links)
|
59
|
+
|
60
|
+
# Delete stale doc links
|
61
|
+
to_delete.each do |seiso_doc_link|
|
62
|
+
href = seiso_doc_link['_links']['self']['href']
|
63
|
+
uri = URI.parse href
|
64
|
+
@rest_connector_v2.delete(uri)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Classify remaining links as post or put
|
68
|
+
to_post = []
|
69
|
+
to_put = []
|
70
|
+
doc_links.each do |sdm_doc_link|
|
71
|
+
sdm_title = sdm_doc_link['title']
|
72
|
+
post_it = true
|
73
|
+
seiso_doc_links.each do |seiso_doc_link|
|
74
|
+
seiso_title = seiso_doc_link['title']
|
75
|
+
if sdm_title == seiso_title
|
76
|
+
href = seiso_doc_link['_links']['self']['href']
|
77
|
+
uri = URI.parse href
|
78
|
+
new_seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
|
79
|
+
to_put << {
|
80
|
+
"uri" => uri,
|
81
|
+
"resource" => new_seiso_doc_link
|
82
|
+
}
|
83
|
+
post_it = false
|
84
|
+
break
|
85
|
+
end
|
86
|
+
end
|
87
|
+
to_post << sdm_doc_link if post_it
|
88
|
+
end
|
89
|
+
|
90
|
+
# Post new links
|
91
|
+
to_post.each do |sdm_doc_link|
|
92
|
+
seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
|
93
|
+
@rest_connector_v2.post(doc_links_uri, seiso_doc_link)
|
94
|
+
end
|
95
|
+
|
96
|
+
# Put existing links
|
97
|
+
@rest_connector_v2.put_all(to_put)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def group_doc_links_by_service(doc_links)
|
102
|
+
doc_links_by_service = {}
|
103
|
+
doc_links.each do |dl|
|
104
|
+
service_key = dl['service']
|
105
|
+
service_doc_links = doc_links_by_service[service_key]
|
106
|
+
if service_doc_links.nil?
|
107
|
+
service_doc_links = []
|
108
|
+
doc_links_by_service[service_key] = service_doc_links
|
109
|
+
end
|
110
|
+
service_doc_links << dl
|
111
|
+
end
|
112
|
+
doc_links_by_service
|
113
|
+
end
|
114
|
+
|
115
|
+
def seiso_doc_links_to_delete(sdm_doc_links, seiso_doc_links)
|
116
|
+
to_delete = []
|
117
|
+
seiso_doc_links.each do |seiso_doc_link|
|
118
|
+
seiso_title = seiso_doc_link['title']
|
119
|
+
delete_it = true
|
120
|
+
sdm_doc_links.each do |sdm_doc_link|
|
121
|
+
sdm_title = sdm_doc_link['title']
|
122
|
+
if sdm_title == seiso_title
|
123
|
+
delete_it = false
|
124
|
+
break
|
125
|
+
end
|
126
|
+
end
|
127
|
+
to_delete << seiso_doc_link if delete_it
|
128
|
+
end
|
129
|
+
to_delete
|
130
|
+
end
|
131
|
+
|
132
|
+
end # class ServiceImporter
|
133
|
+
end # module Importers
|
134
|
+
end # class ImportMaster
|
135
|
+
end # module Seiso
|