geoloader 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +13 -0
- data/.gitignore +17 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +2 -0
- data/README.md +212 -0
- data/Rakefile +10 -0
- data/bin/geoloader +4 -0
- data/config.yaml +18 -0
- data/geoloader.gemspec +37 -0
- data/iso19139.xsl +445 -0
- data/lib/geoloader.rb +48 -0
- data/lib/geoloader/assets/asset.rb +63 -0
- data/lib/geoloader/assets/description.rb +55 -0
- data/lib/geoloader/assets/geonetwork.rb +58 -0
- data/lib/geoloader/assets/geotiff.rb +41 -0
- data/lib/geoloader/assets/shapefile.rb +28 -0
- data/lib/geoloader/assets/solr.rb +23 -0
- data/lib/geoloader/cli/app.rb +26 -0
- data/lib/geoloader/cli/geonetwork.rb +36 -0
- data/lib/geoloader/cli/geoserver.rb +36 -0
- data/lib/geoloader/cli/solr.rb +36 -0
- data/lib/geoloader/loaders/geonetwork.rb +40 -0
- data/lib/geoloader/loaders/geotiff_geoserver.rb +47 -0
- data/lib/geoloader/loaders/geotiff_solr.rb +40 -0
- data/lib/geoloader/loaders/loader.rb +53 -0
- data/lib/geoloader/loaders/shapefile_geoserver.rb +40 -0
- data/lib/geoloader/loaders/shapefile_solr.rb +40 -0
- data/lib/geoloader/services/geonetwork.rb +201 -0
- data/lib/geoloader/services/geoserver.rb +104 -0
- data/lib/geoloader/services/solr.rb +62 -0
- data/lib/geoloader/tasks/geonetwork.rb +28 -0
- data/lib/geoloader/tasks/geoserver.rb +19 -0
- data/lib/geoloader/tasks/solr.rb +19 -0
- data/lib/geoloader/version.rb +4 -0
- data/metadata.md +13 -0
- data/spec/fixtures/geotiff.tfw +6 -0
- data/spec/fixtures/geotiff.tif +0 -0
- data/spec/fixtures/geotiff.tif.aux.xml +1315 -0
- data/spec/fixtures/geotiff.tif.ovr +0 -0
- data/spec/fixtures/geotiff.tif.vat.dbf +0 -0
- data/spec/fixtures/geotiff.tif.xml +2 -0
- data/spec/fixtures/shapefile.dbf +0 -0
- data/spec/fixtures/shapefile.prj +1 -0
- data/spec/fixtures/shapefile.sbn +0 -0
- data/spec/fixtures/shapefile.sbx +0 -0
- data/spec/fixtures/shapefile.shp +0 -0
- data/spec/fixtures/shapefile.shp.xml +51 -0
- data/spec/fixtures/shapefile.shx +0 -0
- data/spec/helpers/fixture.rb +13 -0
- data/spec/loaders/geonetwork.rb +38 -0
- data/spec/loaders/geotiff_geoserver.rb +49 -0
- data/spec/loaders/geotiff_solr.rb +34 -0
- data/spec/loaders/shapefile_geoserver.rb +49 -0
- data/spec/loaders/shapefile_solr.rb +34 -0
- data/spec/spec_helper.rb +6 -0
- metadata +362 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class Geonetwork < Loader
|
6
|
+
|
7
|
+
attr_reader :asset, :geonetwork
|
8
|
+
|
9
|
+
@queue = :geoloader
|
10
|
+
|
11
|
+
#
|
12
|
+
# Configure the asset, connect to Geonetwork.
|
13
|
+
#
|
14
|
+
def initialize(*args)
|
15
|
+
|
16
|
+
super
|
17
|
+
|
18
|
+
# Create and configure the asset.
|
19
|
+
@asset = Geoloader::Assets::Asset.new(@file_path, @workspace, @desc_path)
|
20
|
+
@asset.extend(Geoloader::Assets::Geonetwork)
|
21
|
+
|
22
|
+
# Connect to Geonetwork, create the group.
|
23
|
+
@geonetwork = Geoloader::Services::Geonetwork.new
|
24
|
+
@geonetwork.ensure_group(@workspace)
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Push an asset to Geonetwork.
|
30
|
+
#
|
31
|
+
def load
|
32
|
+
@asset.stage do
|
33
|
+
@geonetwork.create_record(@asset)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class GeotiffGeoserver < Loader
|
6
|
+
|
7
|
+
attr_reader :geotiff, :geoserver
|
8
|
+
|
9
|
+
@queue = :geoloader
|
10
|
+
|
11
|
+
#
|
12
|
+
# Configure the asset, connect to Geoserver.
|
13
|
+
#
|
14
|
+
def initialize(*args)
|
15
|
+
|
16
|
+
super
|
17
|
+
|
18
|
+
# Create and configure the asset.
|
19
|
+
@geotiff = Geoloader::Assets::Asset.new(@file_path, @workspace, @desc_path)
|
20
|
+
@geotiff.extend(Geoloader::Assets::Geotiff)
|
21
|
+
|
22
|
+
# Connect to Geoserver, create the workspace.
|
23
|
+
@geoserver = Geoloader::Services::Geoserver.new
|
24
|
+
@geoserver.ensure_workspace(@workspace)
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Push a GeoTIFF to Geoserver.
|
30
|
+
#
|
31
|
+
def load
|
32
|
+
@geotiff.stage do
|
33
|
+
|
34
|
+
# Prepare the file.
|
35
|
+
@geotiff.make_borders_transparent
|
36
|
+
@geotiff.reproject
|
37
|
+
|
38
|
+
# Push to Geoserver.
|
39
|
+
@geoserver.create_coveragestore(@geotiff)
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class GeotiffSolr < Loader
|
6
|
+
|
7
|
+
attr_reader :geotiff, :solr
|
8
|
+
|
9
|
+
@queue = :geoloader
|
10
|
+
|
11
|
+
#
|
12
|
+
# Configure the asset, connect to Solr.
|
13
|
+
#
|
14
|
+
def initialize(*args)
|
15
|
+
|
16
|
+
super
|
17
|
+
|
18
|
+
# Create and configure the asset.
|
19
|
+
@geotiff = Geoloader::Assets::Asset.new(@file_path, @workspace, @desc_path)
|
20
|
+
@geotiff.extend(Geoloader::Assets::Geotiff)
|
21
|
+
@geotiff.extend(Geoloader::Assets::Solr)
|
22
|
+
|
23
|
+
# Connect to Solr.
|
24
|
+
@solr = Geoloader::Services::Solr.new
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Push a GeoTIFF to Solr.
|
30
|
+
#
|
31
|
+
def load
|
32
|
+
@geotiff.stage do
|
33
|
+
@solr.create_document(@geotiff)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class Loader
|
6
|
+
|
7
|
+
#
|
8
|
+
# Perform or enqueue an upload.
|
9
|
+
#
|
10
|
+
# @param [String] file_path
|
11
|
+
# @param [Hash] options
|
12
|
+
#
|
13
|
+
def self.load_or_enqueue(file_path, options)
|
14
|
+
if options[:queue]
|
15
|
+
Resque.enqueue(self, file_path, options[:workspace], options[:description])
|
16
|
+
else
|
17
|
+
self.perform(file_path, options[:workspace], options[:description])
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
#
|
22
|
+
# Perform an upload (used by Resque).
|
23
|
+
#
|
24
|
+
# @param [String] file_path
|
25
|
+
# @param [String] workspace
|
26
|
+
# @param [String] desc_path
|
27
|
+
#
|
28
|
+
def self.perform(file_path, workspace, desc_path = nil)
|
29
|
+
new(file_path, workspace, desc_path).load
|
30
|
+
puts "Loaded #{File.basename(file_path)}."
|
31
|
+
end
|
32
|
+
|
33
|
+
#
|
34
|
+
# Set the file path and workspace.
|
35
|
+
#
|
36
|
+
# @param [String] file_path
|
37
|
+
# @param [String] workspace
|
38
|
+
# @param [String] desc_path
|
39
|
+
#
|
40
|
+
def initialize(file_path, workspace, desc_path = nil)
|
41
|
+
|
42
|
+
@file_path = file_path
|
43
|
+
@desc_path = desc_path
|
44
|
+
|
45
|
+
# If no workspace is provided, revert to the default.
|
46
|
+
@workspace = (workspace or Geoloader.config.workspaces.production)
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class ShapefileGeoserver < Loader
|
6
|
+
|
7
|
+
attr_reader :shapefile, :geoserver
|
8
|
+
|
9
|
+
@queue = :geoloader
|
10
|
+
|
11
|
+
#
|
12
|
+
# Configure the asset, connect to Geoserver.
|
13
|
+
#
|
14
|
+
def initialize(*args)
|
15
|
+
|
16
|
+
super
|
17
|
+
|
18
|
+
# Create and configure the asset.
|
19
|
+
@shapefile = Geoloader::Assets::Asset.new(@file_path, @workspace, @desc_path)
|
20
|
+
@shapefile.extend(Geoloader::Assets::Shapefile)
|
21
|
+
|
22
|
+
# Connect to Geoserver, create the workspace.
|
23
|
+
@geoserver = Geoloader::Services::Geoserver.new
|
24
|
+
@geoserver.ensure_workspace(@workspace)
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Push a Shapefile to Geoserver.
|
30
|
+
#
|
31
|
+
def load
|
32
|
+
@shapefile.stage do
|
33
|
+
@geoserver.create_datastore(@shapefile)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Loaders
|
4
|
+
|
5
|
+
class ShapefileSolr < Loader
|
6
|
+
|
7
|
+
attr_reader :shapefile, :solr
|
8
|
+
|
9
|
+
@queue = :geoloader
|
10
|
+
|
11
|
+
#
|
12
|
+
# Configure the shapefile, connect to Solr.
|
13
|
+
#
|
14
|
+
def initialize(*args)
|
15
|
+
|
16
|
+
super
|
17
|
+
|
18
|
+
# Create and configure the asset.
|
19
|
+
@shapefile = Geoloader::Assets::Asset.new(@file_path, @workspace, @desc_path)
|
20
|
+
@shapefile.extend(Geoloader::Assets::Shapefile)
|
21
|
+
@shapefile.extend(Geoloader::Assets::Solr)
|
22
|
+
|
23
|
+
# Connect to Solr.
|
24
|
+
@solr = Geoloader::Services::Solr.new
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Push a Shapefile to Solr.
|
30
|
+
#
|
31
|
+
def load
|
32
|
+
@shapefile.stage do
|
33
|
+
@solr.create_document(@shapefile)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
|
2
|
+
require "rest_client"
|
3
|
+
require "builder"
|
4
|
+
require "nokogiri"
|
5
|
+
|
6
|
+
module Geoloader
|
7
|
+
module Services
|
8
|
+
|
9
|
+
class Geonetwork
|
10
|
+
|
11
|
+
attr_reader :resource
|
12
|
+
|
13
|
+
#
|
14
|
+
# Initialize the API wrapper.
|
15
|
+
#
|
16
|
+
def initialize
|
17
|
+
@resource = RestClient::Resource.new(Geoloader.config.geonetwork.url, {
|
18
|
+
:user => Geoloader.config.geonetwork.username,
|
19
|
+
:password => Geoloader.config.geonetwork.password,
|
20
|
+
:headers => { :content_type => :xml }
|
21
|
+
})
|
22
|
+
end
|
23
|
+
|
24
|
+
#
|
25
|
+
# List all groups on the node.
|
26
|
+
#
|
27
|
+
def get_groups
|
28
|
+
@resource["xml.group.list"].get
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Get a group with a given name.
|
33
|
+
#
|
34
|
+
# @param [String] group
|
35
|
+
#
|
36
|
+
def get_group(group)
|
37
|
+
Nokogiri::XML(get_groups).at_xpath("//record[name[text()='#{group}']]")
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
# Does a group with a given name exist?
|
42
|
+
#
|
43
|
+
# @param [String] group
|
44
|
+
#
|
45
|
+
def group_exists?(group)
|
46
|
+
!!get_group(group)
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# Get the id of a group with a given name.
|
51
|
+
#
|
52
|
+
# @param [String] group
|
53
|
+
#
|
54
|
+
def get_group_id(group)
|
55
|
+
get_group(group).at_xpath("id").content.to_i
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# Create a new group with a given name.
|
60
|
+
#
|
61
|
+
# @param [String] group
|
62
|
+
#
|
63
|
+
def create_group(group)
|
64
|
+
post("group.update", self.class.xml_doc.request { |r|
|
65
|
+
r.name group
|
66
|
+
})
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# Check to see if a group exists, and if not, create it.
|
71
|
+
#
|
72
|
+
# @param [String] group
|
73
|
+
#
|
74
|
+
def ensure_group(group)
|
75
|
+
create_group(group) unless group_exists?(group)
|
76
|
+
end
|
77
|
+
|
78
|
+
#
|
79
|
+
# Get a list of the ids of all records in a group.
|
80
|
+
#
|
81
|
+
# @param [String] group
|
82
|
+
#
|
83
|
+
def get_ids_in_group(group)
|
84
|
+
Nokogiri::XML(search_records_by_group(group)).xpath("//metadata//id").map do |m|
|
85
|
+
m.content.to_i
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# Delete group with a given name.
|
91
|
+
#
|
92
|
+
# @param [String] group
|
93
|
+
#
|
94
|
+
def delete_group(group)
|
95
|
+
delete_records_by_group(group)
|
96
|
+
post("group.remove", self.class.xml_doc.request { |r|
|
97
|
+
r.id get_group_id(group)
|
98
|
+
})
|
99
|
+
end
|
100
|
+
|
101
|
+
#
|
102
|
+
# Insert a new record.
|
103
|
+
#
|
104
|
+
# @param [Geoloader::Asset] asset
|
105
|
+
# @param [String] style_sheet
|
106
|
+
# @param [String] category
|
107
|
+
#
|
108
|
+
def create_record(asset, style_sheet = "_none_", category = "_none_")
|
109
|
+
delete_record(asset)
|
110
|
+
post("metadata.insert", self.class.xml_doc.request { |r|
|
111
|
+
r.group get_group_id(asset.workspace)
|
112
|
+
r.data { |d| d.cdata! asset.iso19139_xml }
|
113
|
+
r.category category
|
114
|
+
r.styleSheet style_sheet
|
115
|
+
})
|
116
|
+
end
|
117
|
+
|
118
|
+
#
|
119
|
+
# Get the metadata record for an asset.
|
120
|
+
#
|
121
|
+
# @param [Geoloader::Asset] asset
|
122
|
+
#
|
123
|
+
def get_record(asset)
|
124
|
+
post("xml.metadata.get", self.class.xml_doc.request { |r|
|
125
|
+
r.uuid asset.uuid
|
126
|
+
})
|
127
|
+
end
|
128
|
+
|
129
|
+
#
|
130
|
+
# Get all records in a given group.
|
131
|
+
#
|
132
|
+
# @param [String] group
|
133
|
+
#
|
134
|
+
def search_records_by_group(group)
|
135
|
+
post("xml.search", self.class.xml_doc.request { |r|
|
136
|
+
r.group get_group_id(group)
|
137
|
+
})
|
138
|
+
end
|
139
|
+
|
140
|
+
#
|
141
|
+
# Delete the metadata record for an asset.
|
142
|
+
#
|
143
|
+
# @param [Geoloader::Asset] asset
|
144
|
+
#
|
145
|
+
def delete_record(asset)
|
146
|
+
post("metadata.delete", self.class.xml_doc.request { |r|
|
147
|
+
r.uuid asset.uuid
|
148
|
+
}) rescue nil # Geonetwork 500's if the record doesn't exist...
|
149
|
+
end
|
150
|
+
|
151
|
+
#
|
152
|
+
# Delete a record by id.
|
153
|
+
#
|
154
|
+
# @param [Integer] id
|
155
|
+
#
|
156
|
+
def delete_record_by_id(id)
|
157
|
+
post("metadata.delete", self.class.xml_doc.request { |r|
|
158
|
+
r.id id
|
159
|
+
})
|
160
|
+
end
|
161
|
+
|
162
|
+
#
|
163
|
+
# Delete all records in a group with a given name.
|
164
|
+
#
|
165
|
+
# @param [Integer] group
|
166
|
+
#
|
167
|
+
def delete_records_by_group(group)
|
168
|
+
get_ids_in_group(group).each do |id|
|
169
|
+
delete_record_by_id(id)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
#
|
174
|
+
# POST to an XML service.
|
175
|
+
#
|
176
|
+
# @param [String] service
|
177
|
+
# @param [String] payload
|
178
|
+
#
|
179
|
+
def post(service, payload)
|
180
|
+
@resource[service].post(payload) { |response, request, result, &block|
|
181
|
+
if [301, 302, 307].include?(response.code)
|
182
|
+
response.follow_redirection(request, result, &block)
|
183
|
+
else
|
184
|
+
response.return!(request, result, &block)
|
185
|
+
end
|
186
|
+
}
|
187
|
+
end
|
188
|
+
|
189
|
+
#
|
190
|
+
# Get an XML builder instance.
|
191
|
+
#
|
192
|
+
def self.xml_doc
|
193
|
+
xml = Builder::XmlMarkup.new
|
194
|
+
xml.instruct! :xml, :version => "1.0", :encoding => "UTF-8"
|
195
|
+
xml
|
196
|
+
end
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
201
|
+
end
|