geoloader 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +13 -0
- data/.gitignore +17 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +2 -0
- data/README.md +212 -0
- data/Rakefile +10 -0
- data/bin/geoloader +4 -0
- data/config.yaml +18 -0
- data/geoloader.gemspec +37 -0
- data/iso19139.xsl +445 -0
- data/lib/geoloader.rb +48 -0
- data/lib/geoloader/assets/asset.rb +63 -0
- data/lib/geoloader/assets/description.rb +55 -0
- data/lib/geoloader/assets/geonetwork.rb +58 -0
- data/lib/geoloader/assets/geotiff.rb +41 -0
- data/lib/geoloader/assets/shapefile.rb +28 -0
- data/lib/geoloader/assets/solr.rb +23 -0
- data/lib/geoloader/cli/app.rb +26 -0
- data/lib/geoloader/cli/geonetwork.rb +36 -0
- data/lib/geoloader/cli/geoserver.rb +36 -0
- data/lib/geoloader/cli/solr.rb +36 -0
- data/lib/geoloader/loaders/geonetwork.rb +40 -0
- data/lib/geoloader/loaders/geotiff_geoserver.rb +47 -0
- data/lib/geoloader/loaders/geotiff_solr.rb +40 -0
- data/lib/geoloader/loaders/loader.rb +53 -0
- data/lib/geoloader/loaders/shapefile_geoserver.rb +40 -0
- data/lib/geoloader/loaders/shapefile_solr.rb +40 -0
- data/lib/geoloader/services/geonetwork.rb +201 -0
- data/lib/geoloader/services/geoserver.rb +104 -0
- data/lib/geoloader/services/solr.rb +62 -0
- data/lib/geoloader/tasks/geonetwork.rb +28 -0
- data/lib/geoloader/tasks/geoserver.rb +19 -0
- data/lib/geoloader/tasks/solr.rb +19 -0
- data/lib/geoloader/version.rb +4 -0
- data/metadata.md +13 -0
- data/spec/fixtures/geotiff.tfw +6 -0
- data/spec/fixtures/geotiff.tif +0 -0
- data/spec/fixtures/geotiff.tif.aux.xml +1315 -0
- data/spec/fixtures/geotiff.tif.ovr +0 -0
- data/spec/fixtures/geotiff.tif.vat.dbf +0 -0
- data/spec/fixtures/geotiff.tif.xml +2 -0
- data/spec/fixtures/shapefile.dbf +0 -0
- data/spec/fixtures/shapefile.prj +1 -0
- data/spec/fixtures/shapefile.sbn +0 -0
- data/spec/fixtures/shapefile.sbx +0 -0
- data/spec/fixtures/shapefile.shp +0 -0
- data/spec/fixtures/shapefile.shp.xml +51 -0
- data/spec/fixtures/shapefile.shx +0 -0
- data/spec/helpers/fixture.rb +13 -0
- data/spec/loaders/geonetwork.rb +38 -0
- data/spec/loaders/geotiff_geoserver.rb +49 -0
- data/spec/loaders/geotiff_solr.rb +34 -0
- data/spec/loaders/shapefile_geoserver.rb +49 -0
- data/spec/loaders/shapefile_solr.rb +34 -0
- data/spec/spec_helper.rb +6 -0
- metadata +362 -0
data/lib/geoloader.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
|
2
|
+
require "confstruct"
|
3
|
+
require "require_all"
|
4
|
+
require "yaml"
|
5
|
+
|
6
|
+
module Geoloader
|
7
|
+
|
8
|
+
@config = Confstruct::Configuration.new
|
9
|
+
|
10
|
+
#
|
11
|
+
# Set configuration options.
|
12
|
+
#
|
13
|
+
# @param [Hash] opts
|
14
|
+
#
|
15
|
+
def self.configure(opts)
|
16
|
+
@config.configure(opts)
|
17
|
+
end
|
18
|
+
|
19
|
+
#
|
20
|
+
# Set configuration from a YAML file.
|
21
|
+
#
|
22
|
+
# @param [String] file_path
|
23
|
+
#
|
24
|
+
def self.configure_from_yaml(file_path)
|
25
|
+
@config.configure(YAML::load(File.read(File.expand_path(file_path))))
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Get the root gem directory.
|
30
|
+
#
|
31
|
+
def self.gem_dir
|
32
|
+
File.expand_path("../../", __FILE__)
|
33
|
+
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Get the configuration object.
|
37
|
+
#
|
38
|
+
def self.config
|
39
|
+
@config
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
require_rel("geoloader")
|
45
|
+
|
46
|
+
# Apply default configuration.
|
47
|
+
Geoloader.configure_from_yaml("#{Geoloader.gem_dir}/config.yaml")
|
48
|
+
Geoloader.configure_from_yaml("~/.geoloader.yaml") rescue nil
|
@@ -0,0 +1,63 @@
|
|
1
|
+
|
2
|
+
require "fileutils"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module Assets
|
6
|
+
|
7
|
+
class Asset
|
8
|
+
|
9
|
+
attr_reader :file_path, :file_base, :file_name, :workspace, :uuid
|
10
|
+
|
11
|
+
#
|
12
|
+
# Set the basename and workspace-prefixed uuid, parse the description.
|
13
|
+
#
|
14
|
+
# @param [String] file_path
|
15
|
+
# @param [String] workspace
|
16
|
+
# @param [String] desc_path
|
17
|
+
#
|
18
|
+
def initialize(file_path, workspace, desc_path)
|
19
|
+
|
20
|
+
@file_path = File.expand_path(file_path)
|
21
|
+
@workspace = workspace
|
22
|
+
|
23
|
+
# File name, with and without extension.
|
24
|
+
@file_base = File.basename(@file_path, ".*")
|
25
|
+
@file_name = File.basename(@file_path)
|
26
|
+
|
27
|
+
# Parse the markdown metadata.
|
28
|
+
@description = Description.new(desc_path)
|
29
|
+
|
30
|
+
# Set a workspace-prefixed uuid.
|
31
|
+
@uuid = "#{@workspace}_#{@file_base}"
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Create working copies, yield to a block, remove the copies.
|
37
|
+
#
|
38
|
+
def stage
|
39
|
+
|
40
|
+
@tempdir = Dir.mktmpdir
|
41
|
+
|
42
|
+
begin
|
43
|
+
|
44
|
+
# Copy the assets into the temp dir.
|
45
|
+
files = Dir.glob("#{File.dirname(@file_path)}/#{@file_base}.*")
|
46
|
+
FileUtils.cp(files, @tempdir)
|
47
|
+
|
48
|
+
# Change into the temp dir.
|
49
|
+
FileUtils.cd(@tempdir) do yield end
|
50
|
+
|
51
|
+
ensure
|
52
|
+
|
53
|
+
# Delete the copies.
|
54
|
+
FileUtils.remove_entry @tempdir
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
|
2
|
+
require "redcarpet"
|
3
|
+
require "jekyll"
|
4
|
+
require "nokogiri"
|
5
|
+
require "fileutils"
|
6
|
+
require "ostruct"
|
7
|
+
|
8
|
+
module Geoloader
|
9
|
+
module Assets
|
10
|
+
|
11
|
+
class Description
|
12
|
+
|
13
|
+
attr_reader :title, :abstract, :metadata
|
14
|
+
|
15
|
+
#
|
16
|
+
# Parse the markdown and extract the header.
|
17
|
+
#
|
18
|
+
# @param [String] file_path
|
19
|
+
#
|
20
|
+
def initialize(file_path)
|
21
|
+
|
22
|
+
@title, @abstract = "", ""
|
23
|
+
@metadata = {}
|
24
|
+
|
25
|
+
if file_path
|
26
|
+
|
27
|
+
@file_path = File.expand_path(file_path)
|
28
|
+
|
29
|
+
# Read the YAML front matter.
|
30
|
+
convertible = OpenStruct.new.extend(Jekyll::Convertible)
|
31
|
+
@metadata = convertible.read_yaml(File.dirname(@file_path), File.basename(@file_path))
|
32
|
+
|
33
|
+
# Scrub the YAML out of the markdown.
|
34
|
+
markdown = File.read(@file_path).sub(/---(.|\n)*---/, '')
|
35
|
+
|
36
|
+
# Parse the cleaned markdown.
|
37
|
+
renderer = Redcarpet::Markdown.new(Redcarpet::Render::HTML)
|
38
|
+
document = Nokogiri::HTML::fragment(renderer.render(markdown))
|
39
|
+
|
40
|
+
# Set the title.
|
41
|
+
header = document.at_css('h1')
|
42
|
+
@title = header.text
|
43
|
+
header.remove
|
44
|
+
|
45
|
+
# Set the abstract.
|
46
|
+
@abstract = document.to_s.strip
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
|
2
|
+
require "fileutils"
|
3
|
+
require "cgi"
|
4
|
+
|
5
|
+
module Geoloader
|
6
|
+
module Assets
|
7
|
+
|
8
|
+
module Geonetwork
|
9
|
+
|
10
|
+
#
|
11
|
+
# Form the WMS address.
|
12
|
+
#
|
13
|
+
def wms_address
|
14
|
+
"#{Geoloader.config.geoserver.url}/wms"
|
15
|
+
end
|
16
|
+
|
17
|
+
#
|
18
|
+
# Form the WMS layer string.
|
19
|
+
#
|
20
|
+
def wms_layers
|
21
|
+
"#{@workspace}:#{@file_base}"
|
22
|
+
end
|
23
|
+
|
24
|
+
#
|
25
|
+
# Convert the ESRI XML into a iso19139 record.
|
26
|
+
#
|
27
|
+
def iso19139_xml
|
28
|
+
`saxon #{@file_path}.xml #{Geoloader.gem_dir}/iso19139.xsl #{xslt_params(
|
29
|
+
:identifier => @uuid,
|
30
|
+
:categories => get_list_parameter("categories"),
|
31
|
+
:keywords => get_list_parameter("keywords"),
|
32
|
+
:title => @description.title.to_s,
|
33
|
+
:abstract => @description.abstract.to_s,
|
34
|
+
:wms_address => wms_address,
|
35
|
+
:wms_layers => wms_layers
|
36
|
+
)}`
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# Convert an array metadata attribute to a comma-delimited list.
|
41
|
+
#
|
42
|
+
def get_list_parameter(key)
|
43
|
+
@description.metadata.fetch(key, []).join(",")
|
44
|
+
end
|
45
|
+
|
46
|
+
#
|
47
|
+
# Convert a hash to a Saxon XSLT parameter string.
|
48
|
+
#
|
49
|
+
# @param [Hash] params
|
50
|
+
#
|
51
|
+
def xslt_params(params)
|
52
|
+
params.map { |k, v| "#{k}='#{CGI::escapeHTML(v)}'" }.join(" ")
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
|
2
|
+
require "fileutils"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module Assets
|
6
|
+
|
7
|
+
module Geotiff
|
8
|
+
|
9
|
+
#
|
10
|
+
# Remove the black borders added by ArcMap.
|
11
|
+
#
|
12
|
+
def make_borders_transparent
|
13
|
+
gdal_command("gdalwarp -srcnodata 0 -dstalpha", @file_name)
|
14
|
+
end
|
15
|
+
|
16
|
+
#
|
17
|
+
# (Re)build a EPSG:4326 header.
|
18
|
+
#
|
19
|
+
def reproject
|
20
|
+
srs = Geoloader.config.geoserver.srs
|
21
|
+
gdal_command("gdal_translate -of GTiff -a_srs #{srs}", @file_name)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
#
|
27
|
+
# Run a gdal command on a file, replacing the original file.
|
28
|
+
#
|
29
|
+
# @param [String] command
|
30
|
+
# @param [String] file_path
|
31
|
+
#
|
32
|
+
def gdal_command(command, file_path)
|
33
|
+
`#{command} #{file_path} #{file_path}_`
|
34
|
+
FileUtils.rm(file_path)
|
35
|
+
FileUtils.mv("#{file_path}_", file_path)
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
|
2
|
+
require "zip"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module Assets
|
6
|
+
|
7
|
+
module Shapefile
|
8
|
+
|
9
|
+
#
|
10
|
+
# Zip up the Shapefile and its companion files.
|
11
|
+
#
|
12
|
+
def get_zipfile
|
13
|
+
|
14
|
+
# Create the zipfile.
|
15
|
+
Zip::File.open("#{@file_base}.zip", Zip::File::CREATE) do |zipfile|
|
16
|
+
Dir.glob("#{@file_base}.*") do |file|
|
17
|
+
zipfile.add(File.basename(file), file)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
File.read("#{@file_base}.zip")
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
|
2
|
+
module Geoloader
|
3
|
+
module Assets
|
4
|
+
|
5
|
+
module Solr
|
6
|
+
|
7
|
+
#
|
8
|
+
# Get metadata for Solr document.
|
9
|
+
#
|
10
|
+
def solr_document
|
11
|
+
{
|
12
|
+
:LayerId => @uuid,
|
13
|
+
:LayerDisplayName => @description.title,
|
14
|
+
:Abstract => @description.abstract,
|
15
|
+
:WorkspaceName => @workspace,
|
16
|
+
:Name => @file_base
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
require "thor"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module CLI
|
6
|
+
|
7
|
+
class App < Thor
|
8
|
+
|
9
|
+
desc "solr [SUBCOMMAND]", "Manage Solr documents"
|
10
|
+
subcommand "solr", Solr
|
11
|
+
|
12
|
+
desc "geoserver [SUBCOMMAND]", "Manage Geoserver stores and layers"
|
13
|
+
subcommand "geoserver", Geoserver
|
14
|
+
|
15
|
+
desc "geonetwork [SUBCOMMAND]", "Manage Geonetwork records"
|
16
|
+
subcommand "geonetwork", Geonetwork
|
17
|
+
|
18
|
+
desc "work", "Start a Resque worker"
|
19
|
+
def work
|
20
|
+
Resque::Worker.new("geoloader").work
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
|
2
|
+
require "thor"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module CLI
|
6
|
+
|
7
|
+
class Geonetwork < Thor
|
8
|
+
|
9
|
+
include Tasks
|
10
|
+
|
11
|
+
desc "load [FILES]", "Load Geonetwork metadata records"
|
12
|
+
option :queue, :aliases => "-q", :type => :boolean, :default => false
|
13
|
+
option :description, :aliases => "-d", :type => :string
|
14
|
+
option :workspace, :aliases => "-w", :type => :string
|
15
|
+
def load(*files)
|
16
|
+
|
17
|
+
files.each { |file_path|
|
18
|
+
Geoloader::Loaders::Geonetwork.load_or_enqueue(file_path, options)
|
19
|
+
}
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
desc "publish [WORKSPACE]", "Publish all records in a group"
|
24
|
+
def publish(workspace)
|
25
|
+
Geoloader::Tasks::Geonetwork.publish(workspace)
|
26
|
+
end
|
27
|
+
|
28
|
+
desc "clear [WORKSPACE]", "Clear all records in a group"
|
29
|
+
def clear(workspace)
|
30
|
+
Geoloader::Tasks::Geonetwork.clear(workspace) rescue nil
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
|
2
|
+
require "thor"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module CLI
|
6
|
+
|
7
|
+
class Geoserver < Thor
|
8
|
+
|
9
|
+
include Tasks
|
10
|
+
|
11
|
+
desc "load [FILES]", "Load Geoserver stores and layers"
|
12
|
+
option :queue, :aliases => "-q", :type => :boolean, :default => false
|
13
|
+
option :description, :aliases => "-d", :type => :string
|
14
|
+
option :workspace, :aliases => "-w", :type => :string
|
15
|
+
def load(*files)
|
16
|
+
|
17
|
+
files.each { |file_path|
|
18
|
+
case File.extname(file_path)
|
19
|
+
when ".tif" # GEOTIFF
|
20
|
+
Geoloader::Loaders::GeotiffGeoserver.load_or_enqueue(file_path, options)
|
21
|
+
when ".shp" # SHAPEFILE
|
22
|
+
Geoloader::Loaders::ShapefileGeoserver.load_or_enqueue(file_path, options)
|
23
|
+
end
|
24
|
+
}
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
desc "clear [WORKSPACE]", "Clear all documents in a workspace"
|
29
|
+
def clear(workspace)
|
30
|
+
Geoloader::Tasks::Geoserver.clear(workspace) rescue nil
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
|
2
|
+
require "thor"
|
3
|
+
|
4
|
+
module Geoloader
|
5
|
+
module CLI
|
6
|
+
|
7
|
+
class Solr < Thor
|
8
|
+
|
9
|
+
include Tasks
|
10
|
+
|
11
|
+
desc "load [FILES]", "Load Solr documents"
|
12
|
+
option :queue, :aliases => "-q", :type => :boolean, :default => false
|
13
|
+
option :description, :aliases => "-d", :type => :string
|
14
|
+
option :workspace, :aliases => "-w", :type => :string
|
15
|
+
def load(*files)
|
16
|
+
|
17
|
+
files.each { |file_path|
|
18
|
+
case File.extname(file_path)
|
19
|
+
when ".tif" # GEOTIFF
|
20
|
+
Geoloader::Loaders::GeotiffSolr.load_or_enqueue(file_path, options)
|
21
|
+
when ".shp" # SHAPEFILE
|
22
|
+
Geoloader::Loaders::ShapefileSolr.load_or_enqueue(file_path, options)
|
23
|
+
end
|
24
|
+
}
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
desc "clear [WORKSPACE]", "Clear all documents in a workspace"
|
29
|
+
def clear(workspace)
|
30
|
+
Geoloader::Tasks::Solr.clear(workspace) rescue nil
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|