geoloader 0.1.0

data/lib/geoloader.rb

@@ -0,0 +1,48 @@
+
+require "confstruct"
+require "require_all"
+require "yaml"
+
+module Geoloader
+
+  @config = Confstruct::Configuration.new
+
+  #
+  # Set configuration options.
+  #
+  # @param [Hash] opts
+  #
+  def self.configure(opts)
+    @config.configure(opts)
+  end
+
+  #
+  # Set configuration from a YAML file.
+  #
+  # @param [String] file_path
+  #
+  def self.configure_from_yaml(file_path)
+    @config.configure(YAML::load(File.read(File.expand_path(file_path))))
+  end
+
+  #
+  # Get the root gem directory.
+  #
+  def self.gem_dir
+    File.expand_path("../../", __FILE__)
+  end
+
+  #
+  # Get the configuration object.
+  #
+  def self.config
+    @config
+  end
+
+end
+
+require_rel("geoloader")
+
+# Apply default configuration.
+Geoloader.configure_from_yaml("#{Geoloader.gem_dir}/config.yaml")
+Geoloader.configure_from_yaml("~/.geoloader.yaml") rescue nil

data/lib/geoloader/assets/asset.rb

@@ -0,0 +1,63 @@
+
+require "fileutils"
+
+module Geoloader
+  module Assets
+
+    class Asset
+
+      attr_reader :file_path, :file_base, :file_name, :workspace, :uuid
+
+      #
+      # Set the basename and workspace-prefixed uuid, parse the description.
+      #
+      # @param [String] file_path
+      # @param [String] workspace
+      # @param [String] desc_path
+      #
+      def initialize(file_path, workspace, desc_path)
+
+        @file_path = File.expand_path(file_path)
+        @workspace = workspace
+
+        # File name, with and without extension.
+        @file_base = File.basename(@file_path, ".*")
+        @file_name = File.basename(@file_path)
+
+        # Parse the markdown metadata.
+        @description = Description.new(desc_path)
+
+        # Set a workspace-prefixed uuid.
+        @uuid = "#{@workspace}_#{@file_base}"
+
+      end
+
+      #
+      # Create working copies, yield to a block, remove the copies.
+      #
+      def stage
+
+        @tempdir = Dir.mktmpdir
+
+        begin
+
+          # Copy the assets into the temp dir.
+          files = Dir.glob("#{File.dirname(@file_path)}/#{@file_base}.*")
+          FileUtils.cp(files, @tempdir)
+
+          # Change into the temp dir.
+          FileUtils.cd(@tempdir) do yield end
+
+        ensure
+
+          # Delete the copies.
+          FileUtils.remove_entry @tempdir
+
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/assets/description.rb

@@ -0,0 +1,55 @@
+
+require "redcarpet"
+require "jekyll"
+require "nokogiri"
+require "fileutils"
+require "ostruct"
+
+module Geoloader
+  module Assets
+
+    class Description
+
+      attr_reader :title, :abstract, :metadata
+
+      #
+      # Parse the markdown and extract the header.
+      #
+      # @param [String] file_path
+      #
+      def initialize(file_path)
+
+        @title, @abstract = "", ""
+        @metadata = {}
+
+        if file_path
+
+          @file_path = File.expand_path(file_path)
+
+          # Read the YAML front matter.
+          convertible = OpenStruct.new.extend(Jekyll::Convertible)
+          @metadata = convertible.read_yaml(File.dirname(@file_path), File.basename(@file_path))
+
+          # Scrub the YAML out of the markdown.
+          markdown = File.read(@file_path).sub(/---(.|\n)*---/, '')
+
+          # Parse the cleaned markdown.
+          renderer = Redcarpet::Markdown.new(Redcarpet::Render::HTML)
+          document = Nokogiri::HTML::fragment(renderer.render(markdown))
+
+          # Set the title.
+          header = document.at_css('h1')
+          @title = header.text
+          header.remove
+
+          # Set the abstract.
+          @abstract = document.to_s.strip
+
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/assets/geonetwork.rb

@@ -0,0 +1,58 @@
+
+require "fileutils"
+require "cgi"
+
+module Geoloader
+  module Assets
+
+    module Geonetwork
+
+      #
+      # Form the WMS address.
+      #
+      def wms_address
+        "#{Geoloader.config.geoserver.url}/wms"
+      end
+
+      #
+      # Form the WMS layer string.
+      #
+      def wms_layers
+        "#{@workspace}:#{@file_base}"
+      end
+
+      #
+      # Convert the ESRI XML into a iso19139 record.
+      #
+      def iso19139_xml
+        `saxon #{@file_path}.xml #{Geoloader.gem_dir}/iso19139.xsl #{xslt_params(
+          :identifier   => @uuid,
+          :categories   => get_list_parameter("categories"),
+          :keywords     => get_list_parameter("keywords"),
+          :title        => @description.title.to_s,
+          :abstract     => @description.abstract.to_s,
+          :wms_address  => wms_address,
+          :wms_layers   => wms_layers
+        )}`
+      end
+
+      #
+      # Convert an array metadata attribute to a comma-delimited list.
+      #
+      def get_list_parameter(key)
+        @description.metadata.fetch(key, []).join(",")
+      end
+
+      #
+      # Convert a hash to a Saxon XSLT parameter string.
+      #
+      # @param [Hash] params
+      #
+      def xslt_params(params)
+        params.map { |k, v| "#{k}='#{CGI::escapeHTML(v)}'" }.join(" ")
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/assets/geotiff.rb

@@ -0,0 +1,41 @@
+
+require "fileutils"
+
+module Geoloader
+  module Assets
+
+    module Geotiff
+
+      #
+      # Remove the black borders added by ArcMap.
+      #
+      def make_borders_transparent
+        gdal_command("gdalwarp -srcnodata 0 -dstalpha", @file_name)
+      end
+
+      #
+      # (Re)build a EPSG:4326 header.
+      #
+      def reproject
+        srs = Geoloader.config.geoserver.srs
+        gdal_command("gdal_translate -of GTiff -a_srs #{srs}", @file_name)
+      end
+
+      private
+
+      #
+      # Run a gdal command on a file, replacing the original file.
+      #
+      # @param [String] command
+      # @param [String] file_path
+      #
+      def gdal_command(command, file_path)
+        `#{command} #{file_path} #{file_path}_`
+        FileUtils.rm(file_path)
+        FileUtils.mv("#{file_path}_", file_path)
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/assets/shapefile.rb

@@ -0,0 +1,28 @@
+
+require "zip"
+
+module Geoloader
+  module Assets
+
+    module Shapefile
+
+      #
+      # Zip up the Shapefile and its companion files.
+      #
+      def get_zipfile
+
+        # Create the zipfile.
+        Zip::File.open("#{@file_base}.zip", Zip::File::CREATE) do |zipfile|
+          Dir.glob("#{@file_base}.*") do |file|
+            zipfile.add(File.basename(file), file)
+          end
+        end
+
+        File.read("#{@file_base}.zip")
+
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/assets/solr.rb

@@ -0,0 +1,23 @@
+
+module Geoloader
+  module Assets
+
+    module Solr
+
+      #
+      # Get metadata for Solr document.
+      #
+      def solr_document
+        {
+          :LayerId          => @uuid,
+          :LayerDisplayName => @description.title,
+          :Abstract         => @description.abstract,
+          :WorkspaceName    => @workspace,
+          :Name             => @file_base
+        }
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/cli/app.rb

@@ -0,0 +1,26 @@
+
+require "thor"
+
+module Geoloader
+  module CLI
+
+    class App < Thor
+
+      desc "solr [SUBCOMMAND]", "Manage Solr documents"
+      subcommand "solr", Solr
+
+      desc "geoserver [SUBCOMMAND]", "Manage Geoserver stores and layers"
+      subcommand "geoserver", Geoserver
+
+      desc "geonetwork [SUBCOMMAND]", "Manage Geonetwork records"
+      subcommand "geonetwork", Geonetwork
+
+      desc "work", "Start a Resque worker"
+      def work
+        Resque::Worker.new("geoloader").work
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/cli/geonetwork.rb

@@ -0,0 +1,36 @@
+
+require "thor"
+
+module Geoloader
+  module CLI
+
+    class Geonetwork < Thor
+
+      include Tasks
+
+      desc "load [FILES]", "Load Geonetwork metadata records"
+      option :queue,        :aliases => "-q", :type => :boolean, :default => false
+      option :description,  :aliases => "-d", :type => :string
+      option :workspace,    :aliases => "-w", :type => :string
+      def load(*files)
+
+        files.each { |file_path|
+          Geoloader::Loaders::Geonetwork.load_or_enqueue(file_path, options)
+        }
+
+      end
+
+      desc "publish [WORKSPACE]", "Publish all records in a group"
+      def publish(workspace)
+        Geoloader::Tasks::Geonetwork.publish(workspace)
+      end
+
+      desc "clear [WORKSPACE]", "Clear all records in a group"
+      def clear(workspace)
+        Geoloader::Tasks::Geonetwork.clear(workspace) rescue nil
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/cli/geoserver.rb

@@ -0,0 +1,36 @@
+
+require "thor"
+
+module Geoloader
+  module CLI
+
+    class Geoserver < Thor
+
+      include Tasks
+
+      desc "load [FILES]", "Load Geoserver stores and layers"
+      option :queue,        :aliases => "-q", :type => :boolean, :default => false
+      option :description,  :aliases => "-d", :type => :string
+      option :workspace,    :aliases => "-w", :type => :string
+      def load(*files)
+
+        files.each { |file_path|
+          case File.extname(file_path)
+          when ".tif" # GEOTIFF
+            Geoloader::Loaders::GeotiffGeoserver.load_or_enqueue(file_path, options)
+          when ".shp" # SHAPEFILE
+            Geoloader::Loaders::ShapefileGeoserver.load_or_enqueue(file_path, options)
+          end
+        }
+
+      end
+
+      desc "clear [WORKSPACE]", "Clear all documents in a workspace"
+      def clear(workspace)
+        Geoloader::Tasks::Geoserver.clear(workspace) rescue nil
+      end
+
+    end
+
+  end
+end

data/lib/geoloader/cli/solr.rb

@@ -0,0 +1,36 @@
+
+require "thor"
+
+module Geoloader
+  module CLI
+
+    class Solr < Thor
+
+      include Tasks
+
+      desc "load [FILES]", "Load Solr documents"
+      option :queue,        :aliases => "-q", :type => :boolean, :default => false
+      option :description,  :aliases => "-d", :type => :string
+      option :workspace,    :aliases => "-w", :type => :string
+      def load(*files)
+
+        files.each { |file_path|
+          case File.extname(file_path)
+          when ".tif" # GEOTIFF
+            Geoloader::Loaders::GeotiffSolr.load_or_enqueue(file_path, options)
+          when ".shp" # SHAPEFILE
+            Geoloader::Loaders::ShapefileSolr.load_or_enqueue(file_path, options)
+          end
+        }
+
+      end
+
+      desc "clear [WORKSPACE]", "Clear all documents in a workspace"
+      def clear(workspace)
+        Geoloader::Tasks::Solr.clear(workspace) rescue nil
+      end
+
+    end
+
+  end
+end