spatial_features 1.7.1 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +30 -0
- data/app/models/feature.rb +21 -2
- data/lib/spatial_features/download.rb +32 -0
- data/lib/spatial_features/has_spatial_features/delayed_feature_import.rb +36 -0
- data/lib/spatial_features/has_spatial_features/feature_import.rb +64 -0
- data/lib/spatial_features/has_spatial_features.rb +26 -14
- data/lib/spatial_features/importers/base.rb +51 -0
- data/lib/spatial_features/importers/file.rb +21 -0
- data/lib/spatial_features/importers/geomark.rb +15 -0
- data/lib/spatial_features/importers/kml.rb +26 -0
- data/lib/spatial_features/importers/kml_file.rb +13 -0
- data/lib/spatial_features/importers/kml_file_arcgis.rb +27 -0
- data/lib/spatial_features/importers/shapefile.rb +34 -0
- data/lib/spatial_features/unzip.rb +37 -0
- data/lib/spatial_features/version.rb +1 -1
- data/lib/spatial_features/workers/{arcgis_update_features_job.rb → update_features_job.rb} +2 -2
- data/lib/spatial_features.rb +22 -4
- metadata +63 -17
- data/lib/spatial_features/import/arcgis_kmz_features.rb +0 -144
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee690ccc750978af8db4b018a4564dcad10085cc
|
4
|
+
data.tar.gz: cb9e15cdee984d7c7398f68ec95af49a973a0571
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1816cccfc59bb3d8ad6dfd19467a734e0889fd658224d19099e0c7266dd30b16ecc6ff5c305c557649fe4278c3c0dcab9be4ee8f10d5aa1db37103c60a119d1
|
7
|
+
data.tar.gz: 97f3e7e7eff1d3feac000560651944bd391d5533055a3131b29ad39901ee9442d23d70854f2b10c796006129cd38f8e336e2a5f73d53cffbcb642d3ac43ad5ca
|
data/README.md
CHANGED
@@ -48,3 +48,33 @@ execute("
|
|
48
48
|
);
|
49
49
|
")
|
50
50
|
```
|
51
|
+
|
52
|
+
## Usage
|
53
|
+
|
54
|
+
In your model
|
55
|
+
|
56
|
+
```ruby
|
57
|
+
class Location < ActiveRecord::Base
|
58
|
+
has_spatial_features
|
59
|
+
end
|
60
|
+
|
61
|
+
Person.new(:features => [Feature.new(:geog => 'some binary PostGIS Geography string')])
|
62
|
+
```
|
63
|
+
|
64
|
+
### Import
|
65
|
+
|
66
|
+
You can specify multiple import sources for geometry. Each key is a method that returns the data for the Importer, and
|
67
|
+
each value is the Importer to use to parse the data. See each Importer for more details.
|
68
|
+
```ruby
|
69
|
+
class Location < ActiveRecord::Base
|
70
|
+
has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File' }
|
71
|
+
|
72
|
+
def remote_kml_url
|
73
|
+
"www.test.com/kml/#{id}.kml"
|
74
|
+
end
|
75
|
+
|
76
|
+
def file
|
77
|
+
File.open('local/files/my_kml')
|
78
|
+
end
|
79
|
+
end
|
80
|
+
```
|
data/app/models/feature.rb
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
class Feature < ActiveRecord::Base
|
2
2
|
belongs_to :spatial_model, :polymorphic => :true
|
3
3
|
|
4
|
+
attr_writer :make_valid
|
5
|
+
|
6
|
+
FEATURE_TYPES = %w(polygon point line)
|
7
|
+
|
4
8
|
before_validation :sanitize_feature_type
|
5
9
|
validates_presence_of :geog
|
6
10
|
validate :geometry_is_valid
|
7
|
-
validates_inclusion_of :feature_type, :in =>
|
11
|
+
validates_inclusion_of :feature_type, :in => FEATURE_TYPES
|
12
|
+
before_save :sanitize
|
13
|
+
before_save :make_valid, if: :make_valid?
|
8
14
|
after_save :cache_derivatives
|
9
15
|
|
10
16
|
def self.with_metadata(k, v)
|
@@ -98,8 +104,21 @@ class Feature < ActiveRecord::Base
|
|
98
104
|
return geometry
|
99
105
|
end
|
100
106
|
|
107
|
+
def make_valid?
|
108
|
+
@make_valid
|
109
|
+
end
|
110
|
+
|
101
111
|
private
|
102
112
|
|
113
|
+
def make_valid
|
114
|
+
self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_CollectionExtract(ST_MakeValid('#{sanitize}'),3)")
|
115
|
+
end
|
116
|
+
|
117
|
+
# Use ST_Force_2D to discard z-coordinates that cause failures later in the process
|
118
|
+
def sanitize
|
119
|
+
self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_Force2D('#{geog}')")
|
120
|
+
end
|
121
|
+
|
103
122
|
def self.detect_srid(column_name)
|
104
123
|
connection.select_value("SELECT Find_SRID('public', '#{table_name}', '#{column_name}')")
|
105
124
|
end
|
@@ -116,7 +135,7 @@ class Feature < ActiveRecord::Base
|
|
116
135
|
end
|
117
136
|
|
118
137
|
def sanitize_feature_type
|
119
|
-
self.feature_type = self.feature_type.to_s.strip.downcase
|
138
|
+
self.feature_type = FEATURE_TYPES.find {|type| self.feature_type.to_s.strip.downcase.include?(type) }
|
120
139
|
end
|
121
140
|
|
122
141
|
def sanitize_input_for_sql(input)
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module Download
|
5
|
+
# file can be a url, path, or file, any of which can return be a zipped archive
|
6
|
+
def self.read(file, unzip: nil)
|
7
|
+
file = open(file, unzip: unzip)
|
8
|
+
path = ::File.path(file)
|
9
|
+
return ::File.read(path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.open(file, unzip: nil)
|
13
|
+
file = Kernel.open(file)
|
14
|
+
file = normalize_file(file) if file.is_a?(StringIO)
|
15
|
+
file = find_in_zip(file, unzip) if Unzip.is_zip?(file)
|
16
|
+
return file
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.normalize_file(file)
|
20
|
+
Tempfile.new.tap do |temp|
|
21
|
+
temp.binmode
|
22
|
+
temp.write(file.read)
|
23
|
+
temp.rewind
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.find_in_zip(file, unzip)
|
28
|
+
raise "Must specify an :unzip option if opening a zip file. e.g. open(file, :find => '.shp')" unless unzip
|
29
|
+
return File.open(Unzip.paths(file, :find => unzip))
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module SpatialFeatures
|
2
|
+
module DelayedFeatureImport
|
3
|
+
include FeatureImport
|
4
|
+
|
5
|
+
def queue_feature_update!(options = {})
|
6
|
+
job = UpdateFeaturesJob.new(options.merge :spatial_model_type => self.class, :spatial_model_id => self.id)
|
7
|
+
Delayed::Job.enqueue(job, :queue => delayed_jobs_queue_name)
|
8
|
+
end
|
9
|
+
|
10
|
+
def updating_features?
|
11
|
+
running_feature_update_jobs.exists?
|
12
|
+
end
|
13
|
+
|
14
|
+
def feature_update_error
|
15
|
+
(failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
|
16
|
+
end
|
17
|
+
|
18
|
+
def running_feature_update_jobs
|
19
|
+
feature_update_jobs.where(failed_at: nil)
|
20
|
+
end
|
21
|
+
|
22
|
+
def failed_feature_update_jobs
|
23
|
+
feature_update_jobs.where.not(failed_at: nil)
|
24
|
+
end
|
25
|
+
|
26
|
+
def feature_update_jobs
|
27
|
+
Delayed::Job.where(queue: delayed_jobs_queue_name)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def delayed_jobs_queue_name
|
33
|
+
"#{self.class}/#{self.id}/update_features"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module FeatureImport
|
5
|
+
def update_features!(skip_invalid: false, options: {})
|
6
|
+
options = options.reverse_merge(spatial_features_options).reverse_merge(:import => {})
|
7
|
+
|
8
|
+
ActiveRecord::Base.transaction do
|
9
|
+
imports = spatial_feature_imports(options[:import], options[:make_valid])
|
10
|
+
cache_key = Digest::MD5.hexdigest(imports.collect(&:cache_key).join)
|
11
|
+
|
12
|
+
return if features_cache_key_matches?(cache_key)
|
13
|
+
|
14
|
+
import_features(imports)
|
15
|
+
validate_features!(imports, skip_invalid)
|
16
|
+
set_features_cache_key(cache_key)
|
17
|
+
|
18
|
+
return true
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def spatial_feature_imports(import_options, make_valid)
|
25
|
+
import_options.collect do |data_method, importer_name|
|
26
|
+
data = send(data_method)
|
27
|
+
"SpatialFeatures::Importers::#{importer_name}".constantize.new(data, :make_valid => make_valid) if data.present?
|
28
|
+
end.compact
|
29
|
+
end
|
30
|
+
|
31
|
+
def import_features(imports)
|
32
|
+
self.features.destroy_all
|
33
|
+
self.features = imports.flat_map(&:features)
|
34
|
+
end
|
35
|
+
|
36
|
+
def validate_features!(imports, skip_invalid = false)
|
37
|
+
invalid = features.select {|feature| feature.errors.present? }
|
38
|
+
features.destroy(invalid)
|
39
|
+
|
40
|
+
return if skip_invalid
|
41
|
+
|
42
|
+
errors = imports.flat_map(&:errors)
|
43
|
+
invalid.each do |feature|
|
44
|
+
errors << "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}"
|
45
|
+
end
|
46
|
+
|
47
|
+
if errors.present?
|
48
|
+
raise ImportError, "Error updating #{self.class} #{self.id}. #{errors.to_sentence}"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def features_cache_key_matches?(cache_key)
|
53
|
+
has_spatial_features_hash? && cache_key == features_hash
|
54
|
+
end
|
55
|
+
|
56
|
+
def set_features_cache_key(cache_key)
|
57
|
+
return unless has_spatial_features_hash?
|
58
|
+
self.features_hash = cache_key
|
59
|
+
update_column(:features_hash, cache_key) unless new_record?
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
class ImportError < StandardError; end
|
64
|
+
end
|
@@ -1,29 +1,41 @@
|
|
1
1
|
module SpatialFeatures
|
2
2
|
module ActMethod
|
3
3
|
def has_spatial_features(options = {})
|
4
|
-
|
5
|
-
|
4
|
+
unless acts_like?(:spatial_features)
|
5
|
+
extend ClassMethods
|
6
|
+
include InstanceMethods
|
7
|
+
include DelayedFeatureImport
|
6
8
|
|
7
|
-
|
9
|
+
class_attribute :spatial_features_options
|
10
|
+
self.spatial_features_options = {}
|
8
11
|
|
9
|
-
|
10
|
-
scope :without_features, lambda { joins("LEFT OUTER JOIN features ON features.spatial_model_type = '#{name}' AND features.spatial_model_id = #{table_name}.id").where("features.id IS NULL") }
|
12
|
+
has_many :features, lambda { extending FeaturesAssociationExtensions }, :as => :spatial_model, :dependent => :delete_all
|
11
13
|
|
12
|
-
|
13
|
-
|
14
|
-
scope :with_stale_spatial_cache, lambda { joins(:spatial_cache).where("#{table_name}.features_hash != spatial_caches.features_hash").uniq } if has_spatial_features_hash?
|
14
|
+
scope :with_features, lambda { joins(:features).uniq }
|
15
|
+
scope :without_features, lambda { joins("LEFT OUTER JOIN features ON features.spatial_model_type = '#{name}' AND features.spatial_model_id = #{table_name}.id").where("features.id IS NULL") }
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
scope :with_spatial_cache, lambda {|klass| joins(:spatial_cache).where(:spatial_caches => { :intersection_model_type => klass }).uniq }
|
18
|
+
scope :without_spatial_cache, lambda {|klass| joins("LEFT OUTER JOIN #{SpatialCache.table_name} ON spatial_model_id = #{table_name}.id AND spatial_model_type = '#{name}' and intersection_model_type = '#{klass}'").where('spatial_model_id IS NULL') }
|
19
|
+
scope :with_stale_spatial_cache, lambda { joins(:spatial_cache).where("#{table_name}.features_hash != spatial_caches.features_hash").uniq } if has_spatial_features_hash?
|
19
20
|
|
20
|
-
|
21
|
+
has_many :spatial_cache, :as => :spatial_model, :dependent => :delete_all
|
22
|
+
has_many :model_a_spatial_proximities, :as => :model_a, :class_name => 'SpatialProximity', :dependent => :delete_all
|
23
|
+
has_many :model_b_spatial_proximities, :as => :model_b, :class_name => 'SpatialProximity', :dependent => :delete_all
|
21
24
|
|
22
|
-
|
25
|
+
after_save :update_features_area, :if => :features_hash_changed? if has_features_area? && has_spatial_features_hash?
|
26
|
+
|
27
|
+
delegate :has_spatial_features_hash?, :has_features_area?, :to => self
|
28
|
+
end
|
29
|
+
|
30
|
+
self.spatial_features_options = self.spatial_features_options.merge(options)
|
23
31
|
end
|
24
32
|
end
|
25
33
|
|
26
34
|
module ClassMethods
|
35
|
+
def acts_like_spatial_features?
|
36
|
+
true
|
37
|
+
end
|
38
|
+
|
27
39
|
# Add methods to generate cache keys for a record or all records of this class
|
28
40
|
# NOTE: features are never updated, only deleted and created, therefore we can
|
29
41
|
# tell if they have changed by finding the maximum id and count instead of needing timestamps
|
@@ -164,7 +176,7 @@ module SpatialFeatures
|
|
164
176
|
end
|
165
177
|
|
166
178
|
module InstanceMethods
|
167
|
-
def
|
179
|
+
def acts_like_spatial_features?
|
168
180
|
true
|
169
181
|
end
|
170
182
|
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module Importers
|
5
|
+
class Base
|
6
|
+
attr_reader :errors
|
7
|
+
|
8
|
+
def initialize(data, make_valid: false)
|
9
|
+
@make_valid = make_valid
|
10
|
+
@data = data
|
11
|
+
@errors = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def features
|
15
|
+
@features ||= build_features
|
16
|
+
end
|
17
|
+
|
18
|
+
def cache_key
|
19
|
+
@cache_key ||= Digest::MD5.hexdigest(@data)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def build_features
|
25
|
+
new_features = []
|
26
|
+
|
27
|
+
each_record do |record|
|
28
|
+
begin
|
29
|
+
new_features << build_feature(record)
|
30
|
+
rescue => e
|
31
|
+
@errors << e.message
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
return new_features
|
36
|
+
end
|
37
|
+
|
38
|
+
def each_record(&block)
|
39
|
+
raise NotImplementedError, 'Subclasses should implement this method and yield objects that can be passed to #build_feature'
|
40
|
+
end
|
41
|
+
|
42
|
+
def build_feature(record)
|
43
|
+
Feature.new(:name => record.name, :metadata => record.metadata, :feature_type => record.feature_type, :geog => record.geog, :make_valid => @make_valid)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# EXCEPTIONS
|
49
|
+
|
50
|
+
class ImportError < StandardError; end
|
51
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module Importers
|
5
|
+
class File < SimpleDelegator
|
6
|
+
def initialize(data, *args)
|
7
|
+
file = Download.open(data, unzip: %w(.kml .shp))
|
8
|
+
|
9
|
+
if file.path.end_with? '.kml'
|
10
|
+
__setobj__(KMLFile.new(file, *args))
|
11
|
+
|
12
|
+
elsif file.path.end_with? '.shp'
|
13
|
+
__setobj__(Shapefile.new(file, *args))
|
14
|
+
|
15
|
+
else
|
16
|
+
raise ImportError, "Could not import file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module SpatialFeatures
|
2
|
+
module Importers
|
3
|
+
class Geomark < KMLFile
|
4
|
+
def initialize(geomark, *args)
|
5
|
+
super geomark_url(geomark), *args
|
6
|
+
end
|
7
|
+
|
8
|
+
private
|
9
|
+
|
10
|
+
def geomark_url(geomark)
|
11
|
+
"http://apps.gov.bc.ca/pub/geomark/geomarks/#{geomark}/parts.kml?srid=4326"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module Importers
|
5
|
+
class KML < Base
|
6
|
+
private
|
7
|
+
|
8
|
+
def each_record(&block)
|
9
|
+
Nokogiri::XML(@data).css('Placemark').each do |placemark|
|
10
|
+
name = placemark.css('name').text
|
11
|
+
metadata = {:description => placemark.css('description').text}
|
12
|
+
|
13
|
+
{'Polygon' => 'POLYGON', 'LineString' => 'LINE', 'Point' => 'POINT'}.each do |kml_type, sql_type|
|
14
|
+
placemark.css(kml_type).each do |placemark|
|
15
|
+
yield OpenStruct.new(:feature_type => sql_type, :geog => geom_from_kml(placemark), :name => name, :metadata => metadata)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def geom_from_kml(kml)
|
22
|
+
ActiveRecord::Base.connection.select_value("SELECT ST_GeomFromKML('#{kml}')")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module SpatialFeatures
|
2
|
+
module Importers
|
3
|
+
class KMLFile < KML
|
4
|
+
def initialize(path_or_url, *args)
|
5
|
+
super Download.read(path_or_url, unzip: '.kml'), *args
|
6
|
+
|
7
|
+
rescue SocketError, Errno::ECONNREFUSED, OpenURI::HTTPError
|
8
|
+
url = URI(path_or_url)
|
9
|
+
raise ImportError, "KML server is not responding. Ensure server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module SpatialFeatures
|
4
|
+
module Importers
|
5
|
+
class KMLFileArcGIS < KMLFile
|
6
|
+
def initialize(data, *args)
|
7
|
+
super
|
8
|
+
|
9
|
+
rescue SocketError, Errno::ECONNREFUSED
|
10
|
+
url = URI(data)
|
11
|
+
raise ImportError, "ArcGIS Server is not responding. Ensure ArcGIS Server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
|
12
|
+
rescue OpenURI::HTTPError
|
13
|
+
raise ImportError, "ArcGIS Map Service not found. Ensure ArcGIS Server is running and accessible at #{path_or_url}."
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
# ArcGIS includes metadata as an html table in the description
|
19
|
+
def each_record(&block)
|
20
|
+
super do |record|
|
21
|
+
record.metadata = Hash[Nokogiri::XML(record.metadata[:description]).css('td').collect(&:text).each_slice(2).to_a]
|
22
|
+
yield record
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
require 'digest/md5'
|
3
|
+
|
4
|
+
module SpatialFeatures
|
5
|
+
module Importers
|
6
|
+
class Shapefile < Base
|
7
|
+
def cache_key
|
8
|
+
@cache_key ||= Digest::MD5.hexdigest(features.to_json)
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def each_record(&block)
|
14
|
+
file = Download.open(@data, unzip: '.shp')
|
15
|
+
proj4 = proj4_from_file(file)
|
16
|
+
RGeo::Shapefile::Reader.open(file.path) do |records|
|
17
|
+
records.each do |record|
|
18
|
+
yield OpenStruct.new data_from_wkt(record.geometry.as_text, proj4).merge(:metadata => record.attributes)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def proj4_from_file(file)
|
24
|
+
`gdalsrsinfo "#{file.path}" -o proj4`[/'(.+)'/,1] # Sanitize "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n"
|
25
|
+
end
|
26
|
+
|
27
|
+
def data_from_wkt(wkt, proj4)
|
28
|
+
ActiveRecord::Base.connection.select_one <<-SQL
|
29
|
+
SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog, GeometryType(ST_GeomFromText('#{wkt}')) AS feature_type
|
30
|
+
SQL
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module SpatialFeatures
|
2
|
+
module Unzip
|
3
|
+
def self.paths(file_path, find: nil)
|
4
|
+
dir = Dir.mktmpdir
|
5
|
+
paths = []
|
6
|
+
|
7
|
+
entries(file_path).each do |entry|
|
8
|
+
path = "#{dir}/#{entry.name}"
|
9
|
+
entry.extract(path)
|
10
|
+
paths << path
|
11
|
+
end
|
12
|
+
|
13
|
+
if find = Array.wrap(find).presence
|
14
|
+
paths = paths.detect {|path| find.any? {|pattern| path.include?(pattern) } }
|
15
|
+
raise(ImportError, "No file matched #{find}") unless paths.present?
|
16
|
+
end
|
17
|
+
|
18
|
+
return paths
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.names(file_path)
|
22
|
+
entries(file_path).collect(&:name)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.entries(file_path)
|
26
|
+
Zip::File.open(File.path(file_path))
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.is_zip?(file)
|
30
|
+
zip = file.readline.start_with?('PK')
|
31
|
+
file.rewind
|
32
|
+
return zip
|
33
|
+
rescue EOFError
|
34
|
+
return false
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class
|
1
|
+
class UpdateFeaturesJob < Struct.new(:options)
|
2
2
|
def perform
|
3
3
|
model = options[:spatial_model_type].find(options[:spatial_model_id])
|
4
4
|
|
@@ -22,7 +22,7 @@ class ArcGISUpdateFeaturesJob < Struct.new(:options)
|
|
22
22
|
normalized_messages = []
|
23
23
|
|
24
24
|
if message =~ /invalid KML representation/
|
25
|
-
normalized_messages += invalid_kml_reason(message).presence || ["
|
25
|
+
normalized_messages += invalid_kml_reason(message).presence || ["KML importer received invalid geometry."]
|
26
26
|
end
|
27
27
|
|
28
28
|
if message =~ /Self-intersection/
|
data/lib/spatial_features.rb
CHANGED
@@ -1,13 +1,28 @@
|
|
1
|
+
# Gems
|
2
|
+
require 'rgeo/shapefile'
|
3
|
+
require 'nokogiri'
|
4
|
+
require 'zip'
|
5
|
+
|
1
6
|
# LIB
|
2
7
|
require 'spatial_features/caching'
|
3
8
|
require 'spatial_features/venn_polygons'
|
4
|
-
require 'spatial_features/
|
9
|
+
require 'spatial_features/controller_helpers/spatial_extensions'
|
10
|
+
require 'spatial_features/download'
|
11
|
+
require 'spatial_features/unzip'
|
5
12
|
|
6
|
-
require 'spatial_features/
|
13
|
+
require 'spatial_features/has_spatial_features'
|
14
|
+
require 'spatial_features/has_spatial_features/feature_import'
|
15
|
+
require 'spatial_features/has_spatial_features/delayed_feature_import'
|
7
16
|
|
8
|
-
require 'spatial_features/
|
17
|
+
require 'spatial_features/importers/base'
|
18
|
+
require 'spatial_features/importers/file'
|
19
|
+
require 'spatial_features/importers/kml'
|
20
|
+
require 'spatial_features/importers/kml_file'
|
21
|
+
require 'spatial_features/importers/kml_file_arcgis'
|
22
|
+
require 'spatial_features/importers/geomark'
|
23
|
+
require 'spatial_features/importers/shapefile'
|
9
24
|
|
10
|
-
require 'spatial_features/workers/
|
25
|
+
require 'spatial_features/workers/update_features_job'
|
11
26
|
|
12
27
|
require 'spatial_features/engine'
|
13
28
|
|
@@ -19,3 +34,6 @@ end
|
|
19
34
|
|
20
35
|
# Load the act method
|
21
36
|
ActiveRecord::Base.send :extend, SpatialFeatures::ActMethod
|
37
|
+
|
38
|
+
# Suppress date warnings when unzipping KMZ saved by Google Earth, see https://github.com/rubyzip/rubyzip/issues/112
|
39
|
+
Zip.warn_invalid_date = false
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spatial_features
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Wallace
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-
|
12
|
+
date: 2016-11-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -18,9 +18,6 @@ dependencies:
|
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
20
|
version: '4.2'
|
21
|
-
- - ">="
|
22
|
-
- !ruby/object:Gem::Version
|
23
|
-
version: 4.2.0
|
24
21
|
type: :runtime
|
25
22
|
prerelease: false
|
26
23
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -28,23 +25,62 @@ dependencies:
|
|
28
25
|
- - "~>"
|
29
26
|
- !ruby/object:Gem::Version
|
30
27
|
version: '4.2'
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 4.2.0
|
34
28
|
- !ruby/object:Gem::Dependency
|
35
29
|
name: delayed_job_active_record
|
36
30
|
requirement: !ruby/object:Gem::Requirement
|
37
31
|
requirements:
|
38
32
|
- - "~>"
|
39
33
|
- !ruby/object:Gem::Version
|
40
|
-
version: 4.0
|
34
|
+
version: '4.0'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '4.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rgeo-shapefile
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0.4'
|
41
49
|
type: :runtime
|
42
50
|
prerelease: false
|
43
51
|
version_requirements: !ruby/object:Gem::Requirement
|
44
52
|
requirements:
|
45
53
|
- - "~>"
|
46
54
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
55
|
+
version: '0.4'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rubyzip
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '1.1'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.1'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: nokogiri
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '1.6'
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '1.6'
|
48
84
|
- !ruby/object:Gem::Dependency
|
49
85
|
name: pg
|
50
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -63,16 +99,16 @@ dependencies:
|
|
63
99
|
name: rspec
|
64
100
|
requirement: !ruby/object:Gem::Requirement
|
65
101
|
requirements:
|
66
|
-
- - "
|
102
|
+
- - "~>"
|
67
103
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
104
|
+
version: '3.5'
|
69
105
|
type: :development
|
70
106
|
prerelease: false
|
71
107
|
version_requirements: !ruby/object:Gem::Requirement
|
72
108
|
requirements:
|
73
|
-
- - "
|
109
|
+
- - "~>"
|
74
110
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
111
|
+
version: '3.5'
|
76
112
|
description: Adds spatial methods to a model.
|
77
113
|
email:
|
78
114
|
- contact@culturecode.ca
|
@@ -90,12 +126,22 @@ files:
|
|
90
126
|
- lib/spatial_features.rb
|
91
127
|
- lib/spatial_features/caching.rb
|
92
128
|
- lib/spatial_features/controller_helpers/spatial_extensions.rb
|
129
|
+
- lib/spatial_features/download.rb
|
93
130
|
- lib/spatial_features/engine.rb
|
94
131
|
- lib/spatial_features/has_spatial_features.rb
|
95
|
-
- lib/spatial_features/
|
132
|
+
- lib/spatial_features/has_spatial_features/delayed_feature_import.rb
|
133
|
+
- lib/spatial_features/has_spatial_features/feature_import.rb
|
134
|
+
- lib/spatial_features/importers/base.rb
|
135
|
+
- lib/spatial_features/importers/file.rb
|
136
|
+
- lib/spatial_features/importers/geomark.rb
|
137
|
+
- lib/spatial_features/importers/kml.rb
|
138
|
+
- lib/spatial_features/importers/kml_file.rb
|
139
|
+
- lib/spatial_features/importers/kml_file_arcgis.rb
|
140
|
+
- lib/spatial_features/importers/shapefile.rb
|
141
|
+
- lib/spatial_features/unzip.rb
|
96
142
|
- lib/spatial_features/venn_polygons.rb
|
97
143
|
- lib/spatial_features/version.rb
|
98
|
-
- lib/spatial_features/workers/
|
144
|
+
- lib/spatial_features/workers/update_features_job.rb
|
99
145
|
- lib/tasks/spatial_features_tasks.rake
|
100
146
|
homepage: https://github.com/culturecode/spatial_features
|
101
147
|
licenses:
|
@@ -117,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
117
163
|
version: '0'
|
118
164
|
requirements: []
|
119
165
|
rubyforge_project:
|
120
|
-
rubygems_version: 2.
|
166
|
+
rubygems_version: 2.5.2
|
121
167
|
signing_key:
|
122
168
|
specification_version: 4
|
123
169
|
summary: Adds spatial methods to a model.
|
@@ -1,144 +0,0 @@
|
|
1
|
-
module ArcGISKmzFeatures
|
2
|
-
require 'open-uri'
|
3
|
-
require 'digest/md5'
|
4
|
-
|
5
|
-
def update_features!(options = {})
|
6
|
-
@skip_invalid = options[:skip_invalid]
|
7
|
-
@make_valid = options[:make_valid]
|
8
|
-
@feature_error_messages = []
|
9
|
-
kml_array = []
|
10
|
-
cache_kml = ''
|
11
|
-
|
12
|
-
Array(arcgis_kmz_url).each do |url|
|
13
|
-
kml_array << open_kmz_url(url)
|
14
|
-
cache_kml << kml_array.last.to_s
|
15
|
-
end
|
16
|
-
|
17
|
-
if has_spatial_features_hash?
|
18
|
-
new_features_hash = Digest::MD5.hexdigest(cache_kml) if cache_kml.present?
|
19
|
-
|
20
|
-
if new_features_hash != self.features_hash
|
21
|
-
replace_features(kml_array)
|
22
|
-
update_attributes(:features_hash => new_features_hash)
|
23
|
-
else
|
24
|
-
return false
|
25
|
-
end
|
26
|
-
else
|
27
|
-
replace_features(kml_array)
|
28
|
-
end
|
29
|
-
|
30
|
-
return true
|
31
|
-
end
|
32
|
-
|
33
|
-
def queue_feature_update!(options = {})
|
34
|
-
Delayed::Job.enqueue ArcGISUpdateFeaturesJob.new(options.merge :spatial_model_type => self.class, :spatial_model_id => self.id), :queue => delayed_jobs_queue_name
|
35
|
-
end
|
36
|
-
|
37
|
-
def updating_features?
|
38
|
-
running_feature_update_jobs.exists?
|
39
|
-
end
|
40
|
-
|
41
|
-
def feature_update_error
|
42
|
-
(failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
|
43
|
-
end
|
44
|
-
|
45
|
-
def running_feature_update_jobs
|
46
|
-
feature_update_jobs.where(failed_at: nil)
|
47
|
-
end
|
48
|
-
|
49
|
-
def failed_feature_update_jobs
|
50
|
-
feature_update_jobs.where.not(failed_at: nil)
|
51
|
-
end
|
52
|
-
|
53
|
-
def feature_update_jobs
|
54
|
-
Delayed::Job.where(queue: delayed_jobs_queue_name)
|
55
|
-
end
|
56
|
-
|
57
|
-
private
|
58
|
-
|
59
|
-
def delayed_jobs_queue_name
|
60
|
-
"#{self.class}/#{self.id}/update_features"
|
61
|
-
end
|
62
|
-
|
63
|
-
def replace_features(kml_array)
|
64
|
-
new_features = []
|
65
|
-
kml_array.each {|kml| new_features.concat build_features(kml) }
|
66
|
-
|
67
|
-
ActiveRecord::Base.transaction do
|
68
|
-
self.features.destroy_all
|
69
|
-
new_features.each(&:save)
|
70
|
-
self.clear_association_cache # clear_association_cache so after_feature_update knows about the new features
|
71
|
-
|
72
|
-
@feature_error_messages.concat new_features.collect {|feature| "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}" if feature.errors.present? }.compact.flatten
|
73
|
-
if @feature_error_messages.present? && !@skip_invalid
|
74
|
-
raise UpdateError, "Error updating #{self.class} #{self.id}. #{@feature_error_messages.to_sentence}"
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
def build_features(kml)
|
80
|
-
new_type_features = []
|
81
|
-
|
82
|
-
extract_kml_features(kml) do |feature_type, feature, name, metadata|
|
83
|
-
begin
|
84
|
-
new_type_features << build_feature(feature_type, name, metadata, build_geom(feature))
|
85
|
-
rescue => e
|
86
|
-
@feature_error_messages << e.message
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
return new_type_features
|
91
|
-
end
|
92
|
-
|
93
|
-
# Use ST_Force_2D to discard z-coordinates that cause failures later in the process
|
94
|
-
def build_geom(feature)
|
95
|
-
if make_valid?
|
96
|
-
geom = ActiveRecord::Base.connection.select_value("SELECT ST_CollectionExtract(ST_MakeValid(ST_Force_2D(ST_GeomFromKML('#{feature}'))),3)")
|
97
|
-
else
|
98
|
-
geom = ActiveRecord::Base.connection.select_value("SELECT ST_Force_2D(ST_GeomFromKML('#{feature}'))")
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def extract_kml_features(kml, &block)
|
103
|
-
Nokogiri::XML(kml).css('Placemark').each do |placemark|
|
104
|
-
name = placemark.css('name').text
|
105
|
-
metadata = Hash[Nokogiri::XML(placemark.css('description').text).css('td').collect(&:text).each_slice(2).to_a]
|
106
|
-
|
107
|
-
{'Polygon' => 'POLYGON', 'LineString' => 'LINE', 'Point' => 'POINT'}.each do |kml_type, sql_type|
|
108
|
-
placemark.css(kml_type).each do |feature|
|
109
|
-
yield sql_type, feature, name, metadata
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
def build_feature(feature_type, name, metadata, geom)
|
116
|
-
Feature.new(:spatial_model => self, :name => name, :metadata => metadata, :feature_type => feature_type, :geog => geom)
|
117
|
-
end
|
118
|
-
|
119
|
-
def open_kmz_url(url)
|
120
|
-
url = URI(url)
|
121
|
-
|
122
|
-
Zip::InputStream.open(open(url)) do |io|
|
123
|
-
while (entry = io.get_next_entry)
|
124
|
-
return io.read if entry.name.downcase == 'doc.kml'
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
return nil
|
129
|
-
|
130
|
-
rescue SocketError, Errno::ECONNREFUSED
|
131
|
-
raise UpdateError, "ArcGIS Server is not responding. Ensure ArcGIS Server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
|
132
|
-
rescue OpenURI::HTTPError
|
133
|
-
raise UpdateError, "ArcGIS Map Service not found. Ensure ArcGIS Server is running and accessible at #{url}."
|
134
|
-
rescue => e
|
135
|
-
raise UpdateError, e.message
|
136
|
-
end
|
137
|
-
|
138
|
-
# Can be overridden to use PostGIS to force geometry to be valid
|
139
|
-
def make_valid?
|
140
|
-
!!@make_valid
|
141
|
-
end
|
142
|
-
|
143
|
-
class UpdateError < StandardError; end
|
144
|
-
end
|