spatial_features 2.17.3 → 2.20.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -1
- data/app/models/abstract_feature.rb +5 -2
- data/app/models/aggregate_feature.rb +5 -2
- data/app/models/feature.rb +6 -2
- data/lib/spatial_features/has_spatial_features/feature_import.rb +19 -2
- data/lib/spatial_features/has_spatial_features/queued_spatial_processing.rb +60 -4
- data/lib/spatial_features/importers/base.rb +2 -1
- data/lib/spatial_features/importers/kml.rb +30 -3
- data/lib/spatial_features/importers/kml_file.rb +7 -3
- data/lib/spatial_features/unzip.rb +4 -0
- data/lib/spatial_features/version.rb +1 -1
- metadata +6 -21
- data/config/initializers/chroma_serializers.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 639dee46fedd2cf507642b41119ff99e93506b54d891ab42d21a1f24fcf5a679
|
4
|
+
data.tar.gz: 956175227c6a59acc867810a51796303218eac70c9bc49fe0d0f8222ea8558e6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6a1ac7a0dca15400e0e03f0f1c2192e22a4c1e818edde2a832f925e6ee803a9284c5236684228e25dbeabf2d147aacef4f4c05170216b0857941c87ed9b9a395
|
7
|
+
data.tar.gz: da6fc8be7449202dfe46de18f0829e49664626ba17b90b6e3c1eae7a059f38a10d8310046e4f2bf44fb80bf78f2cdeea26624f429fc0d7b5873775b332086712
|
data/README.md
CHANGED
@@ -109,8 +109,17 @@ Person.new(:features => [Feature.new(:geog => 'some binary PostGIS Geography str
|
|
109
109
|
You can specify multiple import sources for geometry. Each key is a method that returns the data for the Importer, and
|
110
110
|
each value is the Importer to use to parse the data. See each Importer for more details.
|
111
111
|
```ruby
|
112
|
+
def ImageImporter
|
113
|
+
def self.call(feature, image_paths)
|
114
|
+
image_paths.each do |pathname|
|
115
|
+
# ...
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
112
120
|
class Location < ActiveRecord::Base
|
113
|
-
has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File', :geojson => 'GeoJSON' }
|
121
|
+
has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File', :geojson => 'GeoJSON' },
|
122
|
+
:image_handlers => ['ImageImporter']
|
114
123
|
|
115
124
|
def remote_kml_url
|
116
125
|
"www.test.com/kml/#{id}.kml"
|
@@ -49,8 +49,7 @@ class AbstractFeature < ActiveRecord::Base
|
|
49
49
|
where(:feature_type => 'point')
|
50
50
|
end
|
51
51
|
|
52
|
-
def self.
|
53
|
-
# where("ST_DWithin(features.geog, ST_SetSRID( ST_Point( -71.104, 42.315), 4326)::geography, :distance)", :lat => lat, :lng => lng, :distance => distance_in_meters)
|
52
|
+
def self.within_distance_of_point(lat, lng, distance_in_meters)
|
54
53
|
where("ST_DWithin(features.geog, ST_Point(:lng, :lat), :distance)", :lat => lat, :lng => lng, :distance => distance_in_meters)
|
55
54
|
end
|
56
55
|
|
@@ -73,6 +72,10 @@ class AbstractFeature < ActiveRecord::Base
|
|
73
72
|
join_other_features(other).where('ST_Intersects(features.geom_lowres, other_features.geom_lowres)').uniq
|
74
73
|
end
|
75
74
|
|
75
|
+
def self.within_distance(other, distance_in_meters)
|
76
|
+
join_other_features(other).where('ST_DWithin(features.geom_lowres, other_features.geom_lowres, ?)', distance_in_meters).uniq
|
77
|
+
end
|
78
|
+
|
76
79
|
def self.invalid(column = 'geog::geometry')
|
77
80
|
select("features.*, ST_IsValidReason(#{column}) AS invalid_geometry_message").where.not("ST_IsValid(#{column})")
|
78
81
|
end
|
@@ -4,7 +4,11 @@ class AggregateFeature < AbstractFeature
|
|
4
4
|
has_many :features, lambda { |aggregate| where(:spatial_model_type => aggregate.spatial_model_type) }, :foreign_key => :spatial_model_id, :primary_key => :spatial_model_id
|
5
5
|
|
6
6
|
# Aggregate the features for the spatial model into a single feature
|
7
|
-
|
7
|
+
before_validation :set_geog, :on => :create, :unless => :geog?
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def set_geog
|
8
12
|
feature_array_sql = <<~SQL
|
9
13
|
ARRAY[
|
10
14
|
(#{features.select('ST_UNION(ST_CollectionExtract(geog::geometry, 1))').to_sql}),
|
@@ -19,6 +23,5 @@ class AggregateFeature < AbstractFeature
|
|
19
23
|
FROM (SELECT unnest(#{feature_array_sql})) AS features
|
20
24
|
WHERE NOT ST_IsEmpty(unnest)
|
21
25
|
SQL
|
22
|
-
self.save!
|
23
26
|
end
|
24
27
|
end
|
data/app/models/feature.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require_dependency SpatialFeatures::Engine.root.join('app/models/abstract_feature')
|
2
|
+
|
1
3
|
class Feature < AbstractFeature
|
2
4
|
class_attribute :automatically_refresh_aggregate
|
3
5
|
self.automatically_refresh_aggregate = true
|
@@ -13,6 +15,8 @@ class Feature < AbstractFeature
|
|
13
15
|
|
14
16
|
after_save :refresh_aggregate, if: :automatically_refresh_aggregate?
|
15
17
|
|
18
|
+
attr_accessor :importable_image_paths # :nodoc:
|
19
|
+
|
16
20
|
# Features are used for display so we also cache their KML representation
|
17
21
|
def self.cache_derivatives(options = {})
|
18
22
|
super
|
@@ -52,8 +56,8 @@ class Feature < AbstractFeature
|
|
52
56
|
end
|
53
57
|
|
54
58
|
def refresh_aggregate
|
55
|
-
|
56
|
-
|
59
|
+
aggregate_feature&.destroy # Destroy the existing aggregate feature to ensure its cache key changes when it is refreshed
|
60
|
+
create_aggregate_feature!
|
57
61
|
end
|
58
62
|
|
59
63
|
def automatically_refresh_aggregate?
|
@@ -8,7 +8,7 @@ module SpatialFeatures
|
|
8
8
|
included do
|
9
9
|
extend ActiveModel::Callbacks
|
10
10
|
define_model_callbacks :update_features
|
11
|
-
spatial_features_options.reverse_merge!(:import => {}, spatial_cache: [])
|
11
|
+
spatial_features_options.reverse_merge!(:import => {}, :spatial_cache => [], :image_handlers => [])
|
12
12
|
end
|
13
13
|
|
14
14
|
module ClassMethods
|
@@ -78,13 +78,30 @@ module SpatialFeatures
|
|
78
78
|
"SpatialFeatures::Importers::#{importer_name}".constantize
|
79
79
|
end
|
80
80
|
|
81
|
+
def handle_images(feature)
|
82
|
+
return if feature.importable_image_paths.nil? || feature.importable_image_paths.empty?
|
83
|
+
|
84
|
+
Array(spatial_features_options[:image_handlers]).each do |image_handler|
|
85
|
+
image_handler_from_name(image_handler).call(feature, feature.importable_image_paths)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def image_handler_from_name(handler_name)
|
90
|
+
handler_name.to_s.constantize
|
91
|
+
end
|
92
|
+
|
81
93
|
def import_features(imports, skip_invalid)
|
82
94
|
features.delete_all
|
83
95
|
valid, invalid = Feature.defer_aggregate_refresh do
|
84
96
|
Feature.without_caching_derivatives do
|
85
97
|
imports.flat_map(&:features).partition do |feature|
|
86
98
|
feature.spatial_model = self
|
87
|
-
feature.save
|
99
|
+
if feature.save
|
100
|
+
handle_images(feature)
|
101
|
+
true
|
102
|
+
else
|
103
|
+
false
|
104
|
+
end
|
88
105
|
end
|
89
106
|
end
|
90
107
|
end
|
@@ -11,7 +11,22 @@ module SpatialFeatures
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def updating_features?
|
14
|
-
|
14
|
+
case spatial_processing_status(:update_features!)
|
15
|
+
when :queued, :processing
|
16
|
+
true
|
17
|
+
else
|
18
|
+
false
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def updating_features_failed?
|
23
|
+
spatial_processing_status(:update_features!) == :failure
|
24
|
+
end
|
25
|
+
|
26
|
+
def spatial_processing_status(method_name)
|
27
|
+
if has_attribute?(:spatial_processing_status_cache)
|
28
|
+
spatial_processing_status_cache[method_name.to_s]&.to_sym
|
29
|
+
end
|
15
30
|
end
|
16
31
|
|
17
32
|
def feature_update_error
|
@@ -27,17 +42,58 @@ module SpatialFeatures
|
|
27
42
|
end
|
28
43
|
|
29
44
|
def spatial_processing_jobs(suffix = nil)
|
30
|
-
Delayed::Job.where(
|
45
|
+
Delayed::Job.where(:queue => "#{spatial_processing_queue_name}#{suffix}")
|
31
46
|
end
|
32
47
|
|
33
48
|
private
|
34
49
|
|
35
50
|
def queue_spatial_task(method_name, *args)
|
36
|
-
|
51
|
+
Delayed::Job.enqueue SpatialProcessingJob.new(self, method_name, *args), :queue => spatial_processing_queue_name + method_name
|
37
52
|
end
|
38
53
|
|
39
54
|
def spatial_processing_queue_name
|
40
|
-
"#{
|
55
|
+
"#{model_name}/#{id}/"
|
56
|
+
end
|
57
|
+
|
58
|
+
# CLASSES
|
59
|
+
|
60
|
+
class SpatialProcessingJob
|
61
|
+
def initialize(record, method_name, *args)
|
62
|
+
@record = record
|
63
|
+
@method_name = method_name
|
64
|
+
@args = args
|
65
|
+
end
|
66
|
+
|
67
|
+
def enqueue(job)
|
68
|
+
update_cached_status(:queued)
|
69
|
+
end
|
70
|
+
|
71
|
+
def perform
|
72
|
+
update_cached_status(:processing)
|
73
|
+
@record.send(@method_name, *@args)
|
74
|
+
end
|
75
|
+
|
76
|
+
def success(job)
|
77
|
+
update_cached_status(:success)
|
78
|
+
end
|
79
|
+
|
80
|
+
def error(job, exception)
|
81
|
+
update_cached_status(:failure)
|
82
|
+
end
|
83
|
+
|
84
|
+
def failure(job)
|
85
|
+
update_cached_status(:failure)
|
86
|
+
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def update_cached_status(state)
|
91
|
+
if @record.has_attribute?(:spatial_processing_status_cache)
|
92
|
+
cache = @record.spatial_processing_status_cache || {}
|
93
|
+
cache[@method_name] = state
|
94
|
+
@record.update_column(:spatial_processing_status_cache, cache)
|
95
|
+
end
|
96
|
+
end
|
41
97
|
end
|
42
98
|
end
|
43
99
|
end
|
@@ -46,7 +46,8 @@ module SpatialFeatures
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def build_feature(record)
|
49
|
-
|
49
|
+
importable_image_paths = record.importable_image_paths if record.respond_to?(:importable_image_paths)
|
50
|
+
Feature.new(:name => record.name, :metadata => record.metadata, :feature_type => record.feature_type, :geog => record.geog, :importable_image_paths => importable_image_paths, :make_valid => @make_valid)
|
50
51
|
end
|
51
52
|
end
|
52
53
|
end
|
@@ -3,6 +3,14 @@ require 'ostruct'
|
|
3
3
|
module SpatialFeatures
|
4
4
|
module Importers
|
5
5
|
class KML < Base
|
6
|
+
# <SimpleData name> keys that may contain <img> tags
|
7
|
+
IMAGE_METADATA_KEYS = %w[pdfmaps_photos].freeze
|
8
|
+
|
9
|
+
def initialize(data, base_dir: nil, **args)
|
10
|
+
@base_dir = base_dir
|
11
|
+
super data, **args
|
12
|
+
end
|
13
|
+
|
6
14
|
private
|
7
15
|
|
8
16
|
def each_record(&block)
|
@@ -18,10 +26,11 @@ module SpatialFeatures
|
|
18
26
|
next if blank_feature?(feature)
|
19
27
|
|
20
28
|
geog = geom_from_kml(feature)
|
21
|
-
|
22
29
|
next if geog.blank?
|
23
30
|
|
24
|
-
|
31
|
+
importable_image_paths = images_from_metadata(metadata)
|
32
|
+
|
33
|
+
yield OpenStruct.new(:feature_type => sql_type, :geog => geog, :name => name, :metadata => metadata, :importable_image_paths => importable_image_paths)
|
25
34
|
end
|
26
35
|
end
|
27
36
|
end
|
@@ -32,17 +41,35 @@ module SpatialFeatures
|
|
32
41
|
|
33
42
|
def geom_from_kml(kml)
|
34
43
|
geom = nil
|
44
|
+
conn = nil
|
35
45
|
|
36
46
|
# Do query in a new thread so we use a new connection (if the query fails it will poison the transaction of the current connection)
|
47
|
+
#
|
48
|
+
# We manually checkout a new connection since Rails re-uses DB connections across threads.
|
37
49
|
Thread.new do
|
38
|
-
|
50
|
+
conn = ActiveRecord::Base.connection_pool.checkout
|
51
|
+
geom = conn.select_value("SELECT ST_GeomFromKML(#{conn.quote(kml.to_s)})")
|
39
52
|
rescue ActiveRecord::StatementInvalid => e # Discard Invalid KML features
|
40
53
|
geom = nil
|
54
|
+
ensure
|
55
|
+
ActiveRecord::Base.connection_pool.checkin(conn) if conn
|
41
56
|
end.join
|
42
57
|
|
43
58
|
return geom
|
44
59
|
end
|
45
60
|
|
61
|
+
def images_from_metadata(metadata)
|
62
|
+
IMAGE_METADATA_KEYS.flat_map do |key|
|
63
|
+
images = metadata.delete(key)
|
64
|
+
next unless images
|
65
|
+
|
66
|
+
Nokogiri::HTML.fragment(images).css("img").map do |img|
|
67
|
+
next unless (src = img["src"])
|
68
|
+
@base_dir.join(src.downcase)
|
69
|
+
end
|
70
|
+
end.compact
|
71
|
+
end
|
72
|
+
|
46
73
|
def extract_metadata(placemark)
|
47
74
|
metadata = {}
|
48
75
|
metadata.merge! extract_table(placemark)
|
@@ -1,13 +1,17 @@
|
|
1
1
|
module SpatialFeatures
|
2
2
|
module Importers
|
3
3
|
class KMLFile < KML
|
4
|
-
def initialize(path_or_url,
|
5
|
-
|
6
|
-
|
4
|
+
def initialize(path_or_url, **options)
|
5
|
+
path = Download.open_each(path_or_url, unzip: [/\.kml$/], downcase: true).first
|
6
|
+
super ::File.read(path), base_dir: Pathname.new(path).dirname, **options
|
7
7
|
rescue SocketError, Errno::ECONNREFUSED, OpenURI::HTTPError
|
8
8
|
url = URI(path_or_url)
|
9
9
|
raise ImportError, "KML server is not responding. Ensure server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
|
10
10
|
end
|
11
|
+
|
12
|
+
def cache_key
|
13
|
+
@cache_key ||= Digest::MD5.hexdigest(@data)
|
14
|
+
end
|
11
15
|
end
|
12
16
|
end
|
13
17
|
end
|
@@ -2,6 +2,9 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module SpatialFeatures
|
4
4
|
module Unzip
|
5
|
+
# paths containing '__macosx' or beginning with a '.'
|
6
|
+
IGNORED_ENTRY_PATHS = /(\A|\/)(__macosx|\.)/i.freeze
|
7
|
+
|
5
8
|
def self.paths(file_path, find: nil, **extract_options)
|
6
9
|
paths = extract(file_path, **extract_options)
|
7
10
|
|
@@ -16,6 +19,7 @@ module SpatialFeatures
|
|
16
19
|
def self.extract(file_path, output_dir = Dir.mktmpdir, downcase: false)
|
17
20
|
[].tap do |paths|
|
18
21
|
entries(file_path).each do |entry|
|
22
|
+
next if entry.name =~ IGNORED_ENTRY_PATHS
|
19
23
|
output_filename = entry.name
|
20
24
|
output_filename = output_filename.downcase if downcase
|
21
25
|
path = "#{output_dir}/#{output_filename}"
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spatial_features
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.20.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Wallace
|
8
8
|
- Nicholas Jakobsen
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-12-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -87,20 +87,6 @@ dependencies:
|
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '1.6'
|
90
|
-
- !ruby/object:Gem::Dependency
|
91
|
-
name: chroma
|
92
|
-
requirement: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: 0.1.0
|
97
|
-
type: :runtime
|
98
|
-
prerelease: false
|
99
|
-
version_requirements: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - "~>"
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: 0.1.0
|
104
90
|
- !ruby/object:Gem::Dependency
|
105
91
|
name: pg
|
106
92
|
requirement: !ruby/object:Gem::Requirement
|
@@ -144,7 +130,6 @@ files:
|
|
144
130
|
- app/models/feature.rb
|
145
131
|
- app/models/spatial_cache.rb
|
146
132
|
- app/models/spatial_proximity.rb
|
147
|
-
- config/initializers/chroma_serializers.rb
|
148
133
|
- config/initializers/mime_types.rb
|
149
134
|
- config/initializers/register_oids.rb
|
150
135
|
- lib/spatial_features.rb
|
@@ -175,7 +160,7 @@ homepage: https://github.com/culturecode/spatial_features
|
|
175
160
|
licenses:
|
176
161
|
- MIT
|
177
162
|
metadata: {}
|
178
|
-
post_install_message:
|
163
|
+
post_install_message:
|
179
164
|
rdoc_options: []
|
180
165
|
require_paths:
|
181
166
|
- lib
|
@@ -190,8 +175,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
190
175
|
- !ruby/object:Gem::Version
|
191
176
|
version: '0'
|
192
177
|
requirements: []
|
193
|
-
rubygems_version: 3.0.
|
194
|
-
signing_key:
|
178
|
+
rubygems_version: 3.0.3
|
179
|
+
signing_key:
|
195
180
|
specification_version: 4
|
196
181
|
summary: Adds spatial methods to a model.
|
197
182
|
test_files: []
|
@@ -1,15 +0,0 @@
|
|
1
|
-
module Chroma
|
2
|
-
class Color
|
3
|
-
module Serializers
|
4
|
-
# Google's Fusion Table colouring expects the alpha value in the last position, not the first
|
5
|
-
def to_ft_hex
|
6
|
-
[
|
7
|
-
to_2char_hex(@rgb.r),
|
8
|
-
to_2char_hex(@rgb.g),
|
9
|
-
to_2char_hex(@rgb.b),
|
10
|
-
to_2char_hex(alpha * 255)
|
11
|
-
].join('')
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|