logstash-filter-wkt_repair 0.1.11 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -3
- data/lib/logstash/filters/wkt_repair.rb +17 -11
- data/logstash-filter-wkt_repair.gemspec +2 -2
- data/spec/filters/wkt_repair_spec.rb +21 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1630bff071ee79eb0873f68c5133d716f4ac7e0cf8bfcc3e21dabd2229d6e694
|
4
|
+
data.tar.gz: 10786660fbf15502655b667594e5ac8a88f53c2f3a47295562edee2a991c6973
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 931060715411a7ff254aed275865d505eb3973fe48f8e06f48726f3e242edc1749bd5a5ba9c56815286fbcc21cfd6b10eb7cab178535ccdd5cf9810afc66eb92
|
7
|
+
data.tar.gz: 4767a4bb5de36a06b76f4383a1f177b6fea737c8d126fe42aa0e0358367e1d525ea5b1b1d1d7811c03b22a82f59a6d9e0793d1ae8b15c3c8ea779c504a45ac33
|
data/README.md
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
# WKT Repair Logstash Filter
|
1
|
+
# WKT or GeoJson Repair Logstash Filter
|
2
2
|
|
3
3
|
[](https://badge.fury.io/rb/logstash-filter-wkt_repair)
|
4
4
|
|
5
|
-
Repairs WKT Polygons/Multipolygons on a Logstash event that are ambiguous or ill-defined and returns a coherent and clearly defined output.
|
5
|
+
Repairs WKT or GeoJson Polygons/Multipolygons on a Logstash event that are ambiguous or ill-defined and returns a coherent and clearly defined output.
|
6
6
|
|
7
7
|
Examples of ambigous or ill-defined polygons include, but not limited to:
|
8
8
|
|
@@ -18,6 +18,7 @@ Examples of ambigous or ill-defined polygons include, but not limited to:
|
|
18
18
|
wkt_repair {
|
19
19
|
source => "wkt"
|
20
20
|
target => "wkt_repaired"
|
21
|
+
type => "WKT|GeoJson"
|
21
22
|
tag_on_failure => [ "_wkt_repair_failure" ]
|
22
23
|
}
|
23
24
|
}
|
@@ -74,8 +75,9 @@ Add `/opt/prepair` to your $PATH
|
|
74
75
|
### 1. Plugin Developement and Testing
|
75
76
|
|
76
77
|
#### Code
|
77
|
-
- To get started, you'll need JRuby
|
78
|
+
- To get started, you'll need rvm, JRuby and the Bundler gem installed.
|
78
79
|
```sh
|
80
|
+
\curl -sSL https://get.rvm.io | bash -s stable
|
79
81
|
rvm install jruby
|
80
82
|
jruby -S gem install bundler
|
81
83
|
```
|
@@ -10,15 +10,17 @@ class LogStash::Filters::WktRepair < LogStash::Filters::Base
|
|
10
10
|
# [source,ruby]
|
11
11
|
# ----------------------------------
|
12
12
|
# wkt_repair => {
|
13
|
-
# "source" => "name of field containing WKT
|
13
|
+
# "source" => "name of field containing WKT or GeoJson data"
|
14
14
|
# "target" => "name of field to put the repaired WKT"
|
15
|
+
# "type" => "the type of data. WKT or GeoJson"
|
15
16
|
# }
|
16
17
|
# ----------------------------------
|
17
18
|
config_name "wkt_repair"
|
18
19
|
|
19
20
|
config :source, :validate => :string, :required => true
|
20
|
-
config :target, :validate => :string, :default => '
|
21
|
-
config :
|
21
|
+
config :target, :validate => :string, :default => 'data_repaired'
|
22
|
+
config :type, :validate => :string, :default => 'WKT'
|
23
|
+
config :tag_on_failure, :validate => :array, :default => [ '_data_repair_failure' ]
|
22
24
|
|
23
25
|
public
|
24
26
|
def register
|
@@ -28,27 +30,31 @@ class LogStash::Filters::WktRepair < LogStash::Filters::Base
|
|
28
30
|
public
|
29
31
|
def filter(event)
|
30
32
|
|
31
|
-
|
33
|
+
data = event.get(@source)
|
34
|
+
type = @type
|
32
35
|
event_uuid = SecureRandom.uuid
|
33
36
|
random_hex = SecureRandom.hex(10)
|
34
|
-
temp_file = "temp-
|
37
|
+
temp_file = "temp-data_#{event_uuid}_#{random_hex}.txt"
|
35
38
|
|
36
|
-
unless
|
39
|
+
unless data.nil?
|
37
40
|
begin
|
38
|
-
IO.write(temp_file,
|
41
|
+
IO.write(temp_file, data)
|
39
42
|
|
40
43
|
wkt_repair_cmd = "prepair -f #{temp_file}"
|
44
|
+
if type.eql?("GeoJson")
|
45
|
+
wkt_repair_cmd = "prepair --ogr #{temp_file}"
|
46
|
+
end
|
41
47
|
stdout, stderr, status = Open3.capture3("#{wkt_repair_cmd}")
|
42
48
|
|
43
49
|
if (status.success? && !stdout.nil?)
|
44
50
|
event.set(@target, stdout.strip)
|
45
51
|
else
|
46
|
-
@logger.error("
|
47
|
-
@logger.error("
|
52
|
+
@logger.error("#{type} Repair Error: #{stderr}")
|
53
|
+
@logger.error("#{type} Repair Output Message: #{stdout}")
|
48
54
|
@tag_on_failure.each { |tag| event.tag(tag) }
|
49
55
|
end
|
50
56
|
rescue Exception => e
|
51
|
-
@logger.error("
|
57
|
+
@logger.error("#{type} Repair Exception", :exception => e, :stacktrace => e.backtrace.join("\n"))
|
52
58
|
@tag_on_failure.each { |tag| event.tag(tag) }
|
53
59
|
ensure
|
54
60
|
begin
|
@@ -56,7 +62,7 @@ class LogStash::Filters::WktRepair < LogStash::Filters::Base
|
|
56
62
|
File.delete(temp_file)
|
57
63
|
end
|
58
64
|
rescue Exception => e
|
59
|
-
@logger.error("
|
65
|
+
@logger.error("#{type} Repair Exception in ensure block", :exception => e, :stacktrace => e.backtrace.join("\n"))
|
60
66
|
end
|
61
67
|
end
|
62
68
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-wkt_repair'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
|
-
s.summary = 'Logstash filter to repair a WKT shape data.'
|
5
|
+
s.summary = 'Logstash filter to repair a WKT/GeoJson shape data.'
|
6
6
|
s.description = 'It takes ambigous or ill-defined polygons and returns a coherent and clearly defined output.'
|
7
7
|
s.homepage = 'https://github.com/sibenye/logstash-filter-wkt_repair'
|
8
8
|
s.authors = ['Silver Ibenye']
|
@@ -21,6 +21,26 @@ describe LogStash::Filters::WktRepair do
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
+
describe LogStash::Filters::WktRepair do
|
25
|
+
describe "Successfully repair a GeoJson" do
|
26
|
+
let(:config) do <<-CONFIG
|
27
|
+
filter {
|
28
|
+
wkt_repair {
|
29
|
+
source => "geojson"
|
30
|
+
target => "geojson"
|
31
|
+
type => "GeoJson"
|
32
|
+
}
|
33
|
+
}
|
34
|
+
CONFIG
|
35
|
+
end
|
36
|
+
|
37
|
+
sample("geojson" => "{\"type\":\"Polygon\",\"coordinates\":[[[0,0],[0,10],[10,0],[10,10],[0,0]]]}") do
|
38
|
+
expect(subject).to include("geojson")
|
39
|
+
expect(subject.get('geojson')).to eq('MULTIPOLYGON (((0 10,0 0,5 5,0 10)),((5 5,10 0,10 10,5 5)))')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
24
44
|
describe LogStash::Filters::WktRepair do
|
25
45
|
describe "Unsuccessfully repair a WKT" do
|
26
46
|
let(:config) do <<-CONFIG
|
@@ -36,7 +56,7 @@ describe LogStash::Filters::WktRepair do
|
|
36
56
|
sample("geometry" => "POLYGON((0 0, 10 0, 10 11, 11 10, 0 10") do
|
37
57
|
expect(subject).to include("geometry")
|
38
58
|
expect(subject.get('geometry')).to eq('POLYGON((0 0, 10 0, 10 11, 11 10, 0 10')
|
39
|
-
expect(subject.get('tags')).to include("
|
59
|
+
expect(subject.get('tags')).to include("_data_repair_failure")
|
40
60
|
|
41
61
|
end
|
42
62
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-wkt_repair
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Silver Ibenye
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-05-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,7 +86,7 @@ rubyforge_project:
|
|
86
86
|
rubygems_version: 2.6.13
|
87
87
|
signing_key:
|
88
88
|
specification_version: 4
|
89
|
-
summary: Logstash filter to repair a WKT shape data.
|
89
|
+
summary: Logstash filter to repair a WKT/GeoJson shape data.
|
90
90
|
test_files:
|
91
91
|
- spec/filters/wkt_repair_spec.rb
|
92
92
|
- spec/spec_helper.rb
|