logstash-input-yasuri 0.0.2 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +13 -1
- data/lib/logstash/inputs/yasuri.rb +11 -1
- data/logstash-input-yasuri.gemspec +5 -4
- data/spec/inputs/res/parse_tree.json +17 -0
- data/spec/inputs/yasuri_spec.rb +15 -1
- metadata +25 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01cf6f81b0de10a81ff1ddf0cd447e9d9b403fbd
|
4
|
+
data.tar.gz: 25c33d37b927c64bf413709386b500fad9a34ac8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 513917038a3714b3eacd95374446f6ff982c9744101630da21dc350fae34f047ac15ab6eaff80bff00e2d4845c215898ec9dab5685a1baa92794e95b05f02149
|
7
|
+
data.tar.gz: d923a7ffebee5d919046ba0479d9b02eed495307d679ac67d05642df1bf8600b4242a7efb8a5196e0588ded9886673dafb243e3d9aa68a18be777daed6f7afa0
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[](https://travis-ci.org/tac0x2a/logstash-input-yasuri)
|
4
4
|
|
5
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
6
6
|
|
@@ -45,6 +45,18 @@ bundle install
|
|
45
45
|
bundle exec rspec
|
46
46
|
```
|
47
47
|
|
48
|
+
- Build gem
|
49
|
+
|
50
|
+
```sh
|
51
|
+
gem build logstash-input-yasuri
|
52
|
+
# generate logstash-input-yasuri-x.x.x.gem
|
53
|
+
```
|
54
|
+
|
55
|
+
- publish gem
|
56
|
+
```sh
|
57
|
+
bundle exec rake publish_gem
|
58
|
+
```
|
59
|
+
|
48
60
|
### 2. Running your unpublished Plugin in Logstash
|
49
61
|
|
50
62
|
#### 2.1 Run in a local Logstash clone
|
@@ -10,10 +10,16 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
|
|
10
10
|
|
11
11
|
default :codec, "plain"
|
12
12
|
|
13
|
+
# logstash-input-yasuri require option :parse_tree or :parse_tree_path.
|
14
|
+
# If given both, logstash-input-yasuri use :parse_tree.
|
15
|
+
|
13
16
|
# Parse tree as JSON.
|
14
17
|
# Read https://github.com/tac0x2a/yasuri/blob/master/USAGE.md#construct-parse-tree
|
15
18
|
config :parse_tree, :validate => :string
|
16
19
|
|
20
|
+
# Path to parse tree JSON file.
|
21
|
+
config :parse_tree_path, :validate => :string
|
22
|
+
|
17
23
|
# Target web page url.
|
18
24
|
config :url, :validate => :string
|
19
25
|
|
@@ -26,7 +32,11 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
|
|
26
32
|
def register
|
27
33
|
@host = Socket.gethostname
|
28
34
|
@agent = Mechanize.new
|
29
|
-
|
35
|
+
|
36
|
+
# If given both, logstash-input-yasuri use :parse_tree.
|
37
|
+
tree = @parse_tree || File.read(@parse_tree_path)
|
38
|
+
|
39
|
+
@tree = Yasuri.json2tree(tree)
|
30
40
|
end # def register
|
31
41
|
|
32
42
|
def run(queue)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-yasuri'
|
3
|
-
s.version = '
|
3
|
+
s.version = '2.0.4'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Web scraping input plugin for logstash."
|
6
6
|
s.description = "Web scraping input plugin for logstash."
|
@@ -18,10 +18,11 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency
|
22
|
-
s.add_runtime_dependency 'logstash-codec-plain'
|
21
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 1'
|
23
22
|
s.add_runtime_dependency 'stud', '>= 0.0.22'
|
24
|
-
s.add_runtime_dependency 'yasuri', '
|
23
|
+
s.add_runtime_dependency 'yasuri', '~> 1.9'
|
24
|
+
|
25
25
|
s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
|
26
|
+
s.add_development_dependency 'logstash-codec-plain'
|
26
27
|
s.add_development_dependency 'fuubar', '>= 2.2.0'
|
27
28
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
{
|
2
|
+
"node": "struct",
|
3
|
+
"name": "titles",
|
4
|
+
"path": "//td[@class='title'][not(@align)]",
|
5
|
+
"children": [
|
6
|
+
{
|
7
|
+
"node": "text",
|
8
|
+
"name": "title",
|
9
|
+
"path": "./a"
|
10
|
+
},
|
11
|
+
{
|
12
|
+
"node": "text",
|
13
|
+
"name": "url",
|
14
|
+
"path": "./a/@href"
|
15
|
+
}
|
16
|
+
]
|
17
|
+
}
|
data/spec/inputs/yasuri_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe LogStash::Inputs::Yasuri do
|
|
6
6
|
|
7
7
|
let(:input) {
|
8
8
|
input = LogStash::Plugin.lookup("input", "yasuri").new(
|
9
|
-
"
|
9
|
+
"split" => true,
|
10
10
|
"url" => "https://news.ycombinator.com/",
|
11
11
|
"parse_tree" => %Q|
|
12
12
|
{
|
@@ -30,6 +30,14 @@ describe LogStash::Inputs::Yasuri do
|
|
30
30
|
)
|
31
31
|
}
|
32
32
|
|
33
|
+
let(:input_from_parse_tree_file) {
|
34
|
+
input_from_parse_tree_file = LogStash::Plugin.lookup("input", "yasuri").new(
|
35
|
+
"split" => true,
|
36
|
+
"url" => "https://news.ycombinator.com/",
|
37
|
+
"parse_tree_path" => "spec/inputs/res/parse_tree.json"
|
38
|
+
)
|
39
|
+
}
|
40
|
+
|
33
41
|
it "should register" do
|
34
42
|
# register will try to load jars and raise if it cannot find jars or if org.apache.log4j.spi.LoggingEvent class is not present
|
35
43
|
expect {input.register}.to_not raise_error
|
@@ -41,4 +49,10 @@ describe LogStash::Inputs::Yasuri do
|
|
41
49
|
input.inner_run(queue)
|
42
50
|
expect(queue.size).not_to be_zero
|
43
51
|
end
|
52
|
+
|
53
|
+
it "enqueues some events from input_from_parse_tree_file" do
|
54
|
+
input_from_parse_tree_file.register
|
55
|
+
input_from_parse_tree_file.inner_run(queue)
|
56
|
+
expect(queue.size).not_to be_zero
|
57
|
+
end
|
44
58
|
end
|
metadata
CHANGED
@@ -1,91 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-yasuri
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- tac0x2a
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-12-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- - "
|
17
|
-
- !ruby/object:Gem::Version
|
18
|
-
version: 2.0.0
|
19
|
-
- - "<="
|
16
|
+
- - "~>"
|
20
17
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
22
|
-
name: logstash-core
|
18
|
+
version: '1'
|
19
|
+
name: logstash-core-plugin-api
|
23
20
|
prerelease: false
|
24
21
|
type: :runtime
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
26
23
|
requirements:
|
27
|
-
- - "
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: 2.0.0
|
30
|
-
- - "<="
|
24
|
+
- - "~>"
|
31
25
|
- !ruby/object:Gem::Version
|
32
|
-
version:
|
26
|
+
version: '1'
|
33
27
|
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - ">="
|
37
31
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
39
|
-
name:
|
32
|
+
version: 0.0.22
|
33
|
+
name: stud
|
40
34
|
prerelease: false
|
41
35
|
type: :runtime
|
42
36
|
version_requirements: !ruby/object:Gem::Requirement
|
43
37
|
requirements:
|
44
38
|
- - ">="
|
45
39
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
40
|
+
version: 0.0.22
|
47
41
|
- !ruby/object:Gem::Dependency
|
48
42
|
requirement: !ruby/object:Gem::Requirement
|
49
43
|
requirements:
|
50
|
-
- - "
|
44
|
+
- - "~>"
|
51
45
|
- !ruby/object:Gem::Version
|
52
|
-
version:
|
53
|
-
name:
|
46
|
+
version: '1.9'
|
47
|
+
name: yasuri
|
54
48
|
prerelease: false
|
55
49
|
type: :runtime
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
51
|
requirements:
|
58
|
-
- - "
|
52
|
+
- - "~>"
|
59
53
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
54
|
+
version: '1.9'
|
61
55
|
- !ruby/object:Gem::Dependency
|
62
56
|
requirement: !ruby/object:Gem::Requirement
|
63
57
|
requirements:
|
64
58
|
- - ">="
|
65
59
|
- !ruby/object:Gem::Version
|
66
|
-
version: 0.0.
|
67
|
-
name:
|
60
|
+
version: 0.0.16
|
61
|
+
name: logstash-devutils
|
68
62
|
prerelease: false
|
69
|
-
type: :
|
63
|
+
type: :development
|
70
64
|
version_requirements: !ruby/object:Gem::Requirement
|
71
65
|
requirements:
|
72
66
|
- - ">="
|
73
67
|
- !ruby/object:Gem::Version
|
74
|
-
version: 0.0.
|
68
|
+
version: 0.0.16
|
75
69
|
- !ruby/object:Gem::Dependency
|
76
70
|
requirement: !ruby/object:Gem::Requirement
|
77
71
|
requirements:
|
78
72
|
- - ">="
|
79
73
|
- !ruby/object:Gem::Version
|
80
|
-
version: 0
|
81
|
-
name: logstash-
|
74
|
+
version: '0'
|
75
|
+
name: logstash-codec-plain
|
82
76
|
prerelease: false
|
83
77
|
type: :development
|
84
78
|
version_requirements: !ruby/object:Gem::Requirement
|
85
79
|
requirements:
|
86
80
|
- - ">="
|
87
81
|
- !ruby/object:Gem::Version
|
88
|
-
version: 0
|
82
|
+
version: '0'
|
89
83
|
- !ruby/object:Gem::Dependency
|
90
84
|
requirement: !ruby/object:Gem::Requirement
|
91
85
|
requirements:
|
@@ -114,6 +108,7 @@ files:
|
|
114
108
|
- README.md
|
115
109
|
- lib/logstash/inputs/yasuri.rb
|
116
110
|
- logstash-input-yasuri.gemspec
|
111
|
+
- spec/inputs/res/parse_tree.json
|
117
112
|
- spec/inputs/yasuri_spec.rb
|
118
113
|
homepage: https://github.com/tac0x2a/logstash-input-yasuri
|
119
114
|
licenses:
|
@@ -137,9 +132,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
132
|
version: '0'
|
138
133
|
requirements: []
|
139
134
|
rubyforge_project:
|
140
|
-
rubygems_version: 2.6.
|
135
|
+
rubygems_version: 2.6.8
|
141
136
|
signing_key:
|
142
137
|
specification_version: 4
|
143
138
|
summary: Web scraping input plugin for logstash.
|
144
139
|
test_files:
|
140
|
+
- spec/inputs/res/parse_tree.json
|
145
141
|
- spec/inputs/yasuri_spec.rb
|