logstash-input-yasuri 0.0.2 → 2.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +13 -1
  3. data/lib/logstash/inputs/yasuri.rb +11 -1
  4. data/logstash-input-yasuri.gemspec +5 -4
  5. data/spec/inputs/res/parse_tree.json +17 -0
  6. data/spec/inputs/yasuri_spec.rb +15 -1
  7. metadata +25 -29
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 637385a7b928621849907264f3306a3fb66cc00a
4
- data.tar.gz: 880e7867eca00f124d6c5eaad4284e17d20f7cf0
3
+ metadata.gz: 01cf6f81b0de10a81ff1ddf0cd447e9d9b403fbd
4
+ data.tar.gz: 25c33d37b927c64bf413709386b500fad9a34ac8
5
5
  SHA512:
6
- metadata.gz: 9618e43ea302b746791c8e0e68fde846d776694b883871f0d332031205f4b91b8edafdbb62c4889d13efb6521aa613c1980bd7f6b761b6f89d5cc8f0e5b8274d
7
- data.tar.gz: c2bbd5da29a32540e9e8a13c434cd765946509aeded6be8fa23b7d4d0751c8b1287da64d364aa39a00668d255b39ac2334439868a5dccb7b0dd5a3e301a6732c
6
+ metadata.gz: 513917038a3714b3eacd95374446f6ff982c9744101630da21dc350fae34f047ac15ab6eaff80bff00e2d4845c215898ec9dab5685a1baa92794e95b05f02149
7
+ data.tar.gz: d923a7ffebee5d919046ba0479d9b02eed495307d679ac67d05642df1bf8600b4242a7efb8a5196e0588ded9886673dafb243e3d9aa68a18be777daed6f7afa0
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Travis Build Status](https://travis-ci.org/tac0x2a/logstash-input-yasuri.svg)](https://travis-ci.org/tac0x2a/logstash-input-yasuri)
3
+ [![Build Status](https://travis-ci.org/tac0x2a/logstash-input-yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/logstash-input-yasuri)
4
4
 
5
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
6
6
 
@@ -45,6 +45,18 @@ bundle install
45
45
  bundle exec rspec
46
46
  ```
47
47
 
48
+ - Build gem
49
+
50
+ ```sh
51
+ gem build logstash-input-yasuri
52
+ # generate logstash-input-yasuri-x.x.x.gem
53
+ ```
54
+
55
+ - publish gem
56
+ ```sh
57
+ bundle exec rake publish_gem
58
+ ```
59
+
48
60
  ### 2. Running your unpublished Plugin in Logstash
49
61
 
50
62
  #### 2.1 Run in a local Logstash clone
data/lib/logstash/inputs/yasuri.rb CHANGED
@@ -10,10 +10,16 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
10
10
 
11
11
  default :codec, "plain"
12
12
 
13
+ # logstash-input-yasuri require option :parse_tree or :parse_tree_path.
14
+ # If given both, logstash-input-yasuri use :parse_tree.
15
+
13
16
  # Parse tree as JSON.
14
17
  # Read https://github.com/tac0x2a/yasuri/blob/master/USAGE.md#construct-parse-tree
15
18
  config :parse_tree, :validate => :string
16
19
 
20
+ # Path to parse tree JSON file.
21
+ config :parse_tree_path, :validate => :string
22
+
17
23
  # Target web page url.
18
24
  config :url, :validate => :string
19
25
 
@@ -26,7 +32,11 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
26
32
  def register
27
33
  @host = Socket.gethostname
28
34
  @agent = Mechanize.new
29
- @tree = Yasuri.json2tree(@parse_tree)
35
+
36
+ # If given both, logstash-input-yasuri use :parse_tree.
37
+ tree = @parse_tree || File.read(@parse_tree_path)
38
+
39
+ @tree = Yasuri.json2tree(tree)
30
40
  end # def register
31
41
 
32
42
  def run(queue)
data/logstash-input-yasuri.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-yasuri'
3
- s.version = '0.0.2'
3
+ s.version = '2.0.4'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Web scraping input plugin for logstash."
6
6
  s.description = "Web scraping input plugin for logstash."
@@ -18,10 +18,11 @@ Gem::Specification.new do |s|
18
18
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
19
19
 
20
20
  # Gem dependencies
21
- s.add_runtime_dependency "logstash-core", ">= 2.0.0", "<= 5.0.0"
22
- s.add_runtime_dependency 'logstash-codec-plain'
21
+ s.add_runtime_dependency 'logstash-core-plugin-api', '~> 1'
23
22
  s.add_runtime_dependency 'stud', '>= 0.0.22'
24
- s.add_runtime_dependency 'yasuri', '>= 0.0.1'
23
+ s.add_runtime_dependency 'yasuri', '~> 1.9'
24
+
25
25
  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
26
+ s.add_development_dependency 'logstash-codec-plain'
26
27
  s.add_development_dependency 'fuubar', '>= 2.2.0'
27
28
  end
data/spec/inputs/res/parse_tree.json ADDED
@@ -0,0 +1,17 @@
1
+ {
2
+ "node": "struct",
3
+ "name": "titles",
4
+ "path": "//td[@class='title'][not(@align)]",
5
+ "children": [
6
+ {
7
+ "node": "text",
8
+ "name": "title",
9
+ "path": "./a"
10
+ },
11
+ {
12
+ "node": "text",
13
+ "name": "url",
14
+ "path": "./a/@href"
15
+ }
16
+ ]
17
+ }
data/spec/inputs/yasuri_spec.rb CHANGED
@@ -6,7 +6,7 @@ describe LogStash::Inputs::Yasuri do
6
6
 
7
7
  let(:input) {
8
8
  input = LogStash::Plugin.lookup("input", "yasuri").new(
9
- "flatten" => true,
9
+ "split" => true,
10
10
  "url" => "https://news.ycombinator.com/",
11
11
  "parse_tree" => %Q|
12
12
  {
@@ -30,6 +30,14 @@ describe LogStash::Inputs::Yasuri do
30
30
  )
31
31
  }
32
32
 
33
+ let(:input_from_parse_tree_file) {
34
+ input_from_parse_tree_file = LogStash::Plugin.lookup("input", "yasuri").new(
35
+ "split" => true,
36
+ "url" => "https://news.ycombinator.com/",
37
+ "parse_tree_path" => "spec/inputs/res/parse_tree.json"
38
+ )
39
+ }
40
+
33
41
  it "should register" do
34
42
  # register will try to load jars and raise if it cannot find jars or if org.apache.log4j.spi.LoggingEvent class is not present
35
43
  expect {input.register}.to_not raise_error
@@ -41,4 +49,10 @@ describe LogStash::Inputs::Yasuri do
41
49
  input.inner_run(queue)
42
50
  expect(queue.size).not_to be_zero
43
51
  end
52
+
53
+ it "enqueues some events from input_from_parse_tree_file" do
54
+ input_from_parse_tree_file.register
55
+ input_from_parse_tree_file.inner_run(queue)
56
+ expect(queue.size).not_to be_zero
57
+ end
44
58
  end
metadata CHANGED
@@ -1,91 +1,85 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-yasuri
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 2.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - tac0x2a
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-06 00:00:00.000000000 Z
11
+ date: 2016-12-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ">="
17
- - !ruby/object:Gem::Version
18
- version: 2.0.0
19
- - - "<="
16
+ - - "~>"
20
17
  - !ruby/object:Gem::Version
21
- version: 5.0.0
22
- name: logstash-core
18
+ version: '1'
19
+ name: logstash-core-plugin-api
23
20
  prerelease: false
24
21
  type: :runtime
25
22
  version_requirements: !ruby/object:Gem::Requirement
26
23
  requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- version: 2.0.0
30
- - - "<="
24
+ - - "~>"
31
25
  - !ruby/object:Gem::Version
32
- version: 5.0.0
26
+ version: '1'
33
27
  - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - ">="
37
31
  - !ruby/object:Gem::Version
38
- version: '0'
39
- name: logstash-codec-plain
32
+ version: 0.0.22
33
+ name: stud
40
34
  prerelease: false
41
35
  type: :runtime
42
36
  version_requirements: !ruby/object:Gem::Requirement
43
37
  requirements:
44
38
  - - ">="
45
39
  - !ruby/object:Gem::Version
46
- version: '0'
40
+ version: 0.0.22
47
41
  - !ruby/object:Gem::Dependency
48
42
  requirement: !ruby/object:Gem::Requirement
49
43
  requirements:
50
- - - ">="
44
+ - - "~>"
51
45
  - !ruby/object:Gem::Version
52
- version: 0.0.22
53
- name: stud
46
+ version: '1.9'
47
+ name: yasuri
54
48
  prerelease: false
55
49
  type: :runtime
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
51
  requirements:
58
- - - ">="
52
+ - - "~>"
59
53
  - !ruby/object:Gem::Version
60
- version: 0.0.22
54
+ version: '1.9'
61
55
  - !ruby/object:Gem::Dependency
62
56
  requirement: !ruby/object:Gem::Requirement
63
57
  requirements:
64
58
  - - ">="
65
59
  - !ruby/object:Gem::Version
66
- version: 0.0.1
67
- name: yasuri
60
+ version: 0.0.16
61
+ name: logstash-devutils
68
62
  prerelease: false
69
- type: :runtime
63
+ type: :development
70
64
  version_requirements: !ruby/object:Gem::Requirement
71
65
  requirements:
72
66
  - - ">="
73
67
  - !ruby/object:Gem::Version
74
- version: 0.0.1
68
+ version: 0.0.16
75
69
  - !ruby/object:Gem::Dependency
76
70
  requirement: !ruby/object:Gem::Requirement
77
71
  requirements:
78
72
  - - ">="
79
73
  - !ruby/object:Gem::Version
80
- version: 0.0.16
81
- name: logstash-devutils
74
+ version: '0'
75
+ name: logstash-codec-plain
82
76
  prerelease: false
83
77
  type: :development
84
78
  version_requirements: !ruby/object:Gem::Requirement
85
79
  requirements:
86
80
  - - ">="
87
81
  - !ruby/object:Gem::Version
88
- version: 0.0.16
82
+ version: '0'
89
83
  - !ruby/object:Gem::Dependency
90
84
  requirement: !ruby/object:Gem::Requirement
91
85
  requirements:
@@ -114,6 +108,7 @@ files:
114
108
  - README.md
115
109
  - lib/logstash/inputs/yasuri.rb
116
110
  - logstash-input-yasuri.gemspec
111
+ - spec/inputs/res/parse_tree.json
117
112
  - spec/inputs/yasuri_spec.rb
118
113
  homepage: https://github.com/tac0x2a/logstash-input-yasuri
119
114
  licenses:
@@ -137,9 +132,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
137
132
  version: '0'
138
133
  requirements: []
139
134
  rubyforge_project:
140
- rubygems_version: 2.6.6
135
+ rubygems_version: 2.6.8
141
136
  signing_key:
142
137
  specification_version: 4
143
138
  summary: Web scraping input plugin for logstash.
144
139
  test_files:
140
+ - spec/inputs/res/parse_tree.json
145
141
  - spec/inputs/yasuri_spec.rb