logstash-input-yasuri 0.0.2 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +13 -1
  3. data/lib/logstash/inputs/yasuri.rb +11 -1
  4. data/logstash-input-yasuri.gemspec +5 -4
  5. data/spec/inputs/res/parse_tree.json +17 -0
  6. data/spec/inputs/yasuri_spec.rb +15 -1
  7. metadata +25 -29
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 637385a7b928621849907264f3306a3fb66cc00a
4
- data.tar.gz: 880e7867eca00f124d6c5eaad4284e17d20f7cf0
3
+ metadata.gz: 01cf6f81b0de10a81ff1ddf0cd447e9d9b403fbd
4
+ data.tar.gz: 25c33d37b927c64bf413709386b500fad9a34ac8
5
5
  SHA512:
6
- metadata.gz: 9618e43ea302b746791c8e0e68fde846d776694b883871f0d332031205f4b91b8edafdbb62c4889d13efb6521aa613c1980bd7f6b761b6f89d5cc8f0e5b8274d
7
- data.tar.gz: c2bbd5da29a32540e9e8a13c434cd765946509aeded6be8fa23b7d4d0751c8b1287da64d364aa39a00668d255b39ac2334439868a5dccb7b0dd5a3e301a6732c
6
+ metadata.gz: 513917038a3714b3eacd95374446f6ff982c9744101630da21dc350fae34f047ac15ab6eaff80bff00e2d4845c215898ec9dab5685a1baa92794e95b05f02149
7
+ data.tar.gz: d923a7ffebee5d919046ba0479d9b02eed495307d679ac67d05642df1bf8600b4242a7efb8a5196e0588ded9886673dafb243e3d9aa68a18be777daed6f7afa0
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Travis Build Status](https://travis-ci.org/tac0x2a/logstash-input-yasuri.svg)](https://travis-ci.org/tac0x2a/logstash-input-yasuri)
3
+ [![Build Status](https://travis-ci.org/tac0x2a/logstash-input-yasuri.svg?branch=master)](https://travis-ci.org/tac0x2a/logstash-input-yasuri)
4
4
 
5
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
6
6
 
@@ -45,6 +45,18 @@ bundle install
45
45
  bundle exec rspec
46
46
  ```
47
47
 
48
+ - Build gem
49
+
50
+ ```sh
51
+ gem build logstash-input-yasuri
52
+ # generate logstash-input-yasuri-x.x.x.gem
53
+ ```
54
+
55
+ - publish gem
56
+ ```sh
57
+ bundle exec rake publish_gem
58
+ ```
59
+
48
60
  ### 2. Running your unpublished Plugin in Logstash
49
61
 
50
62
  #### 2.1 Run in a local Logstash clone
data/lib/logstash/inputs/yasuri.rb CHANGED
@@ -10,10 +10,16 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
10
10
 
11
11
  default :codec, "plain"
12
12
 
13
+ # logstash-input-yasuri require option :parse_tree or :parse_tree_path.
14
+ # If given both, logstash-input-yasuri use :parse_tree.
15
+
13
16
  # Parse tree as JSON.
14
17
  # Read https://github.com/tac0x2a/yasuri/blob/master/USAGE.md#construct-parse-tree
15
18
  config :parse_tree, :validate => :string
16
19
 
20
+ # Path to parse tree JSON file.
21
+ config :parse_tree_path, :validate => :string
22
+
17
23
  # Target web page url.
18
24
  config :url, :validate => :string
19
25
 
@@ -26,7 +32,11 @@ class LogStash::Inputs::Yasuri < LogStash::Inputs::Base
26
32
  def register
27
33
  @host = Socket.gethostname
28
34
  @agent = Mechanize.new
29
- @tree = Yasuri.json2tree(@parse_tree)
35
+
36
+ # If given both, logstash-input-yasuri use :parse_tree.
37
+ tree = @parse_tree || File.read(@parse_tree_path)
38
+
39
+ @tree = Yasuri.json2tree(tree)
30
40
  end # def register
31
41
 
32
42
  def run(queue)
data/logstash-input-yasuri.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-yasuri'
3
- s.version = '0.0.2'
3
+ s.version = '2.0.4'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Web scraping input plugin for logstash."
6
6
  s.description = "Web scraping input plugin for logstash."
@@ -18,10 +18,11 @@ Gem::Specification.new do |s|
18
18
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
19
19
 
20
20
  # Gem dependencies
21
- s.add_runtime_dependency "logstash-core", ">= 2.0.0", "<= 5.0.0"
22
- s.add_runtime_dependency 'logstash-codec-plain'
21
+ s.add_runtime_dependency 'logstash-core-plugin-api', '~> 1'
23
22
  s.add_runtime_dependency 'stud', '>= 0.0.22'
24
- s.add_runtime_dependency 'yasuri', '>= 0.0.1'
23
+ s.add_runtime_dependency 'yasuri', '~> 1.9'
24
+
25
25
  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
26
+ s.add_development_dependency 'logstash-codec-plain'
26
27
  s.add_development_dependency 'fuubar', '>= 2.2.0'
27
28
  end
data/spec/inputs/res/parse_tree.json ADDED
@@ -0,0 +1,17 @@
1
+ {
2
+ "node": "struct",
3
+ "name": "titles",
4
+ "path": "//td[@class='title'][not(@align)]",
5
+ "children": [
6
+ {
7
+ "node": "text",
8
+ "name": "title",
9
+ "path": "./a"
10
+ },
11
+ {
12
+ "node": "text",
13
+ "name": "url",
14
+ "path": "./a/@href"
15
+ }
16
+ ]
17
+ }
data/spec/inputs/yasuri_spec.rb CHANGED
@@ -6,7 +6,7 @@ describe LogStash::Inputs::Yasuri do
6
6
 
7
7
  let(:input) {
8
8
  input = LogStash::Plugin.lookup("input", "yasuri").new(
9
- "flatten" => true,
9
+ "split" => true,
10
10
  "url" => "https://news.ycombinator.com/",
11
11
  "parse_tree" => %Q|
12
12
  {
@@ -30,6 +30,14 @@ describe LogStash::Inputs::Yasuri do
30
30
  )
31
31
  }
32
32
 
33
+ let(:input_from_parse_tree_file) {
34
+ input_from_parse_tree_file = LogStash::Plugin.lookup("input", "yasuri").new(
35
+ "split" => true,
36
+ "url" => "https://news.ycombinator.com/",
37
+ "parse_tree_path" => "spec/inputs/res/parse_tree.json"
38
+ )
39
+ }
40
+
33
41
  it "should register" do
34
42
  # register will try to load jars and raise if it cannot find jars or if org.apache.log4j.spi.LoggingEvent class is not present
35
43
  expect {input.register}.to_not raise_error
@@ -41,4 +49,10 @@ describe LogStash::Inputs::Yasuri do
41
49
  input.inner_run(queue)
42
50
  expect(queue.size).not_to be_zero
43
51
  end
52
+
53
+ it "enqueues some events from input_from_parse_tree_file" do
54
+ input_from_parse_tree_file.register
55
+ input_from_parse_tree_file.inner_run(queue)
56
+ expect(queue.size).not_to be_zero
57
+ end
44
58
  end
metadata CHANGED
@@ -1,91 +1,85 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-yasuri
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 2.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - tac0x2a
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-06 00:00:00.000000000 Z
11
+ date: 2016-12-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ">="
17
- - !ruby/object:Gem::Version
18
- version: 2.0.0
19
- - - "<="
16
+ - - "~>"
20
17
  - !ruby/object:Gem::Version
21
- version: 5.0.0
22
- name: logstash-core
18
+ version: '1'
19
+ name: logstash-core-plugin-api
23
20
  prerelease: false
24
21
  type: :runtime
25
22
  version_requirements: !ruby/object:Gem::Requirement
26
23
  requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- version: 2.0.0
30
- - - "<="
24
+ - - "~>"
31
25
  - !ruby/object:Gem::Version
32
- version: 5.0.0
26
+ version: '1'
33
27
  - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - ">="
37
31
  - !ruby/object:Gem::Version
38
- version: '0'
39
- name: logstash-codec-plain
32
+ version: 0.0.22
33
+ name: stud
40
34
  prerelease: false
41
35
  type: :runtime
42
36
  version_requirements: !ruby/object:Gem::Requirement
43
37
  requirements:
44
38
  - - ">="
45
39
  - !ruby/object:Gem::Version
46
- version: '0'
40
+ version: 0.0.22
47
41
  - !ruby/object:Gem::Dependency
48
42
  requirement: !ruby/object:Gem::Requirement
49
43
  requirements:
50
- - - ">="
44
+ - - "~>"
51
45
  - !ruby/object:Gem::Version
52
- version: 0.0.22
53
- name: stud
46
+ version: '1.9'
47
+ name: yasuri
54
48
  prerelease: false
55
49
  type: :runtime
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
51
  requirements:
58
- - - ">="
52
+ - - "~>"
59
53
  - !ruby/object:Gem::Version
60
- version: 0.0.22
54
+ version: '1.9'
61
55
  - !ruby/object:Gem::Dependency
62
56
  requirement: !ruby/object:Gem::Requirement
63
57
  requirements:
64
58
  - - ">="
65
59
  - !ruby/object:Gem::Version
66
- version: 0.0.1
67
- name: yasuri
60
+ version: 0.0.16
61
+ name: logstash-devutils
68
62
  prerelease: false
69
- type: :runtime
63
+ type: :development
70
64
  version_requirements: !ruby/object:Gem::Requirement
71
65
  requirements:
72
66
  - - ">="
73
67
  - !ruby/object:Gem::Version
74
- version: 0.0.1
68
+ version: 0.0.16
75
69
  - !ruby/object:Gem::Dependency
76
70
  requirement: !ruby/object:Gem::Requirement
77
71
  requirements:
78
72
  - - ">="
79
73
  - !ruby/object:Gem::Version
80
- version: 0.0.16
81
- name: logstash-devutils
74
+ version: '0'
75
+ name: logstash-codec-plain
82
76
  prerelease: false
83
77
  type: :development
84
78
  version_requirements: !ruby/object:Gem::Requirement
85
79
  requirements:
86
80
  - - ">="
87
81
  - !ruby/object:Gem::Version
88
- version: 0.0.16
82
+ version: '0'
89
83
  - !ruby/object:Gem::Dependency
90
84
  requirement: !ruby/object:Gem::Requirement
91
85
  requirements:
@@ -114,6 +108,7 @@ files:
114
108
  - README.md
115
109
  - lib/logstash/inputs/yasuri.rb
116
110
  - logstash-input-yasuri.gemspec
111
+ - spec/inputs/res/parse_tree.json
117
112
  - spec/inputs/yasuri_spec.rb
118
113
  homepage: https://github.com/tac0x2a/logstash-input-yasuri
119
114
  licenses:
@@ -137,9 +132,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
137
132
  version: '0'
138
133
  requirements: []
139
134
  rubyforge_project:
140
- rubygems_version: 2.6.6
135
+ rubygems_version: 2.6.8
141
136
  signing_key:
142
137
  specification_version: 4
143
138
  summary: Web scraping input plugin for logstash.
144
139
  test_files:
140
+ - spec/inputs/res/parse_tree.json
145
141
  - spec/inputs/yasuri_spec.rb