logstash-input-multirss 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48be80f785a5000cf522c4717ba16872e8547082844d38d032698b2813ddf2af
-  data.tar.gz: 3e37d5fcddafc3cff0cb9dd6c4d488ae6a1fd40c9aad1219f8938771f7fd1ed2
+  metadata.gz: e1cb1975f468ee41df996eab224de416ee1c1389d4508ab153eba11e3d9ec601
+  data.tar.gz: 66adf18634c47c4a2bf3aebc31b071e112bf54b7f1eb40a34e0f47ecbebcedad
 SHA512:
-  metadata.gz: b2795603d6db7056798272912fc7986e410aad8cabdfda6fdce74a5e7dc26ccbfe621a13817f191ff8a04bfa50ffe193b060a661e80ec9d6ba53cf70acc8f38c
-  data.tar.gz: 1b83d04df5d8f0a35003f4c1287f7f800e1a4a3792c87ac58b7a6ca151f60ef0a31c20d940a663468aae62327820a2540995756e63684d3ad302d7e4e9310ab7
+  metadata.gz: '0401495068a213d7d689859c390b3ec1dd9a95e82c6e3af42d49a39780770eb9558c2558c9cf3b1951d1ba76ee7462d568c39384488b48b16f1ac6cf30219d39'
+  data.tar.gz: a65c76f4dce2c18bf85074328c30c1dc5ffbfaecd127bae27a1664cb1e2ac94c13a79bcd4c77a97c03741c62bdb458b3a506f9b9d8737f8a54c6e1a4488e9d1f

data/README.md

@@ -4,7 +4,9 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
-# Install
+# Developing
+
+## Install
 
 U can install the plugin from https://rubygems.org/gems/logstash-input-multirss , or build it yuouself in a logstash service or container with :
 
@@ -16,15 +18,28 @@ ruby -S gem build logstash-input-multirss.gemspec
 
 logstash-plugin install logstash-input-multirss-[nº_version].gem
 
-# Pipeline Example
+### Pipeline Example
 
 You can see a example in https://github.com/felixramirezgarcia/logstash-input-multirss/blob/master/example-pipeline.conf
 
 The difference between the attributes multi_feed and one_feed is that the multi_feed is the URI of the parent address where several rss (xml) are found. For the case where you want to explore only one of those links you can use the one_feed attribute. A visual example can be seen by visiting the following links:
 
 Father (multi_feed) => http://rss.elmundo.es/rss/
+
 Son (one_feed) => http://estaticos.elmundo.es/elmundo/rss/portada.xml
 
+All the params are :
+
+    1) multi_feed => [array] URI parent with more rss links inside , something like this: http://rss.elmundo.es/rss/  
+    
+    2) one_feed => [array] childs URIS with XML content inside , something like this: http://estaticos.elmundo.es/elmundo/rss/portada.xml 
+    
+    3) blacklist => [array] strings , links, text ... what you dont want explored
+    
+    4) Interval => [int] Set the Stoppable_sleep interval for the pipe
+    
+    5) keywords => [array] If you use this parameter will only compile those news that contain in any of its attributes a word from this array
+
 ## Documentation
 
 Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
@@ -36,37 +51,9 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
-## Developing
-
-### 1. Plugin Developement and Testing
-
-#### Code
-- To get started, you'll need JRuby with the Bundler gem installed.
-
-- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
-
-- Install dependencies
-```sh
-bundle install
-```
-
-#### Test
-
-- Update your dependencies
-
-```sh
-bundle install
-```
-
-- Run tests
-
-```sh
-bundle exec rspec
-```
-
-### 2. Running your unpublished Plugin in Logstash
+# Running your unpublished Plugin in Logstash
 
-#### 2.1 Run in a local Logstash clone
+## Run in a local Logstash clone
 
 - Edit Logstash `Gemfile` and add the local plugin path, for example:
 ```ruby
@@ -82,9 +69,9 @@ bin/logstash -e 'filter {awesome {}}'
 ```
 At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
 
-#### 2.2 Run in an installed Logstash
+## Run in an installed Logstash
 
-You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+You can use the same method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
 
 - Build your plugin gem
 ```sh

data/lib/logstash/inputs/multirss.rb

@@ -7,31 +7,37 @@ require "uri"
 require "mechanize"
 require "rss"
 require "nokogiri"
+require "fileutils"
 
 # if you want to debug it you just have to uncomment the puts and build the gem with 
 #   ruby -S gem build logstash-input-multirss.gemspec
-# and install the gem in a logstash service with
+# and install the gem in a logstash service or container with
 #   logstash-plugin install logstash-input-multirss-x.x.x.gem
 
 class LogStash::Inputs::Multirss < LogStash::Inputs::Base
-  config_name "multirss"
+  config_name "multirss"  #Plugin name
 
-  default :codec, "plain"
+  default :codec, "plain" #Codec
 
-  # The rss array list to use in the pipe
-  config :multi_feed, :validate => :array, :required => true
+  # The rss parent array list to use in the pipe (link with a lot rss links inside)
+  config :multi_feed, :validate => :array, :default => []
 
-  # The rss array list to use in the pipe
+  # The rss childs array list to use in the pipe (simple rss link)
   config :one_feed, :validate => :array, :default => []
 
   #Set de interval for stoppable_sleep
   config :interval, :validate => :number, :default => 3600
   
-  #Set de black list to forget read
+  #Set de black list to forget read and get content 
   config :blacklist, :validate => :array, :default => []
 
+  #Set de keywords to ONLY get content whit it
+  config :keywords, :validate => :array, :default => []
+
   public
-   def register
+   
+   def register #initialize
+    #Mechanize agent
     @agent = Mechanize.new
     @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end # def register
@@ -41,11 +47,13 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
     # we can abort the loop if stop? becomes true
     urls = []
 
+    #Don't stop, keep going.
     while !stop?
-      
-      @multi_feed.each do |rss|
-        str = "Read parent: " + rss
-        #puts str
+
+      manage_tempdir
+
+      @multi_feed.each do |rss|     #get the father's children
+        #puts "Read parent: " + rss
         begin
           page = @agent.get(rss)
           page.links.each do |link|
@@ -62,11 +70,9 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
         links.each do |link|
           begin
             response_link(link,queue)
-            str = "Read clidren: " + link
-            #puts str
+            #puts "Read clidren: " + link
           rescue
-            str = "Fail to get " + link + " children"
-            #puts str
+            #puts "Fail to get " + link + " children"
             next
           end # end begin
         end # end each links
@@ -82,17 +88,23 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       all_links.each do |link|
         begin
           response_link(link,queue)
-          str = "Read clidren: " + link
-          #puts str
+          #puts "Read clidren: " + link
         rescue
-          str = "Fail to get " + link
-          #puts str
+          #puts "Fail to get " + link
           next
         end # begin
       end # all_links loop
 
       urls.clear
 
+      # Remove the tempfiles
+      if (File::directory?(@d))
+        ENV.delete("TMPDIR")
+        FileUtils.rm_rf @d
+        #puts "Remove temp dir"
+      end
+
+    #Stoppable_sleep interval
     Stud.stoppable_sleep(@interval) { stop? }
     end # end while
   end # end def run
@@ -129,19 +141,53 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
 
   def link_rss_response(queue, item)
       event = LogStash::Event.new()
-      item.element_children.each do |x| 
-        if x.inner_html.to_s.chars.first(9).join == "<![CDATA["
-          eve = LogStash::Event.new( x.name => x.inner_html.to_s[9..x.inner_html.to_s.length-4])
-          event.append( eve )
-        else
-          eve = LogStash::Event.new( x.name => x.inner_html.to_s )
-          event.append( eve )
-        end # end if
-      end # end loop
-      decorate(event)
-      queue << event
+
+      if @keywords.size.to_s.to_i > 0 # "Have keywords
+        haskey = false
+
+        item.element_children.each do |x|   
+            if include_keywords(x.inner_html.to_s)
+              #puts "--------------Finded notice with the keyword---------------"
+              haskey = true
+            end
+        end # end loop
+
+        if haskey == true
+          item.element_children.each do |x|
+            #puts "The notice " + x.name + " is " + x.inner_html.to_s
+            if x.inner_html.to_s.chars.first(9).join == "<![CDATA["
+              eve = LogStash::Event.new( x.name => x.inner_html.to_s[9..x.inner_html.to_s.length-4] )
+              event.append( eve )
+            else
+              eve = LogStash::Event.new( x.name => x.inner_html.to_s )
+              event.append( eve )
+            end # end if else       
+          end # end loop
+        elsif haskey == false # havent haskey
+          event = nil
+        end # if haskey
+
+      else # havent keywords!
+        #puts "Havent keywords, go to get all items"
+        item.element_children.each do |x|
+          if x.inner_html.to_s.chars.first(9).join == "<![CDATA["
+            eve = LogStash::Event.new( x.name => x.inner_html.to_s[9..x.inner_html.to_s.length-4])
+            event.append( eve )
+          else
+            eve = LogStash::Event.new( x.name => x.inner_html.to_s )
+            event.append( eve )
+          end # end if
+        end # end loop
+      end # end if have keywords
+
+      if event != nil
+        decorate(event)
+        queue << event
+      end # end if
+      
   end # def link_rss_response
 
+
   def not_include_blacklist(link) 
       for i in 0..@blacklist.length-1
         if link.href.include?(@blacklist[i])
@@ -152,4 +198,33 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   end # def not_include_blacklist
 
 
+  def include_keywords(key) 
+    for i in 0..@keywords.length-1
+      if key.include?(@keywords[i])
+        return true
+      end # end if
+    end # end for
+    return false
+  end # def include_keywords
+  
+
+  def manage_tempdir
+    #set the tempfile to openUri output
+    @d = "#{Dir.home}/.tmp"
+    #if exists
+    if (File::directory?(@d))
+      #puts "Dir exists , removed and create again"
+      ENV.delete("TMPDIR")
+      FileUtils.rm_rf @d
+      #create new
+      Dir.mkdir @d    #create in /usr/share/logstash
+      ENV["TMPDIR"] = @d
+    else
+      Dir.mkdir @d    #create in /usr/share/logstash
+      ENV["TMPDIR"] = @d
+      #puts "Dir no exist , created...."
+    end
+  end
+
+
 end # class LogStash::Inputs::Crawler

data/logstash-input-multirss.gemspec

@@ -1,9 +1,14 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '1.1.0'
+  s.version       = '1.2.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
-  s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'
+  s.description   = 'This plugin get the feed rss content (being able to use keywords to get the feed) , the params are: 
+                      1) multi_feed => [array] URI parent with more rss links inside , something like this: http://rss.elmundo.es/rss/  
+                      2) one_feed => [array] (optionally) childs URIS with XML content inside , something like this: http://estaticos.elmundo.es/elmundo/rss/portada.xml 
+                      3) blacklist => [array] (optionally) strings , links, text ... what you dont want explored
+                      4) Interval => [int] Set the Stoppable_sleep interval for the pipe
+                      5) keywords => [array] if you use this parameter will only compile those news that contain in any of its attributes a word from this array'
   s.homepage      = 'https://github.com/felixramirezgarcia/logstash-input-multirss'
   s.authors       = ['Felix R G']
   s.email         = 'felixramirezgarcia@correo.ugr.es'
@@ -24,4 +29,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
   s.add_runtime_dependency "mechanize"
   s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "fileutils"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Felix R G
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2018-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,8 +94,30 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This plugin needs a list of links of different rss. Get all the links
-  of the main feed  pages and get all the content of each of the links.
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: fileutils
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "This plugin get the feed rss content (being able to use keywords to\
+  \ get the feed) , the params are: \n                      1) multi_feed => [array]\
+  \ URI parent with more rss links inside , something like this: http://rss.elmundo.es/rss/\
+  \  \n                      2) one_feed => [array] (optionally) childs URIS with\
+  \ XML content inside , something like this: http://estaticos.elmundo.es/elmundo/rss/portada.xml\
+  \ \n                      3) blacklist => [array] (optionally) strings , links,\
+  \ text ... what you dont want explored\n                      4) Interval => [int]\
+  \ Set the Stoppable_sleep interval for the pipe\n                      5) keywords\
+  \ => [array] if you use this parameter will only compile those news that contain\
+  \ in any of its attributes a word from this array"
 email: felixramirezgarcia@correo.ugr.es
 executables: []
 extensions: []