logstash-input-multirss 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39f775e02e37041ac3235927b046b8c20633cfcda938336bcfcfe22d30832352
-  data.tar.gz: a95dd45d23116dfa3aeccb1f699cbfb84b6bdd70e39321f215a6836323a44a38
+  metadata.gz: 48be80f785a5000cf522c4717ba16872e8547082844d38d032698b2813ddf2af
+  data.tar.gz: 3e37d5fcddafc3cff0cb9dd6c4d488ae6a1fd40c9aad1219f8938771f7fd1ed2
 SHA512:
-  metadata.gz: f4e00ebcbb6b1652efcc4ed5c1c913c57cc3fb044482c03d79620373e6a54df7ae2f160341b960baad5483b6bc3654f42d35eb449a3cac0cf5f30b5de60a63fa
-  data.tar.gz: a76cee32268fd1bfefdbd8d7db4584d2c280efd2c95493abd92fa0e805d54c43fbfeaa6be10005dbd89768e32295de96c6307ba95995c9f5dcb4588d899b434c
+  metadata.gz: b2795603d6db7056798272912fc7986e410aad8cabdfda6fdce74a5e7dc26ccbfe621a13817f191ff8a04bfa50ffe193b060a661e80ec9d6ba53cf70acc8f38c
+  data.tar.gz: 1b83d04df5d8f0a35003f4c1287f7f800e1a4a3792c87ac58b7a6ca151f60ef0a31c20d940a663468aae62327820a2540995756e63684d3ad302d7e4e9310ab7

data/README.md

@@ -23,7 +23,6 @@ You can see a example in https://github.com/felixramirezgarcia/logstash-input-mu
 The difference between the attributes multi_feed and one_feed is that the multi_feed is the URI of the parent address where several rss (xml) are found. For the case where you want to explore only one of those links you can use the one_feed attribute. A visual example can be seen by visiting the following links:
 
 Father (multi_feed) => http://rss.elmundo.es/rss/
-
 Son (one_feed) => http://estaticos.elmundo.es/elmundo/rss/portada.xml
 
 ## Documentation

data/lib/logstash/inputs/multirss.rb

@@ -8,7 +8,10 @@ require "mechanize"
 require "rss"
 require "nokogiri"
 
-#if you want to debug it you just have to uncomment the puts
+# if you want to debug it you just have to uncomment the puts and build the gem with 
+#   ruby -S gem build logstash-input-multirss.gemspec
+# and install the gem in a logstash service with
+#   logstash-plugin install logstash-input-multirss-x.x.x.gem
 
 class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   config_name "multirss"
@@ -41,7 +44,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
     while !stop?
       
       @multi_feed.each do |rss|
-        puts "Read parent: " + rss
+        str = "Read parent: " + rss
+        #puts str
         begin
           page = @agent.get(rss)
           page.links.each do |link|
@@ -49,22 +53,25 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
                 urls << link.href
               end
           end
-        rescue 
-          puts "Fail to get " + rss + " childrens links"
-        end
+        rescue
+          str = "Fail to get " + rss + " childrens links"
+          #puts str
+        end # end begin
          
         links = urls.uniq
         links.each do |link|
           begin
             response_link(link,queue)
-            puts "Read clidren: " + link
+            str = "Read clidren: " + link
+            #puts str
           rescue
-            puts "Fail to get " + link + " children"
+            str = "Fail to get " + link + " children"
+            #puts str
             next
-          end
-        end
-        urls.clear
+          end # end begin
+        end # end each links
 
+        urls.clear
       end # multi_feed loop
 
       @one_feed.each do |feed|
@@ -75,9 +82,11 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       all_links.each do |link|
         begin
           response_link(link,queue)
-          puts "Read clidren: " + link
+          str = "Read clidren: " + link
+          #puts str
         rescue
-          puts "Fail to get " + link
+          str = "Fail to get " + link
+          #puts str
           next
         end # begin
       end # all_links loop
@@ -85,9 +94,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       urls.clear
 
     Stud.stoppable_sleep(@interval) { stop? }
-
-    end # loop while
-  end # def run
+    end # end while
+  end # end def run
 
 
   def stop
@@ -96,10 +104,10 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   def response_link(link, queue)
     tried = 2
     begin
-      page = Nokogiri::XML(open(link,&:read))
+      page = Nokogiri::XML(open(link,&:read)) # [&:read] -> no OpenURI outputs in /tmp
       page.search('item').each do |item| 
         link_rss_response(queue, item)
-      end
+      end # end each page
     rescue => ex
       if link.chars.first(1).join == "/" && link.chars.first(2).join != "//"
         link = "http:/" + link
@@ -107,19 +115,17 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       elsif link.chars.first(1).join == "/" && link.chars.first(2).join == "//"
         link = "http:" + link
         retry
-      end
-
+      end # end if elsif
       if link.chars.first(4).join == "http" && link.chars.first(5).join != "https"
         link = link.sub('http','https')
         tried = tried - 1
         retry if (tried > 0)
-      end 
- 
-      @logger.error("Error with #{link}: ", :exception => ex)
+      end # end if 
+      #@logger.error("Error : ", :exception => ex)
     rescue => exc
-      @logger.error("Uknown error while parsing the #{link} feed", :exception => exc)
-    end # begin
-  end # def response_link
+      @logger.error("Uknown error while parsing the feed", :exception => exc)
+    end # end begin
+  end # end def response_link
 
   def link_rss_response(queue, item)
       event = LogStash::Event.new()
@@ -130,8 +136,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
         else
           eve = LogStash::Event.new( x.name => x.inner_html.to_s )
           event.append( eve )
-        end # if
-      end # loop
+        end # end if
+      end # end loop
       decorate(event)
       queue << event
   end # def link_rss_response
@@ -140,8 +146,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       for i in 0..@blacklist.length-1
         if link.href.include?(@blacklist[i])
           return false
-        end
-      end
+        end # end if
+      end # end for
       return true
   end # def not_include_blacklist
 

data/logstash-input-multirss.gemspec

@@ -1,12 +1,9 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '1.0.5'
+  s.version       = '1.1.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
-  s.description   = 'This plugin get the feed content and works with : 
-                      1) multi_feed => [array] URI parent with more rss links inside , something like this: http://rss.elmundo.es/rss/  
-                      2) one_feed => [array] (optionally) childs URIS with XML content inside , something like this: http://estaticos.elmundo.es/elmundo/rss/portada.xml 
-                      3) blacklist => [array] (optionally) strings , links, text ... what you dont want explored'
+  s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'
   s.homepage      = 'https://github.com/felixramirezgarcia/logstash-input-multirss'
   s.authors       = ['Felix R G']
   s.email         = 'felixramirezgarcia@correo.ugr.es'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
 platform: ruby
 authors:
 - Felix R G
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2018-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,13 +94,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: "This plugin get the feed content and works with : \n               \
-  \       1) multi_feed => [array] URI parent with more rss links inside , something\
-  \ like this: http://rss.elmundo.es/rss/  \n                      2) one_feed =>\
-  \ [array] (optionally) childs URIS with XML content inside , something like this:\
-  \ http://estaticos.elmundo.es/elmundo/rss/portada.xml \n                      3)\
-  \ blacklist => [array] (optionally) strings , links, text ... what you dont want\
-  \ explored"
+description: This plugin needs a list of links of different rss. Get all the links
+  of the main feed  pages and get all the content of each of the links.
 email: felixramirezgarcia@correo.ugr.es
 executables: []
 extensions: []