logstash-input-multirss 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9c37035e249154e18a6791d97ed293dae87c8fe7c50dfe615362df6271be84
-  data.tar.gz: be4e1b502d9c7d6d502e1c56f01e339e250a402885fd03e4824a39f467366682
+  metadata.gz: 13067ead10a7d5f0f79e4917f438fd4ab6bb94dd52ee37d216cecd57fd2b9c01
+  data.tar.gz: 17c13ebebe0b913e9e79d77e1ef3093dc4cc2d3af16ea122e2c9ac6a61315d89
 SHA512:
-  metadata.gz: 25d8d59b55f0f40faab1d0b65bbe06811875516a8ef6547879b188622267983c81ce47c9408c2af2a120b5f3312b2398dc10684dffaef19593e8cf884e813a8e
-  data.tar.gz: 4ffc47877ab4d21928b3d3cb4a96531f1d660d6bb113ac72010a5ee3ee99f647bfeea02398c548a7c3993acc7e77cbc0ad34cef44cea23dcd053a5bbb42400fc
+  metadata.gz: b37d8389d3b92271e53160ed809695fb9dcdc3158f04270f6bf722640837b9982d9933db17a6147c2d8704b25778d752874e4cdcb5c9c58483b41638c8846e7b
+  data.tar.gz: b79f50fad25ebe755440e8b421d47ef6b2dc38b770fbabb03fb13a9d8411d33681995ba7d2e3ab1d3c0820bd1190ab3c9258899e7abfb6f4b14f6d848398f087

data/README.md

@@ -9,38 +9,16 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
 U can install the plugin from https://rubygems.org/gems/logstash-input-multirss , or build it yuouself in a logstash service or container with :
 
 git clone https://github.com/felixramirezgarcia/logstash-input-multirss
+
 rm logstash-input-multirss-[nº_version].gem
+
 ruby -S gem build logstash-input-multirss.gemspec
+
 logstash-plugin install logstash-input-multirss-[nº_version].gem
 
 # Pipeline Example
 
-input {
-	multirss
-	{
-		multi_feed => ['https://www.diaridegirona.cat/servicios/rss/rss.jsp','https://www.elconfidencial.com/rss/','https://www.20minutos.es/sindicacion/','http://www.europapress.es/contenidosrss/','https://www.abc.es/rss/','https://servicios.elpais.com/rss/','http://www.expansion.com/rss/','https://cronicaglobal.elespanol.com/rss.html','https://www.diarioinformacion.com/servicios/rss/rss.jsp']
-
-		one_feed => ['http://www.finanzas.com/rss/noticiasportada.xml','https://www.invertia.com/es/rss/-/rss/getHome','https://www.invertia.com/es/rss/-/rss/getHomeExtra','https://www.invertia.com/es/rss/-/rss/getNews?_rss_WAR_ivrssportlet_category=mercados']
-
-		interval => 1000
-
-		blacklist => ['google.com','yahoo.com','live.com','netvibes.com','bloglines.com','feedly.com','/atom']
-	}
-}
-
-filter {
-
-}
-
-
-output {
-	file {
-		path => "/tmp/rss_links.json"
-		codec => "json"	
-	}
-}
-
-
+You can see a example in https://github.com/felixramirezgarcia/logstash-input-multirss/blob/master/example-pipeline.conf
 
 The difference between the attributes multi_feed and one_feed is that the multi_feed is the URI of the parent address where several rss (xml) are found. For the case where you want to explore only one of those links you can use the one_feed attribute. A visual example can be seen by visiting the following links:
 

data/lib/logstash/inputs/multirss.rb

@@ -5,6 +5,7 @@ require "stud/interval"
 require "net/http"
 require "uri"
 require "mechanize"
+require "rss"
 require "nokogiri"
 
 #if you want to debug it you just have to uncomment the puts
@@ -23,7 +24,7 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   #Set de interval for stoppable_sleep
   config :interval, :validate => :number, :default => 3600
   
-  #Set de black list to forget read 
+  #Set de black list to forget read
   config :blacklist, :validate => :array, :default => []
 
   public
@@ -40,7 +41,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
     while !stop?
       
       @multi_feed.each do |rss|
-        #puts "Read parent: " + rss
+        str = "Read parent: " + rss
+        #puts str
         begin
           page = @agent.get(rss)
           page.links.each do |link|
@@ -49,16 +51,19 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
               end
           end
         rescue
-          #puts "Fail to get " + rss + " childrens links"
+          str = "Fail to get " + rss + " childrens links"
+          #puts str
         end
          
         links = urls.uniq
         links.each do |link|
           begin
             response_link(link,queue)
-            #puts "Read clidren: " + link
+            str = "Read clidren: " + link
+            #puts str
           rescue
-            #puts "Fail to get " + link + " children"
+            str = "Fail to get " + link + " children"
+            #puts str
             next
           end
         end
@@ -74,9 +79,11 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       all_links.each do |link|
         begin
           response_link(link,queue)
-          #puts "Read clidren: " + link
+          str = "Read clidren: " + link
+          #puts str
         rescue
-          #puts "Fail to get " + link
+          str = "Fail to get " + link
+          #puts str
           next
         end # begin
       end # all_links loop
@@ -116,8 +123,7 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
  
       @logger.error("Error : ", :exception => ex)
     rescue => exc
-      #puts "ERROR"
-      #@logger.error("Uknown error while parsing the feed", :exception => exc)
+      @logger.error("Uknown error while parsing the feed", :exception => exc)
     end # begin
   end # def response_link
 

data/logstash-input-multirss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '1.0.2'
+  s.version       = '1.0.3'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
   s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Felix R G