logstash-input-multirss 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3e3215896069197c28eeaf2bc1d97b8a7b922b5f629d33f15c82b6cbe84c20a
-  data.tar.gz: d1391c39b3f2bf48888f93ca26633c821da10f439772408ac61de4ba9a561681
+  metadata.gz: 8e9c37035e249154e18a6791d97ed293dae87c8fe7c50dfe615362df6271be84
+  data.tar.gz: be4e1b502d9c7d6d502e1c56f01e339e250a402885fd03e4824a39f467366682
 SHA512:
-  metadata.gz: acbfa422e853ab42d439c960064f45df5277e372da471b20aa60f6a78d508fd872d9674fb009413fd0aee88b774d88eebe4e68db355c1ae6b7fee21f1d17d0c8
-  data.tar.gz: 0f1cf2179aa9722c65d610b6ea7c266866ad1719984cb89b280607c5fe3f6003688b55fcbbab33cb96255f517200ce5f637d8377c47600de3286b010a9597b7d
+  metadata.gz: 25d8d59b55f0f40faab1d0b65bbe06811875516a8ef6547879b188622267983c81ce47c9408c2af2a120b5f3312b2398dc10684dffaef19593e8cf884e813a8e
+  data.tar.gz: 4ffc47877ab4d21928b3d3cb4a96531f1d660d6bb113ac72010a5ee3ee99f647bfeea02398c548a7c3993acc7e77cbc0ad34cef44cea23dcd053a5bbb42400fc

data/README.md

@@ -4,6 +4,49 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
+# Install
+
+U can install the plugin from https://rubygems.org/gems/logstash-input-multirss , or build it yuouself in a logstash service or container with :
+
+git clone https://github.com/felixramirezgarcia/logstash-input-multirss
+rm logstash-input-multirss-[nº_version].gem
+ruby -S gem build logstash-input-multirss.gemspec
+logstash-plugin install logstash-input-multirss-[nº_version].gem
+
+# Pipeline Example
+
+input {
+	multirss
+	{
+		multi_feed => ['https://www.diaridegirona.cat/servicios/rss/rss.jsp','https://www.elconfidencial.com/rss/','https://www.20minutos.es/sindicacion/','http://www.europapress.es/contenidosrss/','https://www.abc.es/rss/','https://servicios.elpais.com/rss/','http://www.expansion.com/rss/','https://cronicaglobal.elespanol.com/rss.html','https://www.diarioinformacion.com/servicios/rss/rss.jsp']
+
+		one_feed => ['http://www.finanzas.com/rss/noticiasportada.xml','https://www.invertia.com/es/rss/-/rss/getHome','https://www.invertia.com/es/rss/-/rss/getHomeExtra','https://www.invertia.com/es/rss/-/rss/getNews?_rss_WAR_ivrssportlet_category=mercados']
+
+		interval => 1000
+
+		blacklist => ['google.com','yahoo.com','live.com','netvibes.com','bloglines.com','feedly.com','/atom']
+	}
+}
+
+filter {
+
+}
+
+
+output {
+	file {
+		path => "/tmp/rss_links.json"
+		codec => "json"	
+	}
+}
+
+
+
+The difference between the attributes multi_feed and one_feed is that the multi_feed is the URI of the parent address where several rss (xml) are found. For the case where you want to explore only one of those links you can use the one_feed attribute. A visual example can be seen by visiting the following links:
+
+Father (multi_feed) => http://rss.elmundo.es/rss/
+Son (one_feed) => http://estaticos.elmundo.es/elmundo/rss/portada.xml
+
 ## Documentation
 
 Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).

data/lib/logstash/inputs/multirss.rb

@@ -5,9 +5,10 @@ require "stud/interval"
 require "net/http"
 require "uri"
 require "mechanize"
-require "rss"
 require "nokogiri"
 
+#if you want to debug it you just have to uncomment the puts
+
 class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   config_name "multirss"
 
@@ -22,7 +23,7 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   #Set de interval for stoppable_sleep
   config :interval, :validate => :number, :default => 3600
   
-  #Set de black list to forget
+  #Set de black list to forget read 
   config :blacklist, :validate => :array, :default => []
 
   public
@@ -39,7 +40,7 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
     while !stop?
       
       @multi_feed.each do |rss|
-        puts "Read parent: " + rss
+        #puts "Read parent: " + rss
         begin
           page = @agent.get(rss)
           page.links.each do |link|
@@ -48,16 +49,16 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
               end
           end
         rescue
-          puts "Fail to get " + rss + " feed"
+          #puts "Fail to get " + rss + " childrens links"
         end
          
         links = urls.uniq
         links.each do |link|
           begin
             response_link(link,queue)
-            puts "Read clidren: " + link
+            #puts "Read clidren: " + link
           rescue
-            puts "Fail to get " + link + " children"
+            #puts "Fail to get " + link + " children"
             next
           end
         end
@@ -73,9 +74,9 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
       all_links.each do |link|
         begin
           response_link(link,queue)
-          puts "Read clidren: " + link
+          #puts "Read clidren: " + link
         rescue
-          puts "Fail to get " + link
+          #puts "Fail to get " + link
           next
         end # begin
       end # all_links loop
@@ -115,8 +116,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
  
       @logger.error("Error : ", :exception => ex)
     rescue => exc
-      puts "ERROR"
-      @logger.error("Uknown error while parsing the feed", :exception => exc)
+      #puts "ERROR"
+      #@logger.error("Uknown error while parsing the feed", :exception => exc)
     end # begin
   end # def response_link
 

data/logstash-input-multirss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '1.0.1'
+  s.version       = '1.0.2'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
   s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Felix R G
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-16 00:00:00.000000000 Z
+date: 2018-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement