logstash-input-multirss 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cad07458c379cfcd9f3a7cbb9608f693a33efe5dfee0a2087387df491c135218
-  data.tar.gz: a5956c590a9e0d9667b278de7f690d3a55ce9eaf886919f2b355c9e12fff8675
+  metadata.gz: d3e3215896069197c28eeaf2bc1d97b8a7b922b5f629d33f15c82b6cbe84c20a
+  data.tar.gz: d1391c39b3f2bf48888f93ca26633c821da10f439772408ac61de4ba9a561681
 SHA512:
-  metadata.gz: 6611bda686c887dbaebba928718cd6521e4c2076e0c44feabdd1e13e89aef3aeb6a80cf8861dfba40b8b9a442f381262bc72023e6d3b3a14c86b94aadf3611ea
-  data.tar.gz: d73bfa18ec57247ff7b566f94e2d6e6a9e710bd09b5b5563034bb9cc74c46ddc2bc5c9f8e13f268ceb259913195922388f3efe2f0c952689e88cee932cb6a453
+  metadata.gz: acbfa422e853ab42d439c960064f45df5277e372da471b20aa60f6a78d508fd872d9674fb009413fd0aee88b774d88eebe4e68db355c1ae6b7fee21f1d17d0c8
+  data.tar.gz: 0f1cf2179aa9722c65d610b6ea7c266866ad1719984cb89b280607c5fe3f6003688b55fcbbab33cb96255f517200ce5f637d8377c47600de3286b010a9597b7d

data/lib/logstash/inputs/multirss.rb

@@ -20,21 +20,22 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   config :one_feed, :validate => :array, :default => []
 
   #Set de interval for stoppable_sleep
-  config :interval, :validate => :number, :default => 200
+  config :interval, :validate => :number, :default => 3600
   
   #Set de black list to forget
-  config :blacklist, :validate => :array, :default => ['http://fusion.google.com/','yahoo.com','live.com','netvibes.com','bloglines.com']
+  config :blacklist, :validate => :array, :default => []
 
   public
    def register
-    @urls = []
     @agent = Mechanize.new
     @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end # def register
 
 
-   def run(queue)
+  def run(queue)
     # we can abort the loop if stop? becomes true
+    urls = []
+
     while !stop?
       
       @multi_feed.each do |rss|
@@ -42,64 +43,82 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
         begin
           page = @agent.get(rss)
           page.links.each do |link|
-              if link.href.chars.last(3).join == "xml" && not_include_blacklist(link)
-                @urls << link.href
+              if (link.href.chars.last(3).join == "xml" || link.href.include?('/rss') || link.href.include?('/feed')) && not_include_blacklist(link) 
+                urls << link.href
               end
           end
         rescue
           puts "Fail to get " + rss + " feed"
         end
          
-        links = @urls.uniq
+        links = urls.uniq
         links.each do |link|
           begin
             response_link(link,queue)
             puts "Read clidren: " + link
           rescue
-            puts "Fail to get " + link
+            puts "Fail to get " + link + " children"
             next
           end
         end
-        @urls.clear
+        urls.clear
 
-      end
+      end # multi_feed loop
 
       @one_feed.each do |feed|
-        @urls << feed
-      end
-      links_o = @urls.uniq
-      links_o.each do |link|
+        urls << feed
+      end # one_feed loop
+
+      all_links = urls.uniq
+      all_links.each do |link|
         begin
           response_link(link,queue)
           puts "Read clidren: " + link
         rescue
           puts "Fail to get " + link
           next
-        end
-      end
+        end # begin
+      end # all_links loop
 
-      @urls.clear
+      urls.clear
 
     Stud.stoppable_sleep(@interval) { stop? }
-    end # loop
-   end # def run
 
+    end # loop while
+  end # def run
 
-   def stop
- 
-   end
+
+  def stop
+  end #def stop
 
   def response_link(link, queue)
+    tried = 2
     begin
-      page = Nokogiri::XML(open(link))
+      page = Nokogiri::XML(open(link,&:read))
       page.search('item').each do |item| 
         link_rss_response(queue, item)
       end
+    rescue => ex
+      if link.chars.first(1).join == "/" && link.chars.first(2).join != "//"
+        link = "http:/" + link
+        retry
+      elsif link.chars.first(1).join == "/" && link.chars.first(2).join == "//"
+        link = "http:" + link
+        retry
+      end
+
+      if link.chars.first(4).join == "http" && link.chars.first(5).join != "https"
+        link = link.sub('http','https')
+        tried = tried - 1
+        retry if (tried > 0)
+      end 
+ 
+      @logger.error("Error : ", :exception => ex)
     rescue => exc
       puts "ERROR"
       @logger.error("Uknown error while parsing the feed", :exception => exc)
-    end
-  end
+    end # begin
+  end # def response_link
 
   def link_rss_response(queue, item)
       event = LogStash::Event.new()
@@ -110,11 +129,11 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
         else
           eve = LogStash::Event.new( x.name => x.inner_html.to_s )
           event.append( eve )
-        end
-      end
+        end # if
+      end # loop
       decorate(event)
       queue << event
-  end
+  end # def link_rss_response
 
   def not_include_blacklist(link) 
       for i in 0..@blacklist.length-1
@@ -123,7 +142,7 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
         end
       end
       return true
-  end
+  end # def not_include_blacklist
 
 
 end # class LogStash::Inputs::Crawler

data/logstash-input-multirss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '1.0.0'
+  s.version       = '1.0.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
   s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Felix R G
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-14 00:00:00.000000000 Z
+date: 2018-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement