logstash-input-multirss 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: efd60b980a94b246bcc89d1bda92d59e25d59734
-  data.tar.gz: 8f92650a28b9d7af40036a3a0df5d9fd3f5f339e
+SHA256:
+  metadata.gz: cad07458c379cfcd9f3a7cbb9608f693a33efe5dfee0a2087387df491c135218
+  data.tar.gz: a5956c590a9e0d9667b278de7f690d3a55ce9eaf886919f2b355c9e12fff8675
 SHA512:
-  metadata.gz: 7f4d76bf1c671c441a01b0c7b1b6bb64a21ead59bea3d3625a594b02c595688efdf14577350ca5a209fe089a9e4bd01d686e07c99e24b32a06e4173fdc99d033
-  data.tar.gz: e6df04af7ec7b0882ac5882f7692339fbca247b79cc562120d2a672a03331e52a5fa528922d91c267b3eb1cc98d1ac97ac3dced91035110af8052de281461beb
+  metadata.gz: 6611bda686c887dbaebba928718cd6521e4c2076e0c44feabdd1e13e89aef3aeb6a80cf8861dfba40b8b9a442f381262bc72023e6d3b3a14c86b94aadf3611ea
+  data.tar.gz: d73bfa18ec57247ff7b566f94e2d6e6a9e710bd09b5b5563034bb9cc74c46ddc2bc5c9f8e13f268ceb259913195922388f3efe2f0c952689e88cee932cb6a453

data/lib/logstash/inputs/multirss.rb

@@ -6,6 +6,7 @@ require "net/http"
 require "uri"
 require "mechanize"
 require "rss"
+require "nokogiri"
 
 class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   config_name "multirss"
@@ -13,7 +14,10 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   default :codec, "plain"
 
   # The rss array list to use in the pipe
-  config :rss_list, :validate => :array, :required => true
+  config :multi_feed, :validate => :array, :required => true
+
+  # The rss array list to use in the pipe
+  config :one_feed, :validate => :array, :default => []
 
   #Set de interval for stoppable_sleep
   config :interval, :validate => :number, :default => 200
@@ -24,9 +28,8 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
   public
    def register
     @urls = []
-    @list_rss = @rss_list
     @agent = Mechanize.new
-	  @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end # def register
 
 
@@ -34,104 +37,93 @@ class LogStash::Inputs::Multirss < LogStash::Inputs::Base
     # we can abort the loop if stop? becomes true
     while !stop?
       
-      @rss_list.each do |rss|
-        @actual_rss = rss
-        puts "Read parent: " + @actual_rss
+      @multi_feed.each do |rss|
+        puts "Read parent: " + rss
         begin
-          page = @agent.get(@actual_rss)
+          page = @agent.get(rss)
           page.links.each do |link|
               if link.href.chars.last(3).join == "xml" && not_include_blacklist(link)
                 @urls << link.href
               end
           end
         rescue
-          puts "Fail to get " + @actual_rss + "feed"
+          puts "Fail to get " + rss + " feed"
         end
          
         links = @urls.uniq
-  
         links.each do |link|
           begin
-            @agente = Mechanize.new
-            response = @agente.get(link)
-            @agente.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-            handle_response(response, queue)
+            response_link(link,queue)
             puts "Read clidren: " + link
           rescue
             puts "Fail to get " + link
+            next
           end
         end
-
         @urls.clear
 
       end
 
+      @one_feed.each do |feed|
+        @urls << feed
+      end
+      links_o = @urls.uniq
+      links_o.each do |link|
+        begin
+          response_link(link,queue)
+          puts "Read clidren: " + link
+        rescue
+          puts "Fail to get " + link
+          next
+        end
+      end
+
+      @urls.clear
+
     Stud.stoppable_sleep(@interval) { stop? }
     end # loop
    end # def run
 
 
    def stop
-    # nothing to do in this case so it is not necessary to define stop
-    # examples of common "stop" tasks:
-    #  * close sockets (unblocking blocking reads/accets)
-    #  * cleanup temporary files
-    #  * terminate spawned threads
+ 
    end
 
-   def not_include_blacklist(link) 
-      for i in 0..@blacklist.length-1
-        if link.href.include?(@blacklist[i])
-          return false
-        end
-      end
-      return true
-   end
-
-
-  def handle_response(response, queue)
-    body = response.body
+  def response_link(link, queue)
     begin
-      feed = RSS::Parser.parse(body)
-      feed.items.each do |item|
-        if has_enclosure?(item)
-          puts "item have a enclosure field"
-          next
-        else
-          handle_rss_response(queue, item)
-        end
+      page = Nokogiri::XML(open(link))
+      page.search('item').each do |item| 
+        link_rss_response(queue, item)
       end
-    rescue RSS::MissingTagError => e
-      next
-      @logger.error("Invalid RSS feed", :exception => e)
-    rescue RSS::TooMuchTagError => ex
-      next
-      @logger.error("TooMuchTagError feed (have enclosure tag)", :exception => ex)
     rescue => exc
-      next
+      puts "ERROR"
       @logger.error("Uknown error while parsing the feed", :exception => exc)
     end
   end
 
-
-  def handle_rss_response(queue, item)
-        @codec.decode(item.description) do |event|
-          event.set("Feed",  @actual_rss)
-          event.set("published", item.pubDate)
-          event.set("title", item.title)
-          event.set("link", item.link)
-          event.set("author", item.author)
-          decorate(event)
-          queue << event
+  def link_rss_response(queue, item)
+      event = LogStash::Event.new()
+      item.element_children.each do |x| 
+        if x.inner_html.to_s.chars.first(9).join == "<![CDATA["
+          eve = LogStash::Event.new( x.name => x.inner_html.to_s[9..x.inner_html.to_s.length-4])
+          event.append( eve )
+        else
+          eve = LogStash::Event.new( x.name => x.inner_html.to_s )
+          event.append( eve )
         end
+      end
+      decorate(event)
+      queue << event
   end
 
-  def has_enclosure?(item)
-    if item.enclosure 
+  def not_include_blacklist(link) 
+      for i in 0..@blacklist.length-1
+        if link.href.include?(@blacklist[i])
+          return false
+        end
+      end
       return true
-    else
-      return false
-    end
-  end 
+  end
+
 
 end # class LogStash::Inputs::Crawler

data/logstash-input-multirss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirss'
-  s.version       = '0.1.1'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Simple multi rss plugin'
   s.description   = 'This plugin needs a list of links of different rss. Get all the links of the main feed  pages and get all the content of each of the links.'

metadata

@@ -1,94 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirss
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Felix R G
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-07 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-core
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
-  type: :runtime
+  name: stud
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.16
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.16
 - !ruby/object:Gem::Dependency
-  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: mechanize
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: nokogiri
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -108,8 +108,6 @@ files:
 - LICENSE
 - README.md
 - lib/logstash/inputs/multirss.rb
-- lib/logstash/inputs/multirss.rb.bk
-- lib/logstash/inputs/multirss.rb.bk2
 - logstash-input-multirss.gemspec
 - spec/inputs/multirss_spec.rb
 homepage: https://github.com/felixramirezgarcia/logstash-input-multirss
@@ -118,7 +116,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -133,9 +131,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
 specification_version: 4
 summary: Simple multi rss plugin
 test_files:

data/lib/logstash/inputs/multirss.rb.bk

@@ -1,51 +0,0 @@
-# encoding: utf-8
-require "logstash/inputs/base"
-require "logstash/namespace"
-require "stud/interval"
-require "socket" # for Socket.gethostname
-
-# Generate a repeating message.
-#
-# This plugin is intented only as an example.
-
-class LogStash::Inputs::Multirss < LogStash::Inputs::Base
-  config_name "multirss"
-
-  # If undefined, Logstash will complain, even if codec is unused.
-  default :codec, "plain"
-
-  # The message string to use in the event.
-  config :message, :validate => :string, :default => "Hello World!"
-
-  # Set how frequently messages should be sent.
-  #
-  # The default, `1`, means send a message every second.
-  config :interval, :validate => :number, :default => 1
-
-  public
-  def register
-    @host = Socket.gethostname
-  end # def register
-
-  def run(queue)
-    # we can abort the loop if stop? becomes true
-    while !stop?
-      event = LogStash::Event.new("message" => @message, "host" => @host)
-      decorate(event)
-      queue << event
-      # because the sleep interval can be big, when shutdown happens
-      # we want to be able to abort the sleep
-      # Stud.stoppable_sleep will frequently evaluate the given block
-      # and abort the sleep(@interval) if the return value is true
-      Stud.stoppable_sleep(@interval) { stop? }
-    end # loop
-  end # def run
-
-  def stop
-    # nothing to do in this case so it is not necessary to define stop
-    # examples of common "stop" tasks:
-    #  * close sockets (unblocking blocking reads/accepts)
-    #  * cleanup temporary files
-    #  * terminate spawned threads
-  end
-end # class LogStash::Inputs::Multirss

data/lib/logstash/inputs/multirss.rb.bk2

@@ -1,117 +0,0 @@
-# encoding: utf-8
-require "logstash/inputs/base"
-require "logstash/namespace"
-require "stud/interval"
-require "net/http"
-require "uri"
-require "mechanize"
-require "faraday"
-require "rss"
-
-class LogStash::Inputs::Crawler < LogStash::Inputs::Base
-  config_name "multirss"
-
-  # If undefined, Logstash will complain, even if codec is unused.
-  default :codec, "plain"
-
-  # The message string to use in the event.
-  config :urls, :validate => :array, :required => true
-
-  #Set de interval for stoppable_sleep
-  config :interval, :validate => :number, :default => 86400
-
-  #Domains to exclude
-  config :blacklist, :validate => :array , :default => ['http://fusion.google.com/','yahoo.com','live.com','netvibes.com']
-
-  public
-   def register
-    @urls = []
-    puts "**********************************************************"
-    puts "STARTING MULTI-RSS"
-    puts "*******************************************************"
-    @agent = Mechanize.new
-	  @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-   end # def register
-
-
-   def run(queue)
-    # we can abort the loop if stop? becomes true
-    while !stop?
-      puts "********************ENTRA1"
-      @urls.each do |url|
-        page = @agent.get(@url)
-        page.links.each do |link|
-              if link.href.chars.last(3).join == "xml" && valid_link?(link)
-                puts "*******************************************************"
-                puts link.href
-                @new_urls << link.href
-              end
-        end
-      
-        links = @new_urls.uniq
-
-        links.each do |link|
-            puts "********************ENTRA2"
-            response = @agent.get(link)
-            handle_response(response, queue)
-        end
-
-      end #urls.each
-    Stud.stoppable_sleep(@interval) { stop? }
-    end # loop
-   end # def run
-
-
-   def stop
-    # nothing to do in this case so it is not necessary to define stop
-    # examples of common "stop" tasks:
-    #  * close sockets (unblocking blocking reads/accets)
-    #  * cleanup temporary files
-    #  * terminate spawned threads
-   end
-
-   def valid_link?(link)
-    puts "********************ENTRA3"
-      @blacklist.each do |black_link|
-        if link.href.include?(black_link)
-          return false
-        end
-      end
-      return true
-   end
-
-
-   def handle_response(response, queue)
-    puts "********************ENTRA4"
-    body = response.body
-    begin
-      feed = RSS::Parser.parse(body)
-      feed.items.each do |item|
-        # Put each item into an event
-        @logger.debug("Item", :item => item.author)
-        handle_rss_response(queue, item)
-      end
-    rescue RSS::MissingTagError => e
-      @logger.error("Invalid RSS feed", :exception => e)
-    rescue => e
-      @logger.error("Uknown error while parsing the feed", :exception => e)
-    end
-  end
-
-
-  def handle_rss_response(queue, item)
-    puts "********************ENTRA5"
-    @codec.decode(item.description) do |event|
-      event.set("Feed",  @url)
-      event.set("published", item.pubDate)
-      event.set("title", item.title)
-      event.set("link", item.link)
-      event.set("author", item.author)
-      decorate(event)
-      queue << event
-    end
-  end
-
-
-
-end # class LogStash::Inputs::Crawler