carnivore-files 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4150377900b933160fc7b0efb9ad3b022f30d062
-  data.tar.gz: 29eee65845600e46a3b99d3f15fca7c61d54e679
+  metadata.gz: c9393f45222d01b7842f08069591d1e50c3371dc
+  data.tar.gz: 515225fcea7f56ae7c1d4596c951ee3ca827668f
 SHA512:
-  metadata.gz: 35a949382dbeedde31e19f07c1bc9682876446f7b889defb4352c620d95885ca45330d7f86835aaa106f2b77a3315175a7ff4ef93259b6cf9bd3a1e3dec83217
-  data.tar.gz: 8bf78650c8048131e5b64ef7c6816fc5704791986dd62a5217a3863e48f6c8e5942a4c9966e4ea7f8e247381513dc28fb483b6b3544e5d0b1aa8db2c783122cc
+  metadata.gz: 36b7787ad5dbbcff760a296a5dcb405d1998ed87451ecfd43bdd0461623629a8f368c7076f284409ceed821038af270837e886b5bcd803ccc4766d79013275e0
+  data.tar.gz: 435775b1547fa6920900783830c1fd5bbc057606608ed9a7bc7e934a3cf0bfc5790b693ffc41150a14ea0d4cc1573910867835cfc79a0c6eba477616b0814e67

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# v0.2.2
+* Clean up file content fetcher implementations
+
 # v0.2.0
 * Rename Carnivore::Source::File to Carnivore::Source::CarnFile
 

data/carnivore-files.gemspec

@@ -1,6 +1,6 @@
 $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
 require 'carnivore-files/version'
-Gem::Specification.new do |s|
+spec = Gem::Specification.new do |s|
   s.name = 'carnivore-files'
   s.version = Carnivore::Files::VERSION.version
   s.summary = 'Message processing helper'
@@ -11,9 +11,7 @@ Gem::Specification.new do |s|
   s.license = 'Apache 2.0'
   s.require_path = 'lib'
   s.add_dependency 'carnivore', '>= 0.1.8'
-  #  support both but don't want to install both
-  #  update to be setup like carnivore-rabbitmq
-  #  s.add_dependency 'nio4r'
   s.add_dependency 'sleepy_penguin'
+  s.add_dependency 'nio4r'
   s.files = Dir['{lib}/**/**/*'] + %w(carnivore-files.gemspec README.md CHANGELOG.md)
 end

data/lib/carnivore-files/carn_file.rb

@@ -8,7 +8,10 @@ module Carnivore
       # @return [String] path to file
       attr_reader :path
       # @return [Symbol] registry name of fetcher
-      attr_reader :fetcher_name
+      attr_reader :fetcher
+
+      trap_exit :fetcher_failure
+      finalizer :fetcher_destroyer
 
       # Setup source
       #
@@ -16,40 +19,46 @@ module Carnivore
       # @option args [String] :path path to file
       # @option args [Symbol] :foundation underlying file interaction library
       # @option args [Celluloid::Actor] :notify_actor actor to notify on line receive
-      def setup(args={})
+      def setup(*_)
         @path = ::File.expand_path(args[:path])
-        @fetcher_name = "log_fetcher_#{name}".to_sym
         unless(args[:foundation])
-          args[:foundation] = RUBY_ENGINE == 'jruby' ? :nio4r : :penguin
+          args[:foundation] = RUBY_PLATFORM == 'java' ? :nio4r : :penguin
         end
+      end
+
+      # Start the line fetcher
+      def connect
+        @fetcher_name = "log_fetcher_#{name}".to_sym
         case args[:foundation].to_sym
         when :nio, :nio4r
-          callback_supervisor.supervise_as(fetcher_name, Carnivore::Files::Util::Fetcher::Nio,
-            args.merge(:notify_actor => current_actor)
-          )
+          @fetcher = Carnivore::Files::Util::Fetcher::Nio.new(args)
         else
-          callback_supervisor.supervise_as(fetcher_name, Carnivore::Files::Util::Fetcher::Penguin,
-            args.merge(:notify_actor => current_actor)
-          )
+          @fetcher = Carnivore::Files::Util::Fetcher::Penguin.new(args)
         end
+        self.link fetcher
+        fetcher.async.start_fetcher
       end
 
-      # @return [Carnivore::Files::Util::Fetcher] line fetcher
-      def fetcher
-        callback_supervisor[fetcher_name]
+      # Restart file collector if unexpectedly failed
+      #
+      # @param object [Actor] crashed actor
+      # @param reason [Exception, NilClass]
+      def fetcher_failure(object, reason)
+        if(reason && object == fetcher)
+          error "File message collector unexpectedly failed: #{reason} (restarting)"
+          connect
+        end
       end
 
-      # Start the line fetcher
-      def connect
-        fetcher.async.start_fetcher
+      def fetcher_destroyer
+        if(fetcher && fetcher.alive?)
+          fetcher.terminate
+        end
       end
 
       # @return [Array<Hash>] return messages
       def receive(*args)
-        wait(:new_log_lines)
-        fetcher.return_lines.map do |l|
-          format_message(l)
-        end
+        format_message(Celluloid::Future.new{ fetcher.messages.pop }.value)
       end
 
       # Send payload

data/lib/carnivore-files/util/fetcher.rb

@@ -1,3 +1,5 @@
+require 'carnivore-files'
+
 module Carnivore
   module Files
     # Helper utilities
@@ -15,10 +17,8 @@ module Carnivore
         attr_reader :path
         # @return [String] string to split messages on
         attr_reader :delimiter
-        # @return [Celluloid::Actor] actor to notify on new messages
-        attr_reader :notify_actor
 
-        # @return [Array] messages
+        # @return [Queue] messages
         attr_accessor :messages
         # @return [IO] underlying IO instance
         attr_accessor :io
@@ -33,29 +33,12 @@ module Carnivore
           @leftover = ''
           @path = ::File.expand_path(args[:path])
           @delimiter = args.fetch(:delimiter, "\n")
-          @notify_actor = args[:notify_actor]
-          @messages = []
+          @messages = Queue.new
         end
 
         # Start the line fetcher
         def start_fetcher
-          defer do
-            loop do
-              build_socket
-              messages = nil
-              selector.select.each do |mon|
-                self.messages += retrieve_lines
-              end
-              notify_actor.signal(:new_logs_lines) unless self.messages.empty?
-            end
-          end
-        end
-
-        # @return [Array<String>] current lines
-        def return_lines
-          msgs = messages.dup
-          messages.clear
-          msgs
+          raise NotImplementedError
         end
 
         # Write line to IO
@@ -84,6 +67,31 @@ module Carnivore
           end
         end
 
+        # Build the IO and monitor
+        #
+        # @return [TrueClass, FalseClass]
+        def build_io
+          unless(io)
+            if(::File.exists?(path))
+              @io = ::File.open(path, 'r')
+              unless(@waited)
+                @io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
+              else
+                @waited = false
+                retrieve_lines.each do |l|
+                  self.messages << l
+                end
+              end
+            else
+              wait_for_file
+              build_io
+            end
+            true
+          else
+            false
+          end
+        end
+
       end
     end
   end

data/lib/carnivore-files/util/nio.rb

@@ -26,14 +26,13 @@ module Carnivore
 
           # Start the fetcher
           def start_fetcher
-            defer do
-              loop do
-                build_io
-                messages = nil
-                selector.select.each do |mon|
-                  self.messages += retrieve_lines
+            loop do
+              build_io
+              messages = nil
+              Celluloid::Future.new{ selector.select }.value.each do |mon|
+                retrieve_lines.each do |l|
+                  self.messages << l
                 end
-                notify_actor.signal(:new_log_lines) unless self.messages.empty?
               end
             end
           end
@@ -46,6 +45,7 @@ module Carnivore
               begin
                 unless(io.stat.ino == ::File.stat(path).ino)
                   destroy_io
+                  @waited = true
                 end
               rescue Errno::ENOENT
                 destroy_io
@@ -57,19 +57,11 @@ module Carnivore
           #
           # @return [TrueClass]
           def build_io
-            unless(monitor)
-              if(::File.exists?(path))
-                unless(io)
-                  @io = ::File.open(path, 'r')
-                  @io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
-                end
-                @monitor = selector.register(io, :r)
-              else
-                wait_for_file
-                build_io
-              end
+            result = super
+            if(result && @monitor.nil?)
+              @monitor = selector.register(io, :r)
             end
-            true
+            result
           end
 
           # Destroy the IO instance and monitor
@@ -93,6 +85,7 @@ module Carnivore
           def wait_for_file
             warn "Waiting for file to appear (#{path})"
             until(::File.exists?(path))
+              @waited = true
               sleep(5)
             end
             info "File has appeared (#{path})!"

data/lib/carnivore-files/util/penguin.rb

@@ -24,41 +24,41 @@ module Carnivore
 
           # Start the fetcher
           def start_fetcher
-            defer do
-              loop do
-                build_io
-                notify.each do |event|
-                  event.events.each do |ev|
-                    case ev
-                    when :MODIFY
-                      self.messages += retrieve_lines
-                    when :MOVE_SELF, :DELETE_SELF, :ATTRIB
-                      destroy_io
+            loop do
+              build_io
+              notify.each do |event|
+                Celluloid::Future.new{ event.events }.value.each do |ev|
+                  case ev
+                  when :MODIFY
+                    retrieve_lines.each do |l|
+                      self.messages << l
                     end
+                  when :MOVE_SELF, :DELETE_SELF, :ATTRIB
+                    info "Destroying file IO due to FS modification! (#{ev.inspect})"
+                    destroy_io
+                    @waited = true
+                    break
+                  else
+                    debug "Received unhandled event: #{ev.inspect}"
                   end
-                  notify_actor.signal(:new_log_lines) unless messages.empty?
                 end
+                break unless io
               end
             end
           end
 
+
           private
 
           # Build the IO and monitor
           #
-          # @return [TrueClass]
+          # @return [TrueClass, FalseClass]
           def build_io
-            unless(io)
-              if(::File.exists?(path))
-                notify_descriptors[:file_watch] = notify.add_watch(path, :ALL_EVENTS)
-                @io = ::File.open(path, 'r')
-                @io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
-              else
-                wait_for_file
-                build_io
-              end
+            result = super
+            if(result)
+              notify_descriptors[:file_watch] = notify.add_watch(path, :ALL_EVENTS)
             end
-            true
+            result
           end
 
           # Destroy the IO and monitor
@@ -84,13 +84,12 @@ module Carnivore
               notify_descriptors[:file_wait] = notify.add_watch(directory, :OPEN)
               until(notified)
                 warn "Waiting for file to appear (#{path})"
-                event = notify.take
-                if(event.name)
-                  notified = ::File.expand_path(event.name) == path
-                end
+                event = Celluloid::Future.new{ notify.take }.value
+                notified = ::File.exists?(path)
               end
               notify.rm_watch(notify_descriptors.delete(:file_wait))
             end
+            @waited = true
             info "File has appeared (#{path})!"
           end
 

data/lib/carnivore-files/version.rb

@@ -1,6 +1,6 @@
 module Carnivore
   module Files
     # Current version of library
-    VERSION = Gem::Version.new('0.2.0')
+    VERSION = Gem::Version.new('0.2.2')
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: carnivore-files
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Chris Roberts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-11 00:00:00.000000000 Z
+date: 2015-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: carnivore
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Carnivore file source
 email: code@chrisroberts.org
 executables: []