carnivore-files 0.2.2-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: efbd33b4233a2a79fa6084a851e877e7049cb112
+  data.tar.gz: 03990ee4ba3898b06cba3bb5707f0ae2ff384b10
+SHA512:
+  metadata.gz: bdd854f660a2ca204a0445d6c7dd4252be6b7713cdb0544b60e96b3bb62b8ed99b183f409c01101aa90e2d74435036e9a1583efe79d33b646c96bbdfa2d27d20
+  data.tar.gz: 170772e2d4b8f71907e507b2cbfa51e9da6f3b6763e3768e32a98636df1907979ea394958fcc41be3559260e22aede2274d9a68ed2fa41c9e56464612fe7b1c9

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+# v0.2.2
+* Clean up file content fetcher implementations
+
+# v0.2.0
+* Rename Carnivore::Source::File to Carnivore::Source::CarnFile
+
+# v0.1.0
+* Initial release

data/README.md

@@ -0,0 +1,46 @@
+# Carnivore Files
+
+Provides File `Carnivore::Source`
+
+# Usage
+
+```ruby
+require 'carnivore'
+require 'carnivore-files'
+
+Carnivore.configure do
+  source = Carnivore::Source.build(
+    :type => :carn_file, :args => {:path => '/var/log/app.log'}
+  )
+end
+```
+
+The `File` source is built on two "foundations", `sleepy_penguin`
+and `nio4r`. The optimal foundation will be selected based on
+the current Ruby in use (`nio4r` for JRuby, `sleepy_penguin` for
+everything else). If you want to force the foundation:
+
+```ruby
+require 'carnivore'
+require 'carnivore-files'
+
+Carnivore.configure do
+  source = Carnivore::Source.build(
+    :type => :carn_file, :args => {
+      :path => '/var/log/app.log',
+      :foundation => :nio
+    }
+  )
+end
+```
+
+## Important note
+
+The underlying foundations are not installed by this gem. Be sure
+include the dependency within your application dependencies (nio4r
+or sleepy_penguin).
+
+# Info
+* Carnivore: https://github.com/carnivore-rb/carnivore
+* Repository: https://github.com/carnivore-rb/carnivore-files
+* IRC: Freenode @ #carnivore

data/carnivore-files.gemspec

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'carnivore-files/version'
+spec = Gem::Specification.new do |s|
+  s.name = 'carnivore-files'
+  s.version = Carnivore::Files::VERSION.version
+  s.summary = 'Message processing helper'
+  s.author = 'Chris Roberts'
+  s.email = 'code@chrisroberts.org'
+  s.homepage = 'https://github.com/carnivore-rb/carnivore-files'
+  s.description = 'Carnivore file source'
+  s.license = 'Apache 2.0'
+  s.require_path = 'lib'
+  s.add_dependency 'carnivore', '>= 0.1.8'
+  s.add_dependency 'sleepy_penguin'
+  s.add_dependency 'nio4r'
+  s.files = Dir['{lib}/**/**/*'] + %w(carnivore-files.gemspec README.md CHANGELOG.md)
+end

data/lib/carnivore-files/carn_file.rb

@@ -0,0 +1,83 @@
+require 'carnivore'
+
+module Carnivore
+  class Source
+    # Carnivore source for consumption from files
+    class CarnFile < Source
+
+      # @return [String] path to file
+      attr_reader :path
+      # @return [Symbol] registry name of fetcher
+      attr_reader :fetcher
+
+      trap_exit :fetcher_failure
+      finalizer :fetcher_destroyer
+
+      # Setup source
+      #
+      # @param args [Hash]
+      # @option args [String] :path path to file
+      # @option args [Symbol] :foundation underlying file interaction library
+      # @option args [Celluloid::Actor] :notify_actor actor to notify on line receive
+      def setup(*_)
+        @path = ::File.expand_path(args[:path])
+        unless(args[:foundation])
+          args[:foundation] = RUBY_PLATFORM == 'java' ? :nio4r : :penguin
+        end
+      end
+
+      # Start the line fetcher
+      def connect
+        @fetcher_name = "log_fetcher_#{name}".to_sym
+        case args[:foundation].to_sym
+        when :nio, :nio4r
+          @fetcher = Carnivore::Files::Util::Fetcher::Nio.new(args)
+        else
+          @fetcher = Carnivore::Files::Util::Fetcher::Penguin.new(args)
+        end
+        self.link fetcher
+        fetcher.async.start_fetcher
+      end
+
+      # Restart file collector if unexpectedly failed
+      #
+      # @param object [Actor] crashed actor
+      # @param reason [Exception, NilClass]
+      def fetcher_failure(object, reason)
+        if(reason && object == fetcher)
+          error "File message collector unexpectedly failed: #{reason} (restarting)"
+          connect
+        end
+      end
+
+      def fetcher_destroyer
+        if(fetcher && fetcher.alive?)
+          fetcher.terminate
+        end
+      end
+
+      # @return [Array<Hash>] return messages
+      def receive(*args)
+        format_message(Celluloid::Future.new{ fetcher.messages.pop }.value)
+      end
+
+      # Send payload
+      #
+      # @param payload [Object] payload to transmit
+      def transmit(payload, *args)
+        fetcher.write_line(payload)
+      end
+
+      protected
+
+      # Format message into customized Hash
+      #
+      # @param m [Object] payload
+      # @return [Hash]
+      def format_message(m)
+        Smash.new(:path => path, :content => m)
+      end
+
+    end
+  end
+end

data/lib/carnivore-files/util/fetcher.rb

@@ -0,0 +1,98 @@
+require 'carnivore-files'
+
+module Carnivore
+  module Files
+    # Helper utilities
+    module Util
+      # Fetch lines from file
+      class Fetcher
+
+        autoload :Nio, 'carnivore-files/util/nio'
+        autoload :Penguin, 'carnivore-files/util/penguin'
+
+        include Celluloid
+        include Carnivore::Utils::Logging
+
+        # @return [String] path to file
+        attr_reader :path
+        # @return [String] string to split messages on
+        attr_reader :delimiter
+
+        # @return [Queue] messages
+        attr_accessor :messages
+        # @return [IO] underlying IO instance
+        attr_accessor :io
+
+        # Create new instance
+        #
+        # @param args [Hash] initialization args
+        # @option args [String] :path path to file
+        # @option args [String] :delimiter string delimiter to break messages
+        # @option args [Celluloid::Actor] :notify_actor actor to be notified on new messages
+        def initialize(args={})
+          @leftover = ''
+          @path = ::File.expand_path(args[:path])
+          @delimiter = args.fetch(:delimiter, "\n")
+          @messages = Queue.new
+        end
+
+        # Start the line fetcher
+        def start_fetcher
+          raise NotImplementedError
+        end
+
+        # Write line to IO
+        #
+        # @param line [String]
+        # @return [Integer] bytes written
+        def write_line(line)
+          if(io)
+            io.puts(line)
+          else
+            raise 'No IO detected! Failed to write.'
+          end
+        end
+
+        # Retreive lines from file
+        def retrieve_lines
+          if(io)
+            while(data = io.read(4096))
+              @leftover << data
+            end
+            result = @leftover.split(delimiter)
+            @leftover.replace @leftover.end_with?(delimiter) ? '' : result.pop.to_s
+            result
+          else
+            []
+          end
+        end
+
+        # Build the IO and monitor
+        #
+        # @return [TrueClass, FalseClass]
+        def build_io
+          unless(io)
+            if(::File.exists?(path))
+              @io = ::File.open(path, 'r')
+              unless(@waited)
+                @io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
+              else
+                @waited = false
+                retrieve_lines.each do |l|
+                  self.messages << l
+                end
+              end
+            else
+              wait_for_file
+              build_io
+            end
+            true
+          else
+            false
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/carnivore-files/util/nio.rb

@@ -0,0 +1,98 @@
+require 'nio'
+
+module Carnivore
+  module Files
+    module Util
+      class Fetcher
+
+        # NIO based fetcher
+        class Nio < Fetcher
+
+          # @return [NIO::Monitor]
+          attr_accessor :monitor
+          # @return [NIO::Selector]
+          attr_accessor :selector
+
+          # Create new instance
+          #
+          # @param args [Hash] initialization arguments (unused)
+          def initialize(args={})
+            super
+            @selector = NIO::Selector.new
+            every(5) do
+              check_file
+            end
+          end
+
+          # Start the fetcher
+          def start_fetcher
+            loop do
+              build_io
+              messages = nil
+              Celluloid::Future.new{ selector.select }.value.each do |mon|
+                retrieve_lines.each do |l|
+                  self.messages << l
+                end
+              end
+            end
+          end
+
+          private
+
+          # Check for file and destroy monitor if file has changed
+          def check_file
+            if(io)
+              begin
+                unless(io.stat.ino == ::File.stat(path).ino)
+                  destroy_io
+                  @waited = true
+                end
+              rescue Errno::ENOENT
+                destroy_io
+              end
+            end
+          end
+
+          # Build the IO instance if found
+          #
+          # @return [TrueClass]
+          def build_io
+            result = super
+            if(result && @monitor.nil?)
+              @monitor = selector.register(io, :r)
+            end
+            result
+          end
+
+          # Destroy the IO instance and monitor
+          #
+          # @return [TrueClass]
+          def destroy_io
+            if(monitor)
+              selector.deregister(monitor)
+              @monitor = nil
+            end
+            if(io)
+              io.close
+              @io = nil
+            end
+            true
+          end
+
+          # Wait helper for file to appear (5 sleep second intervals)
+          #
+          # @return [TrueClass]
+          def wait_for_file
+            warn "Waiting for file to appear (#{path})"
+            until(::File.exists?(path))
+              @waited = true
+              sleep(5)
+            end
+            info "File has appeared (#{path})!"
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/carnivore-files/util/penguin.rb

@@ -0,0 +1,100 @@
+require 'sleepy_penguin/sp'
+
+module Carnivore
+  module Files
+    module Util
+      class Fetcher
+
+
+        # NIO based fetcher
+        class Penguin < Fetcher
+          # @return [SP::Inotify]
+          attr_accessor :notify
+          # @return [Hash] registered file descriptors
+          attr_accessor :notify_descriptors
+
+          # Create new instance
+          #
+          # @param args [Hash] initialization arguments (unused)
+          def initialize(args={})
+            super
+            @notify = SP::Inotify.new
+            @notify_descriptors = {}
+          end
+
+          # Start the fetcher
+          def start_fetcher
+            loop do
+              build_io
+              notify.each do |event|
+                Celluloid::Future.new{ event.events }.value.each do |ev|
+                  case ev
+                  when :MODIFY
+                    retrieve_lines.each do |l|
+                      self.messages << l
+                    end
+                  when :MOVE_SELF, :DELETE_SELF, :ATTRIB
+                    info "Destroying file IO due to FS modification! (#{ev.inspect})"
+                    destroy_io
+                    @waited = true
+                    break
+                  else
+                    debug "Received unhandled event: #{ev.inspect}"
+                  end
+                end
+                break unless io
+              end
+            end
+          end
+
+
+          private
+
+          # Build the IO and monitor
+          #
+          # @return [TrueClass, FalseClass]
+          def build_io
+            result = super
+            if(result)
+              notify_descriptors[:file_watch] = notify.add_watch(path, :ALL_EVENTS)
+            end
+            result
+          end
+
+          # Destroy the IO and monitor
+          #
+          # @return [TrueClass]
+          def destroy_io
+            if(io)
+              notify.rm_watch(notify_descriptors.delete(:file_watch))
+              @io.close
+              @io = nil
+            end
+            true
+          end
+
+
+          # Wait helper for file to appear (waits for expected notification)
+          #
+          # @return [TrueClass]
+          def wait_for_file
+            until(::File.exists?(path))
+              notified = false
+              directory = ::File.dirname(path)
+              notify_descriptors[:file_wait] = notify.add_watch(directory, :OPEN)
+              until(notified)
+                warn "Waiting for file to appear (#{path})"
+                event = Celluloid::Future.new{ notify.take }.value
+                notified = ::File.exists?(path)
+              end
+              notify.rm_watch(notify_descriptors.delete(:file_wait))
+            end
+            @waited = true
+            info "File has appeared (#{path})!"
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/carnivore-files/version.rb

@@ -0,0 +1,6 @@
+module Carnivore
+  module Files
+    # Current version of library
+    VERSION = Gem::Version.new('0.2.2')
+  end
+end

data/lib/carnivore-files.rb

@@ -0,0 +1,14 @@
+require 'carnivore-files/version'
+require 'carnivore'
+
+module Carnivore
+  # Carnivore source module for files
+  module Files
+    # Utilities for files
+    module Util
+      autoload :Fetcher, 'carnivore-files/util/fetcher'
+    end
+  end
+end
+
+Carnivore::Source.provide(:carn_file, 'carnivore-files/carn_file')

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: carnivore-files
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: java
+authors:
+- Chris Roberts
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-02-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: carnivore
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+description: Carnivore file source
+email: code@chrisroberts.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/carnivore-files.rb
+- lib/carnivore-files/carn_file.rb
+- lib/carnivore-files/version.rb
+- lib/carnivore-files/util/penguin.rb
+- lib/carnivore-files/util/nio.rb
+- lib/carnivore-files/util/fetcher.rb
+- carnivore-files.gemspec
+- README.md
+- CHANGELOG.md
+homepage: https://github.com/carnivore-rb/carnivore-files
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Message processing helper
+test_files: []