carnivore-files 0.2.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: efbd33b4233a2a79fa6084a851e877e7049cb112
4
+ data.tar.gz: 03990ee4ba3898b06cba3bb5707f0ae2ff384b10
5
+ SHA512:
6
+ metadata.gz: bdd854f660a2ca204a0445d6c7dd4252be6b7713cdb0544b60e96b3bb62b8ed99b183f409c01101aa90e2d74435036e9a1583efe79d33b646c96bbdfa2d27d20
7
+ data.tar.gz: 170772e2d4b8f71907e507b2cbfa51e9da6f3b6763e3768e32a98636df1907979ea394958fcc41be3559260e22aede2274d9a68ed2fa41c9e56464612fe7b1c9
data/CHANGELOG.md ADDED
@@ -0,0 +1,8 @@
1
+ # v0.2.2
2
+ * Clean up file content fetcher implementations
3
+
4
+ # v0.2.0
5
+ * Rename Carnivore::Source::File to Carnivore::Source::CarnFile
6
+
7
+ # v0.1.0
8
+ * Initial release
data/README.md ADDED
@@ -0,0 +1,46 @@
1
+ # Carnivore Files
2
+
3
+ Provides File `Carnivore::Source`
4
+
5
+ # Usage
6
+
7
+ ```ruby
8
+ require 'carnivore'
9
+ require 'carnivore-files'
10
+
11
+ Carnivore.configure do
12
+ source = Carnivore::Source.build(
13
+ :type => :carn_file, :args => {:path => '/var/log/app.log'}
14
+ )
15
+ end
16
+ ```
17
+
18
+ The `File` source is built on two "foundations", `sleepy_penguin`
19
+ and `nio4r`. The optimal foundation will be selected based on
20
+ the current Ruby in use (`nio4r` for JRuby, `sleepy_penguin` for
21
+ everything else). If you want to force the foundation:
22
+
23
+ ```ruby
24
+ require 'carnivore'
25
+ require 'carnivore-files'
26
+
27
+ Carnivore.configure do
28
+ source = Carnivore::Source.build(
29
+ :type => :carn_file, :args => {
30
+ :path => '/var/log/app.log',
31
+ :foundation => :nio
32
+ }
33
+ )
34
+ end
35
+ ```
36
+
37
+ ## Important note
38
+
39
+ The underlying foundations are not installed by this gem. Be sure
40
+ include the dependency within your application dependencies (nio4r
41
+ or sleepy_penguin).
42
+
43
+ # Info
44
+ * Carnivore: https://github.com/carnivore-rb/carnivore
45
+ * Repository: https://github.com/carnivore-rb/carnivore-files
46
+ * IRC: Freenode @ #carnivore
@@ -0,0 +1,17 @@
1
+ $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
2
+ require 'carnivore-files/version'
3
+ spec = Gem::Specification.new do |s|
4
+ s.name = 'carnivore-files'
5
+ s.version = Carnivore::Files::VERSION.version
6
+ s.summary = 'Message processing helper'
7
+ s.author = 'Chris Roberts'
8
+ s.email = 'code@chrisroberts.org'
9
+ s.homepage = 'https://github.com/carnivore-rb/carnivore-files'
10
+ s.description = 'Carnivore file source'
11
+ s.license = 'Apache 2.0'
12
+ s.require_path = 'lib'
13
+ s.add_dependency 'carnivore', '>= 0.1.8'
14
+ s.add_dependency 'sleepy_penguin'
15
+ s.add_dependency 'nio4r'
16
+ s.files = Dir['{lib}/**/**/*'] + %w(carnivore-files.gemspec README.md CHANGELOG.md)
17
+ end
@@ -0,0 +1,83 @@
1
+ require 'carnivore'
2
+
3
+ module Carnivore
4
+ class Source
5
+ # Carnivore source for consumption from files
6
+ class CarnFile < Source
7
+
8
+ # @return [String] path to file
9
+ attr_reader :path
10
+ # @return [Symbol] registry name of fetcher
11
+ attr_reader :fetcher
12
+
13
+ trap_exit :fetcher_failure
14
+ finalizer :fetcher_destroyer
15
+
16
+ # Setup source
17
+ #
18
+ # @param args [Hash]
19
+ # @option args [String] :path path to file
20
+ # @option args [Symbol] :foundation underlying file interaction library
21
+ # @option args [Celluloid::Actor] :notify_actor actor to notify on line receive
22
+ def setup(*_)
23
+ @path = ::File.expand_path(args[:path])
24
+ unless(args[:foundation])
25
+ args[:foundation] = RUBY_PLATFORM == 'java' ? :nio4r : :penguin
26
+ end
27
+ end
28
+
29
+ # Start the line fetcher
30
+ def connect
31
+ @fetcher_name = "log_fetcher_#{name}".to_sym
32
+ case args[:foundation].to_sym
33
+ when :nio, :nio4r
34
+ @fetcher = Carnivore::Files::Util::Fetcher::Nio.new(args)
35
+ else
36
+ @fetcher = Carnivore::Files::Util::Fetcher::Penguin.new(args)
37
+ end
38
+ self.link fetcher
39
+ fetcher.async.start_fetcher
40
+ end
41
+
42
+ # Restart file collector if unexpectedly failed
43
+ #
44
+ # @param object [Actor] crashed actor
45
+ # @param reason [Exception, NilClass]
46
+ def fetcher_failure(object, reason)
47
+ if(reason && object == fetcher)
48
+ error "File message collector unexpectedly failed: #{reason} (restarting)"
49
+ connect
50
+ end
51
+ end
52
+
53
+ def fetcher_destroyer
54
+ if(fetcher && fetcher.alive?)
55
+ fetcher.terminate
56
+ end
57
+ end
58
+
59
+ # @return [Array<Hash>] return messages
60
+ def receive(*args)
61
+ format_message(Celluloid::Future.new{ fetcher.messages.pop }.value)
62
+ end
63
+
64
+ # Send payload
65
+ #
66
+ # @param payload [Object] payload to transmit
67
+ def transmit(payload, *args)
68
+ fetcher.write_line(payload)
69
+ end
70
+
71
+ protected
72
+
73
+ # Format message into customized Hash
74
+ #
75
+ # @param m [Object] payload
76
+ # @return [Hash]
77
+ def format_message(m)
78
+ Smash.new(:path => path, :content => m)
79
+ end
80
+
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,98 @@
1
+ require 'carnivore-files'
2
+
3
+ module Carnivore
4
+ module Files
5
+ # Helper utilities
6
+ module Util
7
+ # Fetch lines from file
8
+ class Fetcher
9
+
10
+ autoload :Nio, 'carnivore-files/util/nio'
11
+ autoload :Penguin, 'carnivore-files/util/penguin'
12
+
13
+ include Celluloid
14
+ include Carnivore::Utils::Logging
15
+
16
+ # @return [String] path to file
17
+ attr_reader :path
18
+ # @return [String] string to split messages on
19
+ attr_reader :delimiter
20
+
21
+ # @return [Queue] messages
22
+ attr_accessor :messages
23
+ # @return [IO] underlying IO instance
24
+ attr_accessor :io
25
+
26
+ # Create new instance
27
+ #
28
+ # @param args [Hash] initialization args
29
+ # @option args [String] :path path to file
30
+ # @option args [String] :delimiter string delimiter to break messages
31
+ # @option args [Celluloid::Actor] :notify_actor actor to be notified on new messages
32
+ def initialize(args={})
33
+ @leftover = ''
34
+ @path = ::File.expand_path(args[:path])
35
+ @delimiter = args.fetch(:delimiter, "\n")
36
+ @messages = Queue.new
37
+ end
38
+
39
+ # Start the line fetcher
40
+ def start_fetcher
41
+ raise NotImplementedError
42
+ end
43
+
44
+ # Write line to IO
45
+ #
46
+ # @param line [String]
47
+ # @return [Integer] bytes written
48
+ def write_line(line)
49
+ if(io)
50
+ io.puts(line)
51
+ else
52
+ raise 'No IO detected! Failed to write.'
53
+ end
54
+ end
55
+
56
+ # Retreive lines from file
57
+ def retrieve_lines
58
+ if(io)
59
+ while(data = io.read(4096))
60
+ @leftover << data
61
+ end
62
+ result = @leftover.split(delimiter)
63
+ @leftover.replace @leftover.end_with?(delimiter) ? '' : result.pop.to_s
64
+ result
65
+ else
66
+ []
67
+ end
68
+ end
69
+
70
+ # Build the IO and monitor
71
+ #
72
+ # @return [TrueClass, FalseClass]
73
+ def build_io
74
+ unless(io)
75
+ if(::File.exists?(path))
76
+ @io = ::File.open(path, 'r')
77
+ unless(@waited)
78
+ @io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
79
+ else
80
+ @waited = false
81
+ retrieve_lines.each do |l|
82
+ self.messages << l
83
+ end
84
+ end
85
+ else
86
+ wait_for_file
87
+ build_io
88
+ end
89
+ true
90
+ else
91
+ false
92
+ end
93
+ end
94
+
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,98 @@
1
+ require 'nio'
2
+
3
+ module Carnivore
4
+ module Files
5
+ module Util
6
+ class Fetcher
7
+
8
+ # NIO based fetcher
9
+ class Nio < Fetcher
10
+
11
+ # @return [NIO::Monitor]
12
+ attr_accessor :monitor
13
+ # @return [NIO::Selector]
14
+ attr_accessor :selector
15
+
16
+ # Create new instance
17
+ #
18
+ # @param args [Hash] initialization arguments (unused)
19
+ def initialize(args={})
20
+ super
21
+ @selector = NIO::Selector.new
22
+ every(5) do
23
+ check_file
24
+ end
25
+ end
26
+
27
+ # Start the fetcher
28
+ def start_fetcher
29
+ loop do
30
+ build_io
31
+ messages = nil
32
+ Celluloid::Future.new{ selector.select }.value.each do |mon|
33
+ retrieve_lines.each do |l|
34
+ self.messages << l
35
+ end
36
+ end
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ # Check for file and destroy monitor if file has changed
43
+ def check_file
44
+ if(io)
45
+ begin
46
+ unless(io.stat.ino == ::File.stat(path).ino)
47
+ destroy_io
48
+ @waited = true
49
+ end
50
+ rescue Errno::ENOENT
51
+ destroy_io
52
+ end
53
+ end
54
+ end
55
+
56
+ # Build the IO instance if found
57
+ #
58
+ # @return [TrueClass]
59
+ def build_io
60
+ result = super
61
+ if(result && @monitor.nil?)
62
+ @monitor = selector.register(io, :r)
63
+ end
64
+ result
65
+ end
66
+
67
+ # Destroy the IO instance and monitor
68
+ #
69
+ # @return [TrueClass]
70
+ def destroy_io
71
+ if(monitor)
72
+ selector.deregister(monitor)
73
+ @monitor = nil
74
+ end
75
+ if(io)
76
+ io.close
77
+ @io = nil
78
+ end
79
+ true
80
+ end
81
+
82
+ # Wait helper for file to appear (5 sleep second intervals)
83
+ #
84
+ # @return [TrueClass]
85
+ def wait_for_file
86
+ warn "Waiting for file to appear (#{path})"
87
+ until(::File.exists?(path))
88
+ @waited = true
89
+ sleep(5)
90
+ end
91
+ info "File has appeared (#{path})!"
92
+ end
93
+
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,100 @@
1
+ require 'sleepy_penguin/sp'
2
+
3
+ module Carnivore
4
+ module Files
5
+ module Util
6
+ class Fetcher
7
+
8
+
9
+ # NIO based fetcher
10
+ class Penguin < Fetcher
11
+ # @return [SP::Inotify]
12
+ attr_accessor :notify
13
+ # @return [Hash] registered file descriptors
14
+ attr_accessor :notify_descriptors
15
+
16
+ # Create new instance
17
+ #
18
+ # @param args [Hash] initialization arguments (unused)
19
+ def initialize(args={})
20
+ super
21
+ @notify = SP::Inotify.new
22
+ @notify_descriptors = {}
23
+ end
24
+
25
+ # Start the fetcher
26
+ def start_fetcher
27
+ loop do
28
+ build_io
29
+ notify.each do |event|
30
+ Celluloid::Future.new{ event.events }.value.each do |ev|
31
+ case ev
32
+ when :MODIFY
33
+ retrieve_lines.each do |l|
34
+ self.messages << l
35
+ end
36
+ when :MOVE_SELF, :DELETE_SELF, :ATTRIB
37
+ info "Destroying file IO due to FS modification! (#{ev.inspect})"
38
+ destroy_io
39
+ @waited = true
40
+ break
41
+ else
42
+ debug "Received unhandled event: #{ev.inspect}"
43
+ end
44
+ end
45
+ break unless io
46
+ end
47
+ end
48
+ end
49
+
50
+
51
+ private
52
+
53
+ # Build the IO and monitor
54
+ #
55
+ # @return [TrueClass, FalseClass]
56
+ def build_io
57
+ result = super
58
+ if(result)
59
+ notify_descriptors[:file_watch] = notify.add_watch(path, :ALL_EVENTS)
60
+ end
61
+ result
62
+ end
63
+
64
+ # Destroy the IO and monitor
65
+ #
66
+ # @return [TrueClass]
67
+ def destroy_io
68
+ if(io)
69
+ notify.rm_watch(notify_descriptors.delete(:file_watch))
70
+ @io.close
71
+ @io = nil
72
+ end
73
+ true
74
+ end
75
+
76
+
77
+ # Wait helper for file to appear (waits for expected notification)
78
+ #
79
+ # @return [TrueClass]
80
+ def wait_for_file
81
+ until(::File.exists?(path))
82
+ notified = false
83
+ directory = ::File.dirname(path)
84
+ notify_descriptors[:file_wait] = notify.add_watch(directory, :OPEN)
85
+ until(notified)
86
+ warn "Waiting for file to appear (#{path})"
87
+ event = Celluloid::Future.new{ notify.take }.value
88
+ notified = ::File.exists?(path)
89
+ end
90
+ notify.rm_watch(notify_descriptors.delete(:file_wait))
91
+ end
92
+ @waited = true
93
+ info "File has appeared (#{path})!"
94
+ end
95
+
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,6 @@
1
+ module Carnivore
2
+ module Files
3
+ # Current version of library
4
+ VERSION = Gem::Version.new('0.2.2')
5
+ end
6
+ end
@@ -0,0 +1,14 @@
1
+ require 'carnivore-files/version'
2
+ require 'carnivore'
3
+
4
+ module Carnivore
5
+ # Carnivore source module for files
6
+ module Files
7
+ # Utilities for files
8
+ module Util
9
+ autoload :Fetcher, 'carnivore-files/util/fetcher'
10
+ end
11
+ end
12
+ end
13
+
14
+ Carnivore::Source.provide(:carn_file, 'carnivore-files/carn_file')
metadata ADDED
@@ -0,0 +1,80 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: carnivore-files
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.2
5
+ platform: java
6
+ authors:
7
+ - Chris Roberts
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-02-02 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: carnivore
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 0.1.8
20
+ requirement: !ruby/object:Gem::Requirement
21
+ requirements:
22
+ - - '>='
23
+ - !ruby/object:Gem::Version
24
+ version: 0.1.8
25
+ prerelease: false
26
+ type: :runtime
27
+ - !ruby/object:Gem::Dependency
28
+ name: nio4r
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - '>='
37
+ - !ruby/object:Gem::Version
38
+ version: '0'
39
+ prerelease: false
40
+ type: :runtime
41
+ description: Carnivore file source
42
+ email: code@chrisroberts.org
43
+ executables: []
44
+ extensions: []
45
+ extra_rdoc_files: []
46
+ files:
47
+ - lib/carnivore-files.rb
48
+ - lib/carnivore-files/carn_file.rb
49
+ - lib/carnivore-files/version.rb
50
+ - lib/carnivore-files/util/penguin.rb
51
+ - lib/carnivore-files/util/nio.rb
52
+ - lib/carnivore-files/util/fetcher.rb
53
+ - carnivore-files.gemspec
54
+ - README.md
55
+ - CHANGELOG.md
56
+ homepage: https://github.com/carnivore-rb/carnivore-files
57
+ licenses:
58
+ - Apache 2.0
59
+ metadata: {}
60
+ post_install_message:
61
+ rdoc_options: []
62
+ require_paths:
63
+ - lib
64
+ required_ruby_version: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ required_rubygems_version: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - '>='
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ requirements: []
75
+ rubyforge_project:
76
+ rubygems_version: 2.1.9
77
+ signing_key:
78
+ specification_version: 4
79
+ summary: Message processing helper
80
+ test_files: []