carnivore-files 0.2.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/README.md +46 -0
- data/carnivore-files.gemspec +17 -0
- data/lib/carnivore-files/carn_file.rb +83 -0
- data/lib/carnivore-files/util/fetcher.rb +98 -0
- data/lib/carnivore-files/util/nio.rb +98 -0
- data/lib/carnivore-files/util/penguin.rb +100 -0
- data/lib/carnivore-files/version.rb +6 -0
- data/lib/carnivore-files.rb +14 -0
- metadata +80 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: efbd33b4233a2a79fa6084a851e877e7049cb112
|
4
|
+
data.tar.gz: 03990ee4ba3898b06cba3bb5707f0ae2ff384b10
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bdd854f660a2ca204a0445d6c7dd4252be6b7713cdb0544b60e96b3bb62b8ed99b183f409c01101aa90e2d74435036e9a1583efe79d33b646c96bbdfa2d27d20
|
7
|
+
data.tar.gz: 170772e2d4b8f71907e507b2cbfa51e9da6f3b6763e3768e32a98636df1907979ea394958fcc41be3559260e22aede2274d9a68ed2fa41c9e56464612fe7b1c9
|
data/CHANGELOG.md
ADDED
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# Carnivore Files
|
2
|
+
|
3
|
+
Provides File `Carnivore::Source`
|
4
|
+
|
5
|
+
# Usage
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
require 'carnivore'
|
9
|
+
require 'carnivore-files'
|
10
|
+
|
11
|
+
Carnivore.configure do
|
12
|
+
source = Carnivore::Source.build(
|
13
|
+
:type => :carn_file, :args => {:path => '/var/log/app.log'}
|
14
|
+
)
|
15
|
+
end
|
16
|
+
```
|
17
|
+
|
18
|
+
The `File` source is built on two "foundations", `sleepy_penguin`
|
19
|
+
and `nio4r`. The optimal foundation will be selected based on
|
20
|
+
the current Ruby in use (`nio4r` for JRuby, `sleepy_penguin` for
|
21
|
+
everything else). If you want to force the foundation:
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
require 'carnivore'
|
25
|
+
require 'carnivore-files'
|
26
|
+
|
27
|
+
Carnivore.configure do
|
28
|
+
source = Carnivore::Source.build(
|
29
|
+
:type => :carn_file, :args => {
|
30
|
+
:path => '/var/log/app.log',
|
31
|
+
:foundation => :nio
|
32
|
+
}
|
33
|
+
)
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
## Important note
|
38
|
+
|
39
|
+
The underlying foundations are not installed by this gem. Be sure
|
40
|
+
include the dependency within your application dependencies (nio4r
|
41
|
+
or sleepy_penguin).
|
42
|
+
|
43
|
+
# Info
|
44
|
+
* Carnivore: https://github.com/carnivore-rb/carnivore
|
45
|
+
* Repository: https://github.com/carnivore-rb/carnivore-files
|
46
|
+
* IRC: Freenode @ #carnivore
|
@@ -0,0 +1,17 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
|
2
|
+
require 'carnivore-files/version'
|
3
|
+
spec = Gem::Specification.new do |s|
|
4
|
+
s.name = 'carnivore-files'
|
5
|
+
s.version = Carnivore::Files::VERSION.version
|
6
|
+
s.summary = 'Message processing helper'
|
7
|
+
s.author = 'Chris Roberts'
|
8
|
+
s.email = 'code@chrisroberts.org'
|
9
|
+
s.homepage = 'https://github.com/carnivore-rb/carnivore-files'
|
10
|
+
s.description = 'Carnivore file source'
|
11
|
+
s.license = 'Apache 2.0'
|
12
|
+
s.require_path = 'lib'
|
13
|
+
s.add_dependency 'carnivore', '>= 0.1.8'
|
14
|
+
s.add_dependency 'sleepy_penguin'
|
15
|
+
s.add_dependency 'nio4r'
|
16
|
+
s.files = Dir['{lib}/**/**/*'] + %w(carnivore-files.gemspec README.md CHANGELOG.md)
|
17
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'carnivore'
|
2
|
+
|
3
|
+
module Carnivore
|
4
|
+
class Source
|
5
|
+
# Carnivore source for consumption from files
|
6
|
+
class CarnFile < Source
|
7
|
+
|
8
|
+
# @return [String] path to file
|
9
|
+
attr_reader :path
|
10
|
+
# @return [Symbol] registry name of fetcher
|
11
|
+
attr_reader :fetcher
|
12
|
+
|
13
|
+
trap_exit :fetcher_failure
|
14
|
+
finalizer :fetcher_destroyer
|
15
|
+
|
16
|
+
# Setup source
|
17
|
+
#
|
18
|
+
# @param args [Hash]
|
19
|
+
# @option args [String] :path path to file
|
20
|
+
# @option args [Symbol] :foundation underlying file interaction library
|
21
|
+
# @option args [Celluloid::Actor] :notify_actor actor to notify on line receive
|
22
|
+
def setup(*_)
|
23
|
+
@path = ::File.expand_path(args[:path])
|
24
|
+
unless(args[:foundation])
|
25
|
+
args[:foundation] = RUBY_PLATFORM == 'java' ? :nio4r : :penguin
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Start the line fetcher
|
30
|
+
def connect
|
31
|
+
@fetcher_name = "log_fetcher_#{name}".to_sym
|
32
|
+
case args[:foundation].to_sym
|
33
|
+
when :nio, :nio4r
|
34
|
+
@fetcher = Carnivore::Files::Util::Fetcher::Nio.new(args)
|
35
|
+
else
|
36
|
+
@fetcher = Carnivore::Files::Util::Fetcher::Penguin.new(args)
|
37
|
+
end
|
38
|
+
self.link fetcher
|
39
|
+
fetcher.async.start_fetcher
|
40
|
+
end
|
41
|
+
|
42
|
+
# Restart file collector if unexpectedly failed
|
43
|
+
#
|
44
|
+
# @param object [Actor] crashed actor
|
45
|
+
# @param reason [Exception, NilClass]
|
46
|
+
def fetcher_failure(object, reason)
|
47
|
+
if(reason && object == fetcher)
|
48
|
+
error "File message collector unexpectedly failed: #{reason} (restarting)"
|
49
|
+
connect
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def fetcher_destroyer
|
54
|
+
if(fetcher && fetcher.alive?)
|
55
|
+
fetcher.terminate
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# @return [Array<Hash>] return messages
|
60
|
+
def receive(*args)
|
61
|
+
format_message(Celluloid::Future.new{ fetcher.messages.pop }.value)
|
62
|
+
end
|
63
|
+
|
64
|
+
# Send payload
|
65
|
+
#
|
66
|
+
# @param payload [Object] payload to transmit
|
67
|
+
def transmit(payload, *args)
|
68
|
+
fetcher.write_line(payload)
|
69
|
+
end
|
70
|
+
|
71
|
+
protected
|
72
|
+
|
73
|
+
# Format message into customized Hash
|
74
|
+
#
|
75
|
+
# @param m [Object] payload
|
76
|
+
# @return [Hash]
|
77
|
+
def format_message(m)
|
78
|
+
Smash.new(:path => path, :content => m)
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require 'carnivore-files'
|
2
|
+
|
3
|
+
module Carnivore
|
4
|
+
module Files
|
5
|
+
# Helper utilities
|
6
|
+
module Util
|
7
|
+
# Fetch lines from file
|
8
|
+
class Fetcher
|
9
|
+
|
10
|
+
autoload :Nio, 'carnivore-files/util/nio'
|
11
|
+
autoload :Penguin, 'carnivore-files/util/penguin'
|
12
|
+
|
13
|
+
include Celluloid
|
14
|
+
include Carnivore::Utils::Logging
|
15
|
+
|
16
|
+
# @return [String] path to file
|
17
|
+
attr_reader :path
|
18
|
+
# @return [String] string to split messages on
|
19
|
+
attr_reader :delimiter
|
20
|
+
|
21
|
+
# @return [Queue] messages
|
22
|
+
attr_accessor :messages
|
23
|
+
# @return [IO] underlying IO instance
|
24
|
+
attr_accessor :io
|
25
|
+
|
26
|
+
# Create new instance
|
27
|
+
#
|
28
|
+
# @param args [Hash] initialization args
|
29
|
+
# @option args [String] :path path to file
|
30
|
+
# @option args [String] :delimiter string delimiter to break messages
|
31
|
+
# @option args [Celluloid::Actor] :notify_actor actor to be notified on new messages
|
32
|
+
def initialize(args={})
|
33
|
+
@leftover = ''
|
34
|
+
@path = ::File.expand_path(args[:path])
|
35
|
+
@delimiter = args.fetch(:delimiter, "\n")
|
36
|
+
@messages = Queue.new
|
37
|
+
end
|
38
|
+
|
39
|
+
# Start the line fetcher
|
40
|
+
def start_fetcher
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
|
44
|
+
# Write line to IO
|
45
|
+
#
|
46
|
+
# @param line [String]
|
47
|
+
# @return [Integer] bytes written
|
48
|
+
def write_line(line)
|
49
|
+
if(io)
|
50
|
+
io.puts(line)
|
51
|
+
else
|
52
|
+
raise 'No IO detected! Failed to write.'
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Retreive lines from file
|
57
|
+
def retrieve_lines
|
58
|
+
if(io)
|
59
|
+
while(data = io.read(4096))
|
60
|
+
@leftover << data
|
61
|
+
end
|
62
|
+
result = @leftover.split(delimiter)
|
63
|
+
@leftover.replace @leftover.end_with?(delimiter) ? '' : result.pop.to_s
|
64
|
+
result
|
65
|
+
else
|
66
|
+
[]
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Build the IO and monitor
|
71
|
+
#
|
72
|
+
# @return [TrueClass, FalseClass]
|
73
|
+
def build_io
|
74
|
+
unless(io)
|
75
|
+
if(::File.exists?(path))
|
76
|
+
@io = ::File.open(path, 'r')
|
77
|
+
unless(@waited)
|
78
|
+
@io.seek(0, ::IO::SEEK_END) # fast-forward to EOF
|
79
|
+
else
|
80
|
+
@waited = false
|
81
|
+
retrieve_lines.each do |l|
|
82
|
+
self.messages << l
|
83
|
+
end
|
84
|
+
end
|
85
|
+
else
|
86
|
+
wait_for_file
|
87
|
+
build_io
|
88
|
+
end
|
89
|
+
true
|
90
|
+
else
|
91
|
+
false
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require 'nio'
|
2
|
+
|
3
|
+
module Carnivore
|
4
|
+
module Files
|
5
|
+
module Util
|
6
|
+
class Fetcher
|
7
|
+
|
8
|
+
# NIO based fetcher
|
9
|
+
class Nio < Fetcher
|
10
|
+
|
11
|
+
# @return [NIO::Monitor]
|
12
|
+
attr_accessor :monitor
|
13
|
+
# @return [NIO::Selector]
|
14
|
+
attr_accessor :selector
|
15
|
+
|
16
|
+
# Create new instance
|
17
|
+
#
|
18
|
+
# @param args [Hash] initialization arguments (unused)
|
19
|
+
def initialize(args={})
|
20
|
+
super
|
21
|
+
@selector = NIO::Selector.new
|
22
|
+
every(5) do
|
23
|
+
check_file
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Start the fetcher
|
28
|
+
def start_fetcher
|
29
|
+
loop do
|
30
|
+
build_io
|
31
|
+
messages = nil
|
32
|
+
Celluloid::Future.new{ selector.select }.value.each do |mon|
|
33
|
+
retrieve_lines.each do |l|
|
34
|
+
self.messages << l
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
# Check for file and destroy monitor if file has changed
|
43
|
+
def check_file
|
44
|
+
if(io)
|
45
|
+
begin
|
46
|
+
unless(io.stat.ino == ::File.stat(path).ino)
|
47
|
+
destroy_io
|
48
|
+
@waited = true
|
49
|
+
end
|
50
|
+
rescue Errno::ENOENT
|
51
|
+
destroy_io
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Build the IO instance if found
|
57
|
+
#
|
58
|
+
# @return [TrueClass]
|
59
|
+
def build_io
|
60
|
+
result = super
|
61
|
+
if(result && @monitor.nil?)
|
62
|
+
@monitor = selector.register(io, :r)
|
63
|
+
end
|
64
|
+
result
|
65
|
+
end
|
66
|
+
|
67
|
+
# Destroy the IO instance and monitor
|
68
|
+
#
|
69
|
+
# @return [TrueClass]
|
70
|
+
def destroy_io
|
71
|
+
if(monitor)
|
72
|
+
selector.deregister(monitor)
|
73
|
+
@monitor = nil
|
74
|
+
end
|
75
|
+
if(io)
|
76
|
+
io.close
|
77
|
+
@io = nil
|
78
|
+
end
|
79
|
+
true
|
80
|
+
end
|
81
|
+
|
82
|
+
# Wait helper for file to appear (5 sleep second intervals)
|
83
|
+
#
|
84
|
+
# @return [TrueClass]
|
85
|
+
def wait_for_file
|
86
|
+
warn "Waiting for file to appear (#{path})"
|
87
|
+
until(::File.exists?(path))
|
88
|
+
@waited = true
|
89
|
+
sleep(5)
|
90
|
+
end
|
91
|
+
info "File has appeared (#{path})!"
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'sleepy_penguin/sp'
|
2
|
+
|
3
|
+
module Carnivore
|
4
|
+
module Files
|
5
|
+
module Util
|
6
|
+
class Fetcher
|
7
|
+
|
8
|
+
|
9
|
+
# NIO based fetcher
|
10
|
+
class Penguin < Fetcher
|
11
|
+
# @return [SP::Inotify]
|
12
|
+
attr_accessor :notify
|
13
|
+
# @return [Hash] registered file descriptors
|
14
|
+
attr_accessor :notify_descriptors
|
15
|
+
|
16
|
+
# Create new instance
|
17
|
+
#
|
18
|
+
# @param args [Hash] initialization arguments (unused)
|
19
|
+
def initialize(args={})
|
20
|
+
super
|
21
|
+
@notify = SP::Inotify.new
|
22
|
+
@notify_descriptors = {}
|
23
|
+
end
|
24
|
+
|
25
|
+
# Start the fetcher
|
26
|
+
def start_fetcher
|
27
|
+
loop do
|
28
|
+
build_io
|
29
|
+
notify.each do |event|
|
30
|
+
Celluloid::Future.new{ event.events }.value.each do |ev|
|
31
|
+
case ev
|
32
|
+
when :MODIFY
|
33
|
+
retrieve_lines.each do |l|
|
34
|
+
self.messages << l
|
35
|
+
end
|
36
|
+
when :MOVE_SELF, :DELETE_SELF, :ATTRIB
|
37
|
+
info "Destroying file IO due to FS modification! (#{ev.inspect})"
|
38
|
+
destroy_io
|
39
|
+
@waited = true
|
40
|
+
break
|
41
|
+
else
|
42
|
+
debug "Received unhandled event: #{ev.inspect}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
break unless io
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
# Build the IO and monitor
|
54
|
+
#
|
55
|
+
# @return [TrueClass, FalseClass]
|
56
|
+
def build_io
|
57
|
+
result = super
|
58
|
+
if(result)
|
59
|
+
notify_descriptors[:file_watch] = notify.add_watch(path, :ALL_EVENTS)
|
60
|
+
end
|
61
|
+
result
|
62
|
+
end
|
63
|
+
|
64
|
+
# Destroy the IO and monitor
|
65
|
+
#
|
66
|
+
# @return [TrueClass]
|
67
|
+
def destroy_io
|
68
|
+
if(io)
|
69
|
+
notify.rm_watch(notify_descriptors.delete(:file_watch))
|
70
|
+
@io.close
|
71
|
+
@io = nil
|
72
|
+
end
|
73
|
+
true
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
# Wait helper for file to appear (waits for expected notification)
|
78
|
+
#
|
79
|
+
# @return [TrueClass]
|
80
|
+
def wait_for_file
|
81
|
+
until(::File.exists?(path))
|
82
|
+
notified = false
|
83
|
+
directory = ::File.dirname(path)
|
84
|
+
notify_descriptors[:file_wait] = notify.add_watch(directory, :OPEN)
|
85
|
+
until(notified)
|
86
|
+
warn "Waiting for file to appear (#{path})"
|
87
|
+
event = Celluloid::Future.new{ notify.take }.value
|
88
|
+
notified = ::File.exists?(path)
|
89
|
+
end
|
90
|
+
notify.rm_watch(notify_descriptors.delete(:file_wait))
|
91
|
+
end
|
92
|
+
@waited = true
|
93
|
+
info "File has appeared (#{path})!"
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'carnivore-files/version'
|
2
|
+
require 'carnivore'
|
3
|
+
|
4
|
+
module Carnivore
|
5
|
+
# Carnivore source module for files
|
6
|
+
module Files
|
7
|
+
# Utilities for files
|
8
|
+
module Util
|
9
|
+
autoload :Fetcher, 'carnivore-files/util/fetcher'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
Carnivore::Source.provide(:carn_file, 'carnivore-files/carn_file')
|
metadata
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: carnivore-files
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.2
|
5
|
+
platform: java
|
6
|
+
authors:
|
7
|
+
- Chris Roberts
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-02-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: carnivore
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.1.8
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
21
|
+
requirements:
|
22
|
+
- - '>='
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.1.8
|
25
|
+
prerelease: false
|
26
|
+
type: :runtime
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: nio4r
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
prerelease: false
|
40
|
+
type: :runtime
|
41
|
+
description: Carnivore file source
|
42
|
+
email: code@chrisroberts.org
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- lib/carnivore-files.rb
|
48
|
+
- lib/carnivore-files/carn_file.rb
|
49
|
+
- lib/carnivore-files/version.rb
|
50
|
+
- lib/carnivore-files/util/penguin.rb
|
51
|
+
- lib/carnivore-files/util/nio.rb
|
52
|
+
- lib/carnivore-files/util/fetcher.rb
|
53
|
+
- carnivore-files.gemspec
|
54
|
+
- README.md
|
55
|
+
- CHANGELOG.md
|
56
|
+
homepage: https://github.com/carnivore-rb/carnivore-files
|
57
|
+
licenses:
|
58
|
+
- Apache 2.0
|
59
|
+
metadata: {}
|
60
|
+
post_install_message:
|
61
|
+
rdoc_options: []
|
62
|
+
require_paths:
|
63
|
+
- lib
|
64
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
requirements: []
|
75
|
+
rubyforge_project:
|
76
|
+
rubygems_version: 2.1.9
|
77
|
+
signing_key:
|
78
|
+
specification_version: 4
|
79
|
+
summary: Message processing helper
|
80
|
+
test_files: []
|