observability 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.document +5 -0
- data/.rdoc_options +16 -0
- data/.simplecov +9 -0
- data/ChangeLog +139 -0
- data/DevNotes.md +103 -0
- data/History.md +4 -0
- data/LICENSE.txt +20 -0
- data/Manifest.txt +31 -0
- data/README.md +93 -0
- data/Rakefile +102 -0
- data/bin/observability-collector +16 -0
- data/examples/basic-usage.rb +18 -0
- data/lib/observability.rb +122 -0
- data/lib/observability/collector.rb +61 -0
- data/lib/observability/collector/timescale.rb +140 -0
- data/lib/observability/event.rb +103 -0
- data/lib/observability/observer.rb +296 -0
- data/lib/observability/observer_hooks.rb +28 -0
- data/lib/observability/sender.rb +127 -0
- data/lib/observability/sender/logger.rb +37 -0
- data/lib/observability/sender/null.rb +30 -0
- data/lib/observability/sender/testing.rb +56 -0
- data/lib/observability/sender/udp.rb +88 -0
- data/spec/observability/event_spec.rb +106 -0
- data/spec/observability/observer_hooks_spec.rb +47 -0
- data/spec/observability/observer_spec.rb +292 -0
- data/spec/observability/sender/logger_spec.rb +28 -0
- data/spec/observability/sender/udp_spec.rb +86 -0
- data/spec/observability/sender_spec.rb +77 -0
- data/spec/observability_spec.rb +132 -0
- data/spec/spec_helper.rb +155 -0
- metadata +325 -0
- metadata.gz.sig +1 -0
data/Rakefile
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'hoe'
|
5
|
+
rescue LoadError
|
6
|
+
abort "This Rakefile requires hoe (gem install hoe)"
|
7
|
+
end
|
8
|
+
|
9
|
+
GEMSPEC = 'observability.gemspec'
|
10
|
+
|
11
|
+
|
12
|
+
Hoe.plugin :mercurial
|
13
|
+
Hoe.plugin :signing
|
14
|
+
Hoe.plugin :deveiate
|
15
|
+
|
16
|
+
Hoe.plugins.delete :rubyforge
|
17
|
+
|
18
|
+
hoespec = Hoe.spec 'observability' do |spec|
|
19
|
+
spec.readme_file = 'README.md'
|
20
|
+
spec.history_file = 'History.md'
|
21
|
+
|
22
|
+
spec.extra_rdoc_files = FileList[ '*.rdoc', '*.md' ]
|
23
|
+
spec.license 'BSD-3-Clause'
|
24
|
+
spec.urls = {
|
25
|
+
home: 'http://bitbucket.org/ged/observability',
|
26
|
+
code: 'http://bitbucket.org/ged/observability',
|
27
|
+
docs: 'http://deveiate.org/code/observability',
|
28
|
+
github: 'http://github.com/ged/observability',
|
29
|
+
}
|
30
|
+
|
31
|
+
spec.developer 'Michael Granger', 'ged@FaerieMUD.org'
|
32
|
+
|
33
|
+
spec.dependency 'concurrent-ruby', '~> 1.1.5'
|
34
|
+
spec.dependency 'concurrent-ruby-ext', '~> 1.1.5'
|
35
|
+
spec.dependency 'loggability', '~> 0.11'
|
36
|
+
spec.dependency 'configurability', '~> 3.3'
|
37
|
+
spec.dependency 'pluggability', '~> 0.6'
|
38
|
+
spec.dependency 'msgpack', '~> 1.3'
|
39
|
+
|
40
|
+
spec.dependency 'timecop', '~> 0.9', :developer
|
41
|
+
spec.dependency 'hoe-deveiate', '~> 0.3', :developer
|
42
|
+
spec.dependency 'simplecov', '~> 0.7', :developer
|
43
|
+
spec.dependency 'rdoc-generator-fivefish', '~> 0.1', :developer
|
44
|
+
|
45
|
+
spec.require_ruby_version( '>=2.4.0' )
|
46
|
+
spec.hg_sign_tags = true if spec.respond_to?( :hg_sign_tags= )
|
47
|
+
spec.check_history_on_release = true if spec.respond_to?( :check_history_on_release= )
|
48
|
+
|
49
|
+
self.rdoc_locations << "deveiate:/usr/local/www/public/code/#{remote_rdoc_dir}"
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
ENV['VERSION'] ||= hoespec.spec.version.to_s
|
54
|
+
|
55
|
+
# Run the tests before checking in
|
56
|
+
task 'hg:precheckin' => [ :check_history, :check_manifest, :gemspec, :spec ]
|
57
|
+
|
58
|
+
task :test => :spec
|
59
|
+
|
60
|
+
# Rebuild the ChangeLog immediately before release
|
61
|
+
task :prerelease => 'ChangeLog'
|
62
|
+
CLOBBER.include( 'ChangeLog' )
|
63
|
+
|
64
|
+
desc "Build a coverage report"
|
65
|
+
task :coverage do
|
66
|
+
ENV["COVERAGE"] = 'yes'
|
67
|
+
Rake::Task[:spec].invoke
|
68
|
+
end
|
69
|
+
CLOBBER.include( 'coverage' )
|
70
|
+
|
71
|
+
|
72
|
+
# Use the fivefish formatter for docs generated from development checkout
|
73
|
+
if File.directory?( '.hg' )
|
74
|
+
require 'rdoc/task'
|
75
|
+
|
76
|
+
Rake::Task[ 'docs' ].clear
|
77
|
+
RDoc::Task.new( 'docs' ) do |rdoc|
|
78
|
+
rdoc.main = "README.rdoc"
|
79
|
+
rdoc.markup = 'markdown'
|
80
|
+
rdoc.rdoc_files.include( "*.rdoc", "ChangeLog", "lib/**/*.rb" )
|
81
|
+
rdoc.generator = :fivefish
|
82
|
+
rdoc.title = 'Observability'
|
83
|
+
rdoc.rdoc_dir = 'doc'
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
task :gemspec => GEMSPEC
|
88
|
+
file GEMSPEC => __FILE__
|
89
|
+
task GEMSPEC do |task|
|
90
|
+
spec = $hoespec.spec
|
91
|
+
spec.files.delete( '.gemtest' )
|
92
|
+
spec.signing_key = nil
|
93
|
+
spec.cert_chain = ['certs/ged.pem']
|
94
|
+
spec.version = "#{spec.version.bump}.0.pre#{Time.now.strftime("%Y%m%d%H%M%S")}"
|
95
|
+
File.open( task.name, 'w' ) do |fh|
|
96
|
+
fh.write( spec.to_ruby )
|
97
|
+
end
|
98
|
+
end
|
99
|
+
CLOBBER.include( GEMSPEC.to_s )
|
100
|
+
|
101
|
+
task :default => :gemspec
|
102
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'configurability'
|
4
|
+
require 'observability'
|
5
|
+
require 'observability/collector'
|
6
|
+
|
7
|
+
|
8
|
+
if ( configfile = ARGV.first )
|
9
|
+
config = Configurability::Config.load( configfile )
|
10
|
+
else
|
11
|
+
config = Configurability.default_config
|
12
|
+
end
|
13
|
+
|
14
|
+
config.install
|
15
|
+
Observability::Collector.start
|
16
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'observability'
|
5
|
+
|
6
|
+
|
7
|
+
class Server
|
8
|
+
extend Observability
|
9
|
+
|
10
|
+
|
11
|
+
observer 'udp://10.2.0.250:11311'
|
12
|
+
|
13
|
+
observe_around :handle_request
|
14
|
+
observe_around :handle_
|
15
|
+
|
16
|
+
|
17
|
+
end # class Server
|
18
|
+
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'concurrent'
|
5
|
+
require 'concurrent/configuration'
|
6
|
+
require 'configurability'
|
7
|
+
require 'loggability'
|
8
|
+
|
9
|
+
|
10
|
+
# A mixin that adds effortless Observability to your systems.
|
11
|
+
module Observability
|
12
|
+
extend Loggability
|
13
|
+
|
14
|
+
|
15
|
+
# Package version
|
16
|
+
VERSION = '0.1.0'
|
17
|
+
|
18
|
+
# Version control revision
|
19
|
+
REVISION = %q$Revision$
|
20
|
+
|
21
|
+
|
22
|
+
# Loggability -- Create a logger
|
23
|
+
log_as :observability
|
24
|
+
Concurrent.global_logger = lambda do |loglevel, progname, message=nil, &block|
|
25
|
+
Observability.logger.add( loglevel, message, progname, &block )
|
26
|
+
end
|
27
|
+
|
28
|
+
autoload :Collector, 'observability/collector'
|
29
|
+
autoload :Event, 'observability/event'
|
30
|
+
autoload :ObserverHooks, 'observability/observer_hooks'
|
31
|
+
autoload :Observer, 'observability/observer'
|
32
|
+
autoload :Sender, 'observability/sender'
|
33
|
+
|
34
|
+
|
35
|
+
@observer_hooks = Concurrent::Map.new
|
36
|
+
singleton_class.attr_reader :observer_hooks
|
37
|
+
|
38
|
+
@observer = Concurrent::IVar.new
|
39
|
+
|
40
|
+
|
41
|
+
### Get the observer hooks for the specified +mod+.
|
42
|
+
def self::[]( mod )
|
43
|
+
mod = mod.class unless mod.is_a?( Module )
|
44
|
+
return self.observer_hooks[ mod ]
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
### Extension callback
|
49
|
+
def self::extended( mod )
|
50
|
+
super
|
51
|
+
|
52
|
+
Observability.observer_hooks.compute_if_absent( mod ) do
|
53
|
+
observer_hooks = Observability::ObserverHooks.dup
|
54
|
+
mod.prepend( observer_hooks )
|
55
|
+
observer_hooks
|
56
|
+
end
|
57
|
+
|
58
|
+
mod.singleton_class.extend( Observability ) unless mod.singleton_class?
|
59
|
+
end
|
60
|
+
|
61
|
+
|
62
|
+
### Return the current Observer, creating it if necessary.
|
63
|
+
def self::observer
|
64
|
+
unless @observer.complete?
|
65
|
+
self.log.debug "Creating the observer agent."
|
66
|
+
@observer.try_set do
|
67
|
+
obs = Observability::Observer.new
|
68
|
+
obs.start
|
69
|
+
obs
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
return @observer.value!
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
### Reset all per-process Observability state. This should be called, for instance,
|
78
|
+
### after a fork or between tests.
|
79
|
+
def self::reset
|
80
|
+
@observer.value.stop if @observer.complete?
|
81
|
+
@observer = Concurrent::IVar.new
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
### Make a body for a wrapping method for the method with the given +name+ and
|
86
|
+
### +context+, passing the given +options+.
|
87
|
+
def self::make_wrapped_method( name, context, options, &callback )
|
88
|
+
return Proc.new do |*m_args, **m_options, &block|
|
89
|
+
Loggability[ Observability ].debug "Wrapped method %p: %p" %
|
90
|
+
[ name, context ]
|
91
|
+
Observability.observer.event( context, **options ) do
|
92
|
+
# :TODO: Freeze or dup the arguments to prevent accidental modification?
|
93
|
+
callback.call( *m_args, **m_options, &block ) if callback
|
94
|
+
super( *m_args, **m_options, &block )
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
#
|
100
|
+
# DSL Methods
|
101
|
+
#
|
102
|
+
|
103
|
+
### Wrap an instance method in an observer call.
|
104
|
+
def observe_method( method_name, *details, **options, &callback )
|
105
|
+
hooks = Observability.observer_hooks[ self ] or
|
106
|
+
raise "No observer hooks installed for %p?!" % [ self ]
|
107
|
+
|
108
|
+
context = self.instance_method( method_name )
|
109
|
+
context = [ context, *details ]
|
110
|
+
method_body = Observability.make_wrapped_method( method_name, context, options, &callback )
|
111
|
+
|
112
|
+
hooks.define_method( method_name, &method_body )
|
113
|
+
end
|
114
|
+
|
115
|
+
|
116
|
+
### Wrap a class method in an observer call.
|
117
|
+
def observe_class_method( method_name, *details, **options, &callback )
|
118
|
+
self.singleton_class.observe_method( method_name, *details, **options, &callback )
|
119
|
+
end
|
120
|
+
|
121
|
+
end # module Observability
|
122
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'socket'
|
5
|
+
require 'pluggability'
|
6
|
+
require 'loggability'
|
7
|
+
|
8
|
+
|
9
|
+
require 'observability' unless defined?( Observability )
|
10
|
+
|
11
|
+
|
12
|
+
class Observability::Collector
|
13
|
+
extend Loggability,
|
14
|
+
Pluggability,
|
15
|
+
Configurability
|
16
|
+
|
17
|
+
# Loggability API -- log to the Observability logger
|
18
|
+
log_to :observability
|
19
|
+
|
20
|
+
# Set the directories to search for concrete subclassse
|
21
|
+
plugin_prefixes 'observability/collector'
|
22
|
+
|
23
|
+
# Configurability -- declare config settings and defaults
|
24
|
+
configurability( 'observability.collector' ) do
|
25
|
+
|
26
|
+
setting :type, default: 'timescale'
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
# Prevent direct instantiation
|
32
|
+
private_class_method :new
|
33
|
+
|
34
|
+
|
35
|
+
### Let subclasses be inherited
|
36
|
+
def self::inherited( subclass )
|
37
|
+
super
|
38
|
+
subclass.public_class_method( :new )
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
### Create an instance of the configured type of collector and return it.
|
43
|
+
def self::configured_type
|
44
|
+
return self.create( self.type )
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
### Start a collector of the specified +type+, returning only when it shuts down.
|
49
|
+
def self::start
|
50
|
+
instance = self.configured_type
|
51
|
+
instance.start
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
### Start the collector.
|
56
|
+
def start
|
57
|
+
# No-op
|
58
|
+
end
|
59
|
+
|
60
|
+
end # class Observability::Collector
|
61
|
+
|
@@ -0,0 +1,140 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'sequel'
|
5
|
+
require 'configurability'
|
6
|
+
|
7
|
+
require 'observability/collector' unless defined?( Observability::Collector )
|
8
|
+
|
9
|
+
|
10
|
+
class Observability::Collector::Timescale < Observability::Collector
|
11
|
+
extend Configurability,
|
12
|
+
Loggability
|
13
|
+
|
14
|
+
Sequel.extension( :pg_json )
|
15
|
+
|
16
|
+
|
17
|
+
# The maximum size of event messages
|
18
|
+
MAX_EVENT_BYTES = 64 * 1024
|
19
|
+
|
20
|
+
# The number of seconds to wait between IO loops
|
21
|
+
LOOP_TIMER = 0.25
|
22
|
+
|
23
|
+
# The config to pass to JSON.parse
|
24
|
+
JSON_CONFIG = {
|
25
|
+
object_class: Sequel::Postgres::JSONHash,
|
26
|
+
array_class: Sequel::Postgres::JSONArray
|
27
|
+
}.freeze
|
28
|
+
|
29
|
+
|
30
|
+
log_to :observability
|
31
|
+
|
32
|
+
configurability( 'observability.collector.timescale' ) do
|
33
|
+
|
34
|
+
##
|
35
|
+
# The host to bind to
|
36
|
+
setting :host, default: 'localhost'
|
37
|
+
|
38
|
+
##
|
39
|
+
# The port to bind to
|
40
|
+
setting :port, default: 15775
|
41
|
+
|
42
|
+
##
|
43
|
+
# The URL of the timescale DB to store events in
|
44
|
+
setting :db, default: 'postgres:/observability'
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
### Create a new UDP collector
|
49
|
+
def initialize
|
50
|
+
super
|
51
|
+
|
52
|
+
@socket = UDPSocket.new
|
53
|
+
@db = nil
|
54
|
+
@cursor = nil
|
55
|
+
@processing = false
|
56
|
+
end
|
57
|
+
|
58
|
+
|
59
|
+
######
|
60
|
+
public
|
61
|
+
######
|
62
|
+
|
63
|
+
### Start receiving events.
|
64
|
+
def start
|
65
|
+
self.log.info "Starting up."
|
66
|
+
@db = Sequel.connect( self.class.db )
|
67
|
+
@db.extension( :pg_json )
|
68
|
+
# @cursor = @db[ :events ].prepare( :insert, :insert_new_event,
|
69
|
+
# time: :$time,
|
70
|
+
# type: :$type,
|
71
|
+
# version: :$version,
|
72
|
+
# data: :$data
|
73
|
+
# )
|
74
|
+
|
75
|
+
@socket.bind( self.class.host, self.class.port )
|
76
|
+
|
77
|
+
self.start_processing
|
78
|
+
end
|
79
|
+
|
80
|
+
|
81
|
+
### Stop receiving events.
|
82
|
+
def stop
|
83
|
+
self.stop_processing
|
84
|
+
|
85
|
+
@cursor = nil
|
86
|
+
@db.disconnct
|
87
|
+
end
|
88
|
+
|
89
|
+
|
90
|
+
### Start consuming incoming events and storing them.
|
91
|
+
def start_processing
|
92
|
+
@processing = true
|
93
|
+
while @processing
|
94
|
+
event = self.read_next_event or next
|
95
|
+
self.log.debug "Read event: %p" % [ event ]
|
96
|
+
self.store_event( event )
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
|
101
|
+
### Stop consuming events.
|
102
|
+
def stop_processing
|
103
|
+
@processing = false
|
104
|
+
end
|
105
|
+
|
106
|
+
|
107
|
+
### Read the next event from the socket
|
108
|
+
def read_next_event
|
109
|
+
self.log.debug "Reading next event."
|
110
|
+
data = @socket.recv_nonblock( MAX_EVENT_BYTES, exception: false )
|
111
|
+
|
112
|
+
if data == :wait_readable
|
113
|
+
IO.select( [@socket], nil, nil, LOOP_TIMER )
|
114
|
+
return nil
|
115
|
+
elsif data.empty?
|
116
|
+
return nil
|
117
|
+
else
|
118
|
+
self.log.info "Read %d bytes" % [ data.bytesize ]
|
119
|
+
return JSON.parse( data )
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
### Store the specified +event+.
|
125
|
+
def store_event( event )
|
126
|
+
time = event.delete('@timestamp')
|
127
|
+
type = event.delete('@type')
|
128
|
+
version = event.delete('@version')
|
129
|
+
|
130
|
+
# @cursor.call( time: time, type: type, version: version, data: event )
|
131
|
+
@db[ :events ].insert(
|
132
|
+
time: time,
|
133
|
+
type: type,
|
134
|
+
version: version,
|
135
|
+
data: Sequel.pg_json( event )
|
136
|
+
)
|
137
|
+
end
|
138
|
+
|
139
|
+
end # class Observability::Collector::UDP
|
140
|
+
|