observability 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile ADDED
@@ -0,0 +1,102 @@
1
+ #!/usr/bin/env rake
2
+
3
+ begin
4
+ require 'hoe'
5
+ rescue LoadError
6
+ abort "This Rakefile requires hoe (gem install hoe)"
7
+ end
8
+
9
+ GEMSPEC = 'observability.gemspec'
10
+
11
+
12
+ Hoe.plugin :mercurial
13
+ Hoe.plugin :signing
14
+ Hoe.plugin :deveiate
15
+
16
+ Hoe.plugins.delete :rubyforge
17
+
18
+ hoespec = Hoe.spec 'observability' do |spec|
19
+ spec.readme_file = 'README.md'
20
+ spec.history_file = 'History.md'
21
+
22
+ spec.extra_rdoc_files = FileList[ '*.rdoc', '*.md' ]
23
+ spec.license 'BSD-3-Clause'
24
+ spec.urls = {
25
+ home: 'http://bitbucket.org/ged/observability',
26
+ code: 'http://bitbucket.org/ged/observability',
27
+ docs: 'http://deveiate.org/code/observability',
28
+ github: 'http://github.com/ged/observability',
29
+ }
30
+
31
+ spec.developer 'Michael Granger', 'ged@FaerieMUD.org'
32
+
33
+ spec.dependency 'concurrent-ruby', '~> 1.1.5'
34
+ spec.dependency 'concurrent-ruby-ext', '~> 1.1.5'
35
+ spec.dependency 'loggability', '~> 0.11'
36
+ spec.dependency 'configurability', '~> 3.3'
37
+ spec.dependency 'pluggability', '~> 0.6'
38
+ spec.dependency 'msgpack', '~> 1.3'
39
+
40
+ spec.dependency 'timecop', '~> 0.9', :developer
41
+ spec.dependency 'hoe-deveiate', '~> 0.3', :developer
42
+ spec.dependency 'simplecov', '~> 0.7', :developer
43
+ spec.dependency 'rdoc-generator-fivefish', '~> 0.1', :developer
44
+
45
+ spec.require_ruby_version( '>=2.4.0' )
46
+ spec.hg_sign_tags = true if spec.respond_to?( :hg_sign_tags= )
47
+ spec.check_history_on_release = true if spec.respond_to?( :check_history_on_release= )
48
+
49
+ self.rdoc_locations << "deveiate:/usr/local/www/public/code/#{remote_rdoc_dir}"
50
+ end
51
+
52
+
53
+ ENV['VERSION'] ||= hoespec.spec.version.to_s
54
+
55
+ # Run the tests before checking in
56
+ task 'hg:precheckin' => [ :check_history, :check_manifest, :gemspec, :spec ]
57
+
58
+ task :test => :spec
59
+
60
+ # Rebuild the ChangeLog immediately before release
61
+ task :prerelease => 'ChangeLog'
62
+ CLOBBER.include( 'ChangeLog' )
63
+
64
+ desc "Build a coverage report"
65
+ task :coverage do
66
+ ENV["COVERAGE"] = 'yes'
67
+ Rake::Task[:spec].invoke
68
+ end
69
+ CLOBBER.include( 'coverage' )
70
+
71
+
72
+ # Use the fivefish formatter for docs generated from development checkout
73
+ if File.directory?( '.hg' )
74
+ require 'rdoc/task'
75
+
76
+ Rake::Task[ 'docs' ].clear
77
+ RDoc::Task.new( 'docs' ) do |rdoc|
78
+ rdoc.main = "README.rdoc"
79
+ rdoc.markup = 'markdown'
80
+ rdoc.rdoc_files.include( "*.rdoc", "ChangeLog", "lib/**/*.rb" )
81
+ rdoc.generator = :fivefish
82
+ rdoc.title = 'Observability'
83
+ rdoc.rdoc_dir = 'doc'
84
+ end
85
+ end
86
+
87
+ task :gemspec => GEMSPEC
88
+ file GEMSPEC => __FILE__
89
+ task GEMSPEC do |task|
90
+ spec = $hoespec.spec
91
+ spec.files.delete( '.gemtest' )
92
+ spec.signing_key = nil
93
+ spec.cert_chain = ['certs/ged.pem']
94
+ spec.version = "#{spec.version.bump}.0.pre#{Time.now.strftime("%Y%m%d%H%M%S")}"
95
+ File.open( task.name, 'w' ) do |fh|
96
+ fh.write( spec.to_ruby )
97
+ end
98
+ end
99
+ CLOBBER.include( GEMSPEC.to_s )
100
+
101
+ task :default => :gemspec
102
+
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'configurability'
4
+ require 'observability'
5
+ require 'observability/collector'
6
+
7
+
8
+ if ( configfile = ARGV.first )
9
+ config = Configurability::Config.load( configfile )
10
+ else
11
+ config = Configurability.default_config
12
+ end
13
+
14
+ config.install
15
+ Observability::Collector.start
16
+
@@ -0,0 +1,18 @@
1
+ # -*- ruby -*-
2
+ # frozen_string_literal: true
3
+
4
+ require 'observability'
5
+
6
+
7
+ class Server
8
+ extend Observability
9
+
10
+
11
+ observer 'udp://10.2.0.250:11311'
12
+
13
+ observe_around :handle_request
14
+ observe_around :handle_
15
+
16
+
17
+ end # class Server
18
+
@@ -0,0 +1,122 @@
1
+ # -*- ruby -*-
2
+ # frozen_string_literal: true
3
+
4
+ require 'concurrent'
5
+ require 'concurrent/configuration'
6
+ require 'configurability'
7
+ require 'loggability'
8
+
9
+
10
+ # A mixin that adds effortless Observability to your systems.
11
+ module Observability
12
+ extend Loggability
13
+
14
+
15
+ # Package version
16
+ VERSION = '0.1.0'
17
+
18
+ # Version control revision
19
+ REVISION = %q$Revision$
20
+
21
+
22
+ # Loggability -- Create a logger
23
+ log_as :observability
24
+ Concurrent.global_logger = lambda do |loglevel, progname, message=nil, &block|
25
+ Observability.logger.add( loglevel, message, progname, &block )
26
+ end
27
+
28
+ autoload :Collector, 'observability/collector'
29
+ autoload :Event, 'observability/event'
30
+ autoload :ObserverHooks, 'observability/observer_hooks'
31
+ autoload :Observer, 'observability/observer'
32
+ autoload :Sender, 'observability/sender'
33
+
34
+
35
+ @observer_hooks = Concurrent::Map.new
36
+ singleton_class.attr_reader :observer_hooks
37
+
38
+ @observer = Concurrent::IVar.new
39
+
40
+
41
+ ### Get the observer hooks for the specified +mod+.
42
+ def self::[]( mod )
43
+ mod = mod.class unless mod.is_a?( Module )
44
+ return self.observer_hooks[ mod ]
45
+ end
46
+
47
+
48
+ ### Extension callback
49
+ def self::extended( mod )
50
+ super
51
+
52
+ Observability.observer_hooks.compute_if_absent( mod ) do
53
+ observer_hooks = Observability::ObserverHooks.dup
54
+ mod.prepend( observer_hooks )
55
+ observer_hooks
56
+ end
57
+
58
+ mod.singleton_class.extend( Observability ) unless mod.singleton_class?
59
+ end
60
+
61
+
62
+ ### Return the current Observer, creating it if necessary.
63
+ def self::observer
64
+ unless @observer.complete?
65
+ self.log.debug "Creating the observer agent."
66
+ @observer.try_set do
67
+ obs = Observability::Observer.new
68
+ obs.start
69
+ obs
70
+ end
71
+ end
72
+
73
+ return @observer.value!
74
+ end
75
+
76
+
77
+ ### Reset all per-process Observability state. This should be called, for instance,
78
+ ### after a fork or between tests.
79
+ def self::reset
80
+ @observer.value.stop if @observer.complete?
81
+ @observer = Concurrent::IVar.new
82
+ end
83
+
84
+
85
+ ### Make a body for a wrapping method for the method with the given +name+ and
86
+ ### +context+, passing the given +options+.
87
+ def self::make_wrapped_method( name, context, options, &callback )
88
+ return Proc.new do |*m_args, **m_options, &block|
89
+ Loggability[ Observability ].debug "Wrapped method %p: %p" %
90
+ [ name, context ]
91
+ Observability.observer.event( context, **options ) do
92
+ # :TODO: Freeze or dup the arguments to prevent accidental modification?
93
+ callback.call( *m_args, **m_options, &block ) if callback
94
+ super( *m_args, **m_options, &block )
95
+ end
96
+ end
97
+ end
98
+
99
+ #
100
+ # DSL Methods
101
+ #
102
+
103
+ ### Wrap an instance method in an observer call.
104
+ def observe_method( method_name, *details, **options, &callback )
105
+ hooks = Observability.observer_hooks[ self ] or
106
+ raise "No observer hooks installed for %p?!" % [ self ]
107
+
108
+ context = self.instance_method( method_name )
109
+ context = [ context, *details ]
110
+ method_body = Observability.make_wrapped_method( method_name, context, options, &callback )
111
+
112
+ hooks.define_method( method_name, &method_body )
113
+ end
114
+
115
+
116
+ ### Wrap a class method in an observer call.
117
+ def observe_class_method( method_name, *details, **options, &callback )
118
+ self.singleton_class.observe_method( method_name, *details, **options, &callback )
119
+ end
120
+
121
+ end # module Observability
122
+
@@ -0,0 +1,61 @@
1
+ # -*- ruby -*-
2
+ # frozen_string_literal: true
3
+
4
+ require 'socket'
5
+ require 'pluggability'
6
+ require 'loggability'
7
+
8
+
9
+ require 'observability' unless defined?( Observability )
10
+
11
+
12
+ class Observability::Collector
13
+ extend Loggability,
14
+ Pluggability,
15
+ Configurability
16
+
17
+ # Loggability API -- log to the Observability logger
18
+ log_to :observability
19
+
20
+ # Set the directories to search for concrete subclassse
21
+ plugin_prefixes 'observability/collector'
22
+
23
+ # Configurability -- declare config settings and defaults
24
+ configurability( 'observability.collector' ) do
25
+
26
+ setting :type, default: 'timescale'
27
+
28
+ end
29
+
30
+
31
+ # Prevent direct instantiation
32
+ private_class_method :new
33
+
34
+
35
+ ### Let subclasses be inherited
36
+ def self::inherited( subclass )
37
+ super
38
+ subclass.public_class_method( :new )
39
+ end
40
+
41
+
42
+ ### Create an instance of the configured type of collector and return it.
43
+ def self::configured_type
44
+ return self.create( self.type )
45
+ end
46
+
47
+
48
+ ### Start a collector of the specified +type+, returning only when it shuts down.
49
+ def self::start
50
+ instance = self.configured_type
51
+ instance.start
52
+ end
53
+
54
+
55
+ ### Start the collector.
56
+ def start
57
+ # No-op
58
+ end
59
+
60
+ end # class Observability::Collector
61
+
@@ -0,0 +1,140 @@
1
+ # -*- ruby -*-
2
+ # frozen_string_literal: true
3
+
4
+ require 'sequel'
5
+ require 'configurability'
6
+
7
+ require 'observability/collector' unless defined?( Observability::Collector )
8
+
9
+
10
+ class Observability::Collector::Timescale < Observability::Collector
11
+ extend Configurability,
12
+ Loggability
13
+
14
+ Sequel.extension( :pg_json )
15
+
16
+
17
+ # The maximum size of event messages
18
+ MAX_EVENT_BYTES = 64 * 1024
19
+
20
+ # The number of seconds to wait between IO loops
21
+ LOOP_TIMER = 0.25
22
+
23
+ # The config to pass to JSON.parse
24
+ JSON_CONFIG = {
25
+ object_class: Sequel::Postgres::JSONHash,
26
+ array_class: Sequel::Postgres::JSONArray
27
+ }.freeze
28
+
29
+
30
+ log_to :observability
31
+
32
+ configurability( 'observability.collector.timescale' ) do
33
+
34
+ ##
35
+ # The host to bind to
36
+ setting :host, default: 'localhost'
37
+
38
+ ##
39
+ # The port to bind to
40
+ setting :port, default: 15775
41
+
42
+ ##
43
+ # The URL of the timescale DB to store events in
44
+ setting :db, default: 'postgres:/observability'
45
+ end
46
+
47
+
48
+ ### Create a new UDP collector
49
+ def initialize
50
+ super
51
+
52
+ @socket = UDPSocket.new
53
+ @db = nil
54
+ @cursor = nil
55
+ @processing = false
56
+ end
57
+
58
+
59
+ ######
60
+ public
61
+ ######
62
+
63
+ ### Start receiving events.
64
+ def start
65
+ self.log.info "Starting up."
66
+ @db = Sequel.connect( self.class.db )
67
+ @db.extension( :pg_json )
68
+ # @cursor = @db[ :events ].prepare( :insert, :insert_new_event,
69
+ # time: :$time,
70
+ # type: :$type,
71
+ # version: :$version,
72
+ # data: :$data
73
+ # )
74
+
75
+ @socket.bind( self.class.host, self.class.port )
76
+
77
+ self.start_processing
78
+ end
79
+
80
+
81
+ ### Stop receiving events.
82
+ def stop
83
+ self.stop_processing
84
+
85
+ @cursor = nil
86
+ @db.disconnct
87
+ end
88
+
89
+
90
+ ### Start consuming incoming events and storing them.
91
+ def start_processing
92
+ @processing = true
93
+ while @processing
94
+ event = self.read_next_event or next
95
+ self.log.debug "Read event: %p" % [ event ]
96
+ self.store_event( event )
97
+ end
98
+ end
99
+
100
+
101
+ ### Stop consuming events.
102
+ def stop_processing
103
+ @processing = false
104
+ end
105
+
106
+
107
+ ### Read the next event from the socket
108
+ def read_next_event
109
+ self.log.debug "Reading next event."
110
+ data = @socket.recv_nonblock( MAX_EVENT_BYTES, exception: false )
111
+
112
+ if data == :wait_readable
113
+ IO.select( [@socket], nil, nil, LOOP_TIMER )
114
+ return nil
115
+ elsif data.empty?
116
+ return nil
117
+ else
118
+ self.log.info "Read %d bytes" % [ data.bytesize ]
119
+ return JSON.parse( data )
120
+ end
121
+ end
122
+
123
+
124
+ ### Store the specified +event+.
125
+ def store_event( event )
126
+ time = event.delete('@timestamp')
127
+ type = event.delete('@type')
128
+ version = event.delete('@version')
129
+
130
+ # @cursor.call( time: time, type: type, version: version, data: event )
131
+ @db[ :events ].insert(
132
+ time: time,
133
+ type: type,
134
+ version: version,
135
+ data: Sequel.pg_json( event )
136
+ )
137
+ end
138
+
139
+ end # class Observability::Collector::UDP
140
+