event_tracer 0.2.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 77058f3fa969027f3d51399425d8885c60037ae6250db1cfa686fbafb8524467
4
- data.tar.gz: 40b98f105722939c88bb364a04bfbd708c9fcfc2c76815d41d5b2318994aa9bb
3
+ metadata.gz: '002520584662d6b31ce2f05310a70c4fb8af37c3c865f6bdf2b8ebc44bf16654'
4
+ data.tar.gz: daa3fc169b3daa2771adb7d832872c055b723dbe25d28fff4ba24c618f2b54a2
5
5
  SHA512:
6
- metadata.gz: 31fb42ad32f68d2efd4b78dbfb9d55f16563bc897eaa4c4602c5855eb092bf0f03c4308c22a91ed011663c095548029155a7a795de638405588bb162a93d723b
7
- data.tar.gz: 58c9ec65b708b9b44fb141c9da778fbc6d56fea371a8edc9e0aaf9ef4d4a2b89855e0a88afbafd0a61312c2b8023bd7c13325bc0f7c86be13ee7d87e534dc6ad
6
+ metadata.gz: '038e91c1d03b17070b97c3c170d73992d4050b9cee39f3529fc712a2cfda324d8afdfb10013f6a6838176966f94d6dd17b5f4851a62af6bdc0e6c8ea2b2e95b1'
7
+ data.tar.gz: fdeb28420163369840b82b0a56e6bd83a638fe4c5e7069f280c6807d8f9ad00b9044c642263fc89391628c110af8e527596c2e86c2b759a50691c3d13a7390c5
@@ -2,62 +2,62 @@ require_relative '../event_tracer'
2
2
  require_relative './basic_decorator'
3
3
 
4
4
  # NOTES
5
- # Appsignal interface to send our usual actions
5
+ # Appsignal interface to send our usual actions
6
6
  # BasicDecorator adds a transparent interface on top of the appsignal interface
7
7
  #
8
- # Usage: EventTracer.register :appsignal, EventTracer::AppsignalLogger.new(Appsignal)
9
- # appsignal_logger.info appsignal: { increment_counter: { counter_1: 1, counter_2: 2 }, set_gauge: { gauge_1: 1 } }
10
- # appsignal_logger.info appsignal: { set_gauge: { gauge_1: { value: 1, tags: { region: 'eu' } } } }
8
+ # Usage: EventTracer.register :appsignal,
9
+ # EventTracer::AppsignalLogger.new(Appsignal, allowed_tags: ['tag_1', 'tag_2'])
10
+ #
11
+ # appsignal_logger.info metrics: [:counter_1, :counter_2]
12
+ # appsignal_logger.info metrics: { counter_1: { type: :counter, value: 1 }, gauce_2: { type: :gauce, value: 10 } }
11
13
  module EventTracer
12
14
  class AppsignalLogger < BasicDecorator
13
15
 
14
- SUPPORTED_METRICS ||= %i(increment_counter add_distribution_value set_gauge)
16
+ SUPPORTED_METRIC_TYPES = {
17
+ counter: :increment_counter,
18
+ distribution: :add_distribution_value,
19
+ gauge: :set_gauge
20
+ }
21
+ DEFAULT_METRIC_TYPE = :increment_counter
22
+ DEFAULT_COUNTER = 1
23
+
24
+ def initialize(decoratee, allowed_tags: [])
25
+ super(decoratee)
26
+ @allowed_tags = allowed_tags
27
+ end
15
28
 
16
29
  LOG_TYPES.each do |log_type|
17
30
  define_method log_type do |**args|
18
- return LogResult.new(false, "Invalid appsignal config") unless args[:appsignal] && args[:appsignal].is_a?(Hash)
31
+ metrics = args[:metrics]
32
+
33
+ return fail_result('Invalid appsignal config') unless valid_args?(metrics)
34
+ return success_result if metrics.empty?
19
35
 
20
- applied_metrics(args[:appsignal]).each do |metric|
21
- metric_args = args[:appsignal][metric]
22
- return LogResult.new(false, "Appsignal metric #{metric} invalid") unless metric_args && metric_args.is_a?(Hash)
36
+ tags = args.slice(*allowed_tags)
23
37
 
24
- send_metric metric, metric_args
25
- rescue InvalidTagError => e
26
- return LogResult.new(false, e.message)
38
+ case metrics
39
+ when Array
40
+ metrics.each do |metric|
41
+ appsignal.public_send(DEFAULT_METRIC_TYPE, metric, DEFAULT_COUNTER, tags)
42
+ end
43
+ when Hash
44
+ metrics.each do |metric_name, metric_payload|
45
+ metric_type = SUPPORTED_METRIC_TYPES[metric_payload.fetch(:type).to_sym]
46
+ appsignal.public_send(metric_type, metric_name, metric_payload.fetch(:value), tags) if metric_type
47
+ end
27
48
  end
28
49
 
29
- LogResult.new(true)
50
+ success_result
30
51
  end
31
52
  end
32
53
 
33
54
  private
34
55
 
35
- attr_reader :appsignal, :decoratee
56
+ attr_reader :decoratee, :allowed_tags
36
57
  alias_method :appsignal, :decoratee
37
58
 
38
- def applied_metrics(appsignal_args)
39
- appsignal_args.keys.select { |metric| SUPPORTED_METRICS.include?(metric) }
40
- end
41
-
42
- def send_metric(metric, payload)
43
- payload.each do |increment, attribute|
44
- if attribute.is_a?(Hash)
45
- begin
46
- appsignal.send(
47
- metric,
48
- increment,
49
- attribute.fetch(:value),
50
- attribute.fetch(:tags)
51
- )
52
- rescue KeyError
53
- raise InvalidTagError, "Appsignal payload { #{increment}: #{attribute} } invalid"
54
- end
55
- else
56
- appsignal.send(metric, increment, attribute)
57
- end
58
- end
59
+ def valid_args?(metrics)
60
+ metrics && (metrics.is_a?(Hash) || metrics.is_a?(Array))
59
61
  end
60
62
  end
61
-
62
- class InvalidTagError < StandardError; end
63
63
  end
@@ -7,8 +7,8 @@ module EventTracer
7
7
  class BaseLogger < BasicDecorator
8
8
 
9
9
  LOG_TYPES.each do |log_type|
10
- define_method log_type do |*args|
11
- send_message(log_type, *args)
10
+ define_method log_type do |**args|
11
+ send_message(log_type, **args)
12
12
  LogResult.new(true)
13
13
  end
14
14
  end
@@ -20,6 +20,7 @@ module EventTracer
20
20
 
21
21
  # EventTracer ensures action & message is always populated
22
22
  def send_message(log_method, action:, message:, **args)
23
+ args.delete(:metrics)
23
24
  data = args.merge(action: action, message: message)
24
25
  logger.public_send(log_method, data)
25
26
  end
@@ -1,3 +1,5 @@
1
+ require 'delegate'
2
+
1
3
  module EventTracer
2
4
  class BasicDecorator < Delegator
3
5
 
@@ -14,5 +16,13 @@ module EventTracer
14
16
  @delegate_sd_obj = obj
15
17
  end
16
18
 
19
+ def success_result
20
+ LogResult.new(true)
21
+ end
22
+
23
+ def fail_result(message)
24
+ LogResult.new(false, message)
25
+ end
26
+
17
27
  end
18
- end
28
+ end
@@ -0,0 +1,65 @@
1
+ require 'concurrent'
2
+
3
+ module EventTracer
4
+ # This is an implementation of buffer storage. We use Concurrent::Array underneath
5
+ # to ensure thread-safe behavior. Data is stored until certain size / interval
6
+ # before flushing.
7
+ #
8
+ # Caveats: We should only store non-important data like logs in this buffer
9
+ # because if a process is killed, the data in this buffer is lost.
10
+ class Buffer
11
+ # Buffer can store maximum 10 items.
12
+ # Bigger size requires more memory to store, so choose a reasonable number
13
+ DEFAULT_BUFFER_SIZE = 10
14
+ # An item can live in buffer for at least 10s between each `Buffer#add` if the buffer is not full
15
+ # If there are larger interval between the calls, it can live longer.
16
+ DEFAULT_FLUSH_INTERVAL = 10
17
+
18
+ def initialize(
19
+ buffer_size: DEFAULT_BUFFER_SIZE,
20
+ flush_interval: DEFAULT_FLUSH_INTERVAL
21
+ )
22
+ @buffer_size = buffer_size
23
+ @flush_interval = flush_interval
24
+ @buffer = Concurrent::Array.new
25
+ end
26
+
27
+ # Add an item to buffer
28
+ #
29
+ # @param item: data to be added to buffer
30
+ # @return true if the item can be added, otherwise false
31
+ def add(item)
32
+ if add_item?
33
+ buffer.push({ item: item, created_at: Time.now })
34
+ true
35
+ else
36
+ false
37
+ end
38
+ end
39
+
40
+ # Remove all existing items from buffer
41
+ #
42
+ # @return all items in buffer
43
+ def flush
44
+ data = []
45
+
46
+ data << buffer.shift[:item] until buffer.empty?
47
+
48
+ data
49
+ end
50
+
51
+ # This method is only used to facilitate testing
52
+ def size
53
+ buffer.size
54
+ end
55
+
56
+ private
57
+
58
+ attr_reader :buffer_size, :flush_interval, :buffer
59
+
60
+ def add_item?
61
+ buffer.size < buffer_size &&
62
+ (buffer.empty? || buffer.first[:created_at] > Time.now - flush_interval)
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,10 @@
1
+ require 'dry-configurable'
2
+
3
+ module EventTracer
4
+ class Config
5
+ extend Dry::Configurable
6
+
7
+ setting :app_name, default: 'app_name'
8
+ setting :dynamo_db_table_name, default: 'logs'
9
+ end
10
+ end
@@ -4,72 +4,68 @@ require_relative './basic_decorator'
4
4
  # Datadog interface to send our usual actions
5
5
  # BasicDecorator adds a transparent interface on top of the datadog interface
6
6
  #
7
- # Usage: EventTracer.register :datadog, EventTracer::DataDogLogger.new(DataDog)
8
- # data_dog_logger.info datadog: { count: { counter_1: 1, counter_2: 2 }, set: { gauge_1: 1 } }
9
- # data_dog_logger.info datadog: { count: { counter_1: { value: 1, tags: ['tag1, tag2']} } }
7
+ # Usage: EventTracer.register :datadog,
8
+ # EventTracer::DataDogLogger.new(DataDog, allowed_tags: ['tag_1', 'tag_2'])
9
+ #
10
+ # data_dog_logger.info metrics: [:counter_1, :counter_2]
11
+ # data_dog_logger.info metrics: { counter_1: { type: :counter, value: 1}, gauce_2: { type: :gauce, value: 10 } }
10
12
 
11
13
  module EventTracer
12
14
  class DatadogLogger < BasicDecorator
13
15
 
14
- class InvalidTagError < StandardError; end
16
+ SUPPORTED_METRIC_TYPES = {
17
+ counter: :count,
18
+ distribution: :distribution,
19
+ gauge: :gauge,
20
+ set: :set,
21
+ histogram: :histogram
22
+ }
23
+ DEFAULT_METRIC_TYPE = :count
24
+ DEFAULT_COUNTER = 1
15
25
 
16
- SUPPORTED_METRICS ||= %i[count set distribution gauge histogram].freeze
26
+ def initialize(decoratee, allowed_tags: [])
27
+ super(decoratee)
28
+ @allowed_tags = allowed_tags
29
+ end
17
30
 
18
31
  LOG_TYPES.each do |log_type|
19
32
  define_method log_type do |**args|
20
- return LogResult.new(false, 'Invalid datadog config') unless args[:datadog]&.is_a?(Hash)
33
+ metrics = args[:metrics]
34
+
35
+ return fail_result('Invalid Datadog config') unless valid_args?(metrics)
36
+ return success_result if metrics.empty?
21
37
 
22
- applied_metrics(args[:datadog]).each do |metric|
23
- metric_args = args[:datadog][metric]
24
- return LogResult.new(false, "Datadog metric #{metric} invalid") unless metric_args.is_a?(Hash)
38
+ tags = build_tags(args)
25
39
 
26
- send_metric metric, metric_args
40
+ case metrics
41
+ when Array
42
+ metrics.each do |metric|
43
+ datadog.public_send(DEFAULT_METRIC_TYPE, metric, DEFAULT_COUNTER, tags: tags)
44
+ end
45
+ when Hash
46
+ metrics.each do |metric_name, metric_payload|
47
+ metric_type = SUPPORTED_METRIC_TYPES[metric_payload.fetch(:type).to_sym]
48
+ datadog.public_send(metric_type, metric_name, metric_payload.fetch(:value), tags: tags) if metric_type
49
+ end
27
50
  end
28
51
 
29
- LogResult.new(true)
52
+ success_result
30
53
  end
31
54
  end
32
55
 
33
56
  private
34
57
 
35
- attr_reader :datadog, :decoratee
58
+ attr_reader :decoratee, :allowed_tags
36
59
  alias_method :datadog, :decoratee
37
60
 
38
- def applied_metrics(datadog_args)
39
- datadog_args.keys.select { |metric| SUPPORTED_METRICS.include?(metric) }
61
+ def valid_args?(metrics)
62
+ metrics && (metrics.is_a?(Hash) || metrics.is_a?(Array))
40
63
  end
41
64
 
42
- def send_metric(metric, payload)
43
- payload.each do |increment, attribute|
44
- if attribute.is_a?(Hash)
45
- begin
46
- datadog.public_send(
47
- metric,
48
- increment,
49
- attribute.fetch(:value),
50
- build_options(attribute[:tags])
51
- )
52
- rescue KeyError
53
- raise InvalidTagError, "Datadog payload { #{increment}: #{attribute} } invalid"
54
- end
55
- else
56
- datadog.public_send(metric, increment, attribute)
57
- end
65
+ def build_tags(args)
66
+ args.slice(*allowed_tags).map do |tag, value|
67
+ "#{tag}:#{value}"
58
68
  end
59
69
  end
60
-
61
- def build_options(tags)
62
- return {} unless tags
63
-
64
- formattted_tags =
65
- if tags.is_a?(Array)
66
- tags
67
- else
68
- tags.inject([]) do |acc, (tag, value)|
69
- acc << "#{tag}:#{value}"
70
- end
71
- end
72
- { tags: formattted_tags }
73
- end
74
70
  end
75
71
  end
@@ -0,0 +1,7 @@
1
+ module EventTracer
2
+ class DynamoDBClient
3
+ def self.call
4
+ Aws::DynamoDB::Client.new
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,13 @@
1
+ module EventTracer
2
+ class DynamoDBDefaultProcessor
3
+ def call(log_type, action:, message:, args:)
4
+ args.merge(
5
+ timestamp: Time.now.utc.iso8601(6),
6
+ action: action,
7
+ message: message,
8
+ log_type: log_type,
9
+ app: EventTracer::Config.config.app_name
10
+ )
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'dynamo_db_client'
4
+
5
+ begin
6
+ require 'sidekiq'
7
+ require 'aws-sdk-dynamodb'
8
+ rescue LoadError => e
9
+ puts "Please add the missing gem into your app Gemfile: #{e.message}"
10
+ raise
11
+ end
12
+
13
+ module EventTracer
14
+ class DynamoDBLogWorker
15
+ include ::Sidekiq::Worker
16
+
17
+ sidekiq_options retry: 1, queue: 'low'
18
+
19
+ # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
20
+ MAX_DYNAMO_DB_ITEM_PER_REQUEST = 25
21
+
22
+ def perform(items)
23
+ wrap(items).each_slice(MAX_DYNAMO_DB_ITEM_PER_REQUEST) do |batch|
24
+ data = batch.map do |item|
25
+ { put_request: { item: clean_empty_values(item) } }
26
+ end
27
+
28
+ EventTracer::DynamoDBClient.call.batch_write_item(
29
+ request_items: { EventTracer::Config.config.dynamo_db_table_name => data }
30
+ )
31
+
32
+ rescue Aws::DynamoDB::Errors::ServiceError => e
33
+ EventTracer.error(
34
+ loggers: %i(base),
35
+ action: 'DynamoDBLogWorker',
36
+ app: EventTracer::Config.config.app_name,
37
+ error: e.class.name,
38
+ message: e.message
39
+ )
40
+ end
41
+ end
42
+
43
+ private
44
+
45
+ def wrap(items)
46
+ # NOTE: This allows us to handle either buffered or unbuffered payloads
47
+ if items.is_a?(Hash)
48
+ [items]
49
+ else
50
+ Array(items)
51
+ end
52
+ end
53
+
54
+ # dynamo can't serialise empty strings/ non-zero numerics
55
+ def clean_empty_values(data)
56
+ data.delete_if do |_key, value|
57
+ case value
58
+ when Hash
59
+ clean_empty_values(value)
60
+ false
61
+ when String then value.empty?
62
+ else false
63
+ end
64
+ end
65
+ end
66
+
67
+ end
68
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'time'
4
+ require_relative 'dynamo_db_client'
5
+
6
+ module EventTracer
7
+ class DynamoDBLogger
8
+ def initialize(buffer: Buffer.new(buffer_size: 0), log_processor: EventTracer::DynamoDBDefaultProcessor.new)
9
+ @buffer = buffer
10
+ @log_processor = log_processor
11
+ end
12
+
13
+ EventTracer::LOG_TYPES.each do |log_type|
14
+ define_method log_type do |**args|
15
+ save_message log_type, **args
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ attr_reader :buffer, :log_processor
22
+
23
+ def save_message(log_type, action:, message:, **args)
24
+ payload = log_processor.call(log_type, action: action, message: message, args: args)
25
+
26
+ unless buffer.add(payload)
27
+ all_payloads = buffer.flush + [payload]
28
+ DynamoDBLogWorker.perform_async(all_payloads)
29
+ end
30
+
31
+ LogResult.new(true)
32
+ end
33
+
34
+ end
35
+ end
@@ -1,3 +1,3 @@
1
1
  module EventTracer
2
- VERSION = '0.2.1'.freeze
2
+ VERSION = '0.3.2'.freeze
3
3
  end
data/lib/event_tracer.rb CHANGED
@@ -31,14 +31,10 @@ module EventTracer
31
31
  result = Result.new
32
32
 
33
33
  loggers.each do |code, logger|
34
- begin
35
- if args[:action] && args[:message]
36
- result.record code, logger.send(log_type, filtered_log_arguments(code, args))
37
- else
38
- result.record code, LogResult.new(false, 'Fields action & message need to be populated')
39
- end
40
- rescue Exception => e
41
- result.record code, LogResult.new(false, e.message)
34
+ if args[:action] && args[:message]
35
+ result.record code, logger.send(log_type, **args)
36
+ else
37
+ result.record code, LogResult.new(false, 'Fields action & message need to be populated')
42
38
  end
43
39
  end
44
40
 
@@ -55,14 +51,6 @@ module EventTracer
55
51
  @loggers.select { |code, _logger| selected_codes.include?(code) }
56
52
  end
57
53
 
58
- def self.filtered_log_arguments(logger_code, args)
59
- blacklisted_logger_keys = registered_logger_codes - [logger_code]
60
-
61
- args.reject do |key, _value|
62
- blacklisted_logger_keys.include?(key)
63
- end
64
- end
65
-
66
54
  def self.registered_logger_codes
67
55
  @loggers.keys
68
56
  end
metadata CHANGED
@@ -1,27 +1,55 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: event_tracer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - melvrickgoh
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-11-15 00:00:00.000000000 Z
11
+ date: 2021-10-11 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: concurrent-ruby
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: dry-configurable
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
13
41
  - !ruby/object:Gem::Dependency
14
42
  name: bundler
15
43
  requirement: !ruby/object:Gem::Requirement
16
44
  requirements:
17
- - - "~>"
45
+ - - ">="
18
46
  - !ruby/object:Gem::Version
19
47
  version: 2.1.4
20
48
  type: :development
21
49
  prerelease: false
22
50
  version_requirements: !ruby/object:Gem::Requirement
23
51
  requirements:
24
- - - "~>"
52
+ - - ">="
25
53
  - !ruby/object:Gem::Version
26
54
  version: 2.1.4
27
55
  - !ruby/object:Gem::Dependency
@@ -53,7 +81,7 @@ dependencies:
53
81
  - !ruby/object:Gem::Version
54
82
  version: '3.0'
55
83
  description: 'Thin wrapper for formatted logging/ metric services to be used as a
56
- single service. External service(s) supported: Appsignal'
84
+ single service. External service(s) supported: Appsignal, Datadog, DynamoDB'
57
85
  email:
58
86
  - melvrickgoh@hotmail.com
59
87
  executables: []
@@ -64,7 +92,13 @@ files:
64
92
  - lib/event_tracer/appsignal_logger.rb
65
93
  - lib/event_tracer/base_logger.rb
66
94
  - lib/event_tracer/basic_decorator.rb
95
+ - lib/event_tracer/buffer.rb
96
+ - lib/event_tracer/config.rb
67
97
  - lib/event_tracer/datadog_logger.rb
98
+ - lib/event_tracer/dynamo_db_client.rb
99
+ - lib/event_tracer/dynamo_db_default_processor.rb
100
+ - lib/event_tracer/dynamo_db_log_worker.rb
101
+ - lib/event_tracer/dynamo_db_logger.rb
68
102
  - lib/event_tracer/log_result.rb
69
103
  - lib/event_tracer/version.rb
70
104
  homepage: https://github.com/melvrickgoh/event_tracer
@@ -87,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
87
121
  - !ruby/object:Gem::Version
88
122
  version: '0'
89
123
  requirements: []
90
- rubygems_version: 3.0.8
124
+ rubygems_version: 3.2.28
91
125
  signing_key:
92
126
  specification_version: 4
93
127
  summary: Thin wrapper for formatted logging/ metric services to be used as a single