synced-latency-data-collector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/ci.yml +45 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +84 -0
- data/LICENSE.txt +21 -0
- data/README.md +118 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/synced/latency/data/collector.rb +1 -0
- data/lib/synced_latency_data_collector/configuration.rb +23 -0
- data/lib/synced_latency_data_collector/datadog_collector.rb +112 -0
- data/lib/synced_latency_data_collector/datadog_collector_job.rb +12 -0
- data/lib/synced_latency_data_collector/scheduler.rb +46 -0
- data/lib/synced_latency_data_collector/version.rb +3 -0
- data/lib/synced_latency_data_collector.rb +28 -0
- data/synced-latency-data-collector.gemspec +40 -0
- metadata +211 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 5ea6f9d32bf1d429113b48c5a3b12048ba83c2b511a06d86f2d095c765b61177
|
|
4
|
+
data.tar.gz: 9b2b97f0818e2358622ab3fd3c8359b1e3518235f63110da04486a7017ca5670
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 5feef1a4436ce6395b4285d610796cc299df71fff914a56bd008da52c29ecc865a8fe275984f7c1fdf343c84b5a06f3e348385a70b9386cb92dcb607b8b63bc0
|
|
7
|
+
data.tar.gz: '028cbf09c66551dc241e2700f05bcb0c0f846d910e8e08c88a978e33f8454abe87d8e778de64bbf68a0ae28a9f9f19949d84e802ff208ee594e35794e70b51bf'
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on: [pull_request]
|
|
3
|
+
jobs:
|
|
4
|
+
rspec:
|
|
5
|
+
timeout-minutes: 10
|
|
6
|
+
strategy:
|
|
7
|
+
fail-fast: false
|
|
8
|
+
matrix:
|
|
9
|
+
ruby: ['2.7', '3.0.6', '3.1', '3.2']
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
env:
|
|
12
|
+
DD_PROFILING_NO_EXTENSION: true
|
|
13
|
+
RAILS_ENV: test
|
|
14
|
+
RACK_ENV: test
|
|
15
|
+
REDIS_URL: redis://localhost:6379/1
|
|
16
|
+
POSTGRES_USER: postgres
|
|
17
|
+
POSTGRES_PASSWORD: postgres
|
|
18
|
+
POSTGRES_DB: synced-latency-data-collector-test
|
|
19
|
+
services:
|
|
20
|
+
postgres:
|
|
21
|
+
image: postgres:14
|
|
22
|
+
env:
|
|
23
|
+
POSTGRES_USER: postgres
|
|
24
|
+
POSTGRES_PASSWORD: postgres
|
|
25
|
+
POSTGRES_DB: synced-latency-data-collector-test
|
|
26
|
+
ports:
|
|
27
|
+
- 5432:5432
|
|
28
|
+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
|
29
|
+
redis:
|
|
30
|
+
image: redis
|
|
31
|
+
ports:
|
|
32
|
+
- 6379:6379
|
|
33
|
+
options: >-
|
|
34
|
+
--health-cmd "redis-cli ping"
|
|
35
|
+
--health-interval 10s
|
|
36
|
+
--health-timeout 5s
|
|
37
|
+
--health-retries 5
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v2
|
|
40
|
+
- uses: ruby/setup-ruby@v1
|
|
41
|
+
|
|
42
|
+
with:
|
|
43
|
+
ruby-version: ${{ matrix.ruby }}
|
|
44
|
+
bundler-cache: true
|
|
45
|
+
- run: bundle exec rspec
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
synced-latency-data-collector (0.1.0)
|
|
5
|
+
dogstatsd-ruby (~> 4)
|
|
6
|
+
sidekiq (>= 5, < 7)
|
|
7
|
+
sidekiq-cron (~> 1)
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
activemodel (6.1.3)
|
|
13
|
+
activesupport (= 6.1.3)
|
|
14
|
+
activerecord (6.1.3)
|
|
15
|
+
activemodel (= 6.1.3)
|
|
16
|
+
activesupport (= 6.1.3)
|
|
17
|
+
activesupport (6.1.3)
|
|
18
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
19
|
+
i18n (>= 1.6, < 2)
|
|
20
|
+
minitest (>= 5.1)
|
|
21
|
+
tzinfo (~> 2.0)
|
|
22
|
+
zeitwerk (~> 2.3)
|
|
23
|
+
concurrent-ruby (1.2.2)
|
|
24
|
+
connection_pool (2.4.1)
|
|
25
|
+
diff-lcs (1.3)
|
|
26
|
+
dogstatsd-ruby (4.8.3)
|
|
27
|
+
et-orbi (1.2.7)
|
|
28
|
+
tzinfo
|
|
29
|
+
fugit (1.9.0)
|
|
30
|
+
et-orbi (~> 1, >= 1.2.7)
|
|
31
|
+
raabro (~> 1.4)
|
|
32
|
+
globalid (1.2.1)
|
|
33
|
+
activesupport (>= 6.1)
|
|
34
|
+
i18n (1.14.1)
|
|
35
|
+
concurrent-ruby (~> 1.0)
|
|
36
|
+
minitest (5.20.0)
|
|
37
|
+
pg (1.5.4)
|
|
38
|
+
raabro (1.4.0)
|
|
39
|
+
rack (2.2.8)
|
|
40
|
+
rake (13.0.3)
|
|
41
|
+
redis (4.8.1)
|
|
42
|
+
redis-namespace (1.11.0)
|
|
43
|
+
redis (>= 4)
|
|
44
|
+
rspec (3.9.0)
|
|
45
|
+
rspec-core (~> 3.9.0)
|
|
46
|
+
rspec-expectations (~> 3.9.0)
|
|
47
|
+
rspec-mocks (~> 3.9.0)
|
|
48
|
+
rspec-core (3.9.1)
|
|
49
|
+
rspec-support (~> 3.9.1)
|
|
50
|
+
rspec-expectations (3.9.0)
|
|
51
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
52
|
+
rspec-support (~> 3.9.0)
|
|
53
|
+
rspec-mocks (3.9.1)
|
|
54
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
55
|
+
rspec-support (~> 3.9.0)
|
|
56
|
+
rspec-support (3.9.2)
|
|
57
|
+
sidekiq (6.5.12)
|
|
58
|
+
connection_pool (>= 2.2.5, < 3)
|
|
59
|
+
rack (~> 2.0)
|
|
60
|
+
redis (>= 4.5.0, < 5)
|
|
61
|
+
sidekiq-cron (1.12.0)
|
|
62
|
+
fugit (~> 1.8)
|
|
63
|
+
globalid (>= 1.0.1)
|
|
64
|
+
sidekiq (>= 6)
|
|
65
|
+
timecop (0.9.8)
|
|
66
|
+
tzinfo (2.0.6)
|
|
67
|
+
concurrent-ruby (~> 1.0)
|
|
68
|
+
zeitwerk (2.6.12)
|
|
69
|
+
|
|
70
|
+
PLATFORMS
|
|
71
|
+
ruby
|
|
72
|
+
|
|
73
|
+
DEPENDENCIES
|
|
74
|
+
activerecord (>= 4.2)
|
|
75
|
+
bundler (~> 2.0)
|
|
76
|
+
pg (~> 1)
|
|
77
|
+
rake (~> 13.0)
|
|
78
|
+
redis-namespace (~> 1)
|
|
79
|
+
rspec (~> 3.0)
|
|
80
|
+
synced-latency-data-collector!
|
|
81
|
+
timecop (~> 0.9)
|
|
82
|
+
|
|
83
|
+
BUNDLED WITH
|
|
84
|
+
2.2.33
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Karol Galanciak
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# Synced::Latency::Data::Collector
|
|
2
|
+
|
|
3
|
+
A gem for collecting metrics about synchronization latency.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem "synced-latency-data-collector"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle
|
|
16
|
+
|
|
17
|
+
Or install it yourself as:
|
|
18
|
+
|
|
19
|
+
$ gem install synced-latency-data-collector
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
Add this into an initializer in the Rails app:
|
|
24
|
+
|
|
25
|
+
``` rb
|
|
26
|
+
Rails.application.config.to_prepare do
|
|
27
|
+
SyncedLatencyDataCollector.configure do |config|
|
|
28
|
+
config.datadog_host = ENV.fetch("SYNCED_DATADOG_HOST")
|
|
29
|
+
config.datadog_port = ENV.fetch("SYNCED_DATADOG_PORT")
|
|
30
|
+
config.datadog_namespace = ENV.fetch("SYNCED_DATADOG_NAMESPACE")
|
|
31
|
+
config.active_accounts_scope_proc = -> { Account.active }
|
|
32
|
+
config.account_model_proc = -> { Account }
|
|
33
|
+
config.synced_timestamp_model = Synced::Timestamp
|
|
34
|
+
config.global_models_proc = -> { [Amenity] }
|
|
35
|
+
config.account_scoped_models_proc = -> { [Booking, BookingComment, BookingsFee, BookingsTag, Client, Payment, Photo,
|
|
36
|
+
PreferencesGeneralSetting, Rental, RentalsAmenity, RentalsTag, Review, Source]
|
|
37
|
+
}
|
|
38
|
+
config.non_account_scoped_models_proc = -> { [[Rental, Bedroom], [Rental, Bathroom]] }
|
|
39
|
+
config.active_scope_for_different_parent = :visible
|
|
40
|
+
config.check_timestamps_since_proc = -> { Time.now.utc - (3 * 86_400) }
|
|
41
|
+
config.sidekiq_job_queue = :critical
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
An explanation of the attributes:
|
|
47
|
+
|
|
48
|
+
* datadog_host - most likely "chef-prod.bookingsync.it"
|
|
49
|
+
|
|
50
|
+
* datadog_port - most likely 8125
|
|
51
|
+
|
|
52
|
+
* datadog_namespace - the name of the application and the environment, e.g. "bsa_notifications.production"
|
|
53
|
+
|
|
54
|
+
* active_accounts_scope_proc - since we want to reject synced timestamps that are created for suspended/canceled accounts and models belonging to these accounts, we need to specify the scope of the applicable accounts for which we want to collect data
|
|
55
|
+
|
|
56
|
+
* account_model_proc - in some cases, the account model might not be literally the Account one. In majority of the cases, this is going to be `-> { Account }`, however, in some cases it will be something different, e.g. `-> { Website }`.
|
|
57
|
+
|
|
58
|
+
* synced_timestamp_model - the name of the synced timestamp model, most likely Synced::Timestamp
|
|
59
|
+
|
|
60
|
+
* global_models_proc - the proc returning an array of models that are not scoped by any model
|
|
61
|
+
|
|
62
|
+
* account_scoped_models_proc - the proc returning an array of models that are scoped by Account
|
|
63
|
+
|
|
64
|
+
* non_account_scoped_models_proc - the proc returning an array of models that are not scoped by Account, e.g. by a Rental. Notice that the elements of this array are arrays of two models - a parent and the model for which we are tracking the latency.
|
|
65
|
+
|
|
66
|
+
* active_scope_for_different_parent - the name of the scope that will be applied when searching for valid synced timestamps related to the models specified in `non_account_scoped_models_proc`. A specific example would be tracking latency for bedrooms' sync belonging only to visible rentals.
|
|
67
|
+
|
|
68
|
+
* check_timestamps_since_proc - this proc returns timestamp that will be used to filter out outdated timestamps and hence limit amount of data to be process. Default value is 3 days.
|
|
69
|
+
|
|
70
|
+
* sidekiq_job_queue - the name of the queue for the Sidekiq job
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
Also, add the job to the schedule that will be run every minute. It's recommended to add this to the initializer (ideally, after sidekiq-cron is already configured, to avoid unexpected issues):
|
|
74
|
+
|
|
75
|
+
``` rb
|
|
76
|
+
Sidekiq.configure_server do |config|
|
|
77
|
+
config.on(:startup) do
|
|
78
|
+
SyncedLatencyDataCollector.schedule!
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
That method will add the job to the schedule only if it's not already there yet.
|
|
84
|
+
|
|
85
|
+
The metrics will be available when creating a new notebook on Datadog.
|
|
86
|
+
|
|
87
|
+
Also, make sure that you have the following index for synced timestamps:
|
|
88
|
+
|
|
89
|
+
``` rb
|
|
90
|
+
class AddIndexOnModelClassAndSyncedAtForSyncedTimestamps < ActiveRecord::Migration[5.1]
|
|
91
|
+
disable_ddl_transaction!
|
|
92
|
+
|
|
93
|
+
def change
|
|
94
|
+
add_index :synced_timestamps, [:parent_scope_id, :parent_scope_type, :model_class, :synced_at],
|
|
95
|
+
name: "synced_timestamps_full_index",
|
|
96
|
+
algorithm: :concurrently
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Setting up monitors on Datadog
|
|
103
|
+
|
|
104
|
+
Watch [this tutorial](https://share.getcloudapp.com/lluyJwAv) to learn more.
|
|
105
|
+
|
|
106
|
+
## Development
|
|
107
|
+
|
|
108
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
109
|
+
|
|
110
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
111
|
+
|
|
112
|
+
## Contributing
|
|
113
|
+
|
|
114
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/BookingSync/synced-latency-data-collector.
|
|
115
|
+
|
|
116
|
+
## License
|
|
117
|
+
|
|
118
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "synced/latency/data/collector"
|
|
5
|
+
|
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
8
|
+
|
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
10
|
+
# require "pry"
|
|
11
|
+
# Pry.start
|
|
12
|
+
|
|
13
|
+
require "irb"
|
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
require_relative "../../../synced_latency_data_collector"
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
class SyncedLatencyDataCollector::Configuration
|
|
2
|
+
THREE_DAYS_AGO_PROC = -> { Time.now.utc - (3 * 86_400) }
|
|
3
|
+
|
|
4
|
+
attr_accessor :datadog_host, :datadog_port, :datadog_namespace, :active_accounts_scope_proc, :account_model_proc,
|
|
5
|
+
:synced_timestamp_model, :global_models_proc, :account_scoped_models_proc, :check_timestamps_since_proc,
|
|
6
|
+
:non_account_scoped_models_proc, :active_scope_for_different_parent, :sidekiq_job_queue
|
|
7
|
+
|
|
8
|
+
def global_models_proc
|
|
9
|
+
@global_models_proc || -> { [] }
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def account_scoped_models_proc
|
|
13
|
+
@account_scoped_models_proc || -> { [] }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def non_account_scoped_models_proc
|
|
17
|
+
@non_account_scoped_models_proc || -> { [] }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def check_timestamps_since_proc
|
|
21
|
+
@check_timestamps_since_proc || THREE_DAYS_AGO_PROC
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
require "forwardable"
|
|
2
|
+
|
|
3
|
+
class SyncedLatencyDataCollector
|
|
4
|
+
class DatadogCollector
|
|
5
|
+
extend Forwardable
|
|
6
|
+
|
|
7
|
+
attr_reader :datadog_statsd_client, :configuration
|
|
8
|
+
private :datadog_statsd_client, :configuration
|
|
9
|
+
|
|
10
|
+
METRIC_NAME_PREFIX = "synced".freeze
|
|
11
|
+
METRIC_NAME_SUFFIX = "maximum_sync_latency_in_minutes".freeze
|
|
12
|
+
METRIC_NAME_SEPARATOR = ".".freeze
|
|
13
|
+
private_constant :METRIC_NAME_PREFIX, :METRIC_NAME_SUFFIX, :METRIC_NAME_SEPARATOR
|
|
14
|
+
|
|
15
|
+
def_delegators :configuration, :active_accounts_scope_proc, :datadog_namespace,
|
|
16
|
+
:synced_timestamp_model, :global_models_proc, :account_scoped_models_proc, :account_model_proc,
|
|
17
|
+
:non_account_scoped_models_proc, :active_scope_for_different_parent, :check_timestamps_since_proc
|
|
18
|
+
|
|
19
|
+
def initialize(datadog_statsd_client, configuration)
|
|
20
|
+
@datadog_statsd_client = datadog_statsd_client
|
|
21
|
+
@configuration = configuration
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def collect
|
|
25
|
+
collect_for_global_models
|
|
26
|
+
collect_for_account_scoped_models
|
|
27
|
+
collect_for_differently_scoped_models
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def collect_for_global_models
|
|
33
|
+
global_models_proc.call.each do |model|
|
|
34
|
+
timestamp = synced_timestamp_model
|
|
35
|
+
.where(parent_scope: nil, model_class: model.to_s)
|
|
36
|
+
.order(:synced_at)
|
|
37
|
+
.last
|
|
38
|
+
|
|
39
|
+
register_latency_if_timestamp_exists(model, timestamp)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def collect_for_account_scoped_models
|
|
44
|
+
model_classes = account_scoped_models_proc.call.map(&:to_s)
|
|
45
|
+
models_with_timestamps = fetch_models_with_timestamps_for_parent_and_dependent_models(active_accounts, model_classes)
|
|
46
|
+
|
|
47
|
+
models_with_timestamps.each_pair do |model_name, timestamp|
|
|
48
|
+
register_latency_if_timestamp_exists(infer_model_model_class_from_name(model_name), timestamp)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def collect_for_differently_scoped_models
|
|
53
|
+
account_model_name = account_model_proc.call.model_name.singular
|
|
54
|
+
|
|
55
|
+
non_account_scoped_models_proc.call.each do |parent_model, dependent_model|
|
|
56
|
+
parent_scope = parent_model.where(account_model_name => active_accounts).public_send(active_scope_for_different_parent)
|
|
57
|
+
models_with_timestamps = fetch_models_with_timestamps_for_parent_and_dependent_models(parent_scope, dependent_model.to_s)
|
|
58
|
+
|
|
59
|
+
models_with_timestamps.each_pair do |model_name, timestamp|
|
|
60
|
+
register_latency_if_timestamp_exists(infer_model_model_class_from_name(model_name), timestamp)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def register_latency_if_timestamp_exists(model, timestamp)
|
|
66
|
+
register_latency(model, calculate_latency(timestamp)) if timestamp
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def register_latency(model_klass, latency_in_minutes)
|
|
70
|
+
datadog_statsd_client.gauge(build_metric_name(model_klass), latency_in_minutes)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def calculate_latency(timestamp)
|
|
74
|
+
latency_in_seconds = Time.current - timestamp.synced_at
|
|
75
|
+
(latency_in_seconds / 60).floor
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def active_accounts
|
|
79
|
+
@active_accounts ||= active_accounts_scope_proc.call
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def build_metric_name(model_klass)
|
|
83
|
+
[
|
|
84
|
+
METRIC_NAME_PREFIX,
|
|
85
|
+
datadog_namespace,
|
|
86
|
+
model_klass.model_name.param_key,
|
|
87
|
+
METRIC_NAME_SUFFIX
|
|
88
|
+
].join(METRIC_NAME_SEPARATOR)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def synced_timestamp_table
|
|
92
|
+
@synced_timestamp_table ||= synced_timestamp_model.table_name
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def infer_model_model_class_from_name(name)
|
|
96
|
+
Object.const_get(name)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def fetch_models_with_timestamps_for_parent_and_dependent_models(parent_scope, model_class)
|
|
100
|
+
synced_timestamp_model
|
|
101
|
+
.select("MAX(#{synced_timestamp_table}.synced_at) AS synced_at, #{synced_timestamp_table}.model_class")
|
|
102
|
+
.where(model_class: model_class, parent_scope: parent_scope)
|
|
103
|
+
.where("#{synced_timestamp_table}.synced_at > ?", check_timestamps_since_proc.call)
|
|
104
|
+
.group(:model_class, :parent_scope_id)
|
|
105
|
+
.order(:model_class, :parent_scope_id)
|
|
106
|
+
.group_by(&:model_class)
|
|
107
|
+
.transform_values do |scoped_timestamps|
|
|
108
|
+
scoped_timestamps.min_by { |model_max_timestamp| model_max_timestamp.synced_at }
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
class SyncedLatencyDataCollector
|
|
2
|
+
class Scheduler
|
|
3
|
+
JOB_NAME = "synced_latency_data_collector".freeze
|
|
4
|
+
EVERY_MINUTE_IN_CRON_SYNTAX = "* * * * *".freeze
|
|
5
|
+
JOB_CLASS_NAME = "SyncedLatencyDataCollector::DatadogCollectorJob".freeze
|
|
6
|
+
JOB_DESCRIPTION = "Collect latency metrics from synced".freeze
|
|
7
|
+
|
|
8
|
+
private_constant :JOB_NAME, :EVERY_MINUTE_IN_CRON_SYNTAX, :JOB_CLASS_NAME, :JOB_DESCRIPTION
|
|
9
|
+
|
|
10
|
+
attr_reader :configuration
|
|
11
|
+
private :configuration
|
|
12
|
+
|
|
13
|
+
def initialize(configuration)
|
|
14
|
+
@configuration = configuration
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def schedule!
|
|
18
|
+
find || create
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def find
|
|
24
|
+
Sidekiq::Cron::Job.find(name: JOB_NAME)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def create
|
|
28
|
+
Sidekiq::Cron::Job.create(create_job_arguments)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def create_job_arguments
|
|
32
|
+
{
|
|
33
|
+
name: JOB_NAME,
|
|
34
|
+
cron: EVERY_MINUTE_IN_CRON_SYNTAX,
|
|
35
|
+
class: "SyncedLatencyDataCollector::DatadogCollectorJob",
|
|
36
|
+
queue: configuration.sidekiq_job_queue,
|
|
37
|
+
active_job: false,
|
|
38
|
+
description: JOB_DESCRIPTION
|
|
39
|
+
}
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def every_minute_to_cron_syntax
|
|
43
|
+
EVERY_MINUTE_IN_CRON_SYNTAX
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
require "synced_latency_data_collector/version"
|
|
2
|
+
require "synced_latency_data_collector/configuration"
|
|
3
|
+
require "synced_latency_data_collector/datadog_collector"
|
|
4
|
+
require "synced_latency_data_collector/datadog_collector_job"
|
|
5
|
+
require "synced_latency_data_collector/scheduler"
|
|
6
|
+
require "datadog/statsd"
|
|
7
|
+
|
|
8
|
+
class SyncedLatencyDataCollector
|
|
9
|
+
def self.configuration
|
|
10
|
+
@configuration ||= SyncedLatencyDataCollector::Configuration.new
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.configure
|
|
14
|
+
yield configuration
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.datadog_stats_client
|
|
18
|
+
Datadog::Statsd.new(configuration.datadog_host, configuration.datadog_port, tags: ["host:disabled"])
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.collect
|
|
22
|
+
SyncedLatencyDataCollector::DatadogCollector.new(datadog_stats_client, configuration).collect
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.schedule!
|
|
26
|
+
SyncedLatencyDataCollector::Scheduler.new(configuration).schedule!
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
lib = File.expand_path("lib", __dir__)
|
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
|
+
require "synced_latency_data_collector/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "synced-latency-data-collector"
|
|
7
|
+
spec.version = SyncedLatencyDataCollector::VERSION
|
|
8
|
+
spec.authors = ["Karol Galanciak"]
|
|
9
|
+
spec.email = ["karol.galanciak@gmail.com"]
|
|
10
|
+
|
|
11
|
+
spec.summary = %q{Data collector for synced gem for Datadog.}
|
|
12
|
+
spec.description = %q{Data collector for synced gem for Datadog.}
|
|
13
|
+
spec.homepage = "https://github.com/BookingSync/synced-latency-data-collector"
|
|
14
|
+
spec.license = "MIT"
|
|
15
|
+
|
|
16
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
17
|
+
spec.metadata["source_code_uri"] = "https://github.com/BookingSync/synced-latency-data-collector"
|
|
18
|
+
spec.metadata["changelog_uri"] = "https://github.com/BookingSync/synced-latency-data-collector/blob/master/CHANGELOG.md"
|
|
19
|
+
|
|
20
|
+
# Specify which files should be added to the gem when it is released.
|
|
21
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
22
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
|
23
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
24
|
+
end
|
|
25
|
+
spec.bindir = "exe"
|
|
26
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
27
|
+
spec.require_paths = ["lib"]
|
|
28
|
+
|
|
29
|
+
spec.add_development_dependency "bundler", "~> 2.0"
|
|
30
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
31
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
|
32
|
+
spec.add_development_dependency "activerecord", ">= 4.2"
|
|
33
|
+
spec.add_development_dependency "pg", "~> 1"
|
|
34
|
+
spec.add_development_dependency "redis-namespace", "~> 1"
|
|
35
|
+
spec.add_development_dependency "timecop", "~> 0.9"
|
|
36
|
+
|
|
37
|
+
spec.add_dependency "dogstatsd-ruby", "~> 4"
|
|
38
|
+
spec.add_dependency "sidekiq", ">= 5", "< 7"
|
|
39
|
+
spec.add_dependency "sidekiq-cron", "~> 1"
|
|
40
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: synced-latency-data-collector
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Karol Galanciak
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2023-12-15 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: bundler
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '2.0'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '2.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: rake
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '13.0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '13.0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: rspec
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '3.0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '3.0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: activerecord
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '4.2'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '4.2'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: pg
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '1'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '1'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: redis-namespace
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '1'
|
|
90
|
+
type: :development
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - "~>"
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '1'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: timecop
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0.9'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0.9'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: dogstatsd-ruby
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '4'
|
|
118
|
+
type: :runtime
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '4'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: sidekiq
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - ">="
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '5'
|
|
132
|
+
- - "<"
|
|
133
|
+
- !ruby/object:Gem::Version
|
|
134
|
+
version: '7'
|
|
135
|
+
type: :runtime
|
|
136
|
+
prerelease: false
|
|
137
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
138
|
+
requirements:
|
|
139
|
+
- - ">="
|
|
140
|
+
- !ruby/object:Gem::Version
|
|
141
|
+
version: '5'
|
|
142
|
+
- - "<"
|
|
143
|
+
- !ruby/object:Gem::Version
|
|
144
|
+
version: '7'
|
|
145
|
+
- !ruby/object:Gem::Dependency
|
|
146
|
+
name: sidekiq-cron
|
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
|
148
|
+
requirements:
|
|
149
|
+
- - "~>"
|
|
150
|
+
- !ruby/object:Gem::Version
|
|
151
|
+
version: '1'
|
|
152
|
+
type: :runtime
|
|
153
|
+
prerelease: false
|
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
155
|
+
requirements:
|
|
156
|
+
- - "~>"
|
|
157
|
+
- !ruby/object:Gem::Version
|
|
158
|
+
version: '1'
|
|
159
|
+
description: Data collector for synced gem for Datadog.
|
|
160
|
+
email:
|
|
161
|
+
- karol.galanciak@gmail.com
|
|
162
|
+
executables: []
|
|
163
|
+
extensions: []
|
|
164
|
+
extra_rdoc_files: []
|
|
165
|
+
files:
|
|
166
|
+
- ".github/workflows/ci.yml"
|
|
167
|
+
- ".gitignore"
|
|
168
|
+
- ".rspec"
|
|
169
|
+
- CHANGELOG.md
|
|
170
|
+
- Gemfile
|
|
171
|
+
- Gemfile.lock
|
|
172
|
+
- LICENSE.txt
|
|
173
|
+
- README.md
|
|
174
|
+
- Rakefile
|
|
175
|
+
- bin/console
|
|
176
|
+
- bin/setup
|
|
177
|
+
- lib/synced/latency/data/collector.rb
|
|
178
|
+
- lib/synced_latency_data_collector.rb
|
|
179
|
+
- lib/synced_latency_data_collector/configuration.rb
|
|
180
|
+
- lib/synced_latency_data_collector/datadog_collector.rb
|
|
181
|
+
- lib/synced_latency_data_collector/datadog_collector_job.rb
|
|
182
|
+
- lib/synced_latency_data_collector/scheduler.rb
|
|
183
|
+
- lib/synced_latency_data_collector/version.rb
|
|
184
|
+
- synced-latency-data-collector.gemspec
|
|
185
|
+
homepage: https://github.com/BookingSync/synced-latency-data-collector
|
|
186
|
+
licenses:
|
|
187
|
+
- MIT
|
|
188
|
+
metadata:
|
|
189
|
+
homepage_uri: https://github.com/BookingSync/synced-latency-data-collector
|
|
190
|
+
source_code_uri: https://github.com/BookingSync/synced-latency-data-collector
|
|
191
|
+
changelog_uri: https://github.com/BookingSync/synced-latency-data-collector/blob/master/CHANGELOG.md
|
|
192
|
+
post_install_message:
|
|
193
|
+
rdoc_options: []
|
|
194
|
+
require_paths:
|
|
195
|
+
- lib
|
|
196
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
197
|
+
requirements:
|
|
198
|
+
- - ">="
|
|
199
|
+
- !ruby/object:Gem::Version
|
|
200
|
+
version: '0'
|
|
201
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
202
|
+
requirements:
|
|
203
|
+
- - ">="
|
|
204
|
+
- !ruby/object:Gem::Version
|
|
205
|
+
version: '0'
|
|
206
|
+
requirements: []
|
|
207
|
+
rubygems_version: 3.1.6
|
|
208
|
+
signing_key:
|
|
209
|
+
specification_version: 4
|
|
210
|
+
summary: Data collector for synced gem for Datadog.
|
|
211
|
+
test_files: []
|