phobos_db_checkpoint 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +11 -0
- data/.rubocop.yml +25 -0
- data/.rubocop_common.yml +23 -0
- data/.rubocop_todo.yml +32 -0
- data/.rubosync.yml +2 -0
- data/.ruby-version +1 -1
- data/.travis.yml +35 -0
- data/CHANGELOG.md +7 -0
- data/Dockerfile +18 -0
- data/Gemfile +2 -0
- data/README.md +9 -10
- data/Rakefile +3 -1
- data/bin/console +1 -0
- data/bin/phobos_db_checkpoint +1 -0
- data/docker-compose.yml +9 -0
- data/lib/phobos_db_checkpoint.rb +9 -7
- data/lib/phobos_db_checkpoint/actions/retry_failure.rb +5 -5
- data/lib/phobos_db_checkpoint/cli.rb +10 -14
- data/lib/phobos_db_checkpoint/errors.rb +2 -0
- data/lib/phobos_db_checkpoint/event.rb +3 -1
- data/lib/phobos_db_checkpoint/event_actions.rb +2 -0
- data/lib/phobos_db_checkpoint/event_helper.rb +14 -9
- data/lib/phobos_db_checkpoint/events_api.rb +6 -4
- data/lib/phobos_db_checkpoint/failure.rb +3 -1
- data/lib/phobos_db_checkpoint/handler.rb +39 -41
- data/lib/phobos_db_checkpoint/middleware/database.rb +2 -0
- data/lib/phobos_db_checkpoint/middleware/logger.rb +15 -13
- data/lib/phobos_db_checkpoint/tasks.rb +2 -0
- data/lib/phobos_db_checkpoint/version.rb +3 -1
- data/phobos_db_checkpoint.gemspec +17 -16
- data/templates/config.ru +2 -0
- data/templates/phobos_boot.rb +2 -0
- metadata +50 -57
- data/circle.yml +0 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cda7a0046d7b17d682ad17fb2615b67422c42ad2
|
4
|
+
data.tar.gz: d4bb9fda8a27b46029ba802a43d7fe806e14efbc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9b28fba9214df0f2e183b346eef37e9f0417a20a5e880b30d698120441ae52d8bc13a1d4c8b3c375c70400ef80fd87f915245e5903a4772612348247701b94a
|
7
|
+
data.tar.gz: 6f83144ee24b53cd43495b9aa0d1883931eac7d26e576a1445696c7948ae5f27ad23f56f2429ce870a8878c2a60a2b6e8df9bd0f84dc474020d8ee3016a4adf9
|
data/.dockerignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
###########################
|
2
|
+
# Configuration for rubocop
|
3
|
+
# in .rubocop.yml
|
4
|
+
|
5
|
+
##############
|
6
|
+
# Global rules
|
7
|
+
# see .rubocop_common.yml
|
8
|
+
|
9
|
+
##############
|
10
|
+
# Inherit default rules first, and then override those rules with
|
11
|
+
# our violation whitelist.
|
12
|
+
inherit_from:
|
13
|
+
- .rubocop_common.yml
|
14
|
+
- .rubocop_todo.yml
|
15
|
+
|
16
|
+
##############
|
17
|
+
# Project specific overrides here, example:
|
18
|
+
# Metrics/BlockLength:
|
19
|
+
# Exclude:
|
20
|
+
# - 'tasks/the_huge_task.rake'
|
21
|
+
|
22
|
+
AllCops:
|
23
|
+
Exclude:
|
24
|
+
- phobos_db_checkpoint.gemspec
|
25
|
+
- spec/setup/**/*.rb
|
data/.rubocop_common.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
##############
|
2
|
+
# Global rules
|
3
|
+
AllCops:
|
4
|
+
Exclude:
|
5
|
+
- db/**/*
|
6
|
+
TargetRubyVersion: 2.5
|
7
|
+
|
8
|
+
Rails:
|
9
|
+
Enabled: false
|
10
|
+
|
11
|
+
Style/SymbolArray:
|
12
|
+
EnforcedStyle: brackets
|
13
|
+
|
14
|
+
Metrics/LineLength:
|
15
|
+
Max: 100
|
16
|
+
|
17
|
+
Metrics/BlockLength:
|
18
|
+
Exclude:
|
19
|
+
- '*.gemspec'
|
20
|
+
- 'spec/**/*.rb'
|
21
|
+
|
22
|
+
Documentation:
|
23
|
+
Enabled: false
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2018-05-14 08:11:29 +0200 using RuboCop version 0.55.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 6
|
10
|
+
Metrics/AbcSize:
|
11
|
+
Max: 30
|
12
|
+
|
13
|
+
# Offense count: 1
|
14
|
+
# Configuration parameters: CountComments.
|
15
|
+
Metrics/ClassLength:
|
16
|
+
Max: 107
|
17
|
+
|
18
|
+
# Offense count: 52
|
19
|
+
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
20
|
+
# URISchemes: http, https
|
21
|
+
Metrics/LineLength:
|
22
|
+
Max: 296
|
23
|
+
|
24
|
+
# Offense count: 9
|
25
|
+
# Configuration parameters: CountComments.
|
26
|
+
Metrics/MethodLength:
|
27
|
+
Max: 34
|
28
|
+
|
29
|
+
# Offense count: 1
|
30
|
+
# Configuration parameters: CountKeywordArgs.
|
31
|
+
Metrics/ParameterLists:
|
32
|
+
Max: 7
|
data/.rubosync.yml
ADDED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.5.1
|
data/.travis.yml
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
sudo: required
|
2
|
+
language: ruby
|
3
|
+
rvm:
|
4
|
+
- 2.5.1
|
5
|
+
- 2.4.4
|
6
|
+
- 2.3.7
|
7
|
+
|
8
|
+
services:
|
9
|
+
- docker
|
10
|
+
|
11
|
+
env:
|
12
|
+
global:
|
13
|
+
- CC_TEST_REPORTER_ID=09c6fe3d9f5f4673b7172b7ff0bd9721b5969da49c6c46e976fb89cf83f2a0dd
|
14
|
+
|
15
|
+
before_install:
|
16
|
+
- env
|
17
|
+
- docker-compose --version
|
18
|
+
- docker --version
|
19
|
+
- docker-compose config
|
20
|
+
- docker-compose build test
|
21
|
+
|
22
|
+
before_script:
|
23
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
24
|
+
- chmod +x ./cc-test-reporter
|
25
|
+
- if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter before-build || echo "Skipping CC coverage before-build"; fi
|
26
|
+
- mkdir coverage/
|
27
|
+
- touch ./coverage/.resultset.json
|
28
|
+
|
29
|
+
script:
|
30
|
+
- docker-compose run --rm test rspec
|
31
|
+
|
32
|
+
after_script:
|
33
|
+
- cat ./coverage/.resultset.json | sed "s|/opt/phobos_db_checkpoint|$PWD|" > ./coverage/.newresultset.json
|
34
|
+
- cp ./coverage/.newresultset.json ./coverage/.resultset.json
|
35
|
+
- if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT || echo "Skipping CC coverage after-build"; fi
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/).
|
6
6
|
|
7
|
+
## 3.4.0 (2018-07-22)
|
8
|
+
|
9
|
+
### Changed
|
10
|
+
- Introduce rubocop style guide #37
|
11
|
+
- Use test module from Phobos to run handler specs #39
|
12
|
+
- Accomodate future breaking changes of Phobos `around_consume` being moved to an instance method #38
|
13
|
+
|
7
14
|
## 3.3.0 (2017-10-26)
|
8
15
|
|
9
16
|
- [enhancement] Bump Phobos version to 1.5.0 to better support Avro. #34
|
data/Dockerfile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
FROM ruby:2.4.1-alpine
|
2
|
+
|
3
|
+
RUN apk update && apk upgrade && \
|
4
|
+
apk add --no-cache bash git openssh build-base
|
5
|
+
RUN apk add --no-cache postgresql-dev
|
6
|
+
|
7
|
+
RUN gem install bundler -v 1.16.0
|
8
|
+
|
9
|
+
WORKDIR /opt/phobos_db_checkpoint
|
10
|
+
|
11
|
+
ADD Gemfile Gemfile
|
12
|
+
ADD phobos_db_checkpoint.gemspec phobos_db_checkpoint.gemspec
|
13
|
+
ADD lib/phobos_db_checkpoint/version.rb lib/phobos_db_checkpoint/version.rb
|
14
|
+
|
15
|
+
RUN bundle config build.pg --with-pg-config=/usr/pgsql-9.6/bin/pg_config
|
16
|
+
RUN bundle install
|
17
|
+
|
18
|
+
ADD . .
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
-
](https://travis-ci.org/klarna/phobos_db_checkpoint)
|
2
|
+
[](https://codeclimate.com/github/klarna/phobos_db_checkpoint/maintainability)
|
3
|
+
[](https://codeclimate.com/github/klarna/phobos_db_checkpoint/test_coverage)
|
4
|
+
[](https://discord.gg/rfMUBVD)
|
3
5
|
|
4
6
|
# Phobos DB Checkpoint
|
5
7
|
|
@@ -118,14 +120,14 @@ class MyHandler
|
|
118
120
|
end
|
119
121
|
```
|
120
122
|
|
121
|
-
|
123
|
+
In case your handler returns anything different from an __ack__ it won't be saved to the database.
|
122
124
|
|
123
125
|
Note that the `PhobosDBCheckpoint::Handler` will automatically skip already handled events (i.e. duplicate Kafka messages).
|
124
126
|
|
125
127
|
#### <a name="payload"></a> Payload
|
126
128
|
PhobosDBCheckpoint assumes that the payload received from Phobos is in a JSON format. This means that if your payload is in any other format, for example Avro binary, you need to convert/decode it to JSON.
|
127
129
|
|
128
|
-
To achieve this you can
|
130
|
+
To achieve this you can compose a new handler with `PhobosDBCheckpoint::Handler` using the `#around_consume` method:
|
129
131
|
|
130
132
|
```ruby
|
131
133
|
class MyHandler
|
@@ -133,12 +135,9 @@ class MyHandler
|
|
133
135
|
|
134
136
|
# <-- setup @avro before
|
135
137
|
|
136
|
-
def
|
137
|
-
@avro.decode(payload)
|
138
|
-
|
139
|
-
|
140
|
-
def consume(payload, metadata)
|
141
|
-
# <-- consume your stuff with the decoded payload
|
138
|
+
def around_consume(payload, metadata)
|
139
|
+
decoded_payload = @avro.decode(payload)
|
140
|
+
super(decoded_payload, metadata)
|
142
141
|
end
|
143
142
|
end
|
144
143
|
```
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
data/bin/phobos_db_checkpoint
CHANGED
data/docker-compose.yml
ADDED
data/lib/phobos_db_checkpoint.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'yaml'
|
2
4
|
require 'digest'
|
3
5
|
require 'active_record'
|
@@ -13,10 +15,10 @@ require 'phobos_db_checkpoint/handler'
|
|
13
15
|
require 'phobos_db_checkpoint/actions/retry_failure'
|
14
16
|
|
15
17
|
module PhobosDBCheckpoint
|
16
|
-
DEFAULT_DB_DIR = 'db'
|
18
|
+
DEFAULT_DB_DIR = 'db'
|
17
19
|
DEFAULT_MIGRATION_PATH = File.join(DEFAULT_DB_DIR, 'migrate').freeze
|
18
|
-
DEFAULT_DB_CONFIG_PATH = 'config/database.yml'
|
19
|
-
DEFAULT_POOL_SIZE = 5
|
20
|
+
DEFAULT_DB_CONFIG_PATH = 'config/database.yml'
|
21
|
+
DEFAULT_POOL_SIZE = 5
|
20
22
|
|
21
23
|
class << self
|
22
24
|
attr_reader :db_config
|
@@ -45,14 +47,12 @@ module PhobosDBCheckpoint
|
|
45
47
|
|
46
48
|
@db_config_path ||= ENV['DB_CONFIG'] || DEFAULT_DB_CONFIG_PATH
|
47
49
|
|
48
|
-
configs = YAML.
|
50
|
+
configs = YAML.safe_load(ERB.new(File.read(File.expand_path(@db_config_path))).result, [], [], true)
|
49
51
|
@db_config = configs[env]
|
50
52
|
|
51
53
|
pool_size = @db_config['pool']
|
52
54
|
|
53
|
-
if pool_size.nil? && Phobos.config
|
54
|
-
pool_size = number_of_concurrent_listeners + DEFAULT_POOL_SIZE
|
55
|
-
end
|
55
|
+
pool_size = number_of_concurrent_listeners + DEFAULT_POOL_SIZE if pool_size.nil? && Phobos.config
|
56
56
|
|
57
57
|
@db_config.merge!('pool' => pool_size || DEFAULT_POOL_SIZE)
|
58
58
|
end
|
@@ -61,10 +61,12 @@ module PhobosDBCheckpoint
|
|
61
61
|
ActiveRecord::Base.establish_connection(db_config)
|
62
62
|
end
|
63
63
|
|
64
|
+
# rubocop:disable Lint/HandleExceptions
|
64
65
|
def close_db_connection
|
65
66
|
ActiveRecord::Base.connection_pool.disconnect!
|
66
67
|
rescue ActiveRecord::ConnectionNotEstablished
|
67
68
|
end
|
69
|
+
# rubocop:enable Lint/HandleExceptions
|
68
70
|
|
69
71
|
def load_tasks
|
70
72
|
@db_dir ||= DEFAULT_DB_DIR
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class RetryFailure
|
3
5
|
include PhobosDBCheckpoint::Handler
|
@@ -8,11 +10,9 @@ module PhobosDBCheckpoint
|
|
8
10
|
end
|
9
11
|
|
10
12
|
def perform
|
11
|
-
|
12
|
-
.
|
13
|
-
|
14
|
-
@action_taken = handler.consume(payload, metadata)
|
15
|
-
end
|
13
|
+
around_consume(payload, metadata) do
|
14
|
+
@action_taken = handler.consume(payload, metadata)
|
15
|
+
end
|
16
16
|
|
17
17
|
@failure.destroy
|
18
18
|
@action_taken
|
@@ -1,9 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'thor'
|
2
4
|
require 'fileutils'
|
3
5
|
|
4
6
|
module PhobosDBCheckpoint
|
5
7
|
module CLI
|
6
|
-
|
7
8
|
class Commands < Thor
|
8
9
|
include Thor::Actions
|
9
10
|
|
@@ -39,13 +40,9 @@ module PhobosDBCheckpoint
|
|
39
40
|
default: 'config/database.yml',
|
40
41
|
banner: 'Database configuration relative to your project'
|
41
42
|
def copy_migrations
|
42
|
-
if options[:config]
|
43
|
-
ENV['DB_CONFIG'] = options[:config]
|
44
|
-
end
|
43
|
+
ENV['DB_CONFIG'] = options[:config] if options[:config]
|
45
44
|
|
46
|
-
unless active_connection?
|
47
|
-
PhobosDBCheckpoint.configure
|
48
|
-
end
|
45
|
+
PhobosDBCheckpoint.configure unless active_connection?
|
49
46
|
|
50
47
|
destination_fullpath = File.join(destination_root, options[:destination])
|
51
48
|
generated_migrations = list_migrations(destination_fullpath)
|
@@ -60,7 +57,7 @@ module PhobosDBCheckpoint
|
|
60
57
|
template(template_path, file_path)
|
61
58
|
end
|
62
59
|
end
|
63
|
-
rescue
|
60
|
+
rescue StandardError
|
64
61
|
FileUtils.rm_f(file_path.to_s)
|
65
62
|
raise
|
66
63
|
end
|
@@ -95,16 +92,15 @@ module PhobosDBCheckpoint
|
|
95
92
|
end
|
96
93
|
|
97
94
|
def migration_number(index = 0)
|
98
|
-
[Time.now.utc.strftime('%Y%m%d%H%M%S%6N'), '%.21d'
|
95
|
+
[Time.now.utc.strftime('%Y%m%d%H%M%S%6N'), format('%.21d', index)].max
|
99
96
|
end
|
100
97
|
|
101
98
|
def template_migrations_metadata
|
102
99
|
@template_migrations_metadata ||= begin
|
103
100
|
index = 0
|
104
101
|
template_migrations.map do |path|
|
105
|
-
name = path.split('/').last
|
106
102
|
index += 1
|
107
|
-
{path: path, name: path.gsub(/\.erb$/, ''), number: migration_number(index)}
|
103
|
+
{ path: path, name: path.gsub(/\.erb$/, ''), number: migration_number(index) }
|
108
104
|
end
|
109
105
|
end
|
110
106
|
end
|
@@ -115,7 +111,7 @@ module PhobosDBCheckpoint
|
|
115
111
|
|
116
112
|
def list_migrations(dir)
|
117
113
|
return [] unless Dir.exist?(dir)
|
118
|
-
Dir.entries(dir).select {|f| f =~ /\.rb(\.erb)?$/}
|
114
|
+
Dir.entries(dir).select { |f| f =~ /\.rb(\.erb)?$/ }.sort
|
119
115
|
end
|
120
116
|
|
121
117
|
def migrations_template_dir
|
@@ -133,8 +129,8 @@ module PhobosDBCheckpoint
|
|
133
129
|
def active_connection?
|
134
130
|
ActiveRecord::Base
|
135
131
|
.connection_pool
|
136
|
-
.with_connection
|
137
|
-
rescue
|
132
|
+
.with_connection(&:active?)
|
133
|
+
rescue StandardError
|
138
134
|
false
|
139
135
|
end
|
140
136
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class Event < ActiveRecord::Base
|
3
5
|
include PhobosDBCheckpoint::EventHelper
|
4
6
|
after_initialize :assign_checksum
|
5
7
|
|
6
|
-
scope :order_by_event_time_and_created_at,
|
8
|
+
scope :order_by_event_time_and_created_at, lambda {
|
7
9
|
order('event_time desc nulls last', 'created_at desc nulls last')
|
8
10
|
}
|
9
11
|
|
@@ -1,12 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
module EventHelper
|
3
5
|
def configured_listener
|
4
6
|
listener = Phobos
|
5
|
-
|
6
|
-
|
7
|
-
|
7
|
+
.config
|
8
|
+
.listeners
|
9
|
+
.find { |l| l.group_id == group_id }
|
8
10
|
|
9
|
-
raise(ListenerNotFoundError,
|
11
|
+
raise(ListenerNotFoundError, group_id) unless listener
|
10
12
|
|
11
13
|
listener
|
12
14
|
end
|
@@ -17,15 +19,18 @@ module PhobosDBCheckpoint
|
|
17
19
|
.constantize
|
18
20
|
end
|
19
21
|
|
20
|
-
def method_missing(
|
21
|
-
|
22
|
-
|
23
|
-
if rex
|
22
|
+
def method_missing(method_name, *args, &block)
|
23
|
+
if method_name.to_s =~ /^fetch_(.*)/
|
24
|
+
method = Regexp.last_match(1)
|
24
25
|
handler = configured_handler.new
|
25
|
-
|
26
|
+
handler.send(method, payload) if handler.respond_to?(method)
|
26
27
|
else
|
27
28
|
super
|
28
29
|
end
|
29
30
|
end
|
31
|
+
|
32
|
+
def respond_to_missing?(method_name, include_private = false)
|
33
|
+
method_name.to_s.start_with?('fetch_') || super
|
34
|
+
end
|
30
35
|
end
|
31
36
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'json'
|
2
4
|
require 'rack'
|
3
5
|
require 'sinatra/base'
|
@@ -25,7 +27,7 @@ module PhobosDBCheckpoint
|
|
25
27
|
case exception
|
26
28
|
when ActiveRecord::RecordNotFound
|
27
29
|
status 404
|
28
|
-
type = env['sinatra.route'].match(
|
30
|
+
type = env['sinatra.route'].match(%r{\/.+\/(.+)\/:})[1].chop
|
29
31
|
{ error: true, message: "#{type} not found" }.to_json
|
30
32
|
else
|
31
33
|
status 500
|
@@ -84,8 +86,8 @@ module PhobosDBCheckpoint
|
|
84
86
|
get "/#{VERSION}/failures/count" do
|
85
87
|
content_type :json
|
86
88
|
count = PhobosDBCheckpoint::Failure
|
87
|
-
|
88
|
-
|
89
|
+
.all
|
90
|
+
.count
|
89
91
|
|
90
92
|
{ count: count }.to_json
|
91
93
|
end
|
@@ -116,7 +118,7 @@ module PhobosDBCheckpoint
|
|
116
118
|
PhobosDBCheckpoint::RetryFailure
|
117
119
|
.new(failure)
|
118
120
|
.perform
|
119
|
-
rescue => e
|
121
|
+
rescue StandardError => e
|
120
122
|
status 422
|
121
123
|
return { error: true, message: e.message }.to_json
|
122
124
|
end
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class Failure < ActiveRecord::Base
|
3
5
|
include PhobosDBCheckpoint::EventHelper
|
4
6
|
|
5
|
-
scope :order_by_event_time_and_created_at,
|
7
|
+
scope :order_by_event_time_and_created_at, lambda {
|
6
8
|
order('event_time desc nulls last', 'created_at desc nulls last')
|
7
9
|
}
|
8
10
|
|
@@ -1,6 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
module Handler
|
3
5
|
include Phobos::Handler
|
6
|
+
include Phobos::Instrumentation
|
4
7
|
|
5
8
|
def self.included(base)
|
6
9
|
base.extend(ClassMethods)
|
@@ -10,57 +13,52 @@ module PhobosDBCheckpoint
|
|
10
13
|
PhobosDBCheckpoint::Ack.new(entity_id, event_time, event_type, event_version)
|
11
14
|
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
+
def retry_consume?(_event, event_metadata, _exception)
|
17
|
+
return true unless Phobos.config&.db_checkpoint&.max_retries
|
18
|
+
event_metadata[:retry_count] < Phobos.config&.db_checkpoint&.max_retries
|
19
|
+
end
|
16
20
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
# rubocop:disable Style/RedundantBegin
|
22
|
+
def around_consume(payload, metadata)
|
23
|
+
event = PhobosDBCheckpoint::Event.new(
|
24
|
+
topic: metadata[:topic],
|
25
|
+
group_id: metadata[:group_id],
|
26
|
+
payload: payload
|
27
|
+
)
|
21
28
|
|
22
|
-
|
23
|
-
event = PhobosDBCheckpoint::Event.new(
|
24
|
-
topic: metadata[:topic],
|
25
|
-
group_id: metadata[:group_id],
|
26
|
-
payload: payload
|
27
|
-
)
|
29
|
+
event_metadata = { checksum: event.checksum }.merge(metadata)
|
28
30
|
|
29
|
-
|
31
|
+
instrument('db_checkpoint.around_consume', event_metadata) do
|
32
|
+
event_exists = instrument('db_checkpoint.event_already_exists_check', event_metadata) { event.exists? }
|
33
|
+
if event_exists
|
34
|
+
instrument('db_checkpoint.event_already_consumed', event_metadata)
|
35
|
+
return
|
36
|
+
end
|
30
37
|
|
31
|
-
instrument('db_checkpoint.
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
end
|
38
|
+
event_action = instrument('db_checkpoint.event_action', event_metadata) do
|
39
|
+
begin
|
40
|
+
yield
|
41
|
+
rescue StandardError => e
|
42
|
+
raise e if retry_consume?(event, event_metadata, e)
|
37
43
|
|
38
|
-
|
39
|
-
begin
|
40
|
-
yield
|
41
|
-
rescue => e
|
42
|
-
if retry_consume?(event, event_metadata, e)
|
43
|
-
raise e
|
44
|
-
else
|
45
|
-
Failure.record(event: event, event_metadata: event_metadata, exception: e)
|
46
|
-
end
|
47
|
-
end
|
44
|
+
Failure.record(event: event, event_metadata: event_metadata, exception: e)
|
48
45
|
end
|
46
|
+
end
|
49
47
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
end
|
55
|
-
else
|
56
|
-
instrument('db_checkpoint.event_skipped', event_metadata)
|
48
|
+
case event_action
|
49
|
+
when PhobosDBCheckpoint::Ack
|
50
|
+
instrument('db_checkpoint.event_acknowledged', event_metadata) do
|
51
|
+
event.acknowledge!(event_action)
|
57
52
|
end
|
53
|
+
else
|
54
|
+
instrument('db_checkpoint.event_skipped', event_metadata)
|
58
55
|
end
|
59
|
-
ensure
|
60
|
-
# Returns any connections in use by the current thread back to the pool, and also returns
|
61
|
-
# connections to the pool cached by threads that are no longer alive.
|
62
|
-
ActiveRecord::Base.clear_active_connections!
|
63
56
|
end
|
57
|
+
ensure
|
58
|
+
# Returns any connections in use by the current thread back to the pool, and also returns
|
59
|
+
# connections to the pool cached by threads that are no longer alive.
|
60
|
+
ActiveRecord::Base.clear_active_connections!
|
64
61
|
end
|
62
|
+
# rubocop:enable Style/RedundantBegin
|
65
63
|
end
|
66
64
|
end
|
@@ -1,15 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'rack'
|
2
4
|
|
3
5
|
module PhobosDBCheckpoint
|
4
6
|
module Middleware
|
5
7
|
class Logger
|
6
|
-
RACK_LOGGER = 'rack.logger'
|
7
|
-
SINATRA_ERROR = 'sinatra.error'
|
8
|
-
HTTP_VERSION = 'HTTP_VERSION'
|
9
|
-
PATH_INFO = 'PATH_INFO'
|
10
|
-
REQUEST_METHOD = 'REQUEST_METHOD'
|
11
|
-
QUERY_STRING = 'QUERY_STRING'
|
12
|
-
CONTENT_LENGTH = 'Content-Length'
|
8
|
+
RACK_LOGGER = 'rack.logger'
|
9
|
+
SINATRA_ERROR = 'sinatra.error'
|
10
|
+
HTTP_VERSION = 'HTTP_VERSION'
|
11
|
+
PATH_INFO = 'PATH_INFO'
|
12
|
+
REQUEST_METHOD = 'REQUEST_METHOD'
|
13
|
+
QUERY_STRING = 'QUERY_STRING'
|
14
|
+
CONTENT_LENGTH = 'Content-Length'
|
13
15
|
|
14
16
|
def initialize(app, options = {})
|
15
17
|
@app = app
|
@@ -46,21 +48,21 @@ module PhobosDBCheckpoint
|
|
46
48
|
|
47
49
|
if error
|
48
50
|
Phobos.logger.error(message.merge(
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
exception_class: error.class.to_s,
|
52
|
+
exception_message: error.message,
|
53
|
+
backtrace: error.backtrace
|
54
|
+
))
|
53
55
|
else
|
54
56
|
Phobos.logger.info(message)
|
55
57
|
end
|
56
58
|
end
|
57
59
|
|
58
60
|
def extract_path(request_env)
|
59
|
-
"#{request_env[PATH_INFO]}#{request_env[QUERY_STRING].empty? ?
|
61
|
+
"#{request_env[PATH_INFO]}#{request_env[QUERY_STRING].empty? ? '' : "?#{request_env[QUERY_STRING]}"} #{request_env[HTTP_VERSION]}"
|
60
62
|
end
|
61
63
|
|
62
64
|
def extract_content_length(headers)
|
63
|
-
value = headers[CONTENT_LENGTH]
|
65
|
+
(value = headers[CONTENT_LENGTH]) || return
|
64
66
|
value.to_s == '0' ? nil : value
|
65
67
|
end
|
66
68
|
end
|
@@ -1,5 +1,7 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
lib = File.expand_path('lib', __dir__)
|
3
5
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
6
|
require 'phobos_db_checkpoint/version'
|
5
7
|
|
@@ -15,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
15
17
|
'Francisco Juan',
|
16
18
|
'Tommy Gustafsson'
|
17
19
|
]
|
18
|
-
spec.email
|
20
|
+
spec.email = [
|
19
21
|
'ornelas.tulio@gmail.com',
|
20
22
|
'mathias.klippinge@gmail.com',
|
21
23
|
'sergey.evstifeev@gmail.com',
|
@@ -25,9 +27,9 @@ Gem::Specification.new do |spec|
|
|
25
27
|
'tommydgustafsson@gmail.com'
|
26
28
|
]
|
27
29
|
|
28
|
-
spec.summary =
|
29
|
-
spec.description =
|
30
|
-
spec.homepage =
|
30
|
+
spec.summary = 'Phobos DB Checkpoint is a plugin to Phobos and is meant as a drop in replacement to Phobos::Handler'
|
31
|
+
spec.description = 'Phobos DB Checkpoint is a plugin to Phobos and is meant as a drop in replacement to Phobos::Handler'
|
32
|
+
spec.homepage = 'https://github.com/klarna/phobos_db_checkpoint'
|
31
33
|
spec.license = 'Apache License Version 2.0'
|
32
34
|
|
33
35
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
@@ -44,20 +46,19 @@ Gem::Specification.new do |spec|
|
|
44
46
|
spec.require_paths = ['lib']
|
45
47
|
spec.required_ruby_version = '>= 2.3'
|
46
48
|
|
47
|
-
spec.add_development_dependency 'bundler'
|
49
|
+
spec.add_development_dependency 'bundler'
|
50
|
+
spec.add_development_dependency 'database_cleaner'
|
51
|
+
spec.add_development_dependency 'pg'
|
52
|
+
spec.add_development_dependency 'pry-byebug'
|
53
|
+
spec.add_development_dependency 'rack-test'
|
48
54
|
spec.add_development_dependency 'rake', '~> 10.0'
|
49
55
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
50
|
-
spec.add_development_dependency '
|
51
|
-
spec.add_development_dependency 'coveralls'
|
56
|
+
spec.add_development_dependency 'rubocop_rules'
|
52
57
|
spec.add_development_dependency 'simplecov'
|
53
|
-
spec.add_development_dependency 'pg'
|
54
|
-
spec.add_development_dependency 'database_cleaner'
|
55
|
-
spec.add_development_dependency 'rspec_junit_formatter', '0.2.2'
|
56
|
-
spec.add_development_dependency 'rack-test'
|
57
58
|
|
58
|
-
spec.add_dependency 'thor'
|
59
|
-
spec.add_dependency 'rake'
|
60
59
|
spec.add_dependency 'activerecord', '>= 4.0.0'
|
61
|
-
spec.add_dependency 'phobos', '>= 1.
|
60
|
+
spec.add_dependency 'phobos', '>= 1.8.0'
|
61
|
+
spec.add_dependency 'rake'
|
62
62
|
spec.add_dependency 'sinatra'
|
63
|
+
spec.add_dependency 'thor'
|
63
64
|
end
|
data/templates/config.ru
CHANGED
data/templates/phobos_boot.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: phobos_db_checkpoint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Túlio Ornelas
|
@@ -14,50 +14,50 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2018-
|
17
|
+
date: 2018-07-22 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: bundler
|
21
21
|
requirement: !ruby/object:Gem::Requirement
|
22
22
|
requirements:
|
23
|
-
- - "
|
23
|
+
- - ">="
|
24
24
|
- !ruby/object:Gem::Version
|
25
|
-
version: '
|
25
|
+
version: '0'
|
26
26
|
type: :development
|
27
27
|
prerelease: false
|
28
28
|
version_requirements: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
30
|
-
- - "
|
30
|
+
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '
|
32
|
+
version: '0'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: database_cleaner
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - "
|
37
|
+
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '0'
|
40
40
|
type: :development
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- - "
|
44
|
+
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '0'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
-
name:
|
48
|
+
name: pg
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
|
-
- - "
|
51
|
+
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- - "
|
58
|
+
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: pry-byebug
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,7 +73,7 @@ dependencies:
|
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
|
-
name:
|
76
|
+
name: rack-test
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
79
|
- - ">="
|
@@ -87,35 +87,35 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '0'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
|
-
name:
|
90
|
+
name: rake
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- - "
|
93
|
+
- - "~>"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version: '0'
|
95
|
+
version: '10.0'
|
96
96
|
type: :development
|
97
97
|
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
|
-
- - "
|
100
|
+
- - "~>"
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version: '0'
|
102
|
+
version: '10.0'
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
|
-
name:
|
104
|
+
name: rspec
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
106
106
|
requirements:
|
107
|
-
- - "
|
107
|
+
- - "~>"
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: '0'
|
109
|
+
version: '3.0'
|
110
110
|
type: :development
|
111
111
|
prerelease: false
|
112
112
|
version_requirements: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
|
-
- - "
|
114
|
+
- - "~>"
|
115
115
|
- !ruby/object:Gem::Version
|
116
|
-
version: '0'
|
116
|
+
version: '3.0'
|
117
117
|
- !ruby/object:Gem::Dependency
|
118
|
-
name:
|
118
|
+
name: rubocop_rules
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
121
|
- - ">="
|
@@ -129,21 +129,7 @@ dependencies:
|
|
129
129
|
- !ruby/object:Gem::Version
|
130
130
|
version: '0'
|
131
131
|
- !ruby/object:Gem::Dependency
|
132
|
-
name:
|
133
|
-
requirement: !ruby/object:Gem::Requirement
|
134
|
-
requirements:
|
135
|
-
- - '='
|
136
|
-
- !ruby/object:Gem::Version
|
137
|
-
version: 0.2.2
|
138
|
-
type: :development
|
139
|
-
prerelease: false
|
140
|
-
version_requirements: !ruby/object:Gem::Requirement
|
141
|
-
requirements:
|
142
|
-
- - '='
|
143
|
-
- !ruby/object:Gem::Version
|
144
|
-
version: 0.2.2
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
|
-
name: rack-test
|
132
|
+
name: simplecov
|
147
133
|
requirement: !ruby/object:Gem::Requirement
|
148
134
|
requirements:
|
149
135
|
- - ">="
|
@@ -157,63 +143,63 @@ dependencies:
|
|
157
143
|
- !ruby/object:Gem::Version
|
158
144
|
version: '0'
|
159
145
|
- !ruby/object:Gem::Dependency
|
160
|
-
name:
|
146
|
+
name: activerecord
|
161
147
|
requirement: !ruby/object:Gem::Requirement
|
162
148
|
requirements:
|
163
149
|
- - ">="
|
164
150
|
- !ruby/object:Gem::Version
|
165
|
-
version:
|
151
|
+
version: 4.0.0
|
166
152
|
type: :runtime
|
167
153
|
prerelease: false
|
168
154
|
version_requirements: !ruby/object:Gem::Requirement
|
169
155
|
requirements:
|
170
156
|
- - ">="
|
171
157
|
- !ruby/object:Gem::Version
|
172
|
-
version:
|
158
|
+
version: 4.0.0
|
173
159
|
- !ruby/object:Gem::Dependency
|
174
|
-
name:
|
160
|
+
name: phobos
|
175
161
|
requirement: !ruby/object:Gem::Requirement
|
176
162
|
requirements:
|
177
163
|
- - ">="
|
178
164
|
- !ruby/object:Gem::Version
|
179
|
-
version:
|
165
|
+
version: 1.8.0
|
180
166
|
type: :runtime
|
181
167
|
prerelease: false
|
182
168
|
version_requirements: !ruby/object:Gem::Requirement
|
183
169
|
requirements:
|
184
170
|
- - ">="
|
185
171
|
- !ruby/object:Gem::Version
|
186
|
-
version:
|
172
|
+
version: 1.8.0
|
187
173
|
- !ruby/object:Gem::Dependency
|
188
|
-
name:
|
174
|
+
name: rake
|
189
175
|
requirement: !ruby/object:Gem::Requirement
|
190
176
|
requirements:
|
191
177
|
- - ">="
|
192
178
|
- !ruby/object:Gem::Version
|
193
|
-
version:
|
179
|
+
version: '0'
|
194
180
|
type: :runtime
|
195
181
|
prerelease: false
|
196
182
|
version_requirements: !ruby/object:Gem::Requirement
|
197
183
|
requirements:
|
198
184
|
- - ">="
|
199
185
|
- !ruby/object:Gem::Version
|
200
|
-
version:
|
186
|
+
version: '0'
|
201
187
|
- !ruby/object:Gem::Dependency
|
202
|
-
name:
|
188
|
+
name: sinatra
|
203
189
|
requirement: !ruby/object:Gem::Requirement
|
204
190
|
requirements:
|
205
191
|
- - ">="
|
206
192
|
- !ruby/object:Gem::Version
|
207
|
-
version:
|
193
|
+
version: '0'
|
208
194
|
type: :runtime
|
209
195
|
prerelease: false
|
210
196
|
version_requirements: !ruby/object:Gem::Requirement
|
211
197
|
requirements:
|
212
198
|
- - ">="
|
213
199
|
- !ruby/object:Gem::Version
|
214
|
-
version:
|
200
|
+
version: '0'
|
215
201
|
- !ruby/object:Gem::Dependency
|
216
|
-
name:
|
202
|
+
name: thor
|
217
203
|
requirement: !ruby/object:Gem::Requirement
|
218
204
|
requirements:
|
219
205
|
- - ">="
|
@@ -241,10 +227,17 @@ executables:
|
|
241
227
|
extensions: []
|
242
228
|
extra_rdoc_files: []
|
243
229
|
files:
|
230
|
+
- ".dockerignore"
|
244
231
|
- ".gitignore"
|
245
232
|
- ".rspec"
|
233
|
+
- ".rubocop.yml"
|
234
|
+
- ".rubocop_common.yml"
|
235
|
+
- ".rubocop_todo.yml"
|
236
|
+
- ".rubosync.yml"
|
246
237
|
- ".ruby-version"
|
238
|
+
- ".travis.yml"
|
247
239
|
- CHANGELOG.md
|
240
|
+
- Dockerfile
|
248
241
|
- Gemfile
|
249
242
|
- LICENSE.txt
|
250
243
|
- README.md
|
@@ -252,7 +245,7 @@ files:
|
|
252
245
|
- bin/console
|
253
246
|
- bin/phobos_db_checkpoint
|
254
247
|
- bin/setup
|
255
|
-
-
|
248
|
+
- docker-compose.yml
|
256
249
|
- lib/phobos_db_checkpoint.rb
|
257
250
|
- lib/phobos_db_checkpoint/actions/retry_failure.rb
|
258
251
|
- lib/phobos_db_checkpoint/cli.rb
|
data/circle.yml
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
machine:
|
2
|
-
pre:
|
3
|
-
- curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
|
4
|
-
services:
|
5
|
-
- docker
|
6
|
-
environment:
|
7
|
-
LOG_LEVEL: DEBUG
|
8
|
-
CI: true
|
9
|
-
ruby:
|
10
|
-
version: 2.3.1
|
11
|
-
|
12
|
-
# Ignores circle ci default database setup
|
13
|
-
database:
|
14
|
-
override:
|
15
|
-
- echo "overrides circle CI commands"
|
16
|
-
|
17
|
-
dependencies:
|
18
|
-
pre:
|
19
|
-
- docker -v
|
20
|
-
- docker pull postgres:9.5.4
|
21
|
-
- gem install bundler -v 1.9.5
|
22
|
-
- bundle install
|
23
|
-
|
24
|
-
test:
|
25
|
-
override:
|
26
|
-
- docker run -d -p 5432:5432 --name db postgres; sleep 5
|
27
|
-
- bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
|
28
|
-
post:
|
29
|
-
- cp log/*.log $CIRCLE_ARTIFACTS/ || true
|