phobos_db_checkpoint 3.3.0 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +11 -0
- data/.rubocop.yml +25 -0
- data/.rubocop_common.yml +23 -0
- data/.rubocop_todo.yml +32 -0
- data/.rubosync.yml +2 -0
- data/.ruby-version +1 -1
- data/.travis.yml +35 -0
- data/CHANGELOG.md +7 -0
- data/Dockerfile +18 -0
- data/Gemfile +2 -0
- data/README.md +9 -10
- data/Rakefile +3 -1
- data/bin/console +1 -0
- data/bin/phobos_db_checkpoint +1 -0
- data/docker-compose.yml +9 -0
- data/lib/phobos_db_checkpoint.rb +9 -7
- data/lib/phobos_db_checkpoint/actions/retry_failure.rb +5 -5
- data/lib/phobos_db_checkpoint/cli.rb +10 -14
- data/lib/phobos_db_checkpoint/errors.rb +2 -0
- data/lib/phobos_db_checkpoint/event.rb +3 -1
- data/lib/phobos_db_checkpoint/event_actions.rb +2 -0
- data/lib/phobos_db_checkpoint/event_helper.rb +14 -9
- data/lib/phobos_db_checkpoint/events_api.rb +6 -4
- data/lib/phobos_db_checkpoint/failure.rb +3 -1
- data/lib/phobos_db_checkpoint/handler.rb +39 -41
- data/lib/phobos_db_checkpoint/middleware/database.rb +2 -0
- data/lib/phobos_db_checkpoint/middleware/logger.rb +15 -13
- data/lib/phobos_db_checkpoint/tasks.rb +2 -0
- data/lib/phobos_db_checkpoint/version.rb +3 -1
- data/phobos_db_checkpoint.gemspec +17 -16
- data/templates/config.ru +2 -0
- data/templates/phobos_boot.rb +2 -0
- metadata +50 -57
- data/circle.yml +0 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cda7a0046d7b17d682ad17fb2615b67422c42ad2
|
4
|
+
data.tar.gz: d4bb9fda8a27b46029ba802a43d7fe806e14efbc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9b28fba9214df0f2e183b346eef37e9f0417a20a5e880b30d698120441ae52d8bc13a1d4c8b3c375c70400ef80fd87f915245e5903a4772612348247701b94a
|
7
|
+
data.tar.gz: 6f83144ee24b53cd43495b9aa0d1883931eac7d26e576a1445696c7948ae5f27ad23f56f2429ce870a8878c2a60a2b6e8df9bd0f84dc474020d8ee3016a4adf9
|
data/.dockerignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
###########################
|
2
|
+
# Configuration for rubocop
|
3
|
+
# in .rubocop.yml
|
4
|
+
|
5
|
+
##############
|
6
|
+
# Global rules
|
7
|
+
# see .rubocop_common.yml
|
8
|
+
|
9
|
+
##############
|
10
|
+
# Inherit default rules first, and then override those rules with
|
11
|
+
# our violation whitelist.
|
12
|
+
inherit_from:
|
13
|
+
- .rubocop_common.yml
|
14
|
+
- .rubocop_todo.yml
|
15
|
+
|
16
|
+
##############
|
17
|
+
# Project specific overrides here, example:
|
18
|
+
# Metrics/BlockLength:
|
19
|
+
# Exclude:
|
20
|
+
# - 'tasks/the_huge_task.rake'
|
21
|
+
|
22
|
+
AllCops:
|
23
|
+
Exclude:
|
24
|
+
- phobos_db_checkpoint.gemspec
|
25
|
+
- spec/setup/**/*.rb
|
data/.rubocop_common.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
##############
|
2
|
+
# Global rules
|
3
|
+
AllCops:
|
4
|
+
Exclude:
|
5
|
+
- db/**/*
|
6
|
+
TargetRubyVersion: 2.5
|
7
|
+
|
8
|
+
Rails:
|
9
|
+
Enabled: false
|
10
|
+
|
11
|
+
Style/SymbolArray:
|
12
|
+
EnforcedStyle: brackets
|
13
|
+
|
14
|
+
Metrics/LineLength:
|
15
|
+
Max: 100
|
16
|
+
|
17
|
+
Metrics/BlockLength:
|
18
|
+
Exclude:
|
19
|
+
- '*.gemspec'
|
20
|
+
- 'spec/**/*.rb'
|
21
|
+
|
22
|
+
Documentation:
|
23
|
+
Enabled: false
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2018-05-14 08:11:29 +0200 using RuboCop version 0.55.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 6
|
10
|
+
Metrics/AbcSize:
|
11
|
+
Max: 30
|
12
|
+
|
13
|
+
# Offense count: 1
|
14
|
+
# Configuration parameters: CountComments.
|
15
|
+
Metrics/ClassLength:
|
16
|
+
Max: 107
|
17
|
+
|
18
|
+
# Offense count: 52
|
19
|
+
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
20
|
+
# URISchemes: http, https
|
21
|
+
Metrics/LineLength:
|
22
|
+
Max: 296
|
23
|
+
|
24
|
+
# Offense count: 9
|
25
|
+
# Configuration parameters: CountComments.
|
26
|
+
Metrics/MethodLength:
|
27
|
+
Max: 34
|
28
|
+
|
29
|
+
# Offense count: 1
|
30
|
+
# Configuration parameters: CountKeywordArgs.
|
31
|
+
Metrics/ParameterLists:
|
32
|
+
Max: 7
|
data/.rubosync.yml
ADDED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.5.1
|
data/.travis.yml
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
sudo: required
|
2
|
+
language: ruby
|
3
|
+
rvm:
|
4
|
+
- 2.5.1
|
5
|
+
- 2.4.4
|
6
|
+
- 2.3.7
|
7
|
+
|
8
|
+
services:
|
9
|
+
- docker
|
10
|
+
|
11
|
+
env:
|
12
|
+
global:
|
13
|
+
- CC_TEST_REPORTER_ID=09c6fe3d9f5f4673b7172b7ff0bd9721b5969da49c6c46e976fb89cf83f2a0dd
|
14
|
+
|
15
|
+
before_install:
|
16
|
+
- env
|
17
|
+
- docker-compose --version
|
18
|
+
- docker --version
|
19
|
+
- docker-compose config
|
20
|
+
- docker-compose build test
|
21
|
+
|
22
|
+
before_script:
|
23
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
24
|
+
- chmod +x ./cc-test-reporter
|
25
|
+
- if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter before-build || echo "Skipping CC coverage before-build"; fi
|
26
|
+
- mkdir coverage/
|
27
|
+
- touch ./coverage/.resultset.json
|
28
|
+
|
29
|
+
script:
|
30
|
+
- docker-compose run --rm test rspec
|
31
|
+
|
32
|
+
after_script:
|
33
|
+
- cat ./coverage/.resultset.json | sed "s|/opt/phobos_db_checkpoint|$PWD|" > ./coverage/.newresultset.json
|
34
|
+
- cp ./coverage/.newresultset.json ./coverage/.resultset.json
|
35
|
+
- if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT || echo "Skipping CC coverage after-build"; fi
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/).
|
6
6
|
|
7
|
+
## 3.4.0 (2018-07-22)
|
8
|
+
|
9
|
+
### Changed
|
10
|
+
- Introduce rubocop style guide #37
|
11
|
+
- Use test module from Phobos to run handler specs #39
|
12
|
+
- Accomodate future breaking changes of Phobos `around_consume` being moved to an instance method #38
|
13
|
+
|
7
14
|
## 3.3.0 (2017-10-26)
|
8
15
|
|
9
16
|
- [enhancement] Bump Phobos version to 1.5.0 to better support Avro. #34
|
data/Dockerfile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
FROM ruby:2.4.1-alpine
|
2
|
+
|
3
|
+
RUN apk update && apk upgrade && \
|
4
|
+
apk add --no-cache bash git openssh build-base
|
5
|
+
RUN apk add --no-cache postgresql-dev
|
6
|
+
|
7
|
+
RUN gem install bundler -v 1.16.0
|
8
|
+
|
9
|
+
WORKDIR /opt/phobos_db_checkpoint
|
10
|
+
|
11
|
+
ADD Gemfile Gemfile
|
12
|
+
ADD phobos_db_checkpoint.gemspec phobos_db_checkpoint.gemspec
|
13
|
+
ADD lib/phobos_db_checkpoint/version.rb lib/phobos_db_checkpoint/version.rb
|
14
|
+
|
15
|
+
RUN bundle config build.pg --with-pg-config=/usr/pgsql-9.6/bin/pg_config
|
16
|
+
RUN bundle install
|
17
|
+
|
18
|
+
ADD . .
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
-
![
|
2
|
-
[![
|
1
|
+
[![Build Status](https://travis-ci.org/klarna/phobos_db_checkpoint.svg?branch=travis)](https://travis-ci.org/klarna/phobos_db_checkpoint)
|
2
|
+
[![Maintainability](https://api.codeclimate.com/v1/badges/6f508f2515a418568bd8/maintainability)](https://codeclimate.com/github/klarna/phobos_db_checkpoint/maintainability)
|
3
|
+
[![Test Coverage](https://api.codeclimate.com/v1/badges/6f508f2515a418568bd8/test_coverage)](https://codeclimate.com/github/klarna/phobos_db_checkpoint/test_coverage)
|
4
|
+
[![Chat with us on Discord](https://discordapp.com/api/guilds/379938130326847488/widget.png)](https://discord.gg/rfMUBVD)
|
3
5
|
|
4
6
|
# Phobos DB Checkpoint
|
5
7
|
|
@@ -118,14 +120,14 @@ class MyHandler
|
|
118
120
|
end
|
119
121
|
```
|
120
122
|
|
121
|
-
|
123
|
+
In case your handler returns anything different from an __ack__ it won't be saved to the database.
|
122
124
|
|
123
125
|
Note that the `PhobosDBCheckpoint::Handler` will automatically skip already handled events (i.e. duplicate Kafka messages).
|
124
126
|
|
125
127
|
#### <a name="payload"></a> Payload
|
126
128
|
PhobosDBCheckpoint assumes that the payload received from Phobos is in a JSON format. This means that if your payload is in any other format, for example Avro binary, you need to convert/decode it to JSON.
|
127
129
|
|
128
|
-
To achieve this you can
|
130
|
+
To achieve this you can compose a new handler with `PhobosDBCheckpoint::Handler` using the `#around_consume` method:
|
129
131
|
|
130
132
|
```ruby
|
131
133
|
class MyHandler
|
@@ -133,12 +135,9 @@ class MyHandler
|
|
133
135
|
|
134
136
|
# <-- setup @avro before
|
135
137
|
|
136
|
-
def
|
137
|
-
@avro.decode(payload)
|
138
|
-
|
139
|
-
|
140
|
-
def consume(payload, metadata)
|
141
|
-
# <-- consume your stuff with the decoded payload
|
138
|
+
def around_consume(payload, metadata)
|
139
|
+
decoded_payload = @avro.decode(payload)
|
140
|
+
super(decoded_payload, metadata)
|
142
141
|
end
|
143
142
|
end
|
144
143
|
```
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
data/bin/phobos_db_checkpoint
CHANGED
data/docker-compose.yml
ADDED
data/lib/phobos_db_checkpoint.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'yaml'
|
2
4
|
require 'digest'
|
3
5
|
require 'active_record'
|
@@ -13,10 +15,10 @@ require 'phobos_db_checkpoint/handler'
|
|
13
15
|
require 'phobos_db_checkpoint/actions/retry_failure'
|
14
16
|
|
15
17
|
module PhobosDBCheckpoint
|
16
|
-
DEFAULT_DB_DIR = 'db'
|
18
|
+
DEFAULT_DB_DIR = 'db'
|
17
19
|
DEFAULT_MIGRATION_PATH = File.join(DEFAULT_DB_DIR, 'migrate').freeze
|
18
|
-
DEFAULT_DB_CONFIG_PATH = 'config/database.yml'
|
19
|
-
DEFAULT_POOL_SIZE = 5
|
20
|
+
DEFAULT_DB_CONFIG_PATH = 'config/database.yml'
|
21
|
+
DEFAULT_POOL_SIZE = 5
|
20
22
|
|
21
23
|
class << self
|
22
24
|
attr_reader :db_config
|
@@ -45,14 +47,12 @@ module PhobosDBCheckpoint
|
|
45
47
|
|
46
48
|
@db_config_path ||= ENV['DB_CONFIG'] || DEFAULT_DB_CONFIG_PATH
|
47
49
|
|
48
|
-
configs = YAML.
|
50
|
+
configs = YAML.safe_load(ERB.new(File.read(File.expand_path(@db_config_path))).result, [], [], true)
|
49
51
|
@db_config = configs[env]
|
50
52
|
|
51
53
|
pool_size = @db_config['pool']
|
52
54
|
|
53
|
-
if pool_size.nil? && Phobos.config
|
54
|
-
pool_size = number_of_concurrent_listeners + DEFAULT_POOL_SIZE
|
55
|
-
end
|
55
|
+
pool_size = number_of_concurrent_listeners + DEFAULT_POOL_SIZE if pool_size.nil? && Phobos.config
|
56
56
|
|
57
57
|
@db_config.merge!('pool' => pool_size || DEFAULT_POOL_SIZE)
|
58
58
|
end
|
@@ -61,10 +61,12 @@ module PhobosDBCheckpoint
|
|
61
61
|
ActiveRecord::Base.establish_connection(db_config)
|
62
62
|
end
|
63
63
|
|
64
|
+
# rubocop:disable Lint/HandleExceptions
|
64
65
|
def close_db_connection
|
65
66
|
ActiveRecord::Base.connection_pool.disconnect!
|
66
67
|
rescue ActiveRecord::ConnectionNotEstablished
|
67
68
|
end
|
69
|
+
# rubocop:enable Lint/HandleExceptions
|
68
70
|
|
69
71
|
def load_tasks
|
70
72
|
@db_dir ||= DEFAULT_DB_DIR
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class RetryFailure
|
3
5
|
include PhobosDBCheckpoint::Handler
|
@@ -8,11 +10,9 @@ module PhobosDBCheckpoint
|
|
8
10
|
end
|
9
11
|
|
10
12
|
def perform
|
11
|
-
|
12
|
-
.
|
13
|
-
|
14
|
-
@action_taken = handler.consume(payload, metadata)
|
15
|
-
end
|
13
|
+
around_consume(payload, metadata) do
|
14
|
+
@action_taken = handler.consume(payload, metadata)
|
15
|
+
end
|
16
16
|
|
17
17
|
@failure.destroy
|
18
18
|
@action_taken
|
@@ -1,9 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'thor'
|
2
4
|
require 'fileutils'
|
3
5
|
|
4
6
|
module PhobosDBCheckpoint
|
5
7
|
module CLI
|
6
|
-
|
7
8
|
class Commands < Thor
|
8
9
|
include Thor::Actions
|
9
10
|
|
@@ -39,13 +40,9 @@ module PhobosDBCheckpoint
|
|
39
40
|
default: 'config/database.yml',
|
40
41
|
banner: 'Database configuration relative to your project'
|
41
42
|
def copy_migrations
|
42
|
-
if options[:config]
|
43
|
-
ENV['DB_CONFIG'] = options[:config]
|
44
|
-
end
|
43
|
+
ENV['DB_CONFIG'] = options[:config] if options[:config]
|
45
44
|
|
46
|
-
unless active_connection?
|
47
|
-
PhobosDBCheckpoint.configure
|
48
|
-
end
|
45
|
+
PhobosDBCheckpoint.configure unless active_connection?
|
49
46
|
|
50
47
|
destination_fullpath = File.join(destination_root, options[:destination])
|
51
48
|
generated_migrations = list_migrations(destination_fullpath)
|
@@ -60,7 +57,7 @@ module PhobosDBCheckpoint
|
|
60
57
|
template(template_path, file_path)
|
61
58
|
end
|
62
59
|
end
|
63
|
-
rescue
|
60
|
+
rescue StandardError
|
64
61
|
FileUtils.rm_f(file_path.to_s)
|
65
62
|
raise
|
66
63
|
end
|
@@ -95,16 +92,15 @@ module PhobosDBCheckpoint
|
|
95
92
|
end
|
96
93
|
|
97
94
|
def migration_number(index = 0)
|
98
|
-
[Time.now.utc.strftime('%Y%m%d%H%M%S%6N'), '%.21d'
|
95
|
+
[Time.now.utc.strftime('%Y%m%d%H%M%S%6N'), format('%.21d', index)].max
|
99
96
|
end
|
100
97
|
|
101
98
|
def template_migrations_metadata
|
102
99
|
@template_migrations_metadata ||= begin
|
103
100
|
index = 0
|
104
101
|
template_migrations.map do |path|
|
105
|
-
name = path.split('/').last
|
106
102
|
index += 1
|
107
|
-
{path: path, name: path.gsub(/\.erb$/, ''), number: migration_number(index)}
|
103
|
+
{ path: path, name: path.gsub(/\.erb$/, ''), number: migration_number(index) }
|
108
104
|
end
|
109
105
|
end
|
110
106
|
end
|
@@ -115,7 +111,7 @@ module PhobosDBCheckpoint
|
|
115
111
|
|
116
112
|
def list_migrations(dir)
|
117
113
|
return [] unless Dir.exist?(dir)
|
118
|
-
Dir.entries(dir).select {|f| f =~ /\.rb(\.erb)?$/}
|
114
|
+
Dir.entries(dir).select { |f| f =~ /\.rb(\.erb)?$/ }.sort
|
119
115
|
end
|
120
116
|
|
121
117
|
def migrations_template_dir
|
@@ -133,8 +129,8 @@ module PhobosDBCheckpoint
|
|
133
129
|
def active_connection?
|
134
130
|
ActiveRecord::Base
|
135
131
|
.connection_pool
|
136
|
-
.with_connection
|
137
|
-
rescue
|
132
|
+
.with_connection(&:active?)
|
133
|
+
rescue StandardError
|
138
134
|
false
|
139
135
|
end
|
140
136
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class Event < ActiveRecord::Base
|
3
5
|
include PhobosDBCheckpoint::EventHelper
|
4
6
|
after_initialize :assign_checksum
|
5
7
|
|
6
|
-
scope :order_by_event_time_and_created_at,
|
8
|
+
scope :order_by_event_time_and_created_at, lambda {
|
7
9
|
order('event_time desc nulls last', 'created_at desc nulls last')
|
8
10
|
}
|
9
11
|
|
@@ -1,12 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
module EventHelper
|
3
5
|
def configured_listener
|
4
6
|
listener = Phobos
|
5
|
-
|
6
|
-
|
7
|
-
|
7
|
+
.config
|
8
|
+
.listeners
|
9
|
+
.find { |l| l.group_id == group_id }
|
8
10
|
|
9
|
-
raise(ListenerNotFoundError,
|
11
|
+
raise(ListenerNotFoundError, group_id) unless listener
|
10
12
|
|
11
13
|
listener
|
12
14
|
end
|
@@ -17,15 +19,18 @@ module PhobosDBCheckpoint
|
|
17
19
|
.constantize
|
18
20
|
end
|
19
21
|
|
20
|
-
def method_missing(
|
21
|
-
|
22
|
-
|
23
|
-
if rex
|
22
|
+
def method_missing(method_name, *args, &block)
|
23
|
+
if method_name.to_s =~ /^fetch_(.*)/
|
24
|
+
method = Regexp.last_match(1)
|
24
25
|
handler = configured_handler.new
|
25
|
-
|
26
|
+
handler.send(method, payload) if handler.respond_to?(method)
|
26
27
|
else
|
27
28
|
super
|
28
29
|
end
|
29
30
|
end
|
31
|
+
|
32
|
+
def respond_to_missing?(method_name, include_private = false)
|
33
|
+
method_name.to_s.start_with?('fetch_') || super
|
34
|
+
end
|
30
35
|
end
|
31
36
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'json'
|
2
4
|
require 'rack'
|
3
5
|
require 'sinatra/base'
|
@@ -25,7 +27,7 @@ module PhobosDBCheckpoint
|
|
25
27
|
case exception
|
26
28
|
when ActiveRecord::RecordNotFound
|
27
29
|
status 404
|
28
|
-
type = env['sinatra.route'].match(
|
30
|
+
type = env['sinatra.route'].match(%r{\/.+\/(.+)\/:})[1].chop
|
29
31
|
{ error: true, message: "#{type} not found" }.to_json
|
30
32
|
else
|
31
33
|
status 500
|
@@ -84,8 +86,8 @@ module PhobosDBCheckpoint
|
|
84
86
|
get "/#{VERSION}/failures/count" do
|
85
87
|
content_type :json
|
86
88
|
count = PhobosDBCheckpoint::Failure
|
87
|
-
|
88
|
-
|
89
|
+
.all
|
90
|
+
.count
|
89
91
|
|
90
92
|
{ count: count }.to_json
|
91
93
|
end
|
@@ -116,7 +118,7 @@ module PhobosDBCheckpoint
|
|
116
118
|
PhobosDBCheckpoint::RetryFailure
|
117
119
|
.new(failure)
|
118
120
|
.perform
|
119
|
-
rescue => e
|
121
|
+
rescue StandardError => e
|
120
122
|
status 422
|
121
123
|
return { error: true, message: e.message }.to_json
|
122
124
|
end
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
class Failure < ActiveRecord::Base
|
3
5
|
include PhobosDBCheckpoint::EventHelper
|
4
6
|
|
5
|
-
scope :order_by_event_time_and_created_at,
|
7
|
+
scope :order_by_event_time_and_created_at, lambda {
|
6
8
|
order('event_time desc nulls last', 'created_at desc nulls last')
|
7
9
|
}
|
8
10
|
|
@@ -1,6 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PhobosDBCheckpoint
|
2
4
|
module Handler
|
3
5
|
include Phobos::Handler
|
6
|
+
include Phobos::Instrumentation
|
4
7
|
|
5
8
|
def self.included(base)
|
6
9
|
base.extend(ClassMethods)
|
@@ -10,57 +13,52 @@ module PhobosDBCheckpoint
|
|
10
13
|
PhobosDBCheckpoint::Ack.new(entity_id, event_time, event_type, event_version)
|
11
14
|
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
+
def retry_consume?(_event, event_metadata, _exception)
|
17
|
+
return true unless Phobos.config&.db_checkpoint&.max_retries
|
18
|
+
event_metadata[:retry_count] < Phobos.config&.db_checkpoint&.max_retries
|
19
|
+
end
|
16
20
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
# rubocop:disable Style/RedundantBegin
|
22
|
+
def around_consume(payload, metadata)
|
23
|
+
event = PhobosDBCheckpoint::Event.new(
|
24
|
+
topic: metadata[:topic],
|
25
|
+
group_id: metadata[:group_id],
|
26
|
+
payload: payload
|
27
|
+
)
|
21
28
|
|
22
|
-
|
23
|
-
event = PhobosDBCheckpoint::Event.new(
|
24
|
-
topic: metadata[:topic],
|
25
|
-
group_id: metadata[:group_id],
|
26
|
-
payload: payload
|
27
|
-
)
|
29
|
+
event_metadata = { checksum: event.checksum }.merge(metadata)
|
28
30
|
|
29
|
-
|
31
|
+
instrument('db_checkpoint.around_consume', event_metadata) do
|
32
|
+
event_exists = instrument('db_checkpoint.event_already_exists_check', event_metadata) { event.exists? }
|
33
|
+
if event_exists
|
34
|
+
instrument('db_checkpoint.event_already_consumed', event_metadata)
|
35
|
+
return
|
36
|
+
end
|
30
37
|
|
31
|
-
instrument('db_checkpoint.
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
end
|
38
|
+
event_action = instrument('db_checkpoint.event_action', event_metadata) do
|
39
|
+
begin
|
40
|
+
yield
|
41
|
+
rescue StandardError => e
|
42
|
+
raise e if retry_consume?(event, event_metadata, e)
|
37
43
|
|
38
|
-
|
39
|
-
begin
|
40
|
-
yield
|
41
|
-
rescue => e
|
42
|
-
if retry_consume?(event, event_metadata, e)
|
43
|
-
raise e
|
44
|
-
else
|
45
|
-
Failure.record(event: event, event_metadata: event_metadata, exception: e)
|
46
|
-
end
|
47
|
-
end
|
44
|
+
Failure.record(event: event, event_metadata: event_metadata, exception: e)
|
48
45
|
end
|
46
|
+
end
|
49
47
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
end
|
55
|
-
else
|
56
|
-
instrument('db_checkpoint.event_skipped', event_metadata)
|
48
|
+
case event_action
|
49
|
+
when PhobosDBCheckpoint::Ack
|
50
|
+
instrument('db_checkpoint.event_acknowledged', event_metadata) do
|
51
|
+
event.acknowledge!(event_action)
|
57
52
|
end
|
53
|
+
else
|
54
|
+
instrument('db_checkpoint.event_skipped', event_metadata)
|
58
55
|
end
|
59
|
-
ensure
|
60
|
-
# Returns any connections in use by the current thread back to the pool, and also returns
|
61
|
-
# connections to the pool cached by threads that are no longer alive.
|
62
|
-
ActiveRecord::Base.clear_active_connections!
|
63
56
|
end
|
57
|
+
ensure
|
58
|
+
# Returns any connections in use by the current thread back to the pool, and also returns
|
59
|
+
# connections to the pool cached by threads that are no longer alive.
|
60
|
+
ActiveRecord::Base.clear_active_connections!
|
64
61
|
end
|
62
|
+
# rubocop:enable Style/RedundantBegin
|
65
63
|
end
|
66
64
|
end
|
@@ -1,15 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'rack'
|
2
4
|
|
3
5
|
module PhobosDBCheckpoint
|
4
6
|
module Middleware
|
5
7
|
class Logger
|
6
|
-
RACK_LOGGER = 'rack.logger'
|
7
|
-
SINATRA_ERROR = 'sinatra.error'
|
8
|
-
HTTP_VERSION = 'HTTP_VERSION'
|
9
|
-
PATH_INFO = 'PATH_INFO'
|
10
|
-
REQUEST_METHOD = 'REQUEST_METHOD'
|
11
|
-
QUERY_STRING = 'QUERY_STRING'
|
12
|
-
CONTENT_LENGTH = 'Content-Length'
|
8
|
+
RACK_LOGGER = 'rack.logger'
|
9
|
+
SINATRA_ERROR = 'sinatra.error'
|
10
|
+
HTTP_VERSION = 'HTTP_VERSION'
|
11
|
+
PATH_INFO = 'PATH_INFO'
|
12
|
+
REQUEST_METHOD = 'REQUEST_METHOD'
|
13
|
+
QUERY_STRING = 'QUERY_STRING'
|
14
|
+
CONTENT_LENGTH = 'Content-Length'
|
13
15
|
|
14
16
|
def initialize(app, options = {})
|
15
17
|
@app = app
|
@@ -46,21 +48,21 @@ module PhobosDBCheckpoint
|
|
46
48
|
|
47
49
|
if error
|
48
50
|
Phobos.logger.error(message.merge(
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
exception_class: error.class.to_s,
|
52
|
+
exception_message: error.message,
|
53
|
+
backtrace: error.backtrace
|
54
|
+
))
|
53
55
|
else
|
54
56
|
Phobos.logger.info(message)
|
55
57
|
end
|
56
58
|
end
|
57
59
|
|
58
60
|
def extract_path(request_env)
|
59
|
-
"#{request_env[PATH_INFO]}#{request_env[QUERY_STRING].empty? ?
|
61
|
+
"#{request_env[PATH_INFO]}#{request_env[QUERY_STRING].empty? ? '' : "?#{request_env[QUERY_STRING]}"} #{request_env[HTTP_VERSION]}"
|
60
62
|
end
|
61
63
|
|
62
64
|
def extract_content_length(headers)
|
63
|
-
value = headers[CONTENT_LENGTH]
|
65
|
+
(value = headers[CONTENT_LENGTH]) || return
|
64
66
|
value.to_s == '0' ? nil : value
|
65
67
|
end
|
66
68
|
end
|
@@ -1,5 +1,7 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
lib = File.expand_path('lib', __dir__)
|
3
5
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
6
|
require 'phobos_db_checkpoint/version'
|
5
7
|
|
@@ -15,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
15
17
|
'Francisco Juan',
|
16
18
|
'Tommy Gustafsson'
|
17
19
|
]
|
18
|
-
spec.email
|
20
|
+
spec.email = [
|
19
21
|
'ornelas.tulio@gmail.com',
|
20
22
|
'mathias.klippinge@gmail.com',
|
21
23
|
'sergey.evstifeev@gmail.com',
|
@@ -25,9 +27,9 @@ Gem::Specification.new do |spec|
|
|
25
27
|
'tommydgustafsson@gmail.com'
|
26
28
|
]
|
27
29
|
|
28
|
-
spec.summary =
|
29
|
-
spec.description =
|
30
|
-
spec.homepage =
|
30
|
+
spec.summary = 'Phobos DB Checkpoint is a plugin to Phobos and is meant as a drop in replacement to Phobos::Handler'
|
31
|
+
spec.description = 'Phobos DB Checkpoint is a plugin to Phobos and is meant as a drop in replacement to Phobos::Handler'
|
32
|
+
spec.homepage = 'https://github.com/klarna/phobos_db_checkpoint'
|
31
33
|
spec.license = 'Apache License Version 2.0'
|
32
34
|
|
33
35
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
@@ -44,20 +46,19 @@ Gem::Specification.new do |spec|
|
|
44
46
|
spec.require_paths = ['lib']
|
45
47
|
spec.required_ruby_version = '>= 2.3'
|
46
48
|
|
47
|
-
spec.add_development_dependency 'bundler'
|
49
|
+
spec.add_development_dependency 'bundler'
|
50
|
+
spec.add_development_dependency 'database_cleaner'
|
51
|
+
spec.add_development_dependency 'pg'
|
52
|
+
spec.add_development_dependency 'pry-byebug'
|
53
|
+
spec.add_development_dependency 'rack-test'
|
48
54
|
spec.add_development_dependency 'rake', '~> 10.0'
|
49
55
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
50
|
-
spec.add_development_dependency '
|
51
|
-
spec.add_development_dependency 'coveralls'
|
56
|
+
spec.add_development_dependency 'rubocop_rules'
|
52
57
|
spec.add_development_dependency 'simplecov'
|
53
|
-
spec.add_development_dependency 'pg'
|
54
|
-
spec.add_development_dependency 'database_cleaner'
|
55
|
-
spec.add_development_dependency 'rspec_junit_formatter', '0.2.2'
|
56
|
-
spec.add_development_dependency 'rack-test'
|
57
58
|
|
58
|
-
spec.add_dependency 'thor'
|
59
|
-
spec.add_dependency 'rake'
|
60
59
|
spec.add_dependency 'activerecord', '>= 4.0.0'
|
61
|
-
spec.add_dependency 'phobos', '>= 1.
|
60
|
+
spec.add_dependency 'phobos', '>= 1.8.0'
|
61
|
+
spec.add_dependency 'rake'
|
62
62
|
spec.add_dependency 'sinatra'
|
63
|
+
spec.add_dependency 'thor'
|
63
64
|
end
|
data/templates/config.ru
CHANGED
data/templates/phobos_boot.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: phobos_db_checkpoint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Túlio Ornelas
|
@@ -14,50 +14,50 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2018-
|
17
|
+
date: 2018-07-22 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: bundler
|
21
21
|
requirement: !ruby/object:Gem::Requirement
|
22
22
|
requirements:
|
23
|
-
- - "
|
23
|
+
- - ">="
|
24
24
|
- !ruby/object:Gem::Version
|
25
|
-
version: '
|
25
|
+
version: '0'
|
26
26
|
type: :development
|
27
27
|
prerelease: false
|
28
28
|
version_requirements: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
30
|
-
- - "
|
30
|
+
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '
|
32
|
+
version: '0'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: database_cleaner
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - "
|
37
|
+
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '0'
|
40
40
|
type: :development
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- - "
|
44
|
+
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '0'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
|
-
name:
|
48
|
+
name: pg
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
|
-
- - "
|
51
|
+
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- - "
|
58
|
+
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: pry-byebug
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,7 +73,7 @@ dependencies:
|
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
|
-
name:
|
76
|
+
name: rack-test
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
79
|
- - ">="
|
@@ -87,35 +87,35 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '0'
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
|
-
name:
|
90
|
+
name: rake
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- - "
|
93
|
+
- - "~>"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version: '0'
|
95
|
+
version: '10.0'
|
96
96
|
type: :development
|
97
97
|
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
|
-
- - "
|
100
|
+
- - "~>"
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version: '0'
|
102
|
+
version: '10.0'
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
|
-
name:
|
104
|
+
name: rspec
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
106
106
|
requirements:
|
107
|
-
- - "
|
107
|
+
- - "~>"
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: '0'
|
109
|
+
version: '3.0'
|
110
110
|
type: :development
|
111
111
|
prerelease: false
|
112
112
|
version_requirements: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
|
-
- - "
|
114
|
+
- - "~>"
|
115
115
|
- !ruby/object:Gem::Version
|
116
|
-
version: '0'
|
116
|
+
version: '3.0'
|
117
117
|
- !ruby/object:Gem::Dependency
|
118
|
-
name:
|
118
|
+
name: rubocop_rules
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
121
|
- - ">="
|
@@ -129,21 +129,7 @@ dependencies:
|
|
129
129
|
- !ruby/object:Gem::Version
|
130
130
|
version: '0'
|
131
131
|
- !ruby/object:Gem::Dependency
|
132
|
-
name:
|
133
|
-
requirement: !ruby/object:Gem::Requirement
|
134
|
-
requirements:
|
135
|
-
- - '='
|
136
|
-
- !ruby/object:Gem::Version
|
137
|
-
version: 0.2.2
|
138
|
-
type: :development
|
139
|
-
prerelease: false
|
140
|
-
version_requirements: !ruby/object:Gem::Requirement
|
141
|
-
requirements:
|
142
|
-
- - '='
|
143
|
-
- !ruby/object:Gem::Version
|
144
|
-
version: 0.2.2
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
|
-
name: rack-test
|
132
|
+
name: simplecov
|
147
133
|
requirement: !ruby/object:Gem::Requirement
|
148
134
|
requirements:
|
149
135
|
- - ">="
|
@@ -157,63 +143,63 @@ dependencies:
|
|
157
143
|
- !ruby/object:Gem::Version
|
158
144
|
version: '0'
|
159
145
|
- !ruby/object:Gem::Dependency
|
160
|
-
name:
|
146
|
+
name: activerecord
|
161
147
|
requirement: !ruby/object:Gem::Requirement
|
162
148
|
requirements:
|
163
149
|
- - ">="
|
164
150
|
- !ruby/object:Gem::Version
|
165
|
-
version:
|
151
|
+
version: 4.0.0
|
166
152
|
type: :runtime
|
167
153
|
prerelease: false
|
168
154
|
version_requirements: !ruby/object:Gem::Requirement
|
169
155
|
requirements:
|
170
156
|
- - ">="
|
171
157
|
- !ruby/object:Gem::Version
|
172
|
-
version:
|
158
|
+
version: 4.0.0
|
173
159
|
- !ruby/object:Gem::Dependency
|
174
|
-
name:
|
160
|
+
name: phobos
|
175
161
|
requirement: !ruby/object:Gem::Requirement
|
176
162
|
requirements:
|
177
163
|
- - ">="
|
178
164
|
- !ruby/object:Gem::Version
|
179
|
-
version:
|
165
|
+
version: 1.8.0
|
180
166
|
type: :runtime
|
181
167
|
prerelease: false
|
182
168
|
version_requirements: !ruby/object:Gem::Requirement
|
183
169
|
requirements:
|
184
170
|
- - ">="
|
185
171
|
- !ruby/object:Gem::Version
|
186
|
-
version:
|
172
|
+
version: 1.8.0
|
187
173
|
- !ruby/object:Gem::Dependency
|
188
|
-
name:
|
174
|
+
name: rake
|
189
175
|
requirement: !ruby/object:Gem::Requirement
|
190
176
|
requirements:
|
191
177
|
- - ">="
|
192
178
|
- !ruby/object:Gem::Version
|
193
|
-
version:
|
179
|
+
version: '0'
|
194
180
|
type: :runtime
|
195
181
|
prerelease: false
|
196
182
|
version_requirements: !ruby/object:Gem::Requirement
|
197
183
|
requirements:
|
198
184
|
- - ">="
|
199
185
|
- !ruby/object:Gem::Version
|
200
|
-
version:
|
186
|
+
version: '0'
|
201
187
|
- !ruby/object:Gem::Dependency
|
202
|
-
name:
|
188
|
+
name: sinatra
|
203
189
|
requirement: !ruby/object:Gem::Requirement
|
204
190
|
requirements:
|
205
191
|
- - ">="
|
206
192
|
- !ruby/object:Gem::Version
|
207
|
-
version:
|
193
|
+
version: '0'
|
208
194
|
type: :runtime
|
209
195
|
prerelease: false
|
210
196
|
version_requirements: !ruby/object:Gem::Requirement
|
211
197
|
requirements:
|
212
198
|
- - ">="
|
213
199
|
- !ruby/object:Gem::Version
|
214
|
-
version:
|
200
|
+
version: '0'
|
215
201
|
- !ruby/object:Gem::Dependency
|
216
|
-
name:
|
202
|
+
name: thor
|
217
203
|
requirement: !ruby/object:Gem::Requirement
|
218
204
|
requirements:
|
219
205
|
- - ">="
|
@@ -241,10 +227,17 @@ executables:
|
|
241
227
|
extensions: []
|
242
228
|
extra_rdoc_files: []
|
243
229
|
files:
|
230
|
+
- ".dockerignore"
|
244
231
|
- ".gitignore"
|
245
232
|
- ".rspec"
|
233
|
+
- ".rubocop.yml"
|
234
|
+
- ".rubocop_common.yml"
|
235
|
+
- ".rubocop_todo.yml"
|
236
|
+
- ".rubosync.yml"
|
246
237
|
- ".ruby-version"
|
238
|
+
- ".travis.yml"
|
247
239
|
- CHANGELOG.md
|
240
|
+
- Dockerfile
|
248
241
|
- Gemfile
|
249
242
|
- LICENSE.txt
|
250
243
|
- README.md
|
@@ -252,7 +245,7 @@ files:
|
|
252
245
|
- bin/console
|
253
246
|
- bin/phobos_db_checkpoint
|
254
247
|
- bin/setup
|
255
|
-
-
|
248
|
+
- docker-compose.yml
|
256
249
|
- lib/phobos_db_checkpoint.rb
|
257
250
|
- lib/phobos_db_checkpoint/actions/retry_failure.rb
|
258
251
|
- lib/phobos_db_checkpoint/cli.rb
|
data/circle.yml
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
machine:
|
2
|
-
pre:
|
3
|
-
- curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
|
4
|
-
services:
|
5
|
-
- docker
|
6
|
-
environment:
|
7
|
-
LOG_LEVEL: DEBUG
|
8
|
-
CI: true
|
9
|
-
ruby:
|
10
|
-
version: 2.3.1
|
11
|
-
|
12
|
-
# Ignores circle ci default database setup
|
13
|
-
database:
|
14
|
-
override:
|
15
|
-
- echo "overrides circle CI commands"
|
16
|
-
|
17
|
-
dependencies:
|
18
|
-
pre:
|
19
|
-
- docker -v
|
20
|
-
- docker pull postgres:9.5.4
|
21
|
-
- gem install bundler -v 1.9.5
|
22
|
-
- bundle install
|
23
|
-
|
24
|
-
test:
|
25
|
-
override:
|
26
|
-
- docker run -d -p 5432:5432 --name db postgres; sleep 5
|
27
|
-
- bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
|
28
|
-
post:
|
29
|
-
- cp log/*.log $CIRCLE_ARTIFACTS/ || true
|