active_record_streams 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1 -1
- data/Gemfile.lock +1 -1
- data/README.md +61 -2
- data/lib/active_record_streams/message.rb +14 -0
- data/lib/active_record_streams/message_spec.rb +28 -0
- data/lib/active_record_streams/publishers/http_stream.rb +23 -2
- data/lib/active_record_streams/publishers/http_stream_spec.rb +22 -3
- data/lib/active_record_streams/publishers/kinesis_stream.rb +8 -1
- data/lib/active_record_streams/publishers/kinesis_stream_spec.rb +22 -1
- data/lib/active_record_streams/publishers/sns_stream.rb +8 -1
- data/lib/active_record_streams/publishers/sns_stream_spec.rb +22 -1
- data/lib/active_record_streams/version.rb +1 -1
- data/lib/active_record_streams/version_spec.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4646a534ef38e83da3774fa7c7d17e585f690a3c
|
4
|
+
data.tar.gz: 844d01c83a24ed5027fcd5002719edd3adb3d24c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9221892da39f326d386458b0cebf3ff4803f041b62460e6ae2f549744d448d6ee97dba09e64a0606270866a4b4e2ff176052a621aae33c690cf14f2c64b53427
|
7
|
+
data.tar.gz: b8affbac829c21998eefabfb4a5d8998e690089f17461ebfed5fe9aa9eef454bd7f1a94f0484269b43d0288575da28c7d0d563ded6081b0d83624c284c5f20b5
|
data/CHANGELOG.md
CHANGED
@@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
-
## [0.1.0] -
|
9
|
+
## [0.1.0] - 2019-04-19
|
10
10
|
### Added
|
11
11
|
- Global hooks for ActiveRecord 4.2.10.
|
12
12
|
- Overrides to ActiveRecord::Base, ActiveRecord::Relation, ActiveRecord::Persistence to serve callbacks.
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,14 +1,30 @@
|
|
1
1
|
[![Build Status](https://travis-ci.org/Advanon/active_record_streams.svg?branch=master)](https://travis-ci.org/Advanon/active_record_streams)
|
2
|
+
[![Gem Version](https://badge.fury.io/rb/active_record_streams.svg)](https://badge.fury.io/rb/active_record_streams)
|
2
3
|
|
3
4
|
# Active Record Streams
|
4
5
|
|
5
6
|
A small library to stream ActiveRecord's create/update/delete
|
6
7
|
events to AWS SNS topics, Kinesis streams or HTTP listeners.
|
7
8
|
|
9
|
+
# Table of contents
|
10
|
+
|
11
|
+
* [Version mappings](#version-mappings)
|
12
|
+
* [Warning](#warning)
|
13
|
+
* [How does it work](#how-does-it-work)
|
14
|
+
* [Installation](#installation)
|
15
|
+
* [Usage](#usage)
|
16
|
+
* [Setting up for AWS](#setting-up-for-aws)
|
17
|
+
* [Enabling streams](#enabling-streams)
|
18
|
+
* [Error handling](#error-handling)
|
19
|
+
* [Supported targets](#supported-targets)
|
20
|
+
* [License](#license)
|
21
|
+
* [Development](#development)
|
22
|
+
* [Contributing](#contributing)
|
23
|
+
|
8
24
|
## Version mappings
|
9
25
|
|
10
26
|
```
|
11
|
-
1.
|
27
|
+
0.1.X - ActiveRecord 4.2.10
|
12
28
|
```
|
13
29
|
|
14
30
|
## Warning
|
@@ -95,7 +111,50 @@ ActiveRecordStreams.configure do |config|
|
|
95
111
|
end
|
96
112
|
```
|
97
113
|
|
98
|
-
|
114
|
+
### Error handling
|
115
|
+
|
116
|
+
It might happen that the message delivery fails. In such case you might
|
117
|
+
want to retry sending the message or even use a background processor like
|
118
|
+
`Sidekiq` to perform retires in automated way.
|
119
|
+
|
120
|
+
Every stream has an `error_handler` option which accepts `lambda/proc`
|
121
|
+
with `error_handler :: (Stream, TableName, Message, Error) -> *` signature.
|
122
|
+
|
123
|
+
**NOTE** that your consumer has to take care of duplicated messages.
|
124
|
+
Each message contains `EventID` which may help to identify duplications.
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
# config/initializers/active_record_streams.rb
|
128
|
+
|
129
|
+
require 'active_record_streams'
|
130
|
+
|
131
|
+
class SampleHttpReSender
|
132
|
+
include Sidekiq::Worker
|
133
|
+
|
134
|
+
def perform(table_name, message_json)
|
135
|
+
message = ActiveRecordStreams::Message.from_json(message_json)
|
136
|
+
ActiveRecordStreams.config.streams.each do |stream|
|
137
|
+
stream.publish(table_name, message)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
ActiveRecordStreams.configure do |config|
|
143
|
+
config.streams << ActiveRecordStreams::Publishers::HttpStream.new(
|
144
|
+
url: 'https://posteventshere.test',
|
145
|
+
error_handler: lambda do |stream, table_name, message, error|
|
146
|
+
# Do whatever you want here, you may also try to start a new
|
147
|
+
# thread or re-try publishing directly to stream using
|
148
|
+
# stream.publish(table_name, message)
|
149
|
+
|
150
|
+
# Try to schedule re-publishing with Sidekiq
|
151
|
+
SampleHttpReSender.perform_async(table_name, message.json)
|
152
|
+
end
|
153
|
+
)
|
154
|
+
end
|
155
|
+
```
|
156
|
+
|
157
|
+
## Supported targets
|
99
158
|
|
100
159
|
### ActiveRecordStreams::Publishers::SnsStream
|
101
160
|
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'securerandom'
|
4
|
+
|
3
5
|
module ActiveRecordStreams
|
4
6
|
class Message
|
5
7
|
def initialize(table_name, action_type, old_image, new_image)
|
@@ -9,8 +11,20 @@ module ActiveRecordStreams
|
|
9
11
|
@new_image = new_image
|
10
12
|
end
|
11
13
|
|
14
|
+
def self.from_json(json)
|
15
|
+
parsed_json = JSON.parse(json)
|
16
|
+
|
17
|
+
new(
|
18
|
+
parsed_json['TableName'],
|
19
|
+
parsed_json['ActionType'],
|
20
|
+
parsed_json['OldImage'],
|
21
|
+
parsed_json['NewImage']
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
12
25
|
def json
|
13
26
|
{
|
27
|
+
EventID: SecureRandom.uuid,
|
14
28
|
TableName: @table_name,
|
15
29
|
ActionType: @action_type,
|
16
30
|
OldImage: @old_image,
|
@@ -7,14 +7,42 @@ RSpec.describe ActiveRecordStreams::Message do
|
|
7
7
|
let(:action_type) { :create }
|
8
8
|
let(:old_image) { { 'hello' => 'world' } }
|
9
9
|
let(:new_image) { { 'hello' => 'new world' } }
|
10
|
+
let(:uuid) { '26f4ee2c-20ce-481f-b9f2-833bf7e51c5e' }
|
10
11
|
|
11
12
|
subject do
|
12
13
|
described_class.new(table_name, action_type, old_image, new_image)
|
13
14
|
end
|
14
15
|
|
16
|
+
before do
|
17
|
+
allow(SecureRandom).to receive(:uuid).and_return(uuid)
|
18
|
+
end
|
19
|
+
|
20
|
+
describe '.from_json' do
|
21
|
+
let(:expected_value) do
|
22
|
+
{
|
23
|
+
EventID: uuid,
|
24
|
+
TableName: table_name,
|
25
|
+
ActionType: action_type,
|
26
|
+
OldImage: old_image,
|
27
|
+
NewImage: new_image
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'creates a message from jsom' do
|
32
|
+
expect(described_class.from_json(expected_value.to_json))
|
33
|
+
.to be_a(described_class)
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'creates a right schema from json' do
|
37
|
+
expect(described_class.from_json(expected_value.to_json).json)
|
38
|
+
.to eq(expected_value.to_json)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
15
42
|
describe '#json' do
|
16
43
|
let(:expected_value) do
|
17
44
|
{
|
45
|
+
EventID: uuid,
|
18
46
|
TableName: table_name,
|
19
47
|
ActionType: action_type,
|
20
48
|
OldImage: old_image,
|
@@ -6,18 +6,28 @@ module ActiveRecordStreams
|
|
6
6
|
module Publishers
|
7
7
|
class HttpStream
|
8
8
|
ANY_TABLE = '*'
|
9
|
+
SUCCESSFUL_CODE_REGEX = /\A2\d{2}\z/.freeze
|
9
10
|
DEFAULT_CONTENT_TYPE = 'application/json'
|
10
11
|
|
12
|
+
##
|
13
|
+
# @param [String] url
|
14
|
+
# @param [Hash] headers
|
15
|
+
# @param [String] table_name
|
16
|
+
# @param [Enumerable<String>] ignored_tables
|
17
|
+
# @param [Proc] error_handler
|
18
|
+
|
11
19
|
def initialize(
|
12
20
|
url:,
|
13
21
|
headers: {},
|
14
22
|
table_name: ANY_TABLE,
|
15
|
-
ignored_tables: []
|
23
|
+
ignored_tables: [],
|
24
|
+
error_handler: nil
|
16
25
|
)
|
17
26
|
@url = url
|
18
27
|
@headers = headers
|
19
28
|
@table_name = table_name
|
20
29
|
@ignored_tables = ignored_tables
|
30
|
+
@error_handler = error_handler
|
21
31
|
end
|
22
32
|
|
23
33
|
def publish(table_name, message)
|
@@ -25,7 +35,12 @@ module ActiveRecordStreams
|
|
25
35
|
table_name == @table_name
|
26
36
|
|
27
37
|
request.body = message.json
|
28
|
-
http.request(request)
|
38
|
+
response = http.request(request)
|
39
|
+
assert_response_code(response)
|
40
|
+
rescue StandardError => error
|
41
|
+
raise error unless @error_handler.is_a?(Proc)
|
42
|
+
|
43
|
+
@error_handler.call(self, table_name, message, error)
|
29
44
|
end
|
30
45
|
|
31
46
|
private
|
@@ -53,6 +68,12 @@ module ActiveRecordStreams
|
|
53
68
|
def headers
|
54
69
|
{ 'Content-Type': DEFAULT_CONTENT_TYPE }.merge(@headers)
|
55
70
|
end
|
71
|
+
|
72
|
+
def assert_response_code(response)
|
73
|
+
return if response.code.to_s.match(SUCCESSFUL_CODE_REGEX)
|
74
|
+
|
75
|
+
raise StandardError, response.body
|
76
|
+
end
|
56
77
|
end
|
57
78
|
end
|
58
79
|
end
|
@@ -8,9 +8,11 @@ RSpec.describe ActiveRecordStreams::Publishers::HttpStream do
|
|
8
8
|
let(:desired_table_name) { '*' }
|
9
9
|
let(:actual_table_name) { 'lovely_records' }
|
10
10
|
let(:ignored_tables) { [] }
|
11
|
+
let(:error_handler) { nil }
|
11
12
|
|
12
|
-
let(:request) { double('body=': nil
|
13
|
-
let(:
|
13
|
+
let(:request) { double('body=': nil) }
|
14
|
+
let(:response) { double(code: 200) }
|
15
|
+
let(:http_client) { double('body=': nil, request: response) }
|
14
16
|
let(:message) { double(json: '{}') }
|
15
17
|
|
16
18
|
subject do
|
@@ -18,7 +20,8 @@ RSpec.describe ActiveRecordStreams::Publishers::HttpStream do
|
|
18
20
|
url: url,
|
19
21
|
headers: headers,
|
20
22
|
table_name: desired_table_name,
|
21
|
-
ignored_tables: ignored_tables
|
23
|
+
ignored_tables: ignored_tables,
|
24
|
+
error_handler: error_handler
|
22
25
|
)
|
23
26
|
end
|
24
27
|
|
@@ -69,5 +72,21 @@ RSpec.describe ActiveRecordStreams::Publishers::HttpStream do
|
|
69
72
|
expect(http_client).not_to have_received(:request).with(request)
|
70
73
|
end
|
71
74
|
end
|
75
|
+
|
76
|
+
context 'error response' do
|
77
|
+
let(:response) { double(body: 'Error', code: 400) }
|
78
|
+
let(:error_handler) { Proc.new {} }
|
79
|
+
|
80
|
+
it 'calls error handler' do
|
81
|
+
expect(error_handler).to receive(:call) do |stream, table_name, message, error|
|
82
|
+
expect(stream).to eq(subject)
|
83
|
+
expect(table_name).to eq(actual_table_name)
|
84
|
+
expect(message).to eq(message)
|
85
|
+
expect(error.message).to eq('Error')
|
86
|
+
end
|
87
|
+
|
88
|
+
subject.publish(actual_table_name, message)
|
89
|
+
end
|
90
|
+
end
|
72
91
|
end
|
73
92
|
end
|
@@ -13,17 +13,20 @@ module ActiveRecordStreams
|
|
13
13
|
# @param [String] table_name
|
14
14
|
# @param [Enumerable<String>] ignored_tables
|
15
15
|
# @param [Hash] overrides
|
16
|
+
# @param [Proc] error_handler
|
16
17
|
|
17
18
|
def initialize(
|
18
19
|
stream_name:,
|
19
20
|
table_name: ANY_TABLE,
|
20
21
|
ignored_tables: [],
|
21
|
-
overrides: {}
|
22
|
+
overrides: {},
|
23
|
+
error_handler: nil
|
22
24
|
)
|
23
25
|
@stream_name = stream_name
|
24
26
|
@table_name = table_name
|
25
27
|
@ignored_tables = ignored_tables
|
26
28
|
@overrides = overrides
|
29
|
+
@error_handler = error_handler
|
27
30
|
end
|
28
31
|
|
29
32
|
##
|
@@ -40,6 +43,10 @@ module ActiveRecordStreams
|
|
40
43
|
message.json,
|
41
44
|
@overrides
|
42
45
|
)
|
46
|
+
rescue StandardError => error
|
47
|
+
raise error unless @error_handler.is_a?(Proc)
|
48
|
+
|
49
|
+
@error_handler.call(self, table_name, message, error)
|
43
50
|
end
|
44
51
|
|
45
52
|
private
|
@@ -9,6 +9,7 @@ RSpec.describe ActiveRecordStreams::Publishers::KinesisStream do
|
|
9
9
|
let(:actual_table_name) { 'lovely_records' }
|
10
10
|
let(:ignored_tables) { [] }
|
11
11
|
let(:overrides) { {} }
|
12
|
+
let(:error_handler) { nil }
|
12
13
|
|
13
14
|
let(:message) { double(json: '{}') }
|
14
15
|
let(:kinesis_client) { double(publish: nil) }
|
@@ -18,7 +19,8 @@ RSpec.describe ActiveRecordStreams::Publishers::KinesisStream do
|
|
18
19
|
stream_name: stream_name,
|
19
20
|
table_name: desired_table_name,
|
20
21
|
ignored_tables: ignored_tables,
|
21
|
-
overrides: overrides
|
22
|
+
overrides: overrides,
|
23
|
+
error_handler: error_handler
|
22
24
|
)
|
23
25
|
end
|
24
26
|
|
@@ -89,5 +91,24 @@ RSpec.describe ActiveRecordStreams::Publishers::KinesisStream do
|
|
89
91
|
subject.publish(actual_table_name, message)
|
90
92
|
end
|
91
93
|
end
|
94
|
+
|
95
|
+
context 'delivery error' do
|
96
|
+
let(:error_handler) { Proc.new {} }
|
97
|
+
|
98
|
+
it 'calls error handler' do
|
99
|
+
expect(kinesis_client).to receive(:publish) do
|
100
|
+
raise StandardError, 'Delivery error'
|
101
|
+
end
|
102
|
+
|
103
|
+
expect(error_handler).to receive(:call) do |stream, table_name, message, error|
|
104
|
+
expect(stream).to eq(subject)
|
105
|
+
expect(table_name).to eq(actual_table_name)
|
106
|
+
expect(message).to eq(message)
|
107
|
+
expect(error.message).to eq('Delivery error')
|
108
|
+
end
|
109
|
+
|
110
|
+
subject.publish(actual_table_name, message)
|
111
|
+
end
|
112
|
+
end
|
92
113
|
end
|
93
114
|
end
|
@@ -10,17 +10,20 @@ module ActiveRecordStreams
|
|
10
10
|
# @param [String] table_name
|
11
11
|
# @param [Enumerable<String>] ignored_tables
|
12
12
|
# @param [Hash] overrides
|
13
|
+
# @param [Proc] error_handler
|
13
14
|
|
14
15
|
def initialize(
|
15
16
|
topic_arn:,
|
16
17
|
table_name: ANY_TABLE,
|
17
18
|
ignored_tables: [],
|
18
|
-
overrides: {}
|
19
|
+
overrides: {},
|
20
|
+
error_handler: nil
|
19
21
|
)
|
20
22
|
@topic_arn = topic_arn
|
21
23
|
@table_name = table_name
|
22
24
|
@ignored_tables = ignored_tables
|
23
25
|
@overrides = overrides
|
26
|
+
@error_handler = error_handler
|
24
27
|
end
|
25
28
|
|
26
29
|
##
|
@@ -32,6 +35,10 @@ module ActiveRecordStreams
|
|
32
35
|
table_name == @table_name
|
33
36
|
|
34
37
|
client.publish(@topic_arn, message.json, @overrides)
|
38
|
+
rescue StandardError => error
|
39
|
+
raise error unless @error_handler.is_a?(Proc)
|
40
|
+
|
41
|
+
@error_handler.call(self, table_name, message, error)
|
35
42
|
end
|
36
43
|
|
37
44
|
private
|
@@ -8,6 +8,7 @@ RSpec.describe ActiveRecordStreams::Publishers::SnsStream do
|
|
8
8
|
let(:actual_table_name) { 'lovely_records' }
|
9
9
|
let(:ignored_tables) { [] }
|
10
10
|
let(:overrides) { {} }
|
11
|
+
let(:error_handler) { nil }
|
11
12
|
|
12
13
|
let(:message) { double(json: '{}') }
|
13
14
|
let(:sns_client) { double(publish: nil) }
|
@@ -17,7 +18,8 @@ RSpec.describe ActiveRecordStreams::Publishers::SnsStream do
|
|
17
18
|
topic_arn: topic_arn,
|
18
19
|
table_name: desired_table_name,
|
19
20
|
ignored_tables: ignored_tables,
|
20
|
-
overrides: overrides
|
21
|
+
overrides: overrides,
|
22
|
+
error_handler: error_handler
|
21
23
|
)
|
22
24
|
end
|
23
25
|
|
@@ -85,5 +87,24 @@ RSpec.describe ActiveRecordStreams::Publishers::SnsStream do
|
|
85
87
|
subject.publish(actual_table_name, message)
|
86
88
|
end
|
87
89
|
end
|
90
|
+
|
91
|
+
context 'delivery error' do
|
92
|
+
let(:error_handler) { Proc.new {} }
|
93
|
+
|
94
|
+
it 'calls error handler' do
|
95
|
+
expect(sns_client).to receive(:publish) do
|
96
|
+
raise StandardError, 'Delivery error'
|
97
|
+
end
|
98
|
+
|
99
|
+
expect(error_handler).to receive(:call) do |stream, table_name, message, error|
|
100
|
+
expect(stream).to eq(subject)
|
101
|
+
expect(table_name).to eq(actual_table_name)
|
102
|
+
expect(message).to eq(message)
|
103
|
+
expect(error.message).to eq('Delivery error')
|
104
|
+
end
|
105
|
+
|
106
|
+
subject.publish(actual_table_name, message)
|
107
|
+
end
|
108
|
+
end
|
88
109
|
end
|
89
110
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_streams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Advanon Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-04-
|
11
|
+
date: 2019-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|