bemi 0.0.1 → 0.0.2.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Gemfile +5 -4
- data/README.md +120 -378
- data/app/models/bemi/application_record.rb +3 -0
- data/app/models/bemi/changeset.rb +41 -0
- data/app/models/bemi/context.rb +3 -0
- data/bemi.gemspec +7 -8
- data/config/routes.rb +2 -0
- data/exe/bemi +8 -0
- data/lib/bemi/context_handler.rb +52 -0
- data/lib/bemi/engine.rb +5 -0
- data/lib/bemi/ingester.rb +48 -0
- data/lib/bemi/storage.rb +61 -0
- data/lib/bemi/version.rb +1 -1
- data/lib/bemi.rb +22 -1
- data/lib/tasks/bemi_tasks.rake +2 -0
- metadata +37 -18
- data/spec/bemi_spec.rb +0 -7
- data/spec/spec_helper.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01c8e5628d1f8d178a7d1f460312f876e0c35bc7a8f03bee09572df191aeee27
|
4
|
+
data.tar.gz: 202a9117b07d168606bd67cd68e9edf5d37ce9b354be0739eb3243a1817bf86f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0a68972701ac66a787a1b5f03a8ba68d5ad307ec50e7d09e3f0b21b51d3e159c11c077e08a2eb48311f86aed9fb707bc1422f20ff973dfde14a15ce630511dbe
|
7
|
+
data.tar.gz: fa0929f601f3fdc4802f73b2ed8b0d4b1019a999e4f67c066a7d48ccfc20712a3137a8f2626dcf2a505d1119551da27090398bbc251eb0a033eebdfffa884289
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
source
|
3
|
+
source 'https://rubygems.org'
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in bemi.gemspec
|
6
6
|
gemspec
|
7
7
|
|
8
|
-
gem
|
9
|
-
|
10
|
-
gem
|
8
|
+
gem 'pry-byebug', '~> 3.10'
|
9
|
+
gem 'rake', '~> 13.0'
|
10
|
+
gem 'rspec', '~> 3.0'
|
11
|
+
gem 'sqlite3', '~> 1.6'
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Bemi
|
2
2
|
|
3
|
-
A
|
3
|
+
A suite of tools that allow to reliably track data changes without using extra Rails model callbacks.
|
4
4
|
|
5
5
|
Bemi stands for "beginner mindset" and is pronounced as [ˈbɛmɪ].
|
6
6
|
|
@@ -10,6 +10,8 @@ Bemi stands for "beginner mindset" and is pronounced as [ˈbɛmɪ].
|
|
10
10
|
* [Code example](#code-example)
|
11
11
|
* [Architecture](#architecture)
|
12
12
|
* [Usage](#usage)
|
13
|
+
* [Installation](#installation)
|
14
|
+
* [Database migration](#database-migration)
|
13
15
|
* [Workflows](#workflows)
|
14
16
|
* [Workflow definition](#workflow-definition)
|
15
17
|
* [Workflow validation](#workflow-validation)
|
@@ -21,106 +23,95 @@ Bemi stands for "beginner mindset" and is pronounced as [ˈbɛmɪ].
|
|
21
23
|
* [Action rollback](#action-rollback)
|
22
24
|
* [Action querying](#action-querying)
|
23
25
|
* [Action concurrency](#action-concurrency)
|
24
|
-
* [Installation](#installation)
|
25
26
|
* [Alternatives](#alternatives)
|
26
27
|
* [License](#license)
|
27
28
|
* [Code of Conduct](#code-of-conduct)
|
28
29
|
|
29
30
|
## Overview
|
30
31
|
|
31
|
-
*
|
32
|
-
*
|
33
|
-
*
|
34
|
-
*
|
35
|
-
*
|
36
|
-
*
|
32
|
+
* Automatically storing database changes and any addition context in a structured form
|
33
|
+
* High performance without affecting your code execution with callbacks
|
34
|
+
* 100% reliability by using a design pattern called Change Data Capture
|
35
|
+
* Easy to use, no data engineering knowledge or complex infrastructure is required
|
36
|
+
* Web UI and code tools for inspecting and auditing data changes and user activity
|
37
|
+
* Works with the most popular databases like MySQL, PostgreSQL and MongoDB (soon)
|
37
38
|
|
38
39
|
## Code example
|
39
40
|
|
40
|
-
Here is an example of
|
41
|
+
Here is an example of storing all data changes made when processing an HTTP request:
|
41
42
|
|
42
43
|
```ruby
|
43
|
-
|
44
|
-
|
45
|
-
name :order
|
46
|
-
|
47
|
-
def perform
|
48
|
-
action :process_payment, sync: true
|
49
|
-
action :send_confirmation, wait_for: [:process_payment], async: true
|
50
|
-
action :ship_package, wait_for: [:process_payment], async: { queue: 'warehouse' }
|
51
|
-
action :request_feedback, wait_for: [:ship_package], async: { delay: 7.days.to_i }
|
52
|
-
end
|
53
|
-
end
|
54
|
-
```
|
55
|
-
|
56
|
-
To run an instance of this workflow:
|
57
|
-
|
58
|
-
```ruby
|
59
|
-
# Init a workflow, it will stop at the first action and wait until it is executed synchronously
|
60
|
-
workflow = Bemi.perform_workflow(:order, context: { order_id: params[:order_id], user_id: current_user.id })
|
61
|
-
|
62
|
-
# Process payment by running the first workflow action synchronously
|
63
|
-
Bemi.perform_action(:process_payment, workflow_id: workflow.id, input: { payment_token: params[:payment_token] })
|
64
|
-
|
65
|
-
# Once the payment is processed, the next actions in the workflow
|
66
|
-
# will be executed automatically through background workers
|
67
|
-
```
|
44
|
+
class ApplicationController < ActionController::Base
|
45
|
+
before_action :set_bemi_context
|
68
46
|
|
69
|
-
|
70
|
-
|
71
|
-
```ruby
|
72
|
-
# app/actions/order/process_payment_action.rb
|
73
|
-
class Order::ProcessPaymentAction < Bemi::Action
|
74
|
-
name :process_payment
|
75
|
-
|
76
|
-
def perform
|
77
|
-
payment = PaymentProcessor.pay_for!(workflow.context[:order_id], input[:payment_token])
|
78
|
-
{ payment_id: payment.id }
|
79
|
-
end
|
80
|
-
end
|
81
|
-
```
|
82
|
-
|
83
|
-
```ruby
|
84
|
-
# app/actions/order/send_confirmation_action.rb
|
85
|
-
class Order::SendConfirmationAction < Bemi::Action
|
86
|
-
name :send_confirmation
|
87
|
-
|
88
|
-
def perform
|
89
|
-
payment_output = wait_for(:process_payment).output
|
90
|
-
mail = OrderMailer.send_confirmation(payment_output[:payment_id])
|
91
|
-
{ delivered: mail.delivered? }
|
92
|
-
end
|
93
|
-
end
|
94
|
-
```
|
95
|
-
|
96
|
-
```ruby
|
97
|
-
# ../warehouse/app/actions/order/ship_package_action.rb
|
98
|
-
class Order::ShipPackageAction < Bemi::Action
|
99
|
-
name :ship_package
|
100
|
-
|
101
|
-
def perform
|
102
|
-
# Run a separate "shipment" workflow
|
103
|
-
shipment_workflow = Bemi.perform_workflow(:shipment, context: { order_id: workflow.context[:order_id] })
|
104
|
-
{ shipment_workflow_id: shipment_workflow.id }
|
105
|
-
end
|
106
|
-
end
|
107
|
-
```
|
108
|
-
|
109
|
-
```ruby
|
110
|
-
# app/actions/order/request_feedback_action.rb
|
111
|
-
class Order::RequestFeedbackAction < Bemi::Action
|
112
|
-
name :request_feedback
|
47
|
+
private
|
113
48
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
49
|
+
# Attach any information you want to any subsequent data changes
|
50
|
+
def set_bemi_context
|
51
|
+
Bemi.set_context(
|
52
|
+
user_id: current_user&.id,
|
53
|
+
ip: request.remote_ip,
|
54
|
+
user_agent: request.user_agent,
|
55
|
+
controller: "#{self.class.name}##{action_name}",
|
56
|
+
)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
```
|
60
|
+
|
61
|
+
```ruby
|
62
|
+
class InvoicesController < ApplicationController
|
63
|
+
# Automatically store *any* database changes
|
64
|
+
def update
|
65
|
+
invoice = Invoice.find(params[:id])
|
66
|
+
invoice.update_column(:due_date, params[:due_date])
|
67
|
+
invoice.client.recurring_schedule.delete
|
68
|
+
end
|
69
|
+
end
|
70
|
+
```
|
71
|
+
|
72
|
+
Bemi then allows easily querying data changes:
|
73
|
+
|
74
|
+
```ruby
|
75
|
+
Bemi.activity(ip: '127.0.0.1').map(&:pretty_print)
|
76
|
+
|
77
|
+
# Bemi::Changeset
|
78
|
+
# - id: 2040
|
79
|
+
# - table: "invoices"
|
80
|
+
# - external_id: 43
|
81
|
+
# - action: "update"
|
82
|
+
# - committed_at: Sat, 03 Jun 2023 21:16:22 UTC +00:00
|
83
|
+
# - change:
|
84
|
+
# - updated_at: ["2023-06-03 20:41:35", "2023-06-03 21:16:22"]
|
85
|
+
# - due_date: ["2023-06-03", "2023-06-30"]
|
86
|
+
# - context:
|
87
|
+
# - ip: "127.0.0.1"
|
88
|
+
# - user_id: 3195
|
89
|
+
# - controller: "InvoicesController#update"
|
90
|
+
# - user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
|
91
|
+
#
|
92
|
+
# Bemi::Changeset
|
93
|
+
# - id: 2041
|
94
|
+
# - table: "recurring_schedules"
|
95
|
+
# - external_id: 5
|
96
|
+
# - action: "delete"
|
97
|
+
# - committed_at: Sat, 03 Jun 2023 21:16:22 UTC +00:00
|
98
|
+
# - change:
|
99
|
+
# - id: 5
|
100
|
+
# - frequency: 1
|
101
|
+
# - occurrences: 0
|
102
|
+
# - invoice_id: 43
|
103
|
+
# - created_at: "2023-04-28 20:34:09"
|
104
|
+
# - updated_at: "2023-04-28 20:34:09"
|
105
|
+
# - context:
|
106
|
+
# - ip: "127.0.0.1"
|
107
|
+
# - user_id: 3195
|
108
|
+
# - controller: "InvoicesController#update"
|
109
|
+
# - user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
|
119
110
|
```
|
120
111
|
|
121
112
|
## Architecture
|
122
113
|
|
123
|
-
Bemi is designed to be lightweight and simple to use by default.
|
114
|
+
Bemi is designed to be lightweight, composable, and simple to use by default.
|
124
115
|
|
125
116
|
```
|
126
117
|
/‾‾‾\
|
@@ -128,326 +119,77 @@ Bemi is designed to be lightweight and simple to use by default. As a system dep
|
|
128
119
|
__/ \__
|
129
120
|
/ User \
|
130
121
|
│
|
131
|
-
|
132
|
-
|
133
|
-
╵
|
134
|
-
╵
|
135
|
-
╵
|
136
|
-
╵
|
137
|
-
╵
|
138
|
-
╵
|
139
|
-
╵
|
140
|
-
╵
|
141
|
-
╵
|
142
|
-
╵
|
143
|
-
╵
|
144
|
-
╵
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
Bemi orchestrates workflows by persisting their execution state into PostgreSQL. When connecting to PostgreSQL, Bemi first scans the codebase and registers all workflows uniquely identified by `name`. Workflows describe a sequence of actions by using the DSL written in Ruby that can be run synchronously in the same process or asynchronously and by schedule in workers.
|
162
|
-
|
163
|
-
* Actions
|
164
|
-
|
165
|
-
Actions are also uniquely identified by `name`. They can receive data from an input if ran synchronously, previously executed actions if they depend on them, and the shared workflow execution context. They can be implemented and executed in any service or application as long as it is connected to the same PostgreSQL instance. So, there is no need to deal with message passing by implementing APIs, callbacks, message buses, data serialization, etc.
|
166
|
-
|
167
|
-
* Workers
|
168
|
-
|
169
|
-
Bemi workers allow running actions that are executed asynchronously or by schedule. One worker represents a process with multiple threads to enable concurrency. Workers can process one or more `queues` and execute different actions across different workflows simultaneously if they are assigned to the same workers' queues.
|
170
|
-
|
171
|
-
See the [Alternatives](#alternatives) section that describes how Bemi is different from other tools you might be familiar with.
|
122
|
+
│ Application code
|
123
|
+
- - - - - │ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
124
|
+
╵ │ ╵
|
125
|
+
╵ │ Update invoice ╵
|
126
|
+
╵ ∨ ╵
|
127
|
+
╵ ______________ ______________ ╵
|
128
|
+
╵ ┆ ┆ ┆ ┆ ╵
|
129
|
+
╵ ┆ Rails ┆ Structured changes ┆ Bemi ┆ ╵
|
130
|
+
╵ ┆ server ┆ ╷–––––––––––––––––––––– ┆ process ┆ ╵
|
131
|
+
╵ ┆ ┆ │ ┆ ┆ ╵
|
132
|
+
╵ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾ │ ‾‾‾‾‾‾‾‾‾‾‾‾‾‾ ╵
|
133
|
+
╵ │ │ ⌃ ╵
|
134
|
+
╵ │ Database query │ Replication log │ ╵
|
135
|
+
╵ │ │ │ ╵
|
136
|
+
- - - - - │ - - - - - - - - - - - - - - - │ - - - - - - - - - - - - - - - │ - - - - -╵
|
137
|
+
│ ∨ │
|
138
|
+
│ [‾‾‾‾‾‾‾‾‾‾‾‾] │
|
139
|
+
│ [------------] │
|
140
|
+
╵–––––––––––––––––––––––> [ Database ] –––––––––––––––––––––––╵
|
141
|
+
[------------]
|
142
|
+
[____________]
|
143
|
+
```
|
144
|
+
|
145
|
+
Bemi by reuses the same connection configuration and runs a simple process to process a replication log that databases usually use to communicate within the same cluster:
|
146
|
+
|
147
|
+
* Binary Log for MySQL
|
148
|
+
* Write-Ahead Log for PostgreSQL
|
149
|
+
* Oplog for MongoDB
|
150
|
+
|
151
|
+
By default, it stores the structured data changes in the same database.
|
172
152
|
|
173
153
|
## Usage
|
174
154
|
|
175
|
-
###
|
176
|
-
|
177
|
-
#### Workflow definition
|
178
|
-
|
179
|
-
Workflows declaratively describe actions that can be executed
|
180
|
-
|
181
|
-
* Synchronously
|
182
|
-
|
183
|
-
```ruby
|
184
|
-
class RegistrationWorkflow < Bemi::Workflow
|
185
|
-
name :registration
|
186
|
-
|
187
|
-
def perform
|
188
|
-
action :create_user, sync: true
|
189
|
-
action :send_confirmation_email, sync: true
|
190
|
-
action :confirm_email_address, sync: true
|
191
|
-
end
|
192
|
-
end
|
193
|
-
|
194
|
-
workflow = Bemi.perform_workflow(:registration, context: { email: params[:email] })
|
195
|
-
Bemi.perform_action(:create_user, workflow_id: workflow.id, input: { password: params[:password] })
|
196
|
-
Bemi.perform_action(:send_confirmation_email, workflow_id: workflow.id)
|
197
|
-
Bemi.perform_action(:confirm_email_address, workflow_id: workflow.id, input: { token: params[:token] })
|
198
|
-
```
|
199
|
-
|
200
|
-
* Asynchronously
|
155
|
+
### Installation
|
201
156
|
|
202
|
-
|
203
|
-
class RegistrationWorkflow < Bemi::Workflow
|
204
|
-
name :registration
|
205
|
-
|
206
|
-
def perform
|
207
|
-
action :create_user, async: true
|
208
|
-
action :send_welcome_email, wait_for: [:create_user], async: true
|
209
|
-
action :run_background_check, wait_for: [:send_welcome_email], async: { queue: 'kyc' }
|
210
|
-
end
|
211
|
-
end
|
157
|
+
Add `gem 'bemi'` to your application's Gemfile and execute:
|
212
158
|
|
213
|
-
Bemi.perform_workflow(:registration, context: { email: params[:email] })
|
214
159
|
```
|
215
|
-
|
216
|
-
* By schedule
|
217
|
-
|
218
|
-
```ruby
|
219
|
-
class RegistrationWorkflow < Bemi::Workflow
|
220
|
-
name :registration
|
221
|
-
|
222
|
-
def perform
|
223
|
-
action :create_user, async: { cron: '0 2 * * *' } # daily at 2am
|
224
|
-
action :send_welcome_email, async: { cron: '0 3 * * *', queue: 'emails', priority: 10 }
|
225
|
-
action :run_background_check, async: { delay: 24.hours.to_i },
|
226
|
-
end
|
227
|
-
end
|
228
|
-
|
229
|
-
Bemi.perform_workflow(:registration, context: { email: params[:email] })
|
230
|
-
```
|
231
|
-
|
232
|
-
#### Workflow validation
|
233
|
-
|
234
|
-
Workflow can define the shape of the `context` and validate it against a JSON Schema
|
235
|
-
|
236
|
-
```ruby
|
237
|
-
class RegistrationWorkflow < Bemi::Workflow
|
238
|
-
name :registration
|
239
|
-
context_schema {
|
240
|
-
type: :object,
|
241
|
-
properties: { email: { type: :string } },
|
242
|
-
required: [:email],
|
243
|
-
}
|
244
|
-
end
|
245
|
-
```
|
246
|
-
|
247
|
-
#### Workflow concurrency
|
248
|
-
|
249
|
-
It is possible to set workflow concurrency options if you want to guarantee uniqueness
|
250
|
-
|
251
|
-
```ruby
|
252
|
-
class RegistrationWorkflow < Bemi::Workflow
|
253
|
-
name :registration
|
254
|
-
concurrency limit: 1, on_conflict: :raise # or :reject
|
255
|
-
end
|
256
|
-
|
257
|
-
Bemi.perform_workflow(:registration, context: { email: 'email@example.com' })
|
258
|
-
Bemi.perform_workflow(:registration, context: { email: 'email@example.com' })
|
259
|
-
# => Bemi::ConcurrencyError: cannot run more than 1 'registration' workflow at the same time
|
260
|
-
```
|
261
|
-
|
262
|
-
#### Workflow querying
|
263
|
-
|
264
|
-
You can query workflows if, for example, one of the actions in the middle of the workflow execution needs to be triggered manually
|
265
|
-
|
266
|
-
```ruby
|
267
|
-
workflow = Bemi.find_workflow(:registration, context: { email: 'email@example.com' })
|
268
|
-
|
269
|
-
workflow.canceled?
|
270
|
-
workflow.completed?
|
271
|
-
workflow.failed?
|
272
|
-
workflow.running?
|
273
|
-
workflow.timed_out?
|
274
|
-
|
275
|
-
# Persisted and deserialized from JSON
|
276
|
-
workflow.context
|
277
|
-
|
278
|
-
Bemi.perform_action(:confirm_email_address, workflow_id: workflow.id)
|
279
|
-
```
|
280
|
-
|
281
|
-
### Actions
|
282
|
-
|
283
|
-
#### Action validation
|
284
|
-
|
285
|
-
Bemi allows to define the shape of actions' inputs and outputs and validate it against a JSON Schema
|
286
|
-
|
287
|
-
```ruby
|
288
|
-
class RegistrationWorkflow < Bemi::Workflow
|
289
|
-
name :registration
|
290
|
-
|
291
|
-
def perform
|
292
|
-
action :create_user,
|
293
|
-
sync: true,
|
294
|
-
input_schema: {
|
295
|
-
type: :object, properties: { password: { type: :string } }, required: [:password],
|
296
|
-
},
|
297
|
-
output_schema: {
|
298
|
-
type: :object, properties: { user_id: { type: :integer } }, required: [:user_id],
|
299
|
-
}
|
300
|
-
end
|
301
|
-
end
|
302
|
-
```
|
303
|
-
|
304
|
-
#### Action error handling
|
305
|
-
|
306
|
-
Custom `retry` count
|
307
|
-
|
308
|
-
```ruby
|
309
|
-
class RegistrationWorkflow < Bemi::Workflow
|
310
|
-
name :registration
|
311
|
-
|
312
|
-
def perform
|
313
|
-
action :create_user, async: true, on_error: { retry: :exponential_backoff } # default retry option
|
314
|
-
action :send_welcome_email, async: true, on_error: { retry: 1 }
|
315
|
-
end
|
316
|
-
end
|
317
|
-
```
|
318
|
-
|
319
|
-
Custom error handler with `around_perform`
|
320
|
-
|
321
|
-
```ruby
|
322
|
-
class Registration::SendWelcomeEmailAction < Bemi::Action
|
323
|
-
name :send_welcome_email
|
324
|
-
around_perform :error_handler
|
325
|
-
|
326
|
-
def perform
|
327
|
-
user = User.find(workflow.context[:user_id])
|
328
|
-
context[:email] = user.email
|
329
|
-
mail = UserMailer.welcome(user.id)
|
330
|
-
{ delivered: mail.delivered? }
|
331
|
-
end
|
332
|
-
|
333
|
-
private
|
334
|
-
|
335
|
-
def error_handler(&block)
|
336
|
-
block.call
|
337
|
-
rescue User::InvalidEmail => e
|
338
|
-
add_error!(:email, "Invalid email: #{context[:email]}")
|
339
|
-
fail! # don't retry if there is an application-level error
|
340
|
-
rescue Errno::ECONNRESET => e
|
341
|
-
raise e # retry by raising an exception if there is a temporary system-level error
|
342
|
-
end
|
343
|
-
end
|
160
|
+
$ bundle install
|
344
161
|
```
|
345
162
|
|
346
|
-
|
163
|
+
### Database migration
|
347
164
|
|
348
|
-
|
349
|
-
|
350
|
-
```ruby
|
351
|
-
class Order::ProcessPaymentAction < Bemi::Action
|
352
|
-
name :process_payment
|
353
|
-
around_rollback :rollback_notifier
|
354
|
-
|
355
|
-
def perform
|
356
|
-
payment = PaymentProcessor.pay_for!(workflow.context[:order_id], input[:payment_token])
|
357
|
-
{ payment_id: payment.id }
|
358
|
-
end
|
359
|
-
|
360
|
-
def rollback
|
361
|
-
refund = PaymentProcessor.issue_refund!(output[:payment_id], input[:payment_token])
|
362
|
-
{ refund_id: refund.id }
|
363
|
-
end
|
165
|
+
Create a new database migration to store changeset and context in a structured form:
|
364
166
|
|
365
|
-
def rollback_notifier(&block)
|
366
|
-
OrderMailer.notify_cancelation(output[:payment_id])
|
367
|
-
block.call
|
368
|
-
end
|
369
|
-
end
|
370
167
|
```
|
371
|
-
|
372
|
-
#### Action querying
|
373
|
-
|
374
|
-
```ruby
|
375
|
-
workflow = Bemi.find_workflow(:registration, context: { email: 'email@example.com' })
|
376
|
-
action = Bemi.find_action(:create_user, workflow_id: workflow.id)
|
377
|
-
|
378
|
-
action.canceled?
|
379
|
-
action.completed?
|
380
|
-
action.failed?
|
381
|
-
action.running?
|
382
|
-
action.timed_out?
|
383
|
-
|
384
|
-
action.workflow
|
385
|
-
action.options
|
386
|
-
|
387
|
-
# Persisted and deserialized from JSON
|
388
|
-
action.input
|
389
|
-
action.output
|
390
|
-
action.errors
|
391
|
-
action.rollback_output
|
392
|
-
action.context
|
168
|
+
$ bundle exec rails g migration create_bemi_tables
|
393
169
|
```
|
394
170
|
|
395
|
-
|
396
|
-
|
397
|
-
Custom concurrency `limit`
|
171
|
+
Then paste the following into the created migration file:
|
398
172
|
|
399
173
|
```ruby
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
def perform
|
404
|
-
action :create_user, async: true, concurrency: { limit: 1, on_conflict: :reschedule } # or :raise
|
405
|
-
end
|
406
|
-
end
|
174
|
+
# db/migrate/20230603190131_create_bemi_tables.rb
|
175
|
+
CreateBemiTables = Class.new(Bemi.generate_migration)
|
407
176
|
```
|
408
177
|
|
409
|
-
|
178
|
+
And run:
|
410
179
|
|
411
180
|
```ruby
|
412
|
-
|
413
|
-
name :send_welcome_email
|
414
|
-
|
415
|
-
def perform
|
416
|
-
mail = UserMailer.welcome(workflow.context[:user_id])
|
417
|
-
{ delivered: mail.delivered? }
|
418
|
-
end
|
419
|
-
|
420
|
-
def concurrency_key
|
421
|
-
"#{options[:async][:queue]}-#{input[:user_id]}"
|
422
|
-
end
|
423
|
-
end
|
181
|
+
$ bundle exec rails db:migrate
|
424
182
|
```
|
425
183
|
|
426
|
-
|
427
|
-
|
428
|
-
Add this line to your application's Gemfile:
|
184
|
+
### Bemi process
|
429
185
|
|
430
|
-
```
|
431
|
-
gem 'bemi'
|
432
|
-
```
|
433
186
|
|
434
|
-
And then execute:
|
435
|
-
|
436
|
-
```
|
437
|
-
$ bundle install
|
438
|
-
```
|
439
|
-
|
440
|
-
Or install it yourself as:
|
441
|
-
|
442
|
-
```
|
443
|
-
$ gem install bemi
|
444
|
-
```
|
445
187
|
|
446
188
|
## Alternatives
|
447
189
|
|
448
190
|
#### Background jobs with persistent state
|
449
191
|
|
450
|
-
Tools like Sidekiq, Que, and GoodJob are similar since they execute jobs in background, persist the execution state, retry, etc. These tools, however, focus on executing a single job as a unit of work. Bemi can
|
192
|
+
Tools like Sidekiq, Que, and GoodJob are similar since they execute jobs in background, persist the execution state, retry, etc. These tools, however, focus on executing a single job as a unit of work. Bemi can use these tools to perform single actions when managing chains of actions defined in workflows without a need to use complex callbacks.
|
451
193
|
|
452
194
|
Bemi orchestrates workflows instead of trying to choreograph them. This makes it easy to implement and maintain the code, reduce coordination overhead by having a central coordinator, improve observability, and simplify troubleshooting issues.
|
453
195
|
|
@@ -467,11 +209,11 @@ Bemi orchestrates workflows instead of trying to choreograph them. This makes it
|
|
467
209
|
|
468
210
|
Tools like Temporal, AWS Step Functions, Argo Workflows, and Airflow allow orchestrating workflows, although they use quite different approaches.
|
469
211
|
|
470
|
-
Temporal was born based on challenges faced by big-tech and enterprise companies. As a result, it has a complex architecture with deployed clusters,
|
212
|
+
Temporal was born based on challenges faced by big-tech and enterprise companies. As a result, it has a complex architecture with deployed clusters, support for databases like Cassandra and optional Elasticsearch, and multiple services for frontend, matching, history, etc. Its main differentiator is writing workflows imperatively instead of describing them declaratively (think of state machines). This makes code a lot more complex and forces you to mix business logic with implementation and execution details. Some would argue that Temporal's development and user experience are quite rough. Plus, at the time of this writing, it doesn't have an official stable SDK for our favorite programming language (Ruby).
|
471
213
|
|
472
214
|
AWS Step Functions rely on using AWS Lambda to execute each action in a workflow. For various reasons, not everyone can use AWS and their serverless solution. Additionally, workflows should be defined in JSON by using Amazon States Language instead of using a regular programming language.
|
473
215
|
|
474
|
-
Argo Workflows
|
216
|
+
Argo Workflows rely on using Kubernetes. It is closer to infrastructure-level workflows since it relies on running a container for each workflow action and doesn't provide code-level features and primitives. Additionally, it requires defining workflows in YAML.
|
475
217
|
|
476
218
|
Airflow is a popular tool for data engineering pipelines. Unfortunately, it can work only with Python.
|
477
219
|
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class Bemi::Changeset < Bemi::ApplicationRecord
|
2
|
+
ACTION_DELETE = 'delete'
|
3
|
+
ACTION_UPDATE = 'update'
|
4
|
+
|
5
|
+
belongs_to :context, class_name: 'Bemi::Context'
|
6
|
+
|
7
|
+
def diff
|
8
|
+
values_after.each_with_object({}) do |(k, v), memo|
|
9
|
+
memo[k] = [values_before[k], v] if values_before[k] != v
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def pretty_print
|
14
|
+
formatted_diff =
|
15
|
+
case action
|
16
|
+
when ACTION_DELETE
|
17
|
+
values_before
|
18
|
+
when ACTION_UPDATE
|
19
|
+
diff
|
20
|
+
else
|
21
|
+
values_after
|
22
|
+
end
|
23
|
+
|
24
|
+
formatted_context = <<-EOF
|
25
|
+
- context:
|
26
|
+
#{context.data.map { |k, v| "- #{k}: #{v.inspect}" }.join("\n ")}
|
27
|
+
EOF
|
28
|
+
|
29
|
+
puts <<-EOF
|
30
|
+
Bemi::Changeset
|
31
|
+
- id: #{id.inspect}
|
32
|
+
- table: #{table.inspect}
|
33
|
+
- external_id: #{external_id.inspect}
|
34
|
+
- action: #{action.inspect}
|
35
|
+
- committed_at: #{committed_at.inspect}
|
36
|
+
- change:
|
37
|
+
#{formatted_diff.map { |k, v| "- #{k}: #{v.inspect}" }.join("\n ")}
|
38
|
+
#{context ? formatted_context : ''}
|
39
|
+
EOF
|
40
|
+
end
|
41
|
+
end
|
data/bemi.gemspec
CHANGED
@@ -8,29 +8,28 @@ Gem::Specification.new do |spec|
|
|
8
8
|
spec.authors = ["exAspArk"]
|
9
9
|
spec.email = ["exaspark@gmail.com"]
|
10
10
|
|
11
|
-
spec.summary = "
|
12
|
-
spec.description = "Bemi allows to
|
13
|
-
spec.homepage = "https://github.com/
|
11
|
+
spec.summary = "Reliable audit trail for your application."
|
12
|
+
spec.description = "Bemi allows to reliably track data changes without using model callbacks."
|
13
|
+
spec.homepage = "https://github.com/bemi-io/bemi-ruby"
|
14
14
|
spec.license = "MIT"
|
15
15
|
spec.required_ruby_version = ">= 2.6.0"
|
16
16
|
|
17
17
|
spec.metadata["homepage_uri"] = spec.homepage
|
18
|
-
spec.metadata["source_code_uri"] = "https://github.com/
|
19
|
-
spec.metadata["changelog_uri"] = "https://github.com/
|
18
|
+
spec.metadata["source_code_uri"] = "https://github.com/bemi-io/bemi-ruby"
|
19
|
+
spec.metadata["changelog_uri"] = "https://github.com/bemi-io/bemi-ruby/blob/main/CHANGELOG.md"
|
20
20
|
|
21
21
|
# Specify which files should be added to the gem when it is released.
|
22
22
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
23
23
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
24
24
|
`git ls-files -z`.split("\x0").reject do |f|
|
25
|
-
(f == __FILE__) || f.match(%r{\A(
|
25
|
+
(f == __FILE__) || f.match(%r{\A(images|spec|\.git|\.github)\/|(\.rspec|\.gitignore)})
|
26
26
|
end
|
27
27
|
end
|
28
28
|
spec.bindir = "exe"
|
29
29
|
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
30
30
|
spec.require_paths = ["lib"]
|
31
31
|
|
32
|
-
|
33
|
-
# spec.add_dependency "example-gem", "~> 1.0"
|
32
|
+
spec.add_dependency "rails", ">= 6.0"
|
34
33
|
|
35
34
|
# For more information and examples about making a new gem, check out our
|
36
35
|
# guide at: https://bundler.io/guides/creating_gem.html
|
data/config/routes.rb
ADDED
data/exe/bemi
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
class Bemi::ContextHandler
|
2
|
+
# ActiveRecord::ConnectionAdapters::MySQL::DatabaseStatements.const_get(:READ_QUERY)
|
3
|
+
READ_QUERY = %r{\A(?:[(\s]|(?m-ix:(?:--.*\n)|/\*(?:[^*]|\*[^/])*\*/))*(?-mix:(?i-mx:desc)|(?i-mx:describe)|(?i-mx:set)|(?i-mx:show)|(?i-mx:use)|(?i-mx:begin)|(?i-mx:commit)|(?i-mx:explain)|(?i-mx:release)|(?i-mx:rollback)|(?i-mx:savepoint)|(?i-mx:select)|(?i-mx:with))}
|
4
|
+
IGNORE_TABLES = %r{#{Bemi::Ingester::IGNORE_TABLES.map { |t| "`#{t}`" }.join('|')}}
|
5
|
+
|
6
|
+
class << self
|
7
|
+
def set(data)
|
8
|
+
context = Bemi::Context.new(data: data)
|
9
|
+
Thread.current[:context_handler] = Bemi::ContextHandler.new(context: context)
|
10
|
+
end
|
11
|
+
|
12
|
+
def get
|
13
|
+
Thread.current[:context_handler] ||= Bemi::ContextHandler.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def transform_sql(sql)
|
17
|
+
return sql if !write_query?(sql)
|
18
|
+
|
19
|
+
context_handler = get
|
20
|
+
context_handler.context.save! if context_handler.context && context_handler.context.id.nil?
|
21
|
+
|
22
|
+
context_id = context_handler.context&.id
|
23
|
+
context_id ? "/* #{context_id} */ #{sql}" : sql
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def write_query?(sql)
|
29
|
+
!READ_QUERY.match?(sql) && !IGNORE_TABLES.match?(sql)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
attr_reader :context
|
34
|
+
|
35
|
+
def initialize(context: nil)
|
36
|
+
@context = context
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
module BemiAbstractMysqlAdapter
|
41
|
+
def execute(sql, name = nil) # Rails 5 and 6
|
42
|
+
sql = Bemi::ContextHandler.transform_sql(sql)
|
43
|
+
|
44
|
+
log(sql, name) do
|
45
|
+
ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
|
46
|
+
@connection.query(sql)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
require 'active_record/connection_adapters/abstract_mysql_adapter'
|
52
|
+
ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(BemiAbstractMysqlAdapter)
|
data/lib/bemi/engine.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
class Bemi::Ingester
|
2
|
+
IGNORE_TABLES = %w[bemi_changesets bemi_contexts].freeze
|
3
|
+
|
4
|
+
class << self
|
5
|
+
def daemonize!
|
6
|
+
Open3.popen3(stream_binlong_command) do |stdin, stdout, stderr, thread|
|
7
|
+
stdin.close
|
8
|
+
|
9
|
+
stderr_thread = Thread.new do
|
10
|
+
while line = stderr.gets
|
11
|
+
puts "ERROR: #{line}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
stderr_thread.abort_on_exception = true
|
15
|
+
|
16
|
+
stdout_thread = Thread.new do
|
17
|
+
while line = stdout.gets
|
18
|
+
begin
|
19
|
+
payload = JSON.parse(line)
|
20
|
+
next if IGNORE_TABLES.include?(payload['table'])
|
21
|
+
|
22
|
+
Bemi::Storage.create_changeset!(payload)
|
23
|
+
rescue JSON::ParserError => e
|
24
|
+
puts "ERROR: #{line}"
|
25
|
+
raise "Failed"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
stdout_thread.abort_on_exception = true
|
30
|
+
|
31
|
+
thread.join
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def stream_binlong_command
|
36
|
+
init_position = ''
|
37
|
+
|
38
|
+
if changeset = Bemi::Changeset.last
|
39
|
+
init_position = " --init_position=#{changeset.binlog_position}"
|
40
|
+
puts "Started ingester with: #{changeset.binlog_position}"
|
41
|
+
else
|
42
|
+
puts "Started ingester with no init position"
|
43
|
+
end
|
44
|
+
|
45
|
+
"maxwell --host=127.0.0.1 --user=root --password=mysql --port=3366 --producer=stdout --output_row_query=true --output_binlog_position=true --output_commit_info=false#{init_position}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/bemi/storage.rb
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require_relative '../../app/models/bemi/application_record'
|
3
|
+
require_relative '../../app/models/bemi/changeset'
|
4
|
+
require_relative '../../app/models/bemi/context'
|
5
|
+
|
6
|
+
class Bemi::Storage
|
7
|
+
class << self
|
8
|
+
def create_changeset!(payload)
|
9
|
+
puts payload.inspect
|
10
|
+
action = payload['type']
|
11
|
+
values_before = action == Bemi::Changeset::ACTION_DELETE ? payload['data'] : payload.fetch('old', {})
|
12
|
+
values_after = action == Bemi::Changeset::ACTION_DELETE ? {} : payload['data']
|
13
|
+
fields = (values_before.keys | values_after.keys).sort
|
14
|
+
context_id = payload['query'].split('/* ')[1]&.split(' */')&.first
|
15
|
+
|
16
|
+
Bemi::Changeset.create!(
|
17
|
+
context_id: context_id,
|
18
|
+
external_id: values_after['id'] || values_before['id'],
|
19
|
+
database: payload['database'],
|
20
|
+
table: payload['table'],
|
21
|
+
query: payload['query'],
|
22
|
+
action: action,
|
23
|
+
committed_at: Time.at(payload['ts']),
|
24
|
+
binlog_position: payload['position'],
|
25
|
+
values_before: fields.each_with_object({}) { |f, memo| memo[f] = values_before[f] || (action == Bemi::Changeset::ACTION_UPDATE ? values_after[f] : nil) },
|
26
|
+
values_after: fields.each_with_object({}) { |f, memo| memo[f] = values_after[f] },
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
def generate_migration
|
31
|
+
migration_class = Class.new(ActiveRecord::Migration[6.0])
|
32
|
+
|
33
|
+
migration_class.class_eval do
|
34
|
+
def change
|
35
|
+
create_table :bemi_changesets do |t|
|
36
|
+
t.integer :context_id, index: true
|
37
|
+
t.string :external_id, index: true
|
38
|
+
t.string :database, null: false
|
39
|
+
t.string :table, null: false
|
40
|
+
t.text :query, null: false
|
41
|
+
t.string :action, null: false
|
42
|
+
t.timestamp :committed_at, null: false, index: true
|
43
|
+
t.string :binlog_position, null: false # ?
|
44
|
+
t.json :values_before, null: false
|
45
|
+
t.json :values_after, null: false
|
46
|
+
t.timestamps
|
47
|
+
end
|
48
|
+
|
49
|
+
add_index :bemi_changesets, [:database, :table]
|
50
|
+
|
51
|
+
create_table :bemi_contexts do |t|
|
52
|
+
t.json :data, null: false
|
53
|
+
t.timestamps
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
migration_class
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/lib/bemi/version.rb
CHANGED
data/lib/bemi.rb
CHANGED
@@ -1,6 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative 'bemi/version'
|
4
|
+
require_relative 'bemi/ingester'
|
5
|
+
require_relative 'bemi/context_handler'
|
6
|
+
require_relative 'bemi/engine'
|
7
|
+
require_relative 'bemi/storage'
|
4
8
|
|
5
9
|
class Bemi
|
10
|
+
class << self
|
11
|
+
def set_context(data)
|
12
|
+
Bemi::ContextHandler.set(data)
|
13
|
+
end
|
14
|
+
|
15
|
+
def generate_migration
|
16
|
+
Bemi::Storage.generate_migration
|
17
|
+
end
|
18
|
+
|
19
|
+
def changes(record)
|
20
|
+
Bemi::Changeset.where(external_id: record.id.to_s, table: record.class.table_name).includes(:context)
|
21
|
+
end
|
22
|
+
|
23
|
+
def activity(keyval)
|
24
|
+
Bemi::Changeset.joins(:context).where("JSON_EXTRACT(data, \"$.#{keyval.keys.first}\") = ?", keyval.values.first).includes(:context)
|
25
|
+
end
|
26
|
+
end
|
6
27
|
end
|
metadata
CHANGED
@@ -1,23 +1,34 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bemi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2.alpha1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- exAspArk
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
12
|
-
dependencies:
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
11
|
+
date: 2023-06-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rails
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '6.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '6.0'
|
27
|
+
description: Bemi allows to reliably track data changes without using model callbacks.
|
18
28
|
email:
|
19
29
|
- exaspark@gmail.com
|
20
|
-
executables:
|
30
|
+
executables:
|
31
|
+
- bemi
|
21
32
|
extensions: []
|
22
33
|
extra_rdoc_files: []
|
23
34
|
files:
|
@@ -28,19 +39,27 @@ files:
|
|
28
39
|
- Makefile
|
29
40
|
- README.md
|
30
41
|
- Rakefile
|
42
|
+
- app/models/bemi/application_record.rb
|
43
|
+
- app/models/bemi/changeset.rb
|
44
|
+
- app/models/bemi/context.rb
|
31
45
|
- bemi.gemspec
|
46
|
+
- config/routes.rb
|
47
|
+
- exe/bemi
|
32
48
|
- lib/bemi.rb
|
49
|
+
- lib/bemi/context_handler.rb
|
50
|
+
- lib/bemi/engine.rb
|
51
|
+
- lib/bemi/ingester.rb
|
52
|
+
- lib/bemi/storage.rb
|
33
53
|
- lib/bemi/version.rb
|
54
|
+
- lib/tasks/bemi_tasks.rake
|
34
55
|
- sig/bemi.rbs
|
35
|
-
|
36
|
-
- spec/spec_helper.rb
|
37
|
-
homepage: https://github.com/exAspArk/bemi
|
56
|
+
homepage: https://github.com/bemi-io/bemi-ruby
|
38
57
|
licenses:
|
39
58
|
- MIT
|
40
59
|
metadata:
|
41
|
-
homepage_uri: https://github.com/
|
42
|
-
source_code_uri: https://github.com/
|
43
|
-
changelog_uri: https://github.com/
|
60
|
+
homepage_uri: https://github.com/bemi-io/bemi-ruby
|
61
|
+
source_code_uri: https://github.com/bemi-io/bemi-ruby
|
62
|
+
changelog_uri: https://github.com/bemi-io/bemi-ruby/blob/main/CHANGELOG.md
|
44
63
|
post_install_message:
|
45
64
|
rdoc_options: []
|
46
65
|
require_paths:
|
@@ -52,12 +71,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
52
71
|
version: 2.6.0
|
53
72
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
54
73
|
requirements:
|
55
|
-
- - "
|
74
|
+
- - ">"
|
56
75
|
- !ruby/object:Gem::Version
|
57
|
-
version:
|
76
|
+
version: 1.3.1
|
58
77
|
requirements: []
|
59
78
|
rubygems_version: 3.4.6
|
60
79
|
signing_key:
|
61
80
|
specification_version: 4
|
62
|
-
summary:
|
81
|
+
summary: Reliable audit trail for your application.
|
63
82
|
test_files: []
|
data/spec/bemi_spec.rb
DELETED
data/spec/spec_helper.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "bemi"
|
4
|
-
|
5
|
-
RSpec.configure do |config|
|
6
|
-
# Enable flags like --only-failures and --next-failure
|
7
|
-
config.example_status_persistence_file_path = ".rspec_status"
|
8
|
-
|
9
|
-
# Disable RSpec exposing methods globally on `Module` and `main`
|
10
|
-
config.disable_monkey_patching!
|
11
|
-
|
12
|
-
config.expect_with :rspec do |c|
|
13
|
-
c.syntax = :expect
|
14
|
-
end
|
15
|
-
end
|