sbmt-outbox 6.11.0 → 6.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb +43 -20
- data/app/models/sbmt/outbox/base_item_config.rb +7 -0
- data/lib/sbmt/outbox/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b78c4c7a59f68f29949ef958639b1dd6ae9e30026ff30dd9c431f8e17551c7f
|
4
|
+
data.tar.gz: a2ead04d61fdb3f0ddeb2926f7019566e2c519d89496075c1b2c86b13890a006
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4d57153ed214e2caceaaf46162fc045213884f7d40ada35880d47c7084bce3b67b0725422838baebc750be024a519ba14217dc28ed8db5944118d9c940bd9cfd
|
7
|
+
data.tar.gz: ec87e24359bd4aed28bbef3185e0eedb1704dbd032ce4ab12a0c321742a3eecf50d534b6e6866d53c498785cbfc547b73780c2ec7baf5c253fa4e0070856dfd2
|
data/README.md
CHANGED
@@ -267,7 +267,8 @@ default: &default
|
|
267
267
|
outbox_items: # outbox items section
|
268
268
|
my_outbox_item: # underscored model class name
|
269
269
|
owner: my_outbox_item_team # optional, used in Yabeda metrics
|
270
|
-
retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
270
|
+
retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
271
|
+
retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
271
272
|
max_retries: 3 # default 0, the number of retries before the item will be marked as failed
|
272
273
|
strict_order: false # optional, default
|
273
274
|
transports: # transports section
|
@@ -342,7 +343,8 @@ end
|
|
342
343
|
inbox_items: # inbox items section
|
343
344
|
my_inbox_item: # underscored model class name
|
344
345
|
owner: my_inbox_item_team # optional, used in Yabeda metrics
|
345
|
-
retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
346
|
+
retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
347
|
+
retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
|
346
348
|
max_retries: 3 # default 0, the number of retries before the item will be marked as failed
|
347
349
|
transports: # transports section
|
348
350
|
import_order: # underscored transport class name
|
@@ -13,7 +13,8 @@ module Sbmt
|
|
13
13
|
class << self
|
14
14
|
def enqueue
|
15
15
|
item_classes.each do |item_class|
|
16
|
-
|
16
|
+
delay = rand(15).seconds
|
17
|
+
set(wait: delay).perform_later(item_class.to_s)
|
17
18
|
end
|
18
19
|
end
|
19
20
|
|
@@ -41,12 +42,13 @@ module Sbmt
|
|
41
42
|
|
42
43
|
lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
|
43
44
|
if locked
|
44
|
-
|
45
|
+
duration_failed = item_class.config.retention
|
46
|
+
duration_delivered = item_class.config.retention_delivered_items
|
45
47
|
|
46
|
-
validate_retention!(
|
48
|
+
validate_retention!(duration_failed)
|
47
49
|
|
48
50
|
logger.with_tags(box_type: box_type, box_name: box_name) do
|
49
|
-
delete_stale_items(Time.current -
|
51
|
+
delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
|
50
52
|
end
|
51
53
|
else
|
52
54
|
logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
|
@@ -58,25 +60,25 @@ module Sbmt
|
|
58
60
|
|
59
61
|
private
|
60
62
|
|
61
|
-
def validate_retention!(
|
62
|
-
return if
|
63
|
+
def validate_retention!(duration_failed)
|
64
|
+
return if duration_failed >= MIN_RETENTION_PERIOD
|
63
65
|
|
64
66
|
raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
|
65
67
|
end
|
66
68
|
|
67
|
-
def delete_stale_items(
|
68
|
-
logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{
|
69
|
+
def delete_stale_items(waterline_failed, waterline_delivered)
|
70
|
+
logger.log_info("Start deleting #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
|
69
71
|
|
70
72
|
case database_type
|
71
73
|
when :postgresql
|
72
|
-
postgres_delete_in_batches(
|
74
|
+
postgres_delete_in_batches(waterline_failed, waterline_delivered)
|
73
75
|
when :mysql
|
74
|
-
mysql_delete_in_batches(
|
76
|
+
mysql_delete_in_batches(waterline_failed, waterline_delivered)
|
75
77
|
else
|
76
78
|
raise "Unsupported database type"
|
77
79
|
end
|
78
80
|
|
79
|
-
logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{
|
81
|
+
logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
|
80
82
|
end
|
81
83
|
|
82
84
|
# Deletes stale items from PostgreSQL database in batches
|
@@ -90,12 +92,22 @@ module Sbmt
|
|
90
92
|
# WHERE "items"."id" IN (
|
91
93
|
# SELECT "items"."id"
|
92
94
|
# FROM "items"
|
93
|
-
# WHERE
|
95
|
+
# WHERE (
|
96
|
+
# "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
|
97
|
+
# )
|
94
98
|
# LIMIT 1000
|
95
99
|
# )
|
96
|
-
def postgres_delete_in_batches(
|
100
|
+
def postgres_delete_in_batches(waterline_failed, waterline_delivered)
|
97
101
|
table = item_class.arel_table
|
98
|
-
|
102
|
+
|
103
|
+
status_delivered = item_class.statuses[:delivered]
|
104
|
+
status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
|
105
|
+
|
106
|
+
delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
|
107
|
+
delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
|
108
|
+
end
|
109
|
+
|
110
|
+
def delete_items_in_batches(table, condition)
|
99
111
|
subquery = table
|
100
112
|
.project(table[:id])
|
101
113
|
.where(condition)
|
@@ -129,14 +141,25 @@ module Sbmt
|
|
129
141
|
#
|
130
142
|
# Example SQL generated for deletion:
|
131
143
|
# DELETE FROM `items`
|
132
|
-
# WHERE
|
144
|
+
# WHERE (
|
145
|
+
# `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
|
146
|
+
# )
|
133
147
|
# LIMIT 1000
|
134
|
-
def mysql_delete_in_batches(
|
148
|
+
def mysql_delete_in_batches(waterline_failed, waterline_delivered)
|
149
|
+
status_delivered = item_class.statuses[:delivered]
|
150
|
+
status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
|
151
|
+
|
152
|
+
delete_items_in_batches_mysql(
|
153
|
+
item_class.where(status: status_delivered, created_at: ...waterline_delivered)
|
154
|
+
)
|
155
|
+
delete_items_in_batches_mysql(
|
156
|
+
item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
|
157
|
+
)
|
158
|
+
end
|
159
|
+
|
160
|
+
def delete_items_in_batches_mysql(query)
|
135
161
|
loop do
|
136
|
-
deleted_count =
|
137
|
-
.where(created_at: ...waterline)
|
138
|
-
.limit(BATCH_SIZE)
|
139
|
-
.delete_all
|
162
|
+
deleted_count = query.limit(BATCH_SIZE).delete_all
|
140
163
|
|
141
164
|
logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
|
142
165
|
break if deleted_count == 0
|
@@ -37,6 +37,13 @@ module Sbmt
|
|
37
37
|
@retention ||= ActiveSupport::Duration.parse(options[:retention] || "P1W")
|
38
38
|
end
|
39
39
|
|
40
|
+
def retention_delivered_items
|
41
|
+
@retention_delivered_items ||= begin
|
42
|
+
value = options[:retention_delivered_items] || retention
|
43
|
+
value.is_a?(String) ? ActiveSupport::Duration.parse(value) : value
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
40
47
|
def max_retries
|
41
48
|
@max_retries ||= (options[:max_retries] || 0).to_i
|
42
49
|
end
|
data/lib/sbmt/outbox/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sbmt-outbox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sbermarket Ruby-Platform Team
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-01-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: connection_pool
|