sbmt-outbox 6.11.0 → 6.12.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9e410e7a4507b5470d10986ac8d9949c4c9460a6773a97063b0287d9e777c4d9
4
- data.tar.gz: ef17da6564faeb5f17cee5073cbecebeca083a555a9c046d0794f210f1a1b414
3
+ metadata.gz: 0b78c4c7a59f68f29949ef958639b1dd6ae9e30026ff30dd9c431f8e17551c7f
4
+ data.tar.gz: a2ead04d61fdb3f0ddeb2926f7019566e2c519d89496075c1b2c86b13890a006
5
5
  SHA512:
6
- metadata.gz: 1e5fa1c14bab5bee90e4f5c664c22cdc5e8bea8fcfca391a3eb967ddf1c3b57b96080ab147cf366e5cbc9d2add8d486b8b2ca5ebfe8c9e96d0e989ab27ae685d
7
- data.tar.gz: c47678da77b2f59787dd64cd162974b7cd4f6954fc2fafae54614b930fcf44068498491aaddae77d99b86f50dc67637b2090ae7d155a844ca193b9f011cf7189
6
+ metadata.gz: 4d57153ed214e2caceaaf46162fc045213884f7d40ada35880d47c7084bce3b67b0725422838baebc750be024a519ba14217dc28ed8db5944118d9c940bd9cfd
7
+ data.tar.gz: ec87e24359bd4aed28bbef3185e0eedb1704dbd032ce4ab12a0c321742a3eecf50d534b6e6866d53c498785cbfc547b73780c2ec7baf5c253fa4e0070856dfd2
data/README.md CHANGED
@@ -267,7 +267,8 @@ default: &default
267
267
  outbox_items: # outbox items section
268
268
  my_outbox_item: # underscored model class name
269
269
  owner: my_outbox_item_team # optional, used in Yabeda metrics
270
- retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
270
+ retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
271
+ retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
271
272
  max_retries: 3 # default 0, the number of retries before the item will be marked as failed
272
273
  strict_order: false # optional, default
273
274
  transports: # transports section
@@ -342,7 +343,8 @@ end
342
343
  inbox_items: # inbox items section
343
344
  my_inbox_item: # underscored model class name
344
345
  owner: my_inbox_item_team # optional, used in Yabeda metrics
345
- retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
346
+ retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
347
+ retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
346
348
  max_retries: 3 # default 0, the number of retries before the item will be marked as failed
347
349
  transports: # transports section
348
350
  import_order: # underscored transport class name
@@ -13,7 +13,8 @@ module Sbmt
13
13
  class << self
14
14
  def enqueue
15
15
  item_classes.each do |item_class|
16
- perform_later(item_class.to_s)
16
+ delay = rand(15).seconds
17
+ set(wait: delay).perform_later(item_class.to_s)
17
18
  end
18
19
  end
19
20
 
@@ -41,12 +42,13 @@ module Sbmt
41
42
 
42
43
  lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
43
44
  if locked
44
- duration = item_class.config.retention
45
+ duration_failed = item_class.config.retention
46
+ duration_delivered = item_class.config.retention_delivered_items
45
47
 
46
- validate_retention!(duration)
48
+ validate_retention!(duration_failed)
47
49
 
48
50
  logger.with_tags(box_type: box_type, box_name: box_name) do
49
- delete_stale_items(Time.current - duration)
51
+ delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
50
52
  end
51
53
  else
52
54
  logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
@@ -58,25 +60,25 @@ module Sbmt
58
60
 
59
61
  private
60
62
 
61
- def validate_retention!(duration)
62
- return if duration >= MIN_RETENTION_PERIOD
63
+ def validate_retention!(duration_failed)
64
+ return if duration_failed >= MIN_RETENTION_PERIOD
63
65
 
64
66
  raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
65
67
  end
66
68
 
67
- def delete_stale_items(waterline)
68
- logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{waterline}")
69
+ def delete_stale_items(waterline_failed, waterline_delivered)
70
+ logger.log_info("Start deleting #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
69
71
 
70
72
  case database_type
71
73
  when :postgresql
72
- postgres_delete_in_batches(waterline)
74
+ postgres_delete_in_batches(waterline_failed, waterline_delivered)
73
75
  when :mysql
74
- mysql_delete_in_batches(waterline)
76
+ mysql_delete_in_batches(waterline_failed, waterline_delivered)
75
77
  else
76
78
  raise "Unsupported database type"
77
79
  end
78
80
 
79
- logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
81
+ logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
80
82
  end
81
83
 
82
84
  # Deletes stale items from PostgreSQL database in batches
@@ -90,12 +92,22 @@ module Sbmt
90
92
  # WHERE "items"."id" IN (
91
93
  # SELECT "items"."id"
92
94
  # FROM "items"
93
- # WHERE "items"."created_at" < '2023-05-01 00:00:00'
95
+ # WHERE (
96
+ # "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
97
+ # )
94
98
  # LIMIT 1000
95
99
  # )
96
- def postgres_delete_in_batches(waterline)
100
+ def postgres_delete_in_batches(waterline_failed, waterline_delivered)
97
101
  table = item_class.arel_table
98
- condition = table[:created_at].lt(waterline)
102
+
103
+ status_delivered = item_class.statuses[:delivered]
104
+ status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
105
+
106
+ delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
107
+ delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
108
+ end
109
+
110
+ def delete_items_in_batches(table, condition)
99
111
  subquery = table
100
112
  .project(table[:id])
101
113
  .where(condition)
@@ -129,14 +141,25 @@ module Sbmt
129
141
  #
130
142
  # Example SQL generated for deletion:
131
143
  # DELETE FROM `items`
132
- # WHERE `items`.`created_at` < '2023-05-01 00:00:00'
144
+ # WHERE (
145
+ # `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
146
+ # )
133
147
  # LIMIT 1000
134
- def mysql_delete_in_batches(waterline)
148
+ def mysql_delete_in_batches(waterline_failed, waterline_delivered)
149
+ status_delivered = item_class.statuses[:delivered]
150
+ status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
151
+
152
+ delete_items_in_batches_mysql(
153
+ item_class.where(status: status_delivered, created_at: ...waterline_delivered)
154
+ )
155
+ delete_items_in_batches_mysql(
156
+ item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
157
+ )
158
+ end
159
+
160
+ def delete_items_in_batches_mysql(query)
135
161
  loop do
136
- deleted_count = item_class
137
- .where(created_at: ...waterline)
138
- .limit(BATCH_SIZE)
139
- .delete_all
162
+ deleted_count = query.limit(BATCH_SIZE).delete_all
140
163
 
141
164
  logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
142
165
  break if deleted_count == 0
@@ -37,6 +37,13 @@ module Sbmt
37
37
  @retention ||= ActiveSupport::Duration.parse(options[:retention] || "P1W")
38
38
  end
39
39
 
40
+ def retention_delivered_items
41
+ @retention_delivered_items ||= begin
42
+ value = options[:retention_delivered_items] || retention
43
+ value.is_a?(String) ? ActiveSupport::Duration.parse(value) : value
44
+ end
45
+ end
46
+
40
47
  def max_retries
41
48
  @max_retries ||= (options[:max_retries] || 0).to_i
42
49
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Sbmt
4
4
  module Outbox
5
- VERSION = "6.11.0"
5
+ VERSION = "6.12.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sbmt-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.11.0
4
+ version: 6.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sbermarket Ruby-Platform Team
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-12-27 00:00:00.000000000 Z
11
+ date: 2025-01-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool