codex-notify 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +50 -0
- data/LICENSE +21 -0
- data/README.md +782 -0
- data/bin/codex-notify +6 -0
- data/bin/codex-notify-hook +6 -0
- data/lib/codex_notify/cli.rb +112 -0
- data/lib/codex_notify/config.rb +113 -0
- data/lib/codex_notify/config_diagnostics.rb +61 -0
- data/lib/codex_notify/config_migrator.rb +148 -0
- data/lib/codex_notify/config_support.rb +157 -0
- data/lib/codex_notify/destination_name.rb +18 -0
- data/lib/codex_notify/destination_resolver.rb +68 -0
- data/lib/codex_notify/durable_slack_publisher.rb +70 -0
- data/lib/codex_notify/env_source_loader.rb +91 -0
- data/lib/codex_notify/hook_cli.rb +90 -0
- data/lib/codex_notify/hook_config.rb +110 -0
- data/lib/codex_notify/hook_event.rb +16 -0
- data/lib/codex_notify/hook_formatter.rb +61 -0
- data/lib/codex_notify/hook_input_validator.rb +241 -0
- data/lib/codex_notify/hook_runner.rb +181 -0
- data/lib/codex_notify/hook_store.rb +133 -0
- data/lib/codex_notify/hook_thread_publisher.rb +107 -0
- data/lib/codex_notify/log_event_parser.rb +230 -0
- data/lib/codex_notify/message_formatter.rb +115 -0
- data/lib/codex_notify/outbox_commands.rb +43 -0
- data/lib/codex_notify/secret_protection.rb +37 -0
- data/lib/codex_notify/session_log.rb +63 -0
- data/lib/codex_notify/slack_client.rb +117 -0
- data/lib/codex_notify/slack_delivery_worker.rb +269 -0
- data/lib/codex_notify/slack_outbox.rb +206 -0
- data/lib/codex_notify/stream_processor.rb +141 -0
- data/lib/codex_notify/trusted_config_loader.rb +159 -0
- data/lib/codex_notify/version.rb +5 -0
- data/lib/codex_notify.rb +36 -0
- metadata +96 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CodexNotify
|
|
4
|
+
module MessageFormatter
|
|
5
|
+
SLACK_SAFE_LENGTH = 3500
|
|
6
|
+
CONTINUATION_LABEL = '(cont.)'
|
|
7
|
+
PRESENTATIONS = %i[plain block].freeze
|
|
8
|
+
Message = Data.define(:title, :body, :presentation)
|
|
9
|
+
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def message(title:, body:, presentation:)
|
|
13
|
+
unless PRESENTATIONS.include?(presentation)
|
|
14
|
+
raise ArgumentError, "unsupported message presentation: #{presentation.inspect}"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
Message.new(
|
|
18
|
+
title: title.to_s.dup.freeze,
|
|
19
|
+
body: body.to_s.dup.freeze,
|
|
20
|
+
presentation:
|
|
21
|
+
)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def chunks(message, max_length: SLACK_SAFE_LENGTH)
|
|
25
|
+
return enum_for(__method__, message, max_length:) unless block_given?
|
|
26
|
+
|
|
27
|
+
validate_max_length(max_length, message.presentation)
|
|
28
|
+
body = message.body.gsub("\r\n", "\n")
|
|
29
|
+
index = 0
|
|
30
|
+
chunk_index = 0
|
|
31
|
+
|
|
32
|
+
loop do
|
|
33
|
+
title = fit_title(
|
|
34
|
+
message.title,
|
|
35
|
+
message.presentation,
|
|
36
|
+
max_length,
|
|
37
|
+
body.empty?,
|
|
38
|
+
continuation: chunk_index.positive?
|
|
39
|
+
)
|
|
40
|
+
capacity = max_length - render(title, '', message.presentation).length
|
|
41
|
+
part = body[index, capacity] || ''
|
|
42
|
+
yield render(title, part, message.presentation)
|
|
43
|
+
|
|
44
|
+
index += part.length
|
|
45
|
+
break if index >= body.length
|
|
46
|
+
|
|
47
|
+
chunk_index += 1
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def fmt_block(title, body)
|
|
52
|
+
render(title.to_s, body.to_s, :block)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def build_root_message(title, cwd, user_name: 'user', session_id: nil)
|
|
56
|
+
body = root_body(cwd, user_name:, session_id:)
|
|
57
|
+
message(title:, body:, presentation: :block)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def build_root_text(title, cwd, user_name: 'user', session_id: nil)
|
|
61
|
+
fmt_block(title, root_body(cwd, user_name:, session_id:))
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def fmt_plain(title, body)
|
|
65
|
+
render(title.to_s, body.to_s, :plain)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def root_body(cwd, user_name:, session_id:)
|
|
69
|
+
[
|
|
70
|
+
'Codex log monitoring started.',
|
|
71
|
+
"CWD: #{cwd}",
|
|
72
|
+
"User: #{user_name}",
|
|
73
|
+
"Session ID: #{session_id || 'unknown'}"
|
|
74
|
+
].join("\n")
|
|
75
|
+
end
|
|
76
|
+
private_class_method :root_body
|
|
77
|
+
|
|
78
|
+
def render(title, body, presentation)
|
|
79
|
+
case presentation
|
|
80
|
+
when :plain then "*#{title}*\n#{body}"
|
|
81
|
+
when :block then "*#{title}*\n```#{body}```"
|
|
82
|
+
else raise ArgumentError, "unsupported message presentation: #{presentation.inspect}"
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
private_class_method :render
|
|
86
|
+
|
|
87
|
+
def validate_max_length(max_length, presentation)
|
|
88
|
+
minimum_length = render('', '', presentation).length + 1
|
|
89
|
+
return if max_length >= minimum_length
|
|
90
|
+
|
|
91
|
+
raise ArgumentError, "max_length must be at least #{minimum_length} for #{presentation} messages"
|
|
92
|
+
end
|
|
93
|
+
private_class_method :validate_max_length
|
|
94
|
+
|
|
95
|
+
def fit_title(title, presentation, max_length, empty_body, continuation:)
|
|
96
|
+
body_reserve = empty_body ? 0 : 1
|
|
97
|
+
max_title_length = max_length - render('', '', presentation).length - body_reserve
|
|
98
|
+
return truncate(title, max_title_length) unless continuation
|
|
99
|
+
return truncate(CONTINUATION_LABEL, max_title_length) if max_title_length <= CONTINUATION_LABEL.length
|
|
100
|
+
|
|
101
|
+
base = truncate(title, max_title_length - CONTINUATION_LABEL.length - 1)
|
|
102
|
+
base.empty? ? CONTINUATION_LABEL : "#{base} #{CONTINUATION_LABEL}"
|
|
103
|
+
end
|
|
104
|
+
private_class_method :fit_title
|
|
105
|
+
|
|
106
|
+
def truncate(text, max_length)
|
|
107
|
+
return '' if max_length <= 0
|
|
108
|
+
return text if text.length <= max_length
|
|
109
|
+
return '…' if max_length == 1
|
|
110
|
+
|
|
111
|
+
"#{text[0, max_length - 1]}…"
|
|
112
|
+
end
|
|
113
|
+
private_class_method :truncate
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'pathname'
|
|
5
|
+
require_relative 'hook_store'
|
|
6
|
+
require_relative 'slack_client'
|
|
7
|
+
require_relative 'slack_delivery_worker'
|
|
8
|
+
require_relative 'slack_outbox'
|
|
9
|
+
|
|
10
|
+
module CodexNotify
|
|
11
|
+
module OutboxCommands
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def run(action:, id:, outbox_dir:, token:, channel:, state_file:, stdout:, stderr:)
|
|
15
|
+
outbox = SlackOutbox.new(outbox_dir)
|
|
16
|
+
case action
|
|
17
|
+
when :status
|
|
18
|
+
outbox.status_rows.each { |row| stdout.puts(JSON.generate(row)) }
|
|
19
|
+
0
|
|
20
|
+
when :retry
|
|
21
|
+
outbox.retry(id)
|
|
22
|
+
0
|
|
23
|
+
when :drain
|
|
24
|
+
unless token && channel
|
|
25
|
+
stderr.puts('ERROR: draining the outbox requires Slack token and channel configuration')
|
|
26
|
+
return 2
|
|
27
|
+
end
|
|
28
|
+
store = HookStore.new(state_file)
|
|
29
|
+
result = SlackDeliveryWorker.new(
|
|
30
|
+
outbox:,
|
|
31
|
+
client: SlackClient.new(token:, channel:),
|
|
32
|
+
store:
|
|
33
|
+
).drain(channel:)
|
|
34
|
+
failed = result.failed.any? || result.needs_review.any?
|
|
35
|
+
stderr.puts('ERROR: Slack delivery requires outbox review; run --outbox-status') if failed
|
|
36
|
+
failed ? 1 : 0
|
|
37
|
+
end
|
|
38
|
+
rescue SlackOutbox::Error => e
|
|
39
|
+
stderr.puts("ERROR: #{e.message}")
|
|
40
|
+
1
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CodexNotify
|
|
4
|
+
module SecretProtection
|
|
5
|
+
REDACTED = '[REDACTED]'
|
|
6
|
+
SENSITIVE_NAME = /(?:authorization|proxy[-_]?authorization|token|secret|password|passwd|api[-_]?key)/i
|
|
7
|
+
KNOWN_SECRET_PATTERNS = [
|
|
8
|
+
/\bxox[baprs]-[A-Za-z0-9][A-Za-z0-9-]*\b/,
|
|
9
|
+
/\bxapp-[A-Za-z0-9][A-Za-z0-9-]*\b/,
|
|
10
|
+
/\bgh[pousr]_[A-Za-z0-9]{20,}\b/,
|
|
11
|
+
/\bsk-[A-Za-z0-9_-]{16,}\b/,
|
|
12
|
+
/\bAKIA[0-9A-Z]{16}\b/
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
module_function
|
|
16
|
+
|
|
17
|
+
def redact(value)
|
|
18
|
+
text = value.to_s.dup
|
|
19
|
+
|
|
20
|
+
text.gsub!(/(\b(?:[A-Za-z_][A-Za-z0-9_.-]*)?(?:proxy[-_]?authorization|authorization)[A-Za-z0-9_.-]*\s*[=:]\s*)(?:(?:Bearer|Basic)\s+)?[^\s'",;}]+/i) do
|
|
21
|
+
"#{Regexp.last_match(1)}#{REDACTED}"
|
|
22
|
+
end
|
|
23
|
+
text.gsub!(/(--(?:token|password|passwd|secret|api[-_]?key)(?:=|\s+))(?:"[^"]*"|'[^']*'|[^\s]+)/i) do
|
|
24
|
+
"#{Regexp.last_match(1)}#{REDACTED}"
|
|
25
|
+
end
|
|
26
|
+
text.gsub!(/("[^"]*#{SENSITIVE_NAME.source}[^"]*"\s*:\s*)(?:"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|[^\s,}\]]+)/i) do
|
|
27
|
+
"#{Regexp.last_match(1)}#{REDACTED}"
|
|
28
|
+
end
|
|
29
|
+
text.gsub!(/(\b(?:[A-Za-z_][A-Za-z0-9_.-]*)?#{SENSITIVE_NAME.source}[A-Za-z0-9_.-]*\s*[=:]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}]+)/i) do
|
|
30
|
+
"#{Regexp.last_match(1)}#{REDACTED}"
|
|
31
|
+
end
|
|
32
|
+
KNOWN_SECRET_PATTERNS.each { |pattern| text.gsub!(pattern, REDACTED) }
|
|
33
|
+
|
|
34
|
+
text
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'pathname'
|
|
5
|
+
|
|
6
|
+
module CodexNotify
|
|
7
|
+
module SessionLog
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def iter_follow_lines(path, poll_sec: 1.0, once: false, start_at_end: true, sleep_func: Kernel.method(:sleep))
|
|
11
|
+
return enum_for(__method__, path, poll_sec:, once:, start_at_end:, sleep_func:) unless block_given?
|
|
12
|
+
|
|
13
|
+
Pathname(path).open('r:utf-8') do |handle|
|
|
14
|
+
handle.seek(0, IO::SEEK_END) if start_at_end
|
|
15
|
+
loop do
|
|
16
|
+
line = handle.gets
|
|
17
|
+
if line
|
|
18
|
+
yield line
|
|
19
|
+
next
|
|
20
|
+
end
|
|
21
|
+
break if once
|
|
22
|
+
|
|
23
|
+
sleep_func.call(poll_sec)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def find_latest_session_file(sessions_dir)
|
|
29
|
+
root = Pathname(sessions_dir)
|
|
30
|
+
return nil unless root.exist?
|
|
31
|
+
|
|
32
|
+
files = root.glob('**/*.jsonl').select(&:file?)
|
|
33
|
+
return nil if files.empty?
|
|
34
|
+
|
|
35
|
+
files.max_by { |path| path.stat.mtime.to_f }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def session_id_from_log(path)
|
|
39
|
+
pathname = Pathname(path)
|
|
40
|
+
return nil unless pathname.exist?
|
|
41
|
+
|
|
42
|
+
pathname.open('r:utf-8') do |handle|
|
|
43
|
+
handle.each_line do |raw|
|
|
44
|
+
line = raw.strip
|
|
45
|
+
next if line.empty?
|
|
46
|
+
|
|
47
|
+
event = JSON.parse(line)
|
|
48
|
+
session_id = CodexNotify::LogEventParser.extract_session_id(event)
|
|
49
|
+
return session_id if session_id
|
|
50
|
+
rescue JSON::ParserError
|
|
51
|
+
next
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def session_id_from_path(path)
|
|
59
|
+
pathname = Pathname(path)
|
|
60
|
+
pathname.basename('.jsonl').to_s
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'net/http'
|
|
5
|
+
require 'uri'
|
|
6
|
+
require_relative 'secret_protection'
|
|
7
|
+
|
|
8
|
+
module CodexNotify
|
|
9
|
+
class SlackClient
|
|
10
|
+
class Error < StandardError
|
|
11
|
+
attr_reader :response, :error_code, :http_status, :retry_after
|
|
12
|
+
|
|
13
|
+
def initialize(message, response: nil, error_code: nil, http_status: nil, retry_after: nil,
|
|
14
|
+
retryable: false, ambiguous: false, stale_thread: false)
|
|
15
|
+
super(message)
|
|
16
|
+
@response = response
|
|
17
|
+
@error_code = error_code
|
|
18
|
+
@http_status = http_status
|
|
19
|
+
@retry_after = retry_after
|
|
20
|
+
@retryable = retryable
|
|
21
|
+
@ambiguous = ambiguous
|
|
22
|
+
@stale_thread = stale_thread
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def retryable? = @retryable
|
|
26
|
+
def ambiguous? = @ambiguous
|
|
27
|
+
def stale_thread? = @stale_thread
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
SLACK_API = 'https://slack.com/api/chat.postMessage'
|
|
31
|
+
OPEN_TIMEOUT = 5
|
|
32
|
+
WRITE_TIMEOUT = 10
|
|
33
|
+
READ_TIMEOUT = 20
|
|
34
|
+
RETRYABLE_CODES = %w[ratelimited rate_limited internal_error fatal_error service_unavailable request_timeout].freeze
|
|
35
|
+
AMBIGUOUS_CODES = %w[internal_error fatal_error service_unavailable request_timeout].freeze
|
|
36
|
+
STALE_THREAD_CODES = %w[thread_not_found message_not_found invalid_ts].freeze
|
|
37
|
+
|
|
38
|
+
def initialize(token:, channel:)
|
|
39
|
+
@token = token
|
|
40
|
+
@channel = channel
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def post(text, thread_ts: nil)
|
|
44
|
+
payload = { channel: @channel, text: SecretProtection.redact(text) }
|
|
45
|
+
payload[:thread_ts] = thread_ts.to_s if thread_ts
|
|
46
|
+
|
|
47
|
+
uri = URI(SLACK_API)
|
|
48
|
+
request = Net::HTTP::Post.new(uri)
|
|
49
|
+
request['Content-Type'] = 'application/json; charset=utf-8'
|
|
50
|
+
request['Authorization'] = "Bearer #{@token}"
|
|
51
|
+
request.body = JSON.generate(payload)
|
|
52
|
+
|
|
53
|
+
request_started = false
|
|
54
|
+
response = Net::HTTP.start(
|
|
55
|
+
uri.hostname,
|
|
56
|
+
uri.port,
|
|
57
|
+
use_ssl: true,
|
|
58
|
+
open_timeout: OPEN_TIMEOUT,
|
|
59
|
+
write_timeout: WRITE_TIMEOUT,
|
|
60
|
+
read_timeout: READ_TIMEOUT
|
|
61
|
+
) do |http|
|
|
62
|
+
request_started = true
|
|
63
|
+
http.request(request)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
status = response.respond_to?(:code) ? response.code.to_i : 200
|
|
67
|
+
retry_after = parse_retry_after(response_header(response, 'Retry-After'))
|
|
68
|
+
if status == 429
|
|
69
|
+
raise build_error('Slack rate limited request', http_status: status, retry_after:, retryable: true)
|
|
70
|
+
end
|
|
71
|
+
if status >= 500
|
|
72
|
+
raise build_error('Slack server error', http_status: status, retryable: true, ambiguous: true)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
parsed = JSON.parse(response.body.to_s)
|
|
76
|
+
unless parsed['ok']
|
|
77
|
+
code = parsed['error'].to_s
|
|
78
|
+
raise build_error(
|
|
79
|
+
"Slack API error: #{code.empty? ? 'unknown' : code}",
|
|
80
|
+
response: parsed,
|
|
81
|
+
error_code: code,
|
|
82
|
+
retry_after:,
|
|
83
|
+
retryable: RETRYABLE_CODES.include?(code),
|
|
84
|
+
ambiguous: AMBIGUOUS_CODES.include?(code),
|
|
85
|
+
stale_thread: STALE_THREAD_CODES.include?(code)
|
|
86
|
+
)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
parsed
|
|
90
|
+
rescue Net::OpenTimeout, SocketError => e
|
|
91
|
+
raise build_error("Slack connection failed: #{e.class}", retryable: true)
|
|
92
|
+
rescue OpenSSL::SSL::SSLError, SystemCallError => e
|
|
93
|
+
raise build_error("Slack connection failed: #{e.class}", retryable: true, ambiguous: request_started)
|
|
94
|
+
rescue Net::WriteTimeout, Net::ReadTimeout, EOFError, Errno::ECONNRESET, Errno::EPIPE => e
|
|
95
|
+
raise build_error("Slack request outcome is unknown: #{e.class}", retryable: true, ambiguous: request_started)
|
|
96
|
+
rescue JSON::ParserError
|
|
97
|
+
raise build_error('Slack returned an invalid JSON response', retryable: true, ambiguous: true)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def build_error(message, **attributes)
|
|
103
|
+
Error.new(message, **attributes)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def parse_retry_after(value)
|
|
107
|
+
seconds = Integer(value, exception: false)
|
|
108
|
+
seconds if seconds && seconds >= 0
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def response_header(response, name)
|
|
112
|
+
response[name] if response.respond_to?(:[])
|
|
113
|
+
rescue NameError, IndexError
|
|
114
|
+
nil
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'time'
|
|
4
|
+
require_relative 'slack_client'
|
|
5
|
+
|
|
6
|
+
module CodexNotify
|
|
7
|
+
class SlackDeliveryWorker
|
|
8
|
+
STALE_THREAD_CODES = %w[thread_not_found message_not_found invalid_ts].freeze
|
|
9
|
+
DRAIN_BUDGET = 10.0
|
|
10
|
+
MAX_IMMEDIATE_ATTEMPTS = 3
|
|
11
|
+
MAX_AMBIGUOUS_ATTEMPTS = 3
|
|
12
|
+
BACKOFF_CAP = 8.0
|
|
13
|
+
|
|
14
|
+
Result = Data.define(:delivered, :deferred, :failed, :needs_review)
|
|
15
|
+
|
|
16
|
+
def initialize(outbox:, client:, store:, clock: -> { Time.now.utc }, monotonic_clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
|
|
17
|
+
sleeper: Kernel.method(:sleep), random: Random.new, inter_message_delay: 0.0)
|
|
18
|
+
@outbox = outbox
|
|
19
|
+
@client = client
|
|
20
|
+
@store = store
|
|
21
|
+
@clock = clock
|
|
22
|
+
@monotonic_clock = monotonic_clock
|
|
23
|
+
@sleeper = sleeper
|
|
24
|
+
@random = random
|
|
25
|
+
@inter_message_delay = inter_message_delay
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def drain(channel:, budget: DRAIN_BUDGET)
|
|
29
|
+
counts = { delivered: [], deferred: [], failed: [], needs_review: [] }
|
|
30
|
+
locked = @outbox.try_drain_lock do
|
|
31
|
+
deadline = monotonic_now + budget
|
|
32
|
+
drain_jobs(channel.to_s, deadline, counts)
|
|
33
|
+
end
|
|
34
|
+
return Result.new(**counts) if locked
|
|
35
|
+
|
|
36
|
+
counts[:deferred] = @outbox.jobs.select { |job| job['channel'] == channel.to_s }.map { |job| job['id'] }
|
|
37
|
+
Result.new(**counts)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def drain_jobs(channel, deadline, counts)
|
|
43
|
+
immediate_attempts = Hash.new(0)
|
|
44
|
+
loop do
|
|
45
|
+
job = next_eligible_job(channel)
|
|
46
|
+
break unless job
|
|
47
|
+
break if monotonic_now >= deadline
|
|
48
|
+
|
|
49
|
+
outcome = deliver_with_lock(job)
|
|
50
|
+
if outcome == :delivered
|
|
51
|
+
@outbox.complete(job)
|
|
52
|
+
counts[:delivered] << job['id']
|
|
53
|
+
next
|
|
54
|
+
end
|
|
55
|
+
rescue SlackClient::Error => e
|
|
56
|
+
immediate_attempts[job['id']] += 1
|
|
57
|
+
disposition = handle_delivery_error(job, e, immediate_attempts[job['id']], deadline)
|
|
58
|
+
next if disposition == :retry_now
|
|
59
|
+
|
|
60
|
+
counts[disposition] << job['id'] unless counts[disposition].include?(job['id'])
|
|
61
|
+
break if disposition == :deferred && blocking_head?(job)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def next_eligible_job(channel)
|
|
66
|
+
blocked = (@outbox.jobs(:failed) + @outbox.jobs(:needs_review)).map { |job| job['ordering_key'] }.uniq
|
|
67
|
+
seen = {}
|
|
68
|
+
@outbox.jobs.find do |job|
|
|
69
|
+
key = job['ordering_key']
|
|
70
|
+
next false if seen[key]
|
|
71
|
+
|
|
72
|
+
seen[key] = true
|
|
73
|
+
next false if blocked.include?(key) || job['channel'] != channel
|
|
74
|
+
|
|
75
|
+
due?(job)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def blocking_head?(job)
|
|
80
|
+
@outbox.jobs.none? do |candidate|
|
|
81
|
+
candidate['channel'] == job['channel'] && candidate['ordering_key'] != job['ordering_key'] && due?(candidate)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def due?(job)
|
|
86
|
+
value = job['next_attempt_at']
|
|
87
|
+
value.nil? || Time.iso8601(value) <= now
|
|
88
|
+
rescue ArgumentError
|
|
89
|
+
true
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def deliver_with_lock(job)
|
|
93
|
+
@store.with_session_lock(job['ordering_key']) do
|
|
94
|
+
current = @outbox.jobs.find { |candidate| candidate['id'] == job['id'] }
|
|
95
|
+
return :delivered unless current
|
|
96
|
+
|
|
97
|
+
job.replace(current)
|
|
98
|
+
if @store.generation_for(job['ordering_key']) != job['generation'].to_i
|
|
99
|
+
return :delivered
|
|
100
|
+
end
|
|
101
|
+
if job['resolved_thread_ts'] && job['action'] != 'standalone'
|
|
102
|
+
@store.save_thread_ts(job['ordering_key'], job['resolved_thread_ts'])
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
deliver(job)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def deliver(job)
|
|
110
|
+
case job['phase']
|
|
111
|
+
when 'pending'
|
|
112
|
+
start_job(job)
|
|
113
|
+
when 'posting_root', 'posting_recovery_root'
|
|
114
|
+
post_root(job)
|
|
115
|
+
when 'posting_root_remainder', 'posting_recovery_remainder'
|
|
116
|
+
post_root_remainder(job)
|
|
117
|
+
when 'posting_message'
|
|
118
|
+
post_message(job)
|
|
119
|
+
else
|
|
120
|
+
raise SlackOutbox::Error, "unknown outbox phase: #{job['phase']}"
|
|
121
|
+
end
|
|
122
|
+
rescue SlackClient::Error => e
|
|
123
|
+
stale = e.stale_thread? || STALE_THREAD_CODES.include?(e.error_code)
|
|
124
|
+
already_recovering = job['phase'].to_s.include?('recovery')
|
|
125
|
+
root_not_created = job['phase'] == 'posting_root'
|
|
126
|
+
raise unless stale && job['action'] != 'standalone' && !already_recovering && !root_not_created
|
|
127
|
+
|
|
128
|
+
@store.clear_thread(job['ordering_key'])
|
|
129
|
+
job['phase'] = 'posting_recovery_root'
|
|
130
|
+
job['next_chunk'] = 0
|
|
131
|
+
job['resolved_thread_ts'] = nil
|
|
132
|
+
@outbox.update(job)
|
|
133
|
+
deliver(job)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def start_job(job)
|
|
137
|
+
case job['action']
|
|
138
|
+
when 'standalone'
|
|
139
|
+
transition(job, 'posting_root')
|
|
140
|
+
when 'ensure_thread', 'root_or_reply'
|
|
141
|
+
if (thread_ts = @store.thread_ts_for(job['ordering_key']))
|
|
142
|
+
return :delivered if job['action'] == 'ensure_thread'
|
|
143
|
+
|
|
144
|
+
job['resolved_thread_ts'] = thread_ts
|
|
145
|
+
transition(job, 'posting_message')
|
|
146
|
+
else
|
|
147
|
+
transition(job, 'posting_root')
|
|
148
|
+
end
|
|
149
|
+
when 'reply'
|
|
150
|
+
thread_ts = @store.thread_ts_for(job['ordering_key'])
|
|
151
|
+
return :delivered unless thread_ts
|
|
152
|
+
|
|
153
|
+
job['resolved_thread_ts'] = thread_ts
|
|
154
|
+
transition(job, 'posting_message')
|
|
155
|
+
else
|
|
156
|
+
raise SlackOutbox::Error, "unknown outbox action: #{job['action']}"
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def post_root(job)
|
|
161
|
+
chunks = root_chunks(job)
|
|
162
|
+
return finish_root(job) if chunks.empty?
|
|
163
|
+
|
|
164
|
+
response = post(chunks.fetch(0))
|
|
165
|
+
job['resolved_thread_ts'] = response.fetch('ts').to_s
|
|
166
|
+
job['next_chunk'] = 1
|
|
167
|
+
job['phase'] = job['phase'] == 'posting_recovery_root' ? 'posting_recovery_remainder' : 'posting_root_remainder'
|
|
168
|
+
@outbox.update(job)
|
|
169
|
+
@store.save_thread_ts(job['ordering_key'], job['resolved_thread_ts']) unless job['action'] == 'standalone'
|
|
170
|
+
deliver(job)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def post_root_remainder(job)
|
|
174
|
+
chunks = root_chunks(job)
|
|
175
|
+
while job['next_chunk'] < chunks.length
|
|
176
|
+
post(chunks.fetch(job['next_chunk']), thread_ts: job['resolved_thread_ts'])
|
|
177
|
+
job['next_chunk'] += 1
|
|
178
|
+
@outbox.update(job)
|
|
179
|
+
end
|
|
180
|
+
finish_root(job)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def finish_root(job)
|
|
184
|
+
if job['phase'].to_s.include?('recovery') && job['message_chunks'] != job['recovery_root_chunks']
|
|
185
|
+
job['next_chunk'] = 0
|
|
186
|
+
transition(job, 'posting_message')
|
|
187
|
+
else
|
|
188
|
+
:delivered
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def post_message(job)
|
|
193
|
+
chunks = job['message_chunks']
|
|
194
|
+
while job['next_chunk'] < chunks.length
|
|
195
|
+
post(chunks.fetch(job['next_chunk']), thread_ts: job['resolved_thread_ts'])
|
|
196
|
+
job['next_chunk'] += 1
|
|
197
|
+
@outbox.update(job)
|
|
198
|
+
end
|
|
199
|
+
:delivered
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def root_chunks(job)
|
|
203
|
+
job['phase'].to_s.include?('recovery') ? job['recovery_root_chunks'] : job['message_chunks']
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def transition(job, phase)
|
|
207
|
+
job['phase'] = phase
|
|
208
|
+
job['next_chunk'] = 0
|
|
209
|
+
@outbox.update(job)
|
|
210
|
+
deliver(job)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def post(text, thread_ts: nil)
|
|
214
|
+
response = @client.post(text, thread_ts:)
|
|
215
|
+
@sleeper.call(@inter_message_delay) if @inter_message_delay.positive?
|
|
216
|
+
response
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def handle_delivery_error(job, error, immediate_attempt, deadline)
|
|
220
|
+
job['attempt_count'] += 1
|
|
221
|
+
job['last_error'] = {
|
|
222
|
+
'code' => error.error_code,
|
|
223
|
+
'http_status' => error.http_status,
|
|
224
|
+
'ambiguous' => error.ambiguous?
|
|
225
|
+
}.compact
|
|
226
|
+
|
|
227
|
+
unless error.retryable?
|
|
228
|
+
@outbox.update(job)
|
|
229
|
+
@outbox.move(job, :failed)
|
|
230
|
+
return :failed
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
if error.ambiguous?
|
|
234
|
+
job['ambiguous_attempt_count'] += 1
|
|
235
|
+
@outbox.update(job)
|
|
236
|
+
if job['ambiguous_attempt_count'] >= MAX_AMBIGUOUS_ATTEMPTS
|
|
237
|
+
@outbox.move(job, :needs_review)
|
|
238
|
+
return :needs_review
|
|
239
|
+
end
|
|
240
|
+
schedule(job, backoff(job['attempt_count']))
|
|
241
|
+
return :deferred
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
delay = error.retry_after || backoff(job['attempt_count'])
|
|
245
|
+
if immediate_attempt < MAX_IMMEDIATE_ATTEMPTS && monotonic_now + delay < deadline
|
|
246
|
+
@sleeper.call(delay)
|
|
247
|
+
job['next_attempt_at'] = nil
|
|
248
|
+
@outbox.update(job)
|
|
249
|
+
return :retry_now
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
schedule(job, delay)
|
|
253
|
+
:deferred
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def schedule(job, delay)
|
|
257
|
+
job['next_attempt_at'] = (now + delay).iso8601
|
|
258
|
+
@outbox.update(job)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def backoff(attempt)
|
|
262
|
+
cap = [0.5 * (2**([attempt - 1, 0].max)), BACKOFF_CAP].min
|
|
263
|
+
@random.rand * cap
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def now = @clock.call.utc
|
|
267
|
+
def monotonic_now = @monotonic_clock.call
|
|
268
|
+
end
|
|
269
|
+
end
|