prdigest 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/README.md +331 -149
- data/SECURITY.md +35 -1
- data/configs/config.example.yml +11 -11
- data/lib/prdigest/cli.rb +176 -62
- data/lib/prdigest/collector.rb +38 -0
- data/lib/prdigest/config.rb +96 -29
- data/lib/prdigest/delivery_checkpoint_store.rb +17 -4
- data/lib/prdigest/facts.rb +76 -0
- data/lib/prdigest/facts_runner.rb +32 -0
- data/lib/prdigest/openai_compatible.rb +172 -0
- data/lib/prdigest/prose_renderer.rb +33 -0
- data/lib/prdigest/prose_runner.rb +102 -0
- data/lib/prdigest/telegram.rb +3 -3
- data/lib/prdigest/version.rb +1 -1
- data/lib/prdigest.rb +14 -5
- data/scripts/systemd/prdigest.service +1 -1
- metadata +10 -9
- data/lib/prdigest/renderer.rb +0 -134
- data/lib/prdigest/result.rb +0 -82
- data/lib/prdigest/runner.rb +0 -173
- data/lib/prdigest/schedule.rb +0 -30
- data/lib/prdigest/state.rb +0 -147
data/lib/prdigest/result.rb
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Prdigest
|
|
4
|
-
class Result
|
|
5
|
-
SCHEMA = "prdigest-result"
|
|
6
|
-
SCHEMA_VERSION = 1
|
|
7
|
-
REQUIRED_FIELDS = %i[
|
|
8
|
-
status mode requested_days settled_days skipped_days failed_date remaining_days error
|
|
9
|
-
].freeze
|
|
10
|
-
EXIT_CODES = {
|
|
11
|
-
"config" => 2, "cli" => 2, "refusal" => 2,
|
|
12
|
-
"github" => 3,
|
|
13
|
-
"telegram" => 4, "telegram_refused" => 4, "telegram_permanent" => 4,
|
|
14
|
-
"telegram_ambiguous" => 4, "delivery_checkpoint" => 4,
|
|
15
|
-
"delivery_checkpoint_permanent" => 4,
|
|
16
|
-
"state" => 5
|
|
17
|
-
}.freeze
|
|
18
|
-
|
|
19
|
-
attr_reader(*REQUIRED_FIELDS, :chunks, :delivery)
|
|
20
|
-
|
|
21
|
-
def initialize(status:, mode:, requested_days: [], settled_days: [], skipped_days: [],
|
|
22
|
-
failed_date: nil, remaining_days: [], error: nil, chunks: [], delivery: nil)
|
|
23
|
-
@status = status.to_s.freeze
|
|
24
|
-
@mode = mode.to_s.freeze
|
|
25
|
-
@requested_days = Array(requested_days).freeze
|
|
26
|
-
@settled_days = Array(settled_days).freeze
|
|
27
|
-
@skipped_days = Array(skipped_days).freeze
|
|
28
|
-
@failed_date = failed_date
|
|
29
|
-
@remaining_days = Array(remaining_days).freeze
|
|
30
|
-
@error = error&.transform_keys(&:to_sym)&.freeze
|
|
31
|
-
@chunks = Array(chunks).map(&:to_s).freeze
|
|
32
|
-
@delivery = delivery&.transform_keys(&:to_sym)&.freeze
|
|
33
|
-
freeze
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def self.failure(mode:, error_kind:, message:, requested_days: [], settled_days: [], skipped_days: [],
|
|
37
|
-
failed_date: nil, remaining_days: [], chunks: [], delivery: nil)
|
|
38
|
-
progress = !settled_days.empty? || !skipped_days.empty?
|
|
39
|
-
new(
|
|
40
|
-
status: progress ? "partial_failure" : "failure",
|
|
41
|
-
mode: mode,
|
|
42
|
-
requested_days: requested_days,
|
|
43
|
-
settled_days: settled_days,
|
|
44
|
-
skipped_days: skipped_days,
|
|
45
|
-
failed_date: failed_date,
|
|
46
|
-
remaining_days: remaining_days,
|
|
47
|
-
error: { kind: error_kind.to_s, message: message.to_s },
|
|
48
|
-
chunks: chunks,
|
|
49
|
-
delivery: delivery
|
|
50
|
-
)
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def exit_code
|
|
54
|
-
return 0 if %w[success dry_run].include?(status)
|
|
55
|
-
return 6 if status == "partial_failure"
|
|
56
|
-
EXIT_CODES.fetch(error && error[:kind], 1)
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def to_h
|
|
60
|
-
{
|
|
61
|
-
schema: SCHEMA,
|
|
62
|
-
schema_version: SCHEMA_VERSION,
|
|
63
|
-
status: status,
|
|
64
|
-
mode: mode,
|
|
65
|
-
requested_days: serialize_dates(requested_days),
|
|
66
|
-
settled_days: serialize_dates(settled_days),
|
|
67
|
-
skipped_days: serialize_dates(skipped_days),
|
|
68
|
-
failed_date: failed_date&.to_s,
|
|
69
|
-
remaining_days: serialize_dates(remaining_days),
|
|
70
|
-
error: error,
|
|
71
|
-
chunks: chunks,
|
|
72
|
-
delivery: delivery
|
|
73
|
-
}
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
private
|
|
77
|
-
|
|
78
|
-
def serialize_dates(values)
|
|
79
|
-
values.map(&:to_s)
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
end
|
data/lib/prdigest/runner.rb
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "date"
|
|
4
|
-
|
|
5
|
-
module Prdigest
|
|
6
|
-
class Runner
|
|
7
|
-
def initialize(config:, date: nil, dry_run: false, clock: nil, state_factory: nil,
|
|
8
|
-
github: nil, renderer: nil, telegram_factory: nil, repositories: nil, env: ENV)
|
|
9
|
-
@config = config
|
|
10
|
-
@date = date && Date.iso8601(date.to_s)
|
|
11
|
-
@dry_run = dry_run
|
|
12
|
-
@clock = clock || Clock.new(timezone: config.timezone)
|
|
13
|
-
@state_factory = state_factory || -> { State.new(path: config.state_path, timezone: config.timezone) }
|
|
14
|
-
@env = env
|
|
15
|
-
@repositories = repositories || config.repos
|
|
16
|
-
@github = github || GitHub.new(token: config.github_token(env))
|
|
17
|
-
@renderer = renderer || Renderer.new(send_empty: config.send_empty?, empty_message: config.empty_message)
|
|
18
|
-
@delivery_checkpoint_store_factory = lambda {
|
|
19
|
-
DeliveryCheckpointStore.new(root: config.delivery_state_path)
|
|
20
|
-
}
|
|
21
|
-
@telegram_factory = telegram_factory || lambda {
|
|
22
|
-
Telegram.new(
|
|
23
|
-
token: config.telegram_token(env),
|
|
24
|
-
chat_id: config.chat_id,
|
|
25
|
-
allowlist: config.chat_id_allowlist
|
|
26
|
-
)
|
|
27
|
-
}
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def call
|
|
31
|
-
mode = @date ? "explicit_date_replay" : "scheduled"
|
|
32
|
-
requested = []
|
|
33
|
-
settled = []
|
|
34
|
-
skipped = []
|
|
35
|
-
chunks = []
|
|
36
|
-
failed_date = nil
|
|
37
|
-
delivery = nil
|
|
38
|
-
state = nil
|
|
39
|
-
skip_audit = nil
|
|
40
|
-
|
|
41
|
-
if @dry_run || @date
|
|
42
|
-
requested = [@date || @clock.yesterday]
|
|
43
|
-
else
|
|
44
|
-
state = @state_factory.call
|
|
45
|
-
yesterday = @clock.yesterday
|
|
46
|
-
record = state.read(yesterday: yesterday)
|
|
47
|
-
skip_audit = record.last_skip
|
|
48
|
-
skipped.concat(expand_skip(skip_audit)) if skip_audit && skip_audit[:notice_pending]
|
|
49
|
-
schedule = Schedule.new(
|
|
50
|
-
yesterday: yesterday,
|
|
51
|
-
last_digested_date: record.last_digested_date,
|
|
52
|
-
max_catchup_days: @config.max_catchup_days
|
|
53
|
-
).call
|
|
54
|
-
requested = schedule.requested_days
|
|
55
|
-
unless schedule.skipped_days.empty?
|
|
56
|
-
skip_audit = merge_skip_audit(skip_audit, schedule.skipped_days)
|
|
57
|
-
state.write(last_digested_date: schedule.skipped_days.last, last_skip: skip_audit)
|
|
58
|
-
skipped |= schedule.skipped_days
|
|
59
|
-
end
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
telegram = nil
|
|
63
|
-
requested.each do |date|
|
|
64
|
-
failed_date = date
|
|
65
|
-
window = @clock.window(date)
|
|
66
|
-
digest = if window.zero_length?
|
|
67
|
-
DayDigest.build(date: date, repository_order: @repositories, pulls: [], line_stats: @config.line_stats?)
|
|
68
|
-
else
|
|
69
|
-
@github.fetch(
|
|
70
|
-
date: date,
|
|
71
|
-
window: window,
|
|
72
|
-
repositories: @repositories,
|
|
73
|
-
line_stats: @config.line_stats?
|
|
74
|
-
)
|
|
75
|
-
end
|
|
76
|
-
rendered = @renderer.render(digest)
|
|
77
|
-
if @dry_run
|
|
78
|
-
chunks.concat(rendered.chunks)
|
|
79
|
-
next
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
unless rendered.chunks.empty?
|
|
83
|
-
telegram ||= @telegram_factory.call
|
|
84
|
-
delivery = telegram.deliver(
|
|
85
|
-
rendered.chunks,
|
|
86
|
-
digest_date: date,
|
|
87
|
-
checkpoint_store: @delivery_checkpoint_store_factory.call,
|
|
88
|
-
scope: @repositories
|
|
89
|
-
)
|
|
90
|
-
end
|
|
91
|
-
if state
|
|
92
|
-
if skip_audit
|
|
93
|
-
skip_audit = skip_audit.merge(notice_pending: false)
|
|
94
|
-
end
|
|
95
|
-
state.write(last_digested_date: date, last_skip: skip_audit)
|
|
96
|
-
end
|
|
97
|
-
settled << date
|
|
98
|
-
failed_date = nil
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
Result.new(
|
|
102
|
-
status: @dry_run ? "dry_run" : "success",
|
|
103
|
-
mode: mode,
|
|
104
|
-
requested_days: requested,
|
|
105
|
-
settled_days: settled,
|
|
106
|
-
skipped_days: skipped,
|
|
107
|
-
chunks: chunks,
|
|
108
|
-
delivery: delivery
|
|
109
|
-
)
|
|
110
|
-
rescue StandardError => e
|
|
111
|
-
Result.failure(
|
|
112
|
-
mode: mode || (@date ? "explicit_date_replay" : "scheduled"),
|
|
113
|
-
error_kind: error_kind(e),
|
|
114
|
-
message: safe_message(e),
|
|
115
|
-
requested_days: requested || [],
|
|
116
|
-
settled_days: settled || [],
|
|
117
|
-
skipped_days: skipped || [],
|
|
118
|
-
failed_date: failed_date,
|
|
119
|
-
remaining_days: remaining_days(requested || [], failed_date),
|
|
120
|
-
chunks: chunks || [],
|
|
121
|
-
delivery: e.is_a?(SendError) ? e.delivery : delivery
|
|
122
|
-
)
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
def inspect
|
|
126
|
-
"#<#{self.class} mode=#{@date ? "explicit_date_replay" : "scheduled"} " \
|
|
127
|
-
"dry_run=#{@dry_run} credentials=[REDACTED]>"
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
alias to_s inspect
|
|
131
|
-
|
|
132
|
-
private
|
|
133
|
-
|
|
134
|
-
def expand_skip(audit)
|
|
135
|
-
return [] unless audit
|
|
136
|
-
(audit.fetch(:start_date)..audit.fetch(:end_date)).to_a
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
def merge_skip_audit(existing, dates)
|
|
140
|
-
start_date = if existing && existing[:notice_pending]
|
|
141
|
-
[existing[:start_date], dates.first].min
|
|
142
|
-
else
|
|
143
|
-
dates.first
|
|
144
|
-
end
|
|
145
|
-
{ start_date: start_date, end_date: dates.last, notice_pending: true }
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
def remaining_days(requested, failed_date)
|
|
149
|
-
return requested unless failed_date
|
|
150
|
-
index = requested.index(failed_date)
|
|
151
|
-
index ? requested.drop(index) : [failed_date]
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
def error_kind(error)
|
|
155
|
-
case error
|
|
156
|
-
when ConfigError then "config"
|
|
157
|
-
when FetchError then error.kind
|
|
158
|
-
when SendError then error.kind
|
|
159
|
-
when StateError then "state"
|
|
160
|
-
when RenderError then error.kind
|
|
161
|
-
else "internal"
|
|
162
|
-
end
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
def safe_message(error)
|
|
166
|
-
message = error.is_a?(Error) ? error.message : "unexpected failure (#{error.class})"
|
|
167
|
-
secrets = []
|
|
168
|
-
secrets << @config.github_token(@env) if @config.respond_to?(:github_token)
|
|
169
|
-
secrets << @config.telegram_token(@env) if @config.respond_to?(:telegram_token)
|
|
170
|
-
secrets.compact.reject(&:empty?).reduce(message.dup) { |safe, secret| safe.gsub(secret, "[REDACTED]") }
|
|
171
|
-
end
|
|
172
|
-
end
|
|
173
|
-
end
|
data/lib/prdigest/schedule.rb
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "date"
|
|
4
|
-
|
|
5
|
-
module Prdigest
|
|
6
|
-
class Schedule
|
|
7
|
-
Result = Data.define(:skipped_days, :requested_days)
|
|
8
|
-
|
|
9
|
-
def initialize(yesterday:, last_digested_date:, max_catchup_days:)
|
|
10
|
-
@yesterday = Date.iso8601(yesterday.to_s)
|
|
11
|
-
@last_digested_date = last_digested_date && Date.iso8601(last_digested_date.to_s)
|
|
12
|
-
@max_catchup_days = Integer(max_catchup_days)
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def call
|
|
16
|
-
raise ArgumentError, "max_catchup_days must be from 1 to 30" unless (1..30).cover?(@max_catchup_days)
|
|
17
|
-
raise ArgumentError, "last digested date is in the future" if @last_digested_date && @last_digested_date > @yesterday
|
|
18
|
-
|
|
19
|
-
owed = if @last_digested_date.nil?
|
|
20
|
-
[@yesterday]
|
|
21
|
-
elsif @last_digested_date == @yesterday
|
|
22
|
-
[]
|
|
23
|
-
else
|
|
24
|
-
((@last_digested_date + 1)..@yesterday).to_a
|
|
25
|
-
end
|
|
26
|
-
split = [owed.length - @max_catchup_days, 0].max
|
|
27
|
-
Result.new(owed.first(split).freeze, owed.drop(split).freeze)
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|
data/lib/prdigest/state.rb
DELETED
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "date"
|
|
4
|
-
require "fileutils"
|
|
5
|
-
require "json"
|
|
6
|
-
require "securerandom"
|
|
7
|
-
|
|
8
|
-
module Prdigest
|
|
9
|
-
class State
|
|
10
|
-
VERSION = 1
|
|
11
|
-
Record = Data.define(:last_digested_date, :last_skip)
|
|
12
|
-
|
|
13
|
-
def initialize(path:, timezone:, rename: File.method(:rename), directory_sync: nil)
|
|
14
|
-
@path = File.expand_path(path)
|
|
15
|
-
@timezone = timezone
|
|
16
|
-
@rename = rename
|
|
17
|
-
@directory_sync = directory_sync || method(:sync_directory)
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def read(yesterday: nil)
|
|
21
|
-
return Record.new(nil, nil) unless File.exist?(@path)
|
|
22
|
-
|
|
23
|
-
payload = JSON.parse(File.binread(@path))
|
|
24
|
-
validate_payload!(payload, yesterday: yesterday)
|
|
25
|
-
rescue StateError
|
|
26
|
-
raise
|
|
27
|
-
rescue StandardError => e
|
|
28
|
-
raise StateError, "cannot read state: #{e.class}"
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def write(last_digested_date:, last_skip: nil)
|
|
32
|
-
directory = File.dirname(@path)
|
|
33
|
-
FileUtils.mkdir_p(directory, mode: 0o700)
|
|
34
|
-
temp_path = File.join(directory, ".#{File.basename(@path)}.tmp-#{SecureRandom.hex(8)}")
|
|
35
|
-
payload = JSON.generate(serialize(last_digested_date, last_skip)) << "\n"
|
|
36
|
-
|
|
37
|
-
File.open(temp_path, File::WRONLY | File::CREAT | File::EXCL, 0o600) do |file|
|
|
38
|
-
file.write(payload)
|
|
39
|
-
file.chmod(0o600)
|
|
40
|
-
file.flush
|
|
41
|
-
file.fsync
|
|
42
|
-
end
|
|
43
|
-
rollback_path = preserve_previous_checkpoint(directory)
|
|
44
|
-
@rename.call(temp_path, @path)
|
|
45
|
-
begin
|
|
46
|
-
@directory_sync.call(directory)
|
|
47
|
-
rescue StandardError
|
|
48
|
-
restore_previous_checkpoint(rollback_path)
|
|
49
|
-
raise
|
|
50
|
-
end
|
|
51
|
-
Record.new(Date.iso8601(last_digested_date.to_s), normalize_skip(last_skip))
|
|
52
|
-
rescue StandardError => e
|
|
53
|
-
raise StateError, "cannot write state: #{e.class}"
|
|
54
|
-
ensure
|
|
55
|
-
File.unlink(temp_path) if defined?(temp_path) && temp_path && File.exist?(temp_path)
|
|
56
|
-
discard_rollback(rollback_path) if defined?(rollback_path) && rollback_path
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
private
|
|
60
|
-
|
|
61
|
-
def sync_directory(directory)
|
|
62
|
-
File.open(directory, File::RDONLY) { |dir| dir.fsync }
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
def preserve_previous_checkpoint(directory)
|
|
66
|
-
rollback_path = File.join(directory, ".#{File.basename(@path)}.rollback-#{SecureRandom.hex(8)}")
|
|
67
|
-
File.link(@path, rollback_path)
|
|
68
|
-
rollback_path
|
|
69
|
-
rescue Errno::ENOENT
|
|
70
|
-
nil
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def restore_previous_checkpoint(rollback_path)
|
|
74
|
-
if rollback_path
|
|
75
|
-
File.rename(rollback_path, @path)
|
|
76
|
-
else
|
|
77
|
-
File.unlink(@path)
|
|
78
|
-
end
|
|
79
|
-
rescue Errno::ENOENT
|
|
80
|
-
raise if rollback_path
|
|
81
|
-
nil
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
def discard_rollback(rollback_path)
|
|
85
|
-
FileUtils.rm_f(rollback_path) if rollback_path
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
def validate_payload!(payload, yesterday:)
|
|
89
|
-
raise StateError, "state root must be an object" unless payload.is_a?(Hash)
|
|
90
|
-
raise StateError, "unsupported state version" unless payload["version"] == VERSION
|
|
91
|
-
raise StateError, "state timezone does not match config" unless payload["timezone"] == @timezone
|
|
92
|
-
raise StateError, "state last_digested_date is required" unless payload.key?("last_digested_date")
|
|
93
|
-
|
|
94
|
-
value = payload["last_digested_date"]
|
|
95
|
-
last_date = Date.iso8601(value)
|
|
96
|
-
raise StateError, "state date is in the future" if yesterday && last_date && last_date > Date.iso8601(yesterday.to_s)
|
|
97
|
-
|
|
98
|
-
skip = parse_skip(payload["last_skip"])
|
|
99
|
-
if skip && (skip[:start_date] > skip[:end_date] || skip[:end_date] > last_date)
|
|
100
|
-
raise StateError, "state last_skip range is invalid"
|
|
101
|
-
end
|
|
102
|
-
Record.new(last_date, skip)
|
|
103
|
-
rescue Date::Error, TypeError
|
|
104
|
-
raise StateError, "state contains an invalid date"
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
def parse_skip(value)
|
|
108
|
-
return nil if value.nil?
|
|
109
|
-
raise StateError, "state last_skip is invalid" unless value.is_a?(Hash)
|
|
110
|
-
|
|
111
|
-
pending = value.fetch("notice_pending")
|
|
112
|
-
raise StateError, "state last_skip is invalid" unless pending == true || pending == false
|
|
113
|
-
{
|
|
114
|
-
start_date: Date.iso8601(value.fetch("start_date")),
|
|
115
|
-
end_date: Date.iso8601(value.fetch("end_date")),
|
|
116
|
-
notice_pending: pending
|
|
117
|
-
}
|
|
118
|
-
rescue KeyError
|
|
119
|
-
raise StateError, "state last_skip is invalid"
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
def normalize_skip(skip)
|
|
123
|
-
return nil unless skip
|
|
124
|
-
{
|
|
125
|
-
start_date: Date.iso8601(skip.fetch(:start_date).to_s),
|
|
126
|
-
end_date: Date.iso8601(skip.fetch(:end_date).to_s),
|
|
127
|
-
notice_pending: skip.fetch(:notice_pending) == true
|
|
128
|
-
}
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
def serialize(last_date, last_skip)
|
|
132
|
-
payload = {
|
|
133
|
-
version: VERSION,
|
|
134
|
-
timezone: @timezone,
|
|
135
|
-
last_digested_date: Date.iso8601(last_date.to_s).iso8601
|
|
136
|
-
}
|
|
137
|
-
if (skip = normalize_skip(last_skip))
|
|
138
|
-
payload[:last_skip] = {
|
|
139
|
-
start_date: skip[:start_date].iso8601,
|
|
140
|
-
end_date: skip[:end_date].iso8601,
|
|
141
|
-
notice_pending: skip[:notice_pending]
|
|
142
|
-
}
|
|
143
|
-
end
|
|
144
|
-
payload
|
|
145
|
-
end
|
|
146
|
-
end
|
|
147
|
-
end
|