slk 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +47 -0
- data/README.md +37 -0
- data/lib/slk/api/conversations.rb +6 -2
- data/lib/slk/cli.rb +1 -0
- data/lib/slk/commands/cache.rb +10 -9
- data/lib/slk/commands/deactivations.rb +144 -14
- data/lib/slk/commands/help.rb +2 -0
- data/lib/slk/commands/search.rb +55 -48
- data/lib/slk/commands/sent.rb +288 -0
- data/lib/slk/formatters/attachment_formatter.rb +7 -3
- data/lib/slk/formatters/csv_writer.rb +34 -0
- data/lib/slk/formatters/deactivation_csv.rb +49 -0
- data/lib/slk/formatters/deactivation_formatter.rb +27 -11
- data/lib/slk/formatters/output.rb +39 -0
- data/lib/slk/formatters/search_formatter.rb +25 -11
- data/lib/slk/formatters/sent_formatter.rb +193 -0
- data/lib/slk/models/search_result.rb +2 -0
- data/lib/slk/models/tenure.rb +66 -0
- data/lib/slk/runner.rb +4 -0
- data/lib/slk/services/api_client.rb +12 -1
- data/lib/slk/services/cache_store.rb +19 -0
- data/lib/slk/services/meta_cache.rb +16 -1
- data/lib/slk/services/search_pages.rb +65 -0
- data/lib/slk/services/sent_changes.rb +293 -0
- data/lib/slk/services/sent_channel_label.rb +66 -0
- data/lib/slk/services/sent_conversations.rb +227 -0
- data/lib/slk/services/start_date_field.rb +67 -0
- data/lib/slk/services/start_date_lookup.rb +98 -0
- data/lib/slk/support/check_in_time.rb +73 -0
- data/lib/slk/version.rb +1 -1
- data/lib/slk.rb +13 -0
- metadata +14 -2
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bigdecimal'
|
|
4
|
+
|
|
5
|
+
module Slk
|
|
6
|
+
module Services
|
|
7
|
+
# Stateless diff of watched sent conversations. Search discovers what to
|
|
8
|
+
# watch; exact-ts history/replies determine what actually changed.
|
|
9
|
+
# Orchestration spans workspace watch sets, thread diffs and output windows.
|
|
10
|
+
# rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
|
11
|
+
class SentChanges
|
|
12
|
+
Change = Data.define(:conversation, :new_timestamps, :new_count, :new_from_others)
|
|
13
|
+
Conversation = SentConversations::Conversation
|
|
14
|
+
OPTIONAL_SUBSCRIPTION_ERRORS = %i[missing_scope not_allowed_token_type unknown_method].freeze
|
|
15
|
+
|
|
16
|
+
# rubocop:disable Metrics/ParameterLists
|
|
17
|
+
def initialize(runner:, since:, context: 2, max: 200, before: 5, after_minutes: 30)
|
|
18
|
+
@runner = runner
|
|
19
|
+
@since = since
|
|
20
|
+
@since_ts = Support::CheckInTime.timestamp(since)
|
|
21
|
+
@context = context
|
|
22
|
+
@max = max
|
|
23
|
+
@before = before
|
|
24
|
+
@after_minutes = after_minutes
|
|
25
|
+
end
|
|
26
|
+
# rubocop:enable Metrics/ParameterLists
|
|
27
|
+
|
|
28
|
+
def collect(entries, workspaces:)
|
|
29
|
+
groups = entries.group_by { |workspace, hit| [workspace.name, hit.channel_id] }
|
|
30
|
+
changes = workspaces.flat_map { |workspace| collect_workspace(workspace, groups) }
|
|
31
|
+
changes.sort_by do |change|
|
|
32
|
+
conversation = change.conversation
|
|
33
|
+
# Put the most recently active conversation last. Trailing identity
|
|
34
|
+
# keys make ties deterministic across platforms (including Windows).
|
|
35
|
+
[BigDecimal(conversation.messages.last.ts), conversation.workspace.name.to_s,
|
|
36
|
+
conversation.channel_id.to_s, conversation.thread_ts.to_s]
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def collect_workspace(workspace, groups)
|
|
43
|
+
@workspace = workspace
|
|
44
|
+
@api = @runner.conversations_api(workspace.name)
|
|
45
|
+
@self_id = self_id(workspace, groups)
|
|
46
|
+
local = groups.select { |(name, _id), _hits| name == workspace.name }
|
|
47
|
+
changed_threads = local.flat_map { |(_name, _id), entries| thread_changes(entries.map(&:last)) }
|
|
48
|
+
changed_threads.concat(subscribed_changes(changed_threads, local))
|
|
49
|
+
claimed = changed_threads.flat_map { |change| change.conversation.messages.map(&:ts) }.to_h { |ts| [ts, true] }
|
|
50
|
+
other = local.flat_map do |(_name, _id), entries|
|
|
51
|
+
hits = entries.map(&:last)
|
|
52
|
+
hits.first.dm? ? dm_changes(hits, claimed) : channel_changes(hits, claimed)
|
|
53
|
+
end
|
|
54
|
+
changed_threads + other
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def self_id(workspace, groups)
|
|
58
|
+
entries = groups.find { |(name, _id), rows| name == workspace.name && rows.any? }&.last
|
|
59
|
+
hit = entries&.first&.last
|
|
60
|
+
return hit.user_id if hit&.user_id.to_s.match?(/\A[UW][A-Z0-9]+\z/i)
|
|
61
|
+
|
|
62
|
+
cached = @runner.cache_store.get_meta(workspace.name, 'self_user_id')
|
|
63
|
+
return cached if cached
|
|
64
|
+
|
|
65
|
+
id = @runner.client_api(workspace.name).auth_test['user_id']
|
|
66
|
+
raise ApiError, "Cannot identify sender in #{workspace.name}" unless id
|
|
67
|
+
|
|
68
|
+
@runner.cache_store.set_meta(workspace.name, 'self_user_id', id)
|
|
69
|
+
id
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def thread_changes(hits)
|
|
73
|
+
metadata = root_metadata(hits)
|
|
74
|
+
hits.map { |hit| [hit.thread_ts || hit.ts, hit] }.uniq(&:first).filter_map do |root, seed|
|
|
75
|
+
parent = metadata[root]
|
|
76
|
+
next if parent && !thread_active?(parent)
|
|
77
|
+
|
|
78
|
+
thread_change(seed, root, full_first: parent && parent['latest_reply'])
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Fetch the watched top-level parents in one paginated history scan per
|
|
83
|
+
# channel. Search often omits reply_count; history has latest_reply.
|
|
84
|
+
def root_metadata(hits)
|
|
85
|
+
roots = hits.select { |hit| top_level?(hit) }.map(&:ts).uniq
|
|
86
|
+
return {} if roots.empty?
|
|
87
|
+
|
|
88
|
+
wanted = roots.to_h { |root| [root, true] }
|
|
89
|
+
found = {}
|
|
90
|
+
cursor = nil
|
|
91
|
+
loop do
|
|
92
|
+
response = @api.history(channel: hits.first.channel_id,
|
|
93
|
+
oldest: roots.min_by { |root| BigDecimal(root) }, inclusive: true,
|
|
94
|
+
limit: 200, cursor: cursor)
|
|
95
|
+
response.fetch('messages', []).each do |message|
|
|
96
|
+
found[message['ts']] = message if wanted[message['ts']]
|
|
97
|
+
end
|
|
98
|
+
break if found.size == wanted.size || !response['has_more']
|
|
99
|
+
|
|
100
|
+
cursor = next_cursor!(response, cursor, :history)
|
|
101
|
+
end
|
|
102
|
+
found
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def thread_active?(parent)
|
|
106
|
+
latest = parent['latest_reply']
|
|
107
|
+
return newer?(latest) if latest
|
|
108
|
+
|
|
109
|
+
parent['reply_count'].to_i.positive?
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def thread_change(seed, root, require_participation: false, full_first: false)
|
|
113
|
+
full = page_through(:replies, channel: seed.channel_id, timestamp: root) if full_first
|
|
114
|
+
recent = full || page_through(:replies, channel: seed.channel_id, timestamp: root, oldest: @since_ts)
|
|
115
|
+
fresh = after_cutoff(recent)
|
|
116
|
+
# A new top-level post without replies belongs in its channel/DM
|
|
117
|
+
# window, not in a synthetic one-message thread.
|
|
118
|
+
return if fresh.none? { |row| row['ts'] != root }
|
|
119
|
+
|
|
120
|
+
full ||= page_through(:replies, channel: seed.channel_id, timestamp: root)
|
|
121
|
+
return if require_participation && full.none? { |row| row['user'] == @self_id }
|
|
122
|
+
|
|
123
|
+
context = previous_messages(full, parent_ts: root)
|
|
124
|
+
change(seed, 'thread', root, fresh, context)
|
|
125
|
+
rescue ApiError => e
|
|
126
|
+
raise unless e.code == :thread_not_found
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Slack's thread view exposes unread followed threads, not a complete
|
|
130
|
+
# subscriptions cursor. Union the first page, and verify participation.
|
|
131
|
+
def subscribed_changes(existing, local)
|
|
132
|
+
response = subscribed_view
|
|
133
|
+
return [] unless response
|
|
134
|
+
|
|
135
|
+
known = existing.map { |change| [change.conversation.channel_id, change.conversation.thread_ts] }
|
|
136
|
+
known.concat(local.flat_map do |(_name, channel), entries|
|
|
137
|
+
entries.map { |_workspace, hit| [channel, hit.thread_ts || hit.ts] }
|
|
138
|
+
end)
|
|
139
|
+
response.fetch('threads', []).filter_map do |item|
|
|
140
|
+
root = item['root_msg'] || {}
|
|
141
|
+
channel = root['channel']
|
|
142
|
+
timestamp = root['thread_ts'] || root['ts']
|
|
143
|
+
next unless channel && timestamp && !known.include?([channel, timestamp])
|
|
144
|
+
|
|
145
|
+
seed = subscription_seed(channel, timestamp, root)
|
|
146
|
+
next unless seed
|
|
147
|
+
|
|
148
|
+
thread_change(seed, timestamp, require_participation: true)
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def subscribed_view
|
|
153
|
+
@runner.threads_api(@workspace.name).get_view(limit: 100)
|
|
154
|
+
rescue ApiError => e
|
|
155
|
+
# Only unsupported/unauthorized-to-use subscription views are optional.
|
|
156
|
+
# Network, rate-limit and other failures must not look like a clean check-in.
|
|
157
|
+
raise unless OPTIONAL_SUBSCRIPTION_ERRORS.include?(e.code)
|
|
158
|
+
|
|
159
|
+
nil
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def subscription_seed(channel, timestamp, root)
|
|
163
|
+
info = @api.info(channel: channel).fetch('channel', {})
|
|
164
|
+
type = if info['is_im']
|
|
165
|
+
'im'
|
|
166
|
+
elsif info['is_mpim']
|
|
167
|
+
'mpim'
|
|
168
|
+
else
|
|
169
|
+
'channel'
|
|
170
|
+
end
|
|
171
|
+
name = type == 'im' ? info['user'] : info['name']
|
|
172
|
+
return unless name
|
|
173
|
+
|
|
174
|
+
Models::SearchResult.new(ts: timestamp, user_id: @self_id, username: nil, text: root['text'].to_s,
|
|
175
|
+
channel_id: channel, channel_name: name, channel_type: type,
|
|
176
|
+
thread_ts: timestamp, reply_count: 0, permalink: nil, files: [])
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def dm_changes(hits, claimed)
|
|
180
|
+
seed = hits.first
|
|
181
|
+
probe = @api.history(channel: seed.channel_id, oldest: @since_ts, limit: 1)
|
|
182
|
+
recent = if probe['has_more']
|
|
183
|
+
page_through(:history, channel: seed.channel_id, oldest: @since_ts)
|
|
184
|
+
else
|
|
185
|
+
probe.fetch('messages', [])
|
|
186
|
+
end
|
|
187
|
+
fresh = after_cutoff(recent).reject { |row| claimed[row['ts']] }
|
|
188
|
+
return [] if fresh.empty?
|
|
189
|
+
|
|
190
|
+
earlier = if @context.zero?
|
|
191
|
+
[]
|
|
192
|
+
else
|
|
193
|
+
@api.history(channel: seed.channel_id, latest: @since_ts,
|
|
194
|
+
inclusive: true, limit: @context).fetch('messages', [])
|
|
195
|
+
end
|
|
196
|
+
context = previous_messages(earlier)
|
|
197
|
+
[change(seed, seed.channel_type, nil, fresh, context)]
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def channel_changes(hits, claimed)
|
|
201
|
+
recent = hits.select { |hit| top_level?(hit) && newer?(hit.ts) }
|
|
202
|
+
return [] if recent.empty?
|
|
203
|
+
|
|
204
|
+
windows = SentConversations.new(runner: @runner, start_date: @since.to_date, end_date: Date.today,
|
|
205
|
+
before: @before, after_minutes: @after_minutes, max: 0).collect(
|
|
206
|
+
recent.map { |hit| [@workspace, hit] }
|
|
207
|
+
)
|
|
208
|
+
own_timestamps = recent.to_h { |hit| [hit.ts, true] }
|
|
209
|
+
windows.filter_map do |window|
|
|
210
|
+
next unless window.type == 'channel'
|
|
211
|
+
|
|
212
|
+
fresh = window.messages.select { |msg| own_timestamps[msg.ts] && !claimed[msg.ts] }
|
|
213
|
+
next if fresh.empty?
|
|
214
|
+
|
|
215
|
+
context = previous_messages(window.messages)
|
|
216
|
+
change(hits.first, 'channel', nil, fresh, context)
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def top_level?(hit)
|
|
221
|
+
hit.thread_ts.nil? || hit.thread_ts == hit.ts
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def previous_messages(raw, parent_ts: nil)
|
|
225
|
+
older = raw.select { |row| row_timestamp(row) && !newer?(row_timestamp(row)) }
|
|
226
|
+
older = older.last(@context) unless @context.zero?
|
|
227
|
+
older = [] if @context.zero?
|
|
228
|
+
return older unless parent_ts && @context.positive?
|
|
229
|
+
|
|
230
|
+
parent = raw.find { |row| row_timestamp(row) == parent_ts }
|
|
231
|
+
parent && !newer?(parent_ts) ? ([parent] + older.reject { |row| row == parent }.last(@context - 1)) : older
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Counts and last speaker are calculated before --max hides any output.
|
|
235
|
+
def change(seed, type, root, fresh, context)
|
|
236
|
+
messages = normalize(context + fresh, seed.channel_id)
|
|
237
|
+
new_ts = fresh.map { |row| row_timestamp(row) }.uniq
|
|
238
|
+
new_messages = messages.select { |message| new_ts.include?(message.ts) }
|
|
239
|
+
return if new_messages.empty?
|
|
240
|
+
|
|
241
|
+
kept = @max.zero? ? messages : messages.last(@max)
|
|
242
|
+
conversation = Conversation.new(
|
|
243
|
+
workspace: @workspace, channel_id: seed.channel_id, channel_name: seed.channel_name,
|
|
244
|
+
channel_type: seed.channel_type, type: type, thread_ts: root, first_sent_ts: seed.ts,
|
|
245
|
+
messages: kept, last_speaker_is_me: new_messages.last.user_id == @self_id,
|
|
246
|
+
dropped_messages: messages.size - kept.size, self_user_id: @self_id,
|
|
247
|
+
self_username: seed.username
|
|
248
|
+
)
|
|
249
|
+
others = new_messages.count { |message| message.user_id != @self_id }
|
|
250
|
+
Change.new(conversation: conversation, new_timestamps: new_ts,
|
|
251
|
+
new_count: new_messages.size, new_from_others: others)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def normalize(raw, channel_id)
|
|
255
|
+
raw.map { |row| row.is_a?(Models::Message) ? row : Models::Message.from_api(row, channel_id: channel_id) }
|
|
256
|
+
.uniq(&:ts).sort_by { |message| BigDecimal(message.ts) }
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def after_cutoff(raw)
|
|
260
|
+
raw.select { |row| row['ts'] && newer?(row['ts']) && (row['user'] || row['bot_id'] || row['username']) }
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def row_timestamp(row)
|
|
264
|
+
row.is_a?(Models::Message) ? row.ts : row['ts']
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def newer?(timestamp)
|
|
268
|
+
BigDecimal(timestamp) > BigDecimal(@since_ts)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def page_through(endpoint, **params)
|
|
272
|
+
messages = []
|
|
273
|
+
cursor = nil
|
|
274
|
+
loop do
|
|
275
|
+
response = @api.public_send(endpoint, **params, limit: 200, cursor: cursor)
|
|
276
|
+
messages.concat(response.fetch('messages', []))
|
|
277
|
+
break unless response['has_more']
|
|
278
|
+
|
|
279
|
+
cursor = next_cursor!(response, cursor, endpoint)
|
|
280
|
+
end
|
|
281
|
+
messages
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def next_cursor!(response, previous, endpoint)
|
|
285
|
+
cursor = response.dig('response_metadata', 'next_cursor').to_s
|
|
286
|
+
raise ApiError, "conversations.#{endpoint} has_more without a new cursor" if cursor.empty? || cursor == previous
|
|
287
|
+
|
|
288
|
+
cursor
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
# rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
|
292
|
+
end
|
|
293
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Slk
|
|
4
|
+
module Services
|
|
5
|
+
# Resolves sent-conversation destinations while retaining raw channel IDs
|
|
6
|
+
# and names in JSON. Group DMs need member lookup: their Slack name is an
|
|
7
|
+
# opaque mpdm-* slug, not a user ID that MentionReplacer can resolve.
|
|
8
|
+
class SentChannelLabel
|
|
9
|
+
USER_ID_PATTERN = /\A[UW][A-Z0-9]+\z/
|
|
10
|
+
|
|
11
|
+
def initialize(runner:)
|
|
12
|
+
@runner = runner
|
|
13
|
+
@labels = {}
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# rubocop:disable Metrics/ParameterLists
|
|
17
|
+
def label(workspace:, type:, name:, channel_id:, self_user_id: nil, self_username: nil)
|
|
18
|
+
key = [workspace.name, channel_id, self_user_id]
|
|
19
|
+
@labels[key] ||= if type == 'mpim'
|
|
20
|
+
group_label(workspace, channel_id, name, self_user_id, self_username)
|
|
21
|
+
else
|
|
22
|
+
@runner.search_formatter.channel_label_for(type, name, workspace)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# rubocop:enable Metrics/ParameterLists
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def group_label(workspace, channel_id, name, self_user_id, self_username)
|
|
31
|
+
names = group_member_names(workspace, channel_id, self_user_id)
|
|
32
|
+
return "@#{names.join(', ')}" if names.any?
|
|
33
|
+
|
|
34
|
+
fallback_group_label(name, self_username)
|
|
35
|
+
rescue ApiError
|
|
36
|
+
fallback_group_label(name, self_username)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def group_member_names(workspace, channel_id, self_user_id)
|
|
40
|
+
members = @runner.conversations_api(workspace.name).info(channel: channel_id).dig('channel', 'members') || []
|
|
41
|
+
names = members.reject { |id| id == self_user_id }.map { |id| resolve_name(workspace, id) }
|
|
42
|
+
names.any? { |value| value.match?(USER_ID_PATTERN) } ? [] : names
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def resolve_name(workspace, user_id)
|
|
46
|
+
Services::UserLookup.new(cache_store: @runner.cache_store, workspace: workspace,
|
|
47
|
+
api_client: @runner.api_client).resolve_name_or_bot(user_id) || user_id
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def fallback_group_label(name, self_username)
|
|
51
|
+
handles = name.to_s.sub(/\Ampdm-/, '').sub(/-\d+\z/, '').split('--')
|
|
52
|
+
people = handles.reject { |handle| normalized_handle(handle) == normalized_handle(self_username) }
|
|
53
|
+
people = handles if people.empty?
|
|
54
|
+
"@#{people.map { |handle| pretty_handle(handle) }.join(', ')}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def normalized_handle(handle)
|
|
58
|
+
handle.to_s.downcase.gsub(/[^a-z0-9]/, '')
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def pretty_handle(handle)
|
|
62
|
+
handle.tr('._-', ' ').split.map(&:capitalize).join(' ')
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bigdecimal'
|
|
4
|
+
|
|
5
|
+
module Slk
|
|
6
|
+
module Services
|
|
7
|
+
# Expands indexed from:me hits into bounded channel windows, full DM
|
|
8
|
+
# histories and full threads. Search is the seed, not the context source.
|
|
9
|
+
# rubocop:disable Metrics/ClassLength
|
|
10
|
+
class SentConversations
|
|
11
|
+
Conversation = Data.define(:workspace, :channel_id, :channel_name, :channel_type, :type, :thread_ts,
|
|
12
|
+
:first_sent_ts, :messages, :last_speaker_is_me, :dropped_messages,
|
|
13
|
+
:self_user_id, :self_username)
|
|
14
|
+
DEFAULT_BEFORE = 5
|
|
15
|
+
DEFAULT_AFTER_MINUTES = 30
|
|
16
|
+
DEFAULT_MAX = 200
|
|
17
|
+
|
|
18
|
+
# A date span plus three independent window/output limits.
|
|
19
|
+
# rubocop:disable Metrics/ParameterLists
|
|
20
|
+
def initialize(runner:, start_date:, end_date:, before: DEFAULT_BEFORE,
|
|
21
|
+
after_minutes: DEFAULT_AFTER_MINUTES, max: DEFAULT_MAX)
|
|
22
|
+
@runner = runner
|
|
23
|
+
@start_date = start_date
|
|
24
|
+
@end_date = end_date
|
|
25
|
+
@before = before
|
|
26
|
+
@after_seconds = after_minutes * 60
|
|
27
|
+
@max = max
|
|
28
|
+
end
|
|
29
|
+
# rubocop:enable Metrics/ParameterLists
|
|
30
|
+
|
|
31
|
+
def collect(entries)
|
|
32
|
+
@self_ids = entries.to_h { |workspace, result| [workspace.name, self_id(workspace, result)] }
|
|
33
|
+
groups = entries.group_by { |workspace, result| [workspace.name, result.channel_id] }
|
|
34
|
+
conversations = groups.flat_map { |_key, group| collect_channel(group) }
|
|
35
|
+
# A thread parent can also appear in a channel window / DM history.
|
|
36
|
+
# Thread owns the message so the same (channel, ts) is never repeated.
|
|
37
|
+
ordered_conversations(deduplicate(conversations))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def ordered_conversations(conversations)
|
|
43
|
+
conversations.sort_by do |conversation|
|
|
44
|
+
# Keep Slack's fractional timestamp precise; identity breaks ties regardless of input order or platform.
|
|
45
|
+
[BigDecimal(conversation.first_sent_ts), conversation.workspace.name.to_s,
|
|
46
|
+
conversation.channel_id.to_s, conversation.thread_ts.to_s]
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Partitioning a channel needs both the reply roots and unthreaded hits.
|
|
51
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
52
|
+
def collect_channel(group)
|
|
53
|
+
workspace, seed = group.first
|
|
54
|
+
hits = group.map(&:last)
|
|
55
|
+
@self_id = @self_ids[workspace.name]
|
|
56
|
+
@api = @runner.conversations_api(workspace.name)
|
|
57
|
+
threaded = hits.select { |hit| hit.thread_ts && hit.thread_ts != hit.ts }.group_by(&:thread_ts)
|
|
58
|
+
ordinary = hits.reject { |hit| threaded.key?(hit.thread_ts) || threaded.key?(hit.ts) }
|
|
59
|
+
threads = threaded.map do |root, results|
|
|
60
|
+
parent_hit = hits.find { |hit| hit.ts == root }
|
|
61
|
+
thread_hits = parent_hit ? [parent_hit, *results] : results
|
|
62
|
+
thread_conversation(workspace, seed, root, thread_hits)
|
|
63
|
+
end
|
|
64
|
+
spans = if seed.dm?
|
|
65
|
+
ordinary.empty? ? [] : [dm_conversation(workspace, seed, ordinary)]
|
|
66
|
+
else
|
|
67
|
+
channel_conversations(workspace, seed, ordinary)
|
|
68
|
+
end
|
|
69
|
+
threads + spans
|
|
70
|
+
end
|
|
71
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
72
|
+
|
|
73
|
+
def self_id(workspace, result)
|
|
74
|
+
return result.user_id if result.user_id.to_s.match?(/\A[UW][A-Z0-9]+\z/i)
|
|
75
|
+
|
|
76
|
+
cached = @runner.cache_store.get_meta(workspace.name, 'self_user_id')
|
|
77
|
+
return cached if cached
|
|
78
|
+
|
|
79
|
+
id = @runner.client_api(workspace.name).auth_test['user_id']
|
|
80
|
+
raise ApiError, "Cannot identify sender in #{workspace.name}" unless id
|
|
81
|
+
|
|
82
|
+
@runner.cache_store.set_meta(workspace.name, 'self_user_id', id)
|
|
83
|
+
id
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def thread_conversation(workspace, seed, root, hits)
|
|
87
|
+
raw = thread_messages(seed.channel_id, root)
|
|
88
|
+
build(workspace, seed, 'thread', root, hits, raw)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def dm_conversation(workspace, seed, hits)
|
|
92
|
+
start_at = @start_date.to_time.to_i
|
|
93
|
+
# Slack's latest is exclusive; +1 includes messages in this second.
|
|
94
|
+
end_at = [(@end_date + 1).to_time.to_i, Time.now.to_i + 1].min
|
|
95
|
+
raw = page_through(:history, channel: seed.channel_id, oldest: start_at.to_s, latest: end_at.to_s)
|
|
96
|
+
build(workspace, seed, seed.channel_type, nil, hits, raw, expand_threads: true)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def channel_conversations(workspace, seed, hits)
|
|
100
|
+
merge_windows(hits).map { |window| channel_window(workspace, seed, window) }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def channel_window(workspace, seed, window)
|
|
104
|
+
preceding = preceding_messages(seed.channel_id, window.first.ts)
|
|
105
|
+
following = following_messages(seed.channel_id, window)
|
|
106
|
+
build(workspace, seed, 'channel', nil, window, preceding + following, expand_threads: true)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def preceding_messages(channel_id, timestamp)
|
|
110
|
+
return [] if @before.zero?
|
|
111
|
+
|
|
112
|
+
@api.history(channel: channel_id, latest: timestamp, limit: @before).fetch('messages', [])
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def following_messages(channel_id, window)
|
|
116
|
+
return [] if @after_seconds.zero?
|
|
117
|
+
|
|
118
|
+
end_at = (BigDecimal(window.last.ts) + @after_seconds).to_s('F')
|
|
119
|
+
page_through(:history, channel: channel_id, oldest: window.first.ts, latest: end_at, inclusive: true)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def merge_windows(hits)
|
|
123
|
+
hits.sort_by { |hit| hit.ts.to_f }.each_with_object([]) do |hit, windows|
|
|
124
|
+
if windows.any? && hit.ts.to_f <= windows.last.last.ts.to_f + @after_seconds
|
|
125
|
+
windows.last << hit
|
|
126
|
+
else
|
|
127
|
+
windows << [hit]
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def thread_messages(channel_id, root)
|
|
133
|
+
page_through(:replies, channel: channel_id, timestamp: root)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Cursor follow-up is necessary even when the first page is full.
|
|
137
|
+
# rubocop:disable Metrics/MethodLength
|
|
138
|
+
def page_through(endpoint, **params)
|
|
139
|
+
messages = []
|
|
140
|
+
cursor = nil
|
|
141
|
+
loop do
|
|
142
|
+
response = @api.public_send(endpoint, **params, limit: 200, cursor: cursor)
|
|
143
|
+
messages.concat(response.fetch('messages', []))
|
|
144
|
+
break unless response['has_more']
|
|
145
|
+
|
|
146
|
+
next_cursor = response.dig('response_metadata', 'next_cursor').to_s
|
|
147
|
+
if next_cursor.empty? || next_cursor == cursor
|
|
148
|
+
raise ApiError, "conversations.#{endpoint} has_more without a new cursor"
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
cursor = next_cursor
|
|
152
|
+
end
|
|
153
|
+
messages
|
|
154
|
+
end
|
|
155
|
+
# rubocop:enable Metrics/MethodLength
|
|
156
|
+
|
|
157
|
+
# Keep the search hits as a fallback at Slack's exclusive history bounds.
|
|
158
|
+
# rubocop:disable Metrics/ParameterLists
|
|
159
|
+
def build(workspace, seed, type, root, hits, raw, expand_threads: false)
|
|
160
|
+
# Search's own hits ensure an indexed sent message cannot disappear just
|
|
161
|
+
# because history omits thread replies or returns a window boundary.
|
|
162
|
+
raw += hits.map { |hit| search_message(hit) }
|
|
163
|
+
raw = expand_own_threads(raw, seed, hits) if expand_threads
|
|
164
|
+
messages = normalize(raw, seed.channel_id)
|
|
165
|
+
last_is_me = messages.last&.user_id == @self_id
|
|
166
|
+
Conversation.new(workspace: workspace, channel_id: seed.channel_id, channel_name: seed.channel_name,
|
|
167
|
+
channel_type: seed.channel_type, type: type, thread_ts: root, first_sent_ts: hits.first.ts,
|
|
168
|
+
messages: messages, last_speaker_is_me: last_is_me, dropped_messages: 0,
|
|
169
|
+
self_user_id: @self_id, self_username: hits.first.username)
|
|
170
|
+
end
|
|
171
|
+
# rubocop:enable Metrics/ParameterLists
|
|
172
|
+
|
|
173
|
+
def search_message(hit)
|
|
174
|
+
{ 'ts' => hit.ts, 'user' => @self_id, 'username' => hit.username, 'text' => hit.text,
|
|
175
|
+
'thread_ts' => hit.thread_ts, 'reply_count' => hit.reply_count }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def expand_own_threads(raw, seed, hits)
|
|
179
|
+
own_roots(raw, seed, hits).each { |root| raw.concat(thread_messages(seed.channel_id, root)) }
|
|
180
|
+
raw
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def own_roots(raw, seed, hits)
|
|
184
|
+
# Five preceding channel messages may include an old post of ours;
|
|
185
|
+
# don't expand its unrelated thread into today's conversation.
|
|
186
|
+
earliest = seed.dm? ? @start_date.to_time.to_i : hits.first.ts.to_f
|
|
187
|
+
raw.filter_map { |message| message['ts'] if own_thread_root?(message, earliest) }.uniq
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def own_thread_root?(message, earliest)
|
|
191
|
+
message['user'] == @self_id && message['reply_count'].to_i.positive? &&
|
|
192
|
+
message['ts'].to_f >= earliest
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def normalize(raw, channel_id)
|
|
196
|
+
raw.select { |message| readable?(message) }
|
|
197
|
+
.uniq { |message| message['ts'] }
|
|
198
|
+
.sort_by { |message| message['ts'].to_f }
|
|
199
|
+
.map { |message| Models::Message.from_api(message, channel_id: channel_id) }
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def readable?(message)
|
|
203
|
+
message['ts'] && (message['user'] || message['bot_id'] || message['username'])
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def deduplicate(conversations)
|
|
207
|
+
seen = {}
|
|
208
|
+
conversations.filter_map { |conversation| unique_conversation(conversation, seen) }
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Global dedupe happens before --max so a duplicate cannot consume a slot.
|
|
212
|
+
# rubocop:disable Metrics/AbcSize
|
|
213
|
+
def unique_conversation(conversation, seen)
|
|
214
|
+
key = [conversation.workspace.name, conversation.channel_id]
|
|
215
|
+
unique = conversation.messages.reject { |message| seen[[*key, message.ts]] }
|
|
216
|
+
return if unique.empty?
|
|
217
|
+
|
|
218
|
+
unique.each { |message| seen[[*key, message.ts]] = true }
|
|
219
|
+
kept = @max.zero? ? unique : unique.last(@max)
|
|
220
|
+
conversation.with(messages: kept, dropped_messages: unique.size - kept.size,
|
|
221
|
+
last_speaker_is_me: unique.last.user_id == @self_ids[conversation.workspace.name])
|
|
222
|
+
end
|
|
223
|
+
# rubocop:enable Metrics/AbcSize
|
|
224
|
+
end
|
|
225
|
+
# rubocop:enable Metrics/ClassLength
|
|
226
|
+
end
|
|
227
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Slk
|
|
4
|
+
module Services
|
|
5
|
+
# Which custom profile field holds a start date, if any.
|
|
6
|
+
#
|
|
7
|
+
# Workspaces name and number these fields themselves, so the ID has to be
|
|
8
|
+
# discovered from the team schema rather than hardcoded — and the schema
|
|
9
|
+
# barely changes, so the answer is cached for a week.
|
|
10
|
+
class StartDateField
|
|
11
|
+
CACHE_KEY = 'start_date_field_v1'
|
|
12
|
+
TTL = 604_800 # 7 days
|
|
13
|
+
LABEL = /\Astart date\z/i
|
|
14
|
+
|
|
15
|
+
def initialize(team_api:, workspace_name:, cache_store: nil, on_debug: nil)
|
|
16
|
+
@team_api = team_api
|
|
17
|
+
@workspace_name = workspace_name
|
|
18
|
+
@cache = cache_store
|
|
19
|
+
@on_debug = on_debug
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @return [String, nil] the Xf… field ID, or nil if the workspace has none
|
|
23
|
+
def id
|
|
24
|
+
return @id if defined?(@id)
|
|
25
|
+
|
|
26
|
+
cached = MetaCache.read(@cache, @workspace_name, CACHE_KEY, ttl: TTL)
|
|
27
|
+
@id = cached.is_a?(Hash) ? cached['id'] : discover
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def missing_message
|
|
31
|
+
'This workspace has no "Start Date" profile field, so tenure cannot be worked out. ' \
|
|
32
|
+
'Run `slk debug schema` (an undocumented command) to see the fields it does have.'
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
# A date-typed field wins over a text one with the same label: someone
|
|
38
|
+
# typing "started summer 2019" into a text box is not a date.
|
|
39
|
+
def discover
|
|
40
|
+
id = usable_id(best_match(@team_api.profile_schema.dig('profile', 'fields')))
|
|
41
|
+
@on_debug&.call("start date field: #{id || 'not found in team schema'}")
|
|
42
|
+
remember(id)
|
|
43
|
+
id
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def usable_id(match)
|
|
47
|
+
id = match && match['id']
|
|
48
|
+
id.is_a?(String) && !id.empty? ? id : nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Losing this costs one extra team.profile.get next run, not minutes of
|
|
52
|
+
# rate-limited lookups, so it is noted rather than warned about.
|
|
53
|
+
def remember(id)
|
|
54
|
+
failure = MetaCache.write(@cache, @workspace_name, CACHE_KEY, { 'id' => id })
|
|
55
|
+
@on_debug&.call("start date field cache not written: #{failure.message}") if failure
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A workspace that answers with something other than a list of field
|
|
59
|
+
# hashes has no start date field as far as we are concerned — better a
|
|
60
|
+
# clear "this workspace cannot do tenure" than an unexpected error.
|
|
61
|
+
def best_match(fields)
|
|
62
|
+
labelled = Array(fields).grep(Hash).select { |f| LABEL.match?(f['label'].to_s.strip) }
|
|
63
|
+
labelled.find { |f| f['type'] == 'date' } || labelled.first
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|