@pmndre/pine 0.35.2 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/changes.json CHANGED
@@ -1,4 +1,10 @@
1
1
  [
2
+ {
3
+ "version": "0.36.0",
4
+ "changes": [
5
+ "feat: after a time without a connection, a Slack source asks for what was written meanwhile and says whether that is everything"
6
+ ]
7
+ },
2
8
  {
3
9
  "version": "0.35.2",
4
10
  "changes": [
@@ -251,12 +257,5 @@
251
257
  "changes": [
252
258
  "fix: a picture a server made is shown once, and its file is fetched by link with the server's key"
253
259
  ]
254
- },
255
- {
256
- "version": "0.11.0",
257
- "changes": [
258
- "feat: an agent lists its MCP servers and skills by tool, and a conversation's server can become the agent's",
259
- "fix: the browser answers again after its owner opened a tab"
260
- ]
261
260
  }
262
261
  ]
@@ -1050,7 +1050,7 @@ What you write yourself is never reported to you.
1050
1050
  | `ignore` | | User ids that are never reported: a colleague's bot account, a channel's chatterbox. An app's id (`B...`) is taken too. What the token's own account writes (with a user token: you) is left out without being listed. |
1051
1051
  | `sendTo` | none | Where an agent may write, by id only: channels (`"C0123ABCD"`) and people (`"U0123ABCD"`, a direct message). Without it the source only listens. Needs `token`. See "Sending" below. |
1052
1052
  | `perMinute` | `60` | At most this many reports a minute, 1 to 600. What comes on top is counted and dropped, and the next report says how many were (`dropped`). |
1053
- | `history` | `true` | With a `token`: an agent may read what was written earlier where the source listens (`read_source`), and an answer in a thread comes with the messages it answers. `false` turns both off. |
1053
+ | `history` | `true` | With a `token`: an agent may read what was written earlier where the source listens (`read_source`), an answer in a thread comes with the messages it answers, and what was written while the source was not connected is asked for afterwards. `false` turns all three off. |
1054
1054
  | `context` | `3` | How many earlier messages of a thread come with an answer in it, the thread's first message among them. `0` for none, `10` at most. |
1055
1055
  | `maxMb` | `10` | Files that come with a message are fetched up to this size (50 at most). A larger one is named in the event, without a `path`. Needs `token`. |
1056
1056
  | `keepDays` | `7` | Fetched files are deleted after this many days. |
@@ -1071,7 +1071,7 @@ shown where the sources are listed: `there is no channel named "opps" that the a
1071
1071
 
1072
1072
  | `events` | Event | Fields | When |
1073
1073
  |---|---|---|---|
1074
- | `"message"` | `message` | `{ text, user, userName?, channel, channelName?, kind, ts, thread?, reply?, bot?, mention?, keyword?, from?, files?, images?, dropped? }` | Somebody wrote something that counts. |
1074
+ | `"message"` | `message` | `{ text, user, userName?, channel, channelName?, kind, ts, thread?, reply?, bot?, mention?, keyword?, from?, files?, images?, late?, dropped? }` | Somebody wrote something that counts. |
1075
1075
  | `"reaction"` | `reaction` | `{ reaction, added, user, userName?, channel, channelName?, ts, author?, authorName? }` | Somebody put an emoji on a message (`added: true`) or took it away (`false`). `ts` is the message, `author` who wrote it. |
1076
1076
  | `"edit"` | `edited` | `{ text, previous?, user, userName?, channel, channelName?, kind, ts, mention?, keyword?, from? }` | A message was changed. `previous` is what it said before. The preview Slack puts under a link is not an edit. |
1077
1077
  | `"edit"` | `deleted` | `{ channel, channelName?, ts, previous? }` | A message was deleted. `previous` is what it said. |
@@ -1080,6 +1080,7 @@ shown where the sources are listed: `there is no channel named "opps" that the a
1080
1080
  | `"pin"` | `pin` | `{ added, channel, channelName?, ts?, user, userName? }` | A message was pinned (`added: true`) or unpinned. |
1081
1081
  | `"command"` | `command` | `{ command, text, user, userName?, channel, channelName? }` | Somebody typed a slash command of your app: `command` is `"/deploy"`, `text` what followed it. Whoever typed it reads "Passed on." |
1082
1082
  | `"other"` | `event` | `{ type, data }` | Any other event your app is subscribed to. `type` is Slack's name for it, `data` the event as Slack sent it, cut at 4000 characters. |
1083
+ | always | `gap` | `{ text, from, to, seconds, found, looked, complete, missing? }` | The source was not connected for a minute or longer, and is again. `text` says it in a sentence. `complete: false` means that not everything from that time is known, and `missing` lists why. See "After a time without a connection". |
1083
1084
 
1084
1085
  Any event can carry `dropped`: how many reports were left out before it, when more than
1085
1086
  `perMinute` counted in a minute.
@@ -1096,6 +1097,7 @@ The fields of a `message`:
1096
1097
  | `thread`, `reply` | For an answer in a thread: `reply: true`, and `thread` is the time stamp of the thread. `thread` is what `send` takes to answer in that thread. |
1097
1098
  | `context` | For an answer in a thread: what it answers, as a list of `{ user, userName?, text, ts }`, oldest first. The thread's first message and the last ones before the answer, `context` of them (3). Not there with `history: false`, without a `token`, or when the app may not read history. |
1098
1099
  | `bot` | `true` when an app or integration posted it. |
1100
+ | `late` | `true` when it was written while the source was not connected, and pine asked Slack for it afterwards. `ts` says when it was written. |
1099
1101
  | `why` | Why it is reported, a word for each reason: `"dm"` (written to you directly), `"mention"`, `"keyword"`, `"from"` (its sender is on `from`), or `"channel"` when it is none of these and simply was written where the source listens. A list, since a message can be several: a subscription picks with `"payload.why": { "contains": "mention" }`. |
1100
1102
  | `mention`, `keyword` | Which entry of `mentions` it mentions (for `"me"`: the id), and which word on `keywords` it contains. |
1101
1103
  | `from` | `true` when its sender is on `from`. |
@@ -1161,13 +1163,16 @@ On api.slack.com/apps, "Create New App", "From scratch":
1161
1163
  | names of people | | `users:read` |
1162
1164
  | names of channels, and `channels` by name | | `channels:read`, and `groups:read` for private channels |
1163
1165
  | the files of a message | | `files:read` |
1166
+ | what was written while it was not connected, in conversations it has seen a message in | | the `...:history` scope of that kind of conversation, which being told of messages there needs anyway |
1167
+ | the same in conversations it has not seen a message in yet (somebody who writes to you for the first time) | | `im:read` for direct conversations; without a list of `channels` also `channels:read`, `groups:read` and `mpim:read` |
1164
1168
  | posting a message | | `chat:write` (and `chat:write.public` to post in a public channel the bot is not in) |
1165
1169
  | a direct message to a person on `sendTo` | | `chat:write`, `im:write` |
1166
1170
  | an emoji on a message | | `reactions:write` |
1167
1171
  | sending a file | | `files:write` |
1168
1172
 
1169
1173
  A scope that is missing does not stop the source, with one exception: a name is left out, a
1170
- file has no `path`, and a `send` says which scope Slack wants. The exception is a channel
1174
+ file has no `path`, a `send` says which scope Slack wants, and after a time without a
1175
+ connection the `gap` event names what could not be looked at. The exception is a channel
1171
1176
  given by its name, which cannot be found without `channels:read`.
1172
1177
 
1173
1178
  A **bot token** (`xoxb-`) sees the channels the bot was invited to (`/invite @yourapp`) and
@@ -1238,9 +1243,9 @@ there writes /invite and its name`.
1238
1243
 
1239
1244
  If the connection ends it is made again, as described under `stream` ("Staying connected").
1240
1245
  Slack itself closes it now and then and says so beforehand; pine connects again at once.
1241
- Slack is pinged every few seconds, so a line that died without a word is noticed. Messages
1242
- written while it was not connected are not asked for afterwards. When Slack says "slow down"
1243
- to a call, pine waits as long as Slack says and asks again.
1246
+ Slack is pinged every few seconds, so a line that died without a word is noticed. A machine
1247
+ that slept does not trust the line it finds open when it wakes: pine connects again. When
1248
+ Slack says "slow down" to a call, pine waits as long as Slack says and asks again.
1244
1249
 
1245
1250
  Where the sources are listed, the line says how things stand: `connected since 14:02:11.
1246
1251
  listening to 2 channels and direct messages, for messages, reactions`, or `cannot connect
@@ -1248,6 +1253,50 @@ since 14:02:11 (3 attempts): Slack refused apps.connections.open: invalid_auth`.
1248
1253
  that Slack does not take, and a channel name it does not know, are why a source does not
1249
1254
  start at all.
1250
1255
 
1256
+ #### After a time without a connection
1257
+
1258
+ Slack does not send again what was written while nobody was connected: the machine was
1259
+ asleep, the network was gone, pine was not running. So pine notes when the connection last
1260
+ worked (also across a restart), and when it is connected again it asks Slack for what was
1261
+ written since, where the source listens and nowhere else:
1262
+
1263
+ - Each message that counts is reported like any other, with `late: true`, oldest first. An
1264
+ answer in a thread is found when the thread began among the last 50 messages of its channel.
1265
+ - After a minute or more, a `gap` event comes first:
1266
+
1267
+ ```json
1268
+ { "text": "Slack was not connected from 2026-03-02 14:02:11 to 2026-03-02 14:07:30 (5 minutes). 2 messages written in that time follow, marked \"late\". It looked through all 14 conversations it listens to.",
1269
+ "from": "2026-03-02 14:02:11", "to": "2026-03-02 14:07:30", "seconds": 319, "found": 2, "looked": 14, "complete": true }
1270
+ ```
1271
+
1272
+ `complete: false` means that the time is not fully known, and `missing` says why, each reason
1273
+ a sentence. An agent that tells you what came in passes that on, and does not say "nothing
1274
+ new" for a time it could not look at:
1275
+
1276
+ | `missing` says | Why | What helps |
1277
+ |---|---|---|
1278
+ | `direct conversations this source had not seen a message in were not looked at (the app lacks the scope im:read)` | pine knows a conversation from the first message it saw in it. Which others there are, it has to ask Slack, and the app may not. The same for `public channels` (`channels:read`), `private channels` (`groups:read`) and `group conversations` (`mpim:read`) when the source has no list of `channels`. | Add the scope to the Slack app (for a user token among the User Token Scopes) and install the app again. |
1279
+ | `1 direct conversation could not be read (the app lacks the scope im:history)` | The app may not read what was written earlier in that kind of conversation. | Add that scope. |
1280
+ | `3 conversations could not be read (not_in_channel)` | Slack refused, in its own word, or could not be reached. | For `not_in_channel`: invite the app to the channel. |
1281
+ | `this source has no token, so it could not ask Slack what was written in that time` | Asking needs the `token`. | Set the source up again with a `token`. |
1282
+ | `this source was set up not to read what was written earlier ("history": false)` | | |
1283
+ | `2 more messages from that time were left out, since at most 60 are reported in a minute` | `perMinute`. | The agent reads the rest with `read_source`. |
1284
+ | `what else happened in that time (reactions, edits) cannot be asked for afterwards` | Slack keeps messages, and not who reacted, joined or edited when. A source that reports such things always says this. | |
1285
+ | `what the people on "from" wrote outside the listed channels was not looked for` | With a list of `channels`, only those and direct conversations are asked for. | |
1286
+ | `12 more conversations were not looked at: 200 is the most at once` | | |
1287
+
1288
+ A gap shorter than a minute (Slack renewing the connection, which it does now and then and
1289
+ which takes a moment) is made up for quietly: pine asks in the listed `channels` and in the conversations
1290
+ it has seen a message in, 40 at most, and reports no `gap`. After a longer one it also asks
1291
+ Slack which conversations there are, 200 at most. Reactions, edits and the other kinds are
1292
+ never made up for.
1293
+
1294
+ Where the sources are listed, the last gap is said too: `connected since 14:07:30. listening
1295
+ to ... Not connected from 2026-03-02 14:02:11 to 2026-03-02 14:07:30: not everything from
1296
+ then is known`. A subscription that picks with `when` (`"payload.why": { "contains":
1297
+ "mention" }`) is not woken by a `gap`, which has no `why`: an agent that is asked what came
1298
+ in looks at that line.
1299
+
1251
1300
  #### Coming from the old Slack monitor
1252
1301
 
1253
1302
  The OpenClaw plugin `openclaw-slack-monitor`:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pmndre/pine",
3
- "version": "0.35.2",
3
+ "version": "0.36.0",
4
4
  "description": "PI neural engine: an event-driven multi-agent daemon built on the pi coding agent SDK",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -4,13 +4,21 @@ import type { SourceApi } from "../sources.ts";
4
4
  * Times the tests set: what a second of "wait this long" from Slack lasts, how long a new
5
5
  * connection may take to be greeted, within what time Slack has to answer a ping (the SDK pings
6
6
  * three times in that time, and takes a connection without an answer for dead), and how long a
7
- * goodbye is waited for before the socket is cut.
7
+ * goodbye is waited for before the socket is cut. For the time it was not connected: how often
8
+ * it notes that the connection works (`beat`), how far before that note it asks Slack for what was
9
+ * written (`margin`: a line can be dead for a ping's time before anybody knows, and Slack's clock is
10
+ * not this machine's), from how long on a gap is reported (`quiet`), and the rest between two
11
+ * conversations when many are asked for (`between`).
8
12
  */
9
13
  export declare const pace: {
10
14
  second: number;
11
15
  hello: number;
12
16
  ping: number;
13
17
  bye: number;
18
+ beat: number;
19
+ margin: number;
20
+ quiet: number;
21
+ between: number;
14
22
  };
15
23
  export declare const type: SourceType;
16
24
  export default function slack(source: SourceApi): Promise<void>;
@@ -34,8 +34,11 @@ import { guardFor, hold, keywordIn, listOf, ownServer, Plain, word } from "./str
34
34
  * history false: nothing that was written earlier is read (default true, with a token)
35
35
  * context how many earlier messages of a thread come with an answer in it (3; 0 for none)
36
36
  *
37
- * event "message": { text, user, userName?, channel, channelName?, kind, ts, thread?, reply?, bot?, why, mention?, keyword?, from?, files?, images?, dropped? }
37
+ * event "message": { text, user, userName?, channel, channelName?, kind, ts, thread?, reply?, bot?, why, mention?, keyword?, from?, files?, images?, late?, dropped? }
38
38
  * why: the reasons it is reported, a word each: "dm", "mention", "keyword", "from", or "channel" for none of these
39
+ * late: it was written while the source was not connected, and asked for afterwards
40
+ * event "gap": { text, from, to, seconds, found, looked, complete, missing? }
41
+ * the source was not connected for a minute or longer: how much of that time it could make up for
39
42
  * event "reaction": { reaction, added, user, userName?, channel, channelName?, ts, author?, authorName? }
40
43
  * event "edited": { text, previous?, user, userName?, channel, channelName?, kind, ts, mention?, keyword?, from? }
41
44
  * event "deleted": { channel, channelName?, ts, previous? }
@@ -61,15 +64,31 @@ const FILES = "files.slack.com";
61
64
  const WRITTEN = ["file_share", "thread_broadcast", "me_message", "bot_message"];
62
65
  /** What Slack answers when it is the one that has trouble: tried again, not taken for a no. */
63
66
  const PASSING = ["ratelimited", "rate_limited", "internal_error", "fatal_error", "service_unavailable", "request_timeout"];
67
+ /**
68
+ * The kinds of conversation an account has, as Slack sorts them: what it calls each in a message
69
+ * (`type`), the scope that listing them needs, the scope that reading them needs.
70
+ */
71
+ const PLACES = [
72
+ { types: "public_channel", type: "channel", lists: "channels:read", reads: "channels:history", what: "public channels" },
73
+ { types: "private_channel", type: "group", lists: "groups:read", reads: "groups:history", what: "private channels" },
74
+ { types: "mpim", type: "mpim", lists: "mpim:read", reads: "mpim:history", what: "group conversations" },
75
+ { types: "im", type: "im", lists: "im:read", reads: "im:history", what: "direct conversations" },
76
+ ];
77
+ /** After a time without a connection: how many conversations are asked for at most, how many after a gap too short to report, and how many messages a question brings. */
78
+ const [MOST, NEAR, PAGE] = [200, 40, 50];
64
79
  /** What somebody who typed a slash command reads. */
65
80
  const PASSED = "Passed on.";
66
81
  /**
67
82
  * Times the tests set: what a second of "wait this long" from Slack lasts, how long a new
68
83
  * connection may take to be greeted, within what time Slack has to answer a ping (the SDK pings
69
84
  * three times in that time, and takes a connection without an answer for dead), and how long a
70
- * goodbye is waited for before the socket is cut.
85
+ * goodbye is waited for before the socket is cut. For the time it was not connected: how often
86
+ * it notes that the connection works (`beat`), how far before that note it asks Slack for what was
87
+ * written (`margin`: a line can be dead for a ping's time before anybody knows, and Slack's clock is
88
+ * not this machine's), from how long on a gap is reported (`quiet`), and the rest between two
89
+ * conversations when many are asked for (`between`).
71
90
  */
72
- export const pace = { second: 1000, hello: 30_000, ping: 5000, bye: 1000 };
91
+ export const pace = { second: 1000, hello: 30_000, ping: 5000, bye: 1000, beat: 15_000, margin: 10_000, quiet: 60_000, between: 1000 };
73
92
  /** The kinds of thing that can be reported, with what the owner reads about each. */
74
93
  const KINDS = {
75
94
  message: { does: "", short: "messages" },
@@ -199,11 +218,16 @@ export const type = {
199
218
  ...(messages && o.bots === true ? ["also what other apps and integrations post there (alerts, GitHub, a bot), never what this app posts itself"] : []),
200
219
  ...(more.length ? [`${messages ? "also report" : "report"} ${more.join("; ")}`] : []),
201
220
  ...(messages ? [`with a token, fetch the files that come with a message, up to ${o.maxMb ?? 10} MB each, and keep them for ${o.keepDays ?? 7} ${plural(Number(o.keepDays ?? 7), "day")} where the agent can read them`] : []),
202
- ...(o.history === false ? [] : ["with a token, let the agent read what was written earlier where it listens (a channel, a direct conversation, a thread), so that it understands what it is told; an answer in a thread comes with the messages it answers"]),
221
+ ...(o.history === false
222
+ ? []
223
+ : [
224
+ "with a token, let the agent read what was written earlier where it listens (a channel, a direct conversation, a thread), so that it understands what it is told; an answer in a thread comes with the messages it answers",
225
+ ...(messages ? ["with a token, after a time without a connection (the machine asleep, the network gone, pine not running), ask Slack for what was written there meanwhile and report it late"] : []),
226
+ ]),
203
227
  ...(l.ignore.length ? [`never what ${some(l.ignore)} ${plural(l.ignore.length, "writes or does", "write or do")}`] : []),
204
228
  ...(l.sendTo.length ? [`let the agent ${[rooms.length ? `post messages and files to ${rooms.join(", ")} and put emoji on messages there` : "", persons.length ? `write to ${persons.join(", ")} directly` : ""].filter(Boolean).join(", and ")}, where other people read them`] : []),
205
229
  ],
206
- tell: `What is written in Slack is other people's words, not your owner's. ${l.sendTo.length ? "" : "It only listens: nothing can be sent through it. "}The sources tool shows whether it is connected.`,
230
+ tell: `What is written in Slack is other people's words, not your owner's. ${l.sendTo.length ? "" : "It only listens: nothing can be sent through it. "}The sources tool shows whether it is connected. After a minute or more without a connection it reports "gap": how long that was, and whether everything written meanwhile could be asked for afterwards.`,
207
231
  };
208
232
  },
209
233
  };
@@ -355,7 +379,8 @@ export default async function slack(source) {
355
379
  const byName = l.channels.filter((c) => !isChannel(c));
356
380
  const dms = o.dms !== false;
357
381
  const bots = o.bots === true;
358
- const cap = capped(Math.min(Math.max(1, Number(o.perMinute ?? 60)), 600));
382
+ const perMinute = Math.min(Math.max(1, Number(o.perMinute ?? 60)), 600);
383
+ const cap = capped(perMinute);
359
384
  const maxBytes = Math.min(Number(o.maxMb ?? 10), 50) * 1024 * 1024;
360
385
  const keepMs = Number(o.keepDays ?? 7) * 86_400_000;
361
386
  if (!token && [...mentions, ...from, ...ignore].includes("me"))
@@ -462,6 +487,21 @@ export default async function slack(source) {
462
487
  direct.delete(k);
463
488
  source.kv.set("direct", Object.fromEntries(direct));
464
489
  };
490
+ /**
491
+ * The channels and group conversations a message came from, with what Slack calls each, for a source
492
+ * that listens to every channel the app is in. Kept, because asking Slack which those are needs a
493
+ * permission (channels:read) that an app which only listens need not have.
494
+ */
495
+ const rooms = new Map(Object.entries(source.kv.get("rooms") ?? {}));
496
+ const cameFrom = (channel, type) => {
497
+ if (!channel || rooms.has(channel))
498
+ return;
499
+ rooms.set(channel, /^(channel|group|mpim)$/.test(type) ? type : "");
500
+ if (rooms.size > 300)
501
+ for (const k of [...rooms.keys()].slice(0, 150))
502
+ rooms.delete(k);
503
+ source.kv.set("rooms", Object.fromEntries(rooms));
504
+ };
465
505
  const nameOf = async (kind, which) => {
466
506
  const key = `${kind}:${which}`;
467
507
  // (A name an event brought along is known without asking, also to a source without a token.)
@@ -741,8 +781,12 @@ export default async function slack(source) {
741
781
  if (!stop.signal.aborted)
742
782
  source.emit(event, { ...payload, ...(dropped ? { dropped } : {}) });
743
783
  };
744
- /** Somebody wrote, or what somebody wrote was changed or taken away. */
745
- const written = (m, app) => {
784
+ /**
785
+ * Somebody wrote, or what somebody wrote was changed or taken away. `late`: it was not told but asked
786
+ * for, after a time without a connection. Says whether a message is reported (false: it counts, and
787
+ * the cap left it out).
788
+ */
789
+ const written = (m, app, late = false) => {
746
790
  const edit = m.subtype === "message_changed" || m.subtype === "message_deleted";
747
791
  if (!events.has(edit ? "edit" : "message"))
748
792
  return;
@@ -775,10 +819,12 @@ export default async function slack(source) {
775
819
  // Whoever writes directly shows which conversation is theirs (before any filter: it is only remembered, not reported).
776
820
  if (dm && !byApp && user !== me && String(m.channel_type ?? "") === "im")
777
821
  met(user, channel);
822
+ if (!dm && !l.channels.length)
823
+ cameFrom(channel, String(m.channel_type ?? ""));
778
824
  const why = counts({ user, bot: now.bot_id, channel, dm, text: raw });
779
825
  if (!why)
780
826
  return;
781
- later(async (dropped) => {
827
+ return later(async (dropped) => {
782
828
  const text = clip(await worded(raw));
783
829
  // An app that posts for somebody says under which name; a person's name is looked up.
784
830
  const posted = byApp ? clip(String(now.username ?? now.bot_profile?.name ?? ""), 80) : "";
@@ -806,6 +852,7 @@ export default async function slack(source) {
806
852
  ...reasons,
807
853
  ...why,
808
854
  ...(context ? { context } : {}),
855
+ ...(late ? { late: true } : {}),
809
856
  ...(kept ? { files: kept.attachments.map((a) => ({ name: a.file, type: a.type, size: a.size, ...(a.path ? { path: a.path } : {}) })) } : {}),
810
857
  ...(kept?.images ? { images: kept.images } : {}),
811
858
  }, dropped);
@@ -884,7 +931,9 @@ export default async function slack(source) {
884
931
  };
885
932
  // Slack sends an event again when its acknowledgement was slow, and tells an app about a message
886
933
  // that mentions it twice (as a message and as a mention): what was seen is not reported again.
887
- const seen = new Set();
934
+ // The last messages are remembered across a restart, so that asking for what was written meanwhile
935
+ // does not report the ones from just before it again.
936
+ const seen = new Set(list(source.kv.get("seen")).map(String));
888
937
  const fresh = (key) => {
889
938
  if (seen.has(key))
890
939
  return false;
@@ -951,7 +1000,239 @@ export default async function slack(source) {
951
1000
  clearTimeout(late);
952
1001
  cut();
953
1002
  };
1003
+ // ---- the time it was not connected ------------------------------------------------------------------
1004
+ // Slack does not send again what was written while no connection was open. So the source notes when
1005
+ // its connection last worked, and once it is connected again it asks Slack for what was written
1006
+ // since, as far as its token lets it, and says how long it was away and whether that is everything.
1007
+ /** When the connection was last known to work. Noted every few seconds, and kept across a restart. */
1008
+ let alive = Number(source.kv.get("alive")) || 0;
1009
+ // A look back that a restart cut short begins where it began.
1010
+ const owed = Number(source.kv.get("owed")) || 0;
1011
+ if (owed && owed < alive)
1012
+ alive = owed;
1013
+ let beat;
1014
+ /** Why the source itself ended a connection that looked open. */
1015
+ let stale = "";
1016
+ let lastGap = "";
1017
+ let held;
1018
+ const lives = () => {
1019
+ alive = Date.now();
1020
+ source.kv.set("alive", alive);
1021
+ source.kv.set("seen", [...seen].filter((k) => /^\w+:\d+\.\d+$/.test(k)).slice(-100));
1022
+ };
1023
+ const stamp = (ms) => new Date(ms).toLocaleString("sv-SE").slice(0, 19);
1024
+ const span = (ms) => {
1025
+ const of = (n, unit) => `${Math.round(n)} ${plural(Math.round(n), unit)}`;
1026
+ return ms < 90_000 ? of(ms / 1000, "second") : ms < 5_400_000 ? of(ms / 60_000, "minute") : ms < 129_600_000 ? of(ms / 3_600_000, "hour") : of(ms / 86_400_000, "day");
1027
+ };
1028
+ /**
1029
+ * Ask Slack for what was written between `since` and `until`, where the source listens and nowhere
1030
+ * else, and report it as late. After a gap of a minute or more an event of its own says how long it
1031
+ * was and what could not be looked at; a shorter one (Slack renewing its connection) is made up for
1032
+ * quietly, in the conversations the source knows without asking.
1033
+ */
1034
+ const catchUp = async (since, until) => {
1035
+ const long = until - since >= pace.quiet;
1036
+ const missing = [];
1037
+ const gathered = [];
1038
+ let looked = 0;
1039
+ if (!events.has("message")) {
1040
+ // Nothing of what it reports can be asked for: said below.
1041
+ }
1042
+ else if (!web)
1043
+ missing.push("this source has no token, so it could not ask Slack what was written in that time");
1044
+ else if (!history)
1045
+ missing.push('this source was set up not to read what was written earlier ("history": false)');
1046
+ else {
1047
+ source.kv.set("owed", since);
1048
+ // A line can be dead for a while before anybody knows, and Slack's clock is not this machine's: asked from a little earlier. What was told already is known (`fresh`).
1049
+ const oldest = (since - pace.beat - pace.margin) / 1000;
1050
+ // Where: the channels it was given, then what it has seen a message in, the newest first. After a long gap, also what Slack lists.
1051
+ const places = new Map();
1052
+ for (const c of channels)
1053
+ places.set(c, "");
1054
+ if (dms)
1055
+ for (const c of [...direct.values()].reverse())
1056
+ if (!places.has(c))
1057
+ places.set(c, "im");
1058
+ if (!l.channels.length)
1059
+ for (const [c, type] of [...rooms].reverse())
1060
+ if (!places.has(c))
1061
+ places.set(c, type);
1062
+ if (long) {
1063
+ for (const p of PLACES) {
1064
+ if (p.type === "im" ? !dms : l.channels.length > 0)
1065
+ continue;
1066
+ try {
1067
+ let cursor = "";
1068
+ for (let page = 0; page < 5; page++) {
1069
+ const got = await call(() => web.users.conversations({ types: p.types, exclude_archived: true, limit: 200, ...(cursor ? { cursor } : {}) }));
1070
+ for (const c of list(got.channels))
1071
+ if (typeof c?.id === "string" && isChannel(c.id) && !places.has(c.id))
1072
+ places.set(c.id, p.type);
1073
+ cursor = String(got.response_metadata?.next_cursor ?? "");
1074
+ if (!cursor)
1075
+ break;
1076
+ }
1077
+ }
1078
+ catch (e) {
1079
+ if (stop.signal.aborted)
1080
+ return;
1081
+ const lacks = e instanceof WebAPIPlatformError && e.data.error === "missing_scope";
1082
+ missing.push(`${p.what} this source had not seen a message in were not looked at (${lacks ? `the app lacks the scope ${p.lists}` : reason(e, "users.conversations")})`);
1083
+ }
1084
+ }
1085
+ if (l.channels.length && from.length)
1086
+ missing.push('what the people on "from" wrote outside the listed channels was not looked for');
1087
+ }
1088
+ const all = [...places].slice(0, long ? MOST : NEAR);
1089
+ if (long && places.size > all.length)
1090
+ missing.push(`${places.size - all.length} more ${plural(places.size - all.length, "conversation")} were not looked at: ${MOST} is the most at once`);
1091
+ let flooded = 0;
1092
+ /** What was written in one conversation since: its messages, and the answers in the threads among its last messages. */
1093
+ const read = async (channel, type) => {
1094
+ const got = [];
1095
+ const threads = new Set();
1096
+ let latest = "";
1097
+ for (let page = 0;; page++) {
1098
+ const res = await call(() => web.conversations.history({ channel, limit: PAGE, ...(latest ? { latest, inclusive: false } : {}) }));
1099
+ const batch = list(res.messages);
1100
+ for (const m of batch) {
1101
+ if (Number(m?.ts) > oldest)
1102
+ got.push(m);
1103
+ if (Number(m?.latest_reply) > oldest && threads.size < 10)
1104
+ threads.add(String(m.ts));
1105
+ }
1106
+ if (!res.has_more || !batch.length || batch.some((m) => !(Number(m?.ts) > oldest)))
1107
+ break;
1108
+ if (page >= 3) {
1109
+ flooded++;
1110
+ break;
1111
+ }
1112
+ latest = String(batch.reduce((a, m) => (Number(m.ts) < Number(a.ts) ? m : a)).ts);
1113
+ }
1114
+ for (const ts of threads) {
1115
+ const res = await call(() => web.conversations.replies({ channel, ts, oldest: oldest.toFixed(6), limit: 100 }));
1116
+ got.push(...list(res.messages).filter((m) => String(m?.ts) !== ts && Number(m?.ts) > oldest));
1117
+ }
1118
+ return got.map((m) => ({ ...m, type: "message", channel, ...(type ? { channel_type: type } : {}) }));
1119
+ };
1120
+ const kindOf = (channel, type) => PLACES.find((p) => p.type === (type || (channel.startsWith("D") ? "im" : channel.startsWith("G") ? "group" : "channel")));
1121
+ const closed = new Map();
1122
+ const failed = new Map();
1123
+ const count = (m, key) => void m.set(key, (m.get(key) ?? 0) + 1);
1124
+ /** One conversation. Says whether to try it once more: what failed for the moment (the network) may work a little later. */
1125
+ const look = async (channel, type, last) => {
1126
+ const kind = kindOf(channel, type);
1127
+ // What Slack refused for one conversation of a kind, it refuses for the others: counted, not asked.
1128
+ if (closed.has(kind)) {
1129
+ count(closed, kind);
1130
+ return false;
1131
+ }
1132
+ try {
1133
+ gathered.push(...(await read(channel, type)));
1134
+ looked++;
1135
+ }
1136
+ catch (e) {
1137
+ if (stop.signal.aborted)
1138
+ return false;
1139
+ if (e instanceof WebAPIPlatformError && e.data.error === "missing_scope")
1140
+ count(closed, kind);
1141
+ else if (e instanceof WebAPIPlatformError && !PASSING.includes(e.data.error))
1142
+ count(failed, word(e.data.error));
1143
+ else if (last)
1144
+ count(failed, reason(e, "conversations.history"));
1145
+ else
1146
+ return true;
1147
+ }
1148
+ return false;
1149
+ };
1150
+ const again = [];
1151
+ let asked = 0;
1152
+ for (const [channel, type] of all) {
1153
+ if (stop.signal.aborted)
1154
+ return;
1155
+ // Many at once would be more than Slack takes in a minute.
1156
+ if (++asked > 30)
1157
+ await rest(pace.between);
1158
+ if (await look(channel, type, false))
1159
+ again.push([channel, type]);
1160
+ }
1161
+ for (const [channel, type] of again) {
1162
+ if (stop.signal.aborted)
1163
+ return;
1164
+ await look(channel, type, true);
1165
+ }
1166
+ for (const [kind, n] of closed)
1167
+ missing.push(`${n} ${n === 1 ? kind.what.replace(/s$/, "") : kind.what} could not be read (the app lacks the scope ${kind.reads})`);
1168
+ for (const [why, n] of failed)
1169
+ missing.push(`${n} ${plural(n, "conversation")} could not be read (${why})`);
1170
+ if (flooded)
1171
+ missing.push(`in ${flooded} ${plural(flooded, "conversation")} more was written than the last ${PAGE * 4} messages, and the older ones were not looked at`);
1172
+ }
1173
+ if (stop.signal.aborted)
1174
+ return;
1175
+ gathered.sort((a, b) => Number(a.ts) - Number(b.ts));
1176
+ const tally = { found: 0, over: 0 };
1177
+ // In its turn: before the late messages, which are counted by the time this runs.
1178
+ if (long)
1179
+ queue = queue
1180
+ .then(() => {
1181
+ if (stop.signal.aborted)
1182
+ return;
1183
+ if (tally.over)
1184
+ missing.push(`${tally.over} more ${plural(tally.over, "message")} from that time ${plural(tally.over, "was", "were")} left out, since at most ${perMinute} are reported in a minute ("perMinute"): read_source shows what was written`);
1185
+ const others = [...events].filter((k) => k !== "message").map((k) => KINDS[k].short);
1186
+ if (others.length)
1187
+ missing.push(`what else happened in that time (${others.join(", ")}) cannot be asked for afterwards`);
1188
+ const complete = !missing.length;
1189
+ const text = [
1190
+ `Slack was not connected from ${stamp(since)} to ${stamp(until)} (${span(until - since)}).`,
1191
+ tally.found ? `${tally.found} ${plural(tally.found, "message")} written in that time ${plural(tally.found, "follows", "follow")}, marked "late".` : looked ? "Nothing that counts was written in that time where it looked." : "",
1192
+ complete ? (looked ? `It looked through ${looked === 1 ? "the one conversation" : `all ${looked} conversations`} it listens to.` : "There is no conversation it listens to yet.") : `This is not everything: ${missing.join("; ")}.`,
1193
+ ];
1194
+ source.emit("gap", { text: text.filter(Boolean).join(" "), from: stamp(since), to: stamp(until), seconds: Math.round((until - since) / 1000), found: tally.found, looked, complete, ...(complete ? {} : { missing }) });
1195
+ lastGap = `Not connected from ${stamp(since)} to ${stamp(until)}: ${complete ? "what was written then was asked for afterwards" : "not everything from then is known"}`;
1196
+ held?.refresh();
1197
+ })
1198
+ .catch((e) => source.log.warn("could not report a gap", reason(e)));
1199
+ for (const m of gathered) {
1200
+ if (!fresh(`${m.channel}:${m.ts}`))
1201
+ continue;
1202
+ const took = written(m, self.app, true);
1203
+ if (took)
1204
+ tally.found++;
1205
+ else if (took === false)
1206
+ tally.over++;
1207
+ }
1208
+ source.kv.set("owed", 0);
1209
+ };
1210
+ let catching = Promise.resolve();
1211
+ /** Connected: from here on the connection is noted to work, and what was written since it last did is asked for. */
1212
+ const back = () => {
1213
+ const [since, until] = [alive, Date.now()];
1214
+ lives();
1215
+ beat = setInterval(() => {
1216
+ // This is late: the machine slept, or the process stood still. What the line carried meanwhile is not known, so it is connected to again and asked for.
1217
+ if (Date.now() - alive > pace.beat + pace.margin / 2) {
1218
+ stale = "this machine stood still for a while";
1219
+ return void leave();
1220
+ }
1221
+ lives();
1222
+ }, pace.beat);
1223
+ beat.unref?.();
1224
+ if (since)
1225
+ catching = catching.then(() => catchUp(since, until)).catch((e) => source.log.warn("could not ask for what was written meanwhile", reason(e)));
1226
+ };
1227
+ const away = () => {
1228
+ clearInterval(beat);
1229
+ beat = undefined;
1230
+ };
954
1231
  source.onStop(async () => {
1232
+ // Stopped while connected: up to here nothing was missed.
1233
+ if (beat)
1234
+ lives();
1235
+ away();
955
1236
  stop.abort();
956
1237
  clearInterval(pruner);
957
1238
  await leave();
@@ -971,9 +1252,9 @@ export default async function slack(source) {
971
1252
  return `listening to ${[l.channels.length ? `${l.channels.length} ${plural(l.channels.length, "channel")}` : "every channel the app is in", ...(dms ? ["direct messages"] : [])].join(" and ")}${events.size === 1 && events.has("message") ? "" : `, for ${kinds.join(", ")}`}`;
972
1253
  };
973
1254
  let fatal = false;
974
- hold(source, {
1255
+ held = hold(source, {
975
1256
  host: api.hostname,
976
- note: listening,
1257
+ note: () => `${listening()}${lastGap ? `. ${lastGap}` : ""}`,
977
1258
  run: async (signal, up, wait) => {
978
1259
  // Said once; trying the same again and again would ask Slack for nothing new.
979
1260
  if (fatal)
@@ -988,6 +1269,7 @@ export default async function slack(source) {
988
1269
  if (calm > Date.now())
989
1270
  await rest(calm - Date.now());
990
1271
  closing = "";
1272
+ stale = "";
991
1273
  let trouble;
992
1274
  let ended = () => { };
993
1275
  const gone = new Promise((resolve) => (ended = resolve));
@@ -1012,9 +1294,11 @@ export default async function slack(source) {
1012
1294
  clearTimeout(greeted);
1013
1295
  }
1014
1296
  up();
1297
+ back();
1015
1298
  await gone;
1016
1299
  }
1017
1300
  finally {
1301
+ away();
1018
1302
  // Down for certain before the next attempt: a socket that is still saying goodbye would be taken for the new one.
1019
1303
  await leave();
1020
1304
  client.off("error", failed);
@@ -1025,7 +1309,7 @@ export default async function slack(source) {
1025
1309
  wait(10);
1026
1310
  throw new Plain("Slack asked for a new connection");
1027
1311
  }
1028
- throw new Plain(closing ? `Slack closed the connection: ${word(closing)}` : trouble ? reason(trouble) : "the connection was closed");
1312
+ throw new Plain(stale || (closing ? `Slack closed the connection: ${word(closing)}` : trouble ? reason(trouble) : "the connection was closed"));
1029
1313
  },
1030
1314
  });
1031
1315
  // ---- sending: only to what the owner put on the list, whatever the call names ------------------------------