@tinycloud/cli 1.3.0-beta.44 → 1.3.0-beta.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/courier-entry.js +523 -91
- package/dist/courier-entry.js.map +1 -1
- package/package.json +5 -5
- package/skills/courier/SKILL.md +26 -1
package/dist/courier-entry.js
CHANGED
|
@@ -196,7 +196,7 @@ function parseTaskState(input) {
|
|
|
196
196
|
return full;
|
|
197
197
|
throw new Error(`Unknown task state "${input}". Expected one of: ${TASK_STATES.join(", ")}`);
|
|
198
198
|
}
|
|
199
|
-
var COURIER_NOTICES = ["thread-paused"];
|
|
199
|
+
var COURIER_NOTICES = ["thread-paused", "thread-resumed"];
|
|
200
200
|
function messageRole(input) {
|
|
201
201
|
if (input.refsOwnTask)
|
|
202
202
|
return "ROLE_AGENT";
|
|
@@ -5704,11 +5704,15 @@ function identity(notice) {
|
|
|
5704
5704
|
}
|
|
5705
5705
|
function noticeText(notice, _owner) {
|
|
5706
5706
|
const { peer, thread } = identity(notice);
|
|
5707
|
+
const pending = /^[0-9]{6}$/.test(notice.count ?? "") ? Number(notice.count) : 1;
|
|
5708
|
+
const total = `${pending}${pending === 999999 ? "+" : ""}`;
|
|
5707
5709
|
if (notice.kind === "new-message")
|
|
5708
|
-
return { title: "Courier: needs you", body:
|
|
5710
|
+
return { title: "Courier: needs you", body: `${total} new message(s) from ${peer}; latest thread ${thread}. Start a session, run courier inbox, then courier read and courier orders for this peer.` };
|
|
5709
5711
|
if (notice.kind === "new-approval")
|
|
5710
|
-
return { title: "Courier: decision ready", body:
|
|
5711
|
-
|
|
5712
|
+
return { title: "Courier: decision ready", body: `${total} owner decision(s) with ${peer}; latest thread ${thread}. Start a session and run courier approval with --peek; an approved PR merge uses courier run-approved. No agent was woken.` };
|
|
5713
|
+
const count = /^[0-9]{6}$/.test(notice.repeats ?? "") ? Number(notice.repeats) : 0;
|
|
5714
|
+
const repeated = count ? ` Additional pauses on this thread: ${count}${count === 999999 ? "+" : ""}.` : "";
|
|
5715
|
+
return notice.kind === "paused" ? { title: "Courier: thread paused", body: `Thread ${thread} with ${peer} is paused on this side. Your agent will not wake for it until you resume it. Run courier inbox or courier read in a session you start.${repeated}` } : { title: "Courier: peer paused", body: `The paired peer ${peer} reported thread ${thread} paused. Their agent waits for their human. Run courier inbox or courier read in a session you start.${repeated}` };
|
|
5712
5716
|
}
|
|
5713
5717
|
function appendNotice(home, notice) {
|
|
5714
5718
|
const { peer, thread } = identity(notice);
|
|
@@ -5717,7 +5721,7 @@ function appendNotice(home, notice) {
|
|
|
5717
5721
|
fs8.mkdirSync(home, { recursive: true, mode: 448 });
|
|
5718
5722
|
if (fs8.existsSync(file) && fs8.statSync(file).size >= MAX_NOTICE_LOG_BYTES)
|
|
5719
5723
|
fs8.renameSync(file, `${file}.1`);
|
|
5720
|
-
fs8.appendFileSync(file, `${JSON.stringify({ t: (/* @__PURE__ */ new Date()).toISOString(), kind: notice.kind, peer, thread, title, body })}
|
|
5724
|
+
fs8.appendFileSync(file, `${JSON.stringify({ t: (/* @__PURE__ */ new Date()).toISOString(), kind: notice.kind, peer, thread, ...notice.count ? { count: notice.count } : {}, title, body })}
|
|
5721
5725
|
`, { mode: 384 });
|
|
5722
5726
|
fs8.chmodSync(file, 384);
|
|
5723
5727
|
}
|
|
@@ -5735,7 +5739,7 @@ function recentNotices(home) {
|
|
|
5735
5739
|
const record = JSON.parse(line);
|
|
5736
5740
|
if (!["paused", "peer-paused", "new-message", "new-approval"].includes(record.kind) || typeof record.peer !== "string" || !HANDLE.test(record.peer) || typeof record.thread !== "string" || !(ULID.test(record.thread) || /^sha256:[a-f0-9]{12}$/.test(record.thread)))
|
|
5737
5741
|
continue;
|
|
5738
|
-
records.push({ kind: record.kind, peer: record.peer, thread: record.thread, ...noticeText({ kind: record.kind, peer: record.peer, threadId: record.thread, detail: "" }, "") });
|
|
5742
|
+
records.push({ kind: record.kind, peer: record.peer, thread: record.thread, ...noticeText({ kind: record.kind, peer: record.peer, threadId: record.thread, count: typeof record.count === "string" ? record.count : void 0, detail: "" }, "") });
|
|
5739
5743
|
} catch {
|
|
5740
5744
|
}
|
|
5741
5745
|
}
|
|
@@ -5763,7 +5767,7 @@ function createNotifier(config, deps) {
|
|
|
5763
5767
|
const result = await run2("/bin/sh", ["-c", config.exec], {
|
|
5764
5768
|
cwd: config.wake.cwd,
|
|
5765
5769
|
timeoutMs: config.wake.timeoutMs,
|
|
5766
|
-
env: { ...courierOnlyEnv(env, config.wake.home), COURIER_HOME: config.wake.home, COURIER_NOTICE: notice.kind, COURIER_PEER: peer, COURIER_THREAD: thread, COURIER_NOTICE_TITLE: title, COURIER_NOTICE_TEXT: body }
|
|
5770
|
+
env: { ...courierOnlyEnv(env, config.wake.home), COURIER_HOME: config.wake.home, COURIER_NOTICE: notice.kind, COURIER_NOTICE_COUNT: String(/^[0-9]{6}$/.test(notice.count ?? "") ? Number(notice.count) : 1), COURIER_PEER: peer, COURIER_THREAD: thread, COURIER_NOTICE_TITLE: title, COURIER_NOTICE_TEXT: body }
|
|
5767
5771
|
});
|
|
5768
5772
|
if (result.code !== 0)
|
|
5769
5773
|
throw new Error(`notify command exited ${result.code}`);
|
|
@@ -6814,20 +6818,25 @@ import { spawnSync as spawnSync4 } from "child_process";
|
|
|
6814
6818
|
|
|
6815
6819
|
// ../courier/dist/pause.js
|
|
6816
6820
|
var SAFE = /^[0-9A-Za-z_-]{1,64}$/;
|
|
6817
|
-
async function
|
|
6821
|
+
async function nextNoticeId(kv2, peer, threadId, now = Date.now()) {
|
|
6822
|
+
const prefix = k.msgPrefix(peer, threadId);
|
|
6823
|
+
const ids = (await listAll(kv2, prefix)).map((key) => key.slice(prefix.length).replace(/\.json$/, "")).filter((id) => ULID.test(id)).sort();
|
|
6824
|
+
const latest = ids.at(-1);
|
|
6825
|
+
return ulid(Math.max(now, latest ? decodeTime(latest) + 1 : 0));
|
|
6826
|
+
}
|
|
6827
|
+
async function postPausedInform(kv2, profile, peerHandle, threadId, text, messageId, resumed = false) {
|
|
6828
|
+
const pauseId = messageId ?? await nextNoticeId(kv2, peerHandle, threadId);
|
|
6829
|
+
if (!ULID.test(pauseId))
|
|
6830
|
+
throw new CourierError("usage", "Invalid pause inform id.");
|
|
6818
6831
|
const message = buildMessage({
|
|
6819
6832
|
threadId,
|
|
6820
6833
|
text,
|
|
6821
6834
|
role: "ROLE_AGENT",
|
|
6822
6835
|
intent: "inform",
|
|
6823
|
-
notice: "thread-paused",
|
|
6836
|
+
notice: resumed ? "thread-resumed" : "thread-paused",
|
|
6824
6837
|
from: { owner: profile.ownerDid, agent: profile.courierDid, label: profile.label }
|
|
6825
6838
|
});
|
|
6826
|
-
|
|
6827
|
-
if (!ULID.test(messageId))
|
|
6828
|
-
throw new CourierError("usage", "Invalid pause inform id.");
|
|
6829
|
-
message.messageId = messageId;
|
|
6830
|
-
}
|
|
6839
|
+
message.messageId = pauseId;
|
|
6831
6840
|
try {
|
|
6832
6841
|
await putCreate(kv2, k.msg(peerHandle, threadId, message.messageId), message);
|
|
6833
6842
|
} catch (error) {
|
|
@@ -6867,7 +6876,7 @@ async function cmdPause(args, deps = {}) {
|
|
|
6867
6876
|
if (!exists)
|
|
6868
6877
|
throw new CourierError("not-found", `Thread ${threadId} with ${peer} does not exist.`);
|
|
6869
6878
|
const reason = args.has("reason") && args.get("reason") !== "true" ? args.get("reason") : "paused by its owner";
|
|
6870
|
-
const marker = { v: 1, peer, threadId, by: "human", at: (/* @__PURE__ */ new Date()).toISOString(), reason, informId:
|
|
6879
|
+
const marker = { v: 1, peer, threadId, by: "human", at: (/* @__PURE__ */ new Date()).toISOString(), reason, informId: await nextNoticeId(node.kv, peer, threadId), informSent: false };
|
|
6871
6880
|
const wrote = await writePauseMarker(node.kv, marker);
|
|
6872
6881
|
const stored = await getJson(node.kv, k.paused(peer, threadId));
|
|
6873
6882
|
if (!stored)
|
|
@@ -6878,7 +6887,7 @@ async function cmdPause(args, deps = {}) {
|
|
|
6878
6887
|
}
|
|
6879
6888
|
const saved = stored.value;
|
|
6880
6889
|
if (!saved.informId) {
|
|
6881
|
-
saved.informId =
|
|
6890
|
+
saved.informId = await nextNoticeId(node.kv, peer, threadId);
|
|
6882
6891
|
await putMatch(node.kv, k.paused(peer, threadId), saved, stored.etag);
|
|
6883
6892
|
}
|
|
6884
6893
|
await postPausedInform(node.kv, profile, peer, threadId, "Thread paused by its owner. Its agent waits until its owner resumes it.", saved.informId);
|
|
@@ -6887,15 +6896,20 @@ async function cmdPause(args, deps = {}) {
|
|
|
6887
6896
|
await putMatch(node.kv, k.paused(peer, threadId), { ...fresh.value, informSent: true }, fresh.etag);
|
|
6888
6897
|
console.log(`Paused thread ${threadId} with ${peer}; told ${peer}. Resume: ${cmd(`resume ${peer} ${threadId}`)}`);
|
|
6889
6898
|
}
|
|
6890
|
-
async function cmdResume(args) {
|
|
6899
|
+
async function cmdResume(args, deps = {}) {
|
|
6891
6900
|
const { peer, threadId } = target(args, resumeUsage());
|
|
6892
|
-
const profile = loadProfile();
|
|
6901
|
+
const profile = (deps.profile ?? loadProfile)();
|
|
6893
6902
|
getPeer(profile, peer);
|
|
6894
|
-
const node = await courierNode(profile);
|
|
6895
|
-
|
|
6903
|
+
const node = await (deps.node ?? courierNode)(profile);
|
|
6904
|
+
const stored = await getJson(node.kv, k.paused(peer, threadId));
|
|
6905
|
+
if (!stored) {
|
|
6896
6906
|
console.log(`Thread ${threadId} with ${peer} is not paused.`);
|
|
6897
6907
|
return;
|
|
6898
6908
|
}
|
|
6909
|
+
const resumeInformId = stored.value.resumeInformId ?? await nextNoticeId(node.kv, peer, threadId);
|
|
6910
|
+
if (!stored.value.resumeInformId)
|
|
6911
|
+
await putMatch(node.kv, k.paused(peer, threadId), { ...stored.value, resumeInformId }, stored.etag);
|
|
6912
|
+
await postPausedInform(node.kv, profile, peer, threadId, "Thread resumed by its owner.", resumeInformId, true);
|
|
6899
6913
|
await del(node.kv, k.paused(peer, threadId));
|
|
6900
6914
|
console.log(`Resumed thread ${threadId} with ${peer}. A running watcher picks this up with the next message it sees.`);
|
|
6901
6915
|
}
|
|
@@ -6917,14 +6931,47 @@ var FOREIGN_CLAIM_LEASE_MS = 45 * 6e4;
|
|
|
6917
6931
|
var CLAIM_SKIP_MS = 3e4;
|
|
6918
6932
|
var DAY_MS2 = 24 * 3600 * 1e3;
|
|
6919
6933
|
var NOTICES_PER_PEER_PER_HOUR = 10;
|
|
6934
|
+
var NOTICES_PER_HOUR = 10;
|
|
6935
|
+
var NOTICE_RESERVE_PER_PEER = 3;
|
|
6936
|
+
var MAX_COALESCED_PAUSES = 999999;
|
|
6937
|
+
var MAX_PEER_CLOCK_SKEW_MS = 10 * 6e4;
|
|
6920
6938
|
var MAX_OUTBOX = 50;
|
|
6921
6939
|
var MAX_OUTBOX_TRIES = 8;
|
|
6940
|
+
var MAX_NOTICE_ATTEMPTS = 3;
|
|
6922
6941
|
var MAX_HELD_PER_THREAD = 100;
|
|
6923
6942
|
var MAX_SUBJECT = 200;
|
|
6924
6943
|
var SAFE_ID = /^[0-9A-Za-z_-]{1,64}$/;
|
|
6944
|
+
var emptyNoticeRetry = () => ({ dispatchArrivals: "------", tries: 0, nextAttemptAt: 0, logged: false });
|
|
6945
|
+
var ZERO_ULID = "0".repeat(26);
|
|
6946
|
+
var EPOCH = (/* @__PURE__ */ new Date(0)).toISOString();
|
|
6947
|
+
var needsKind = (kind) => kind === "new-message" ? "message" : kind === "new-approval" ? "approval" : void 0;
|
|
6948
|
+
var emptyNeedsSlot = () => ({
|
|
6949
|
+
threadId: ZERO_ULID,
|
|
6950
|
+
eventId: ZERO_ULID,
|
|
6951
|
+
count: "000000",
|
|
6952
|
+
firstSeenAt: EPOCH,
|
|
6953
|
+
arrivalFirstSeenAt: EPOCH,
|
|
6954
|
+
...emptyNoticeRetry()
|
|
6955
|
+
});
|
|
6956
|
+
var needsKey = (peer, kind) => `needs:${peer}:${kind}`;
|
|
6957
|
+
var emptyEpisode = () => ({ kind: "", eventId: ZERO_ULID, repeats: "000000", ...emptyNoticeRetry() });
|
|
6958
|
+
var incrementCount = (count) => String(Math.min(MAX_COALESCED_PAUSES, Number(count) + 1)).padStart(6, "0");
|
|
6959
|
+
var episodeKey = (key, slot) => `notice:${key}:${slot}`;
|
|
6960
|
+
function threadNoticeState() {
|
|
6961
|
+
return {
|
|
6962
|
+
ourPaused: false,
|
|
6963
|
+
peerPaused: false,
|
|
6964
|
+
lastPeerNoticeId: "0".repeat(26),
|
|
6965
|
+
lastNoticeSentAt: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
6966
|
+
ourMarkerId: "",
|
|
6967
|
+
episodes: [emptyEpisode(), emptyEpisode()],
|
|
6968
|
+
informPending: false,
|
|
6969
|
+
informText: ""
|
|
6970
|
+
};
|
|
6971
|
+
}
|
|
6925
6972
|
var WATCH_DEFAULTS = { debounceMs: 3e4, pollMs: 6e4, ticketSeconds: 25, wakeBudget: 12 };
|
|
6926
6973
|
function emptyWatchState() {
|
|
6927
|
-
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], outbox: [], held: {},
|
|
6974
|
+
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], wakeRetries: {}, approvalRetries: {}, outbox: [], held: {}, threadNotices: {}, peerNotices: {}, noticeGlobalLog: [], noticeLog: {} };
|
|
6928
6975
|
}
|
|
6929
6976
|
function wakeDueAt(pending, debounceMs) {
|
|
6930
6977
|
if (pending.approvals?.length)
|
|
@@ -6970,7 +7017,6 @@ var Watcher = class {
|
|
|
6970
7017
|
pausedKeys = /* @__PURE__ */ new Set();
|
|
6971
7018
|
/** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
|
|
6972
7019
|
pausedLock = Promise.resolve();
|
|
6973
|
-
wakeFailures = /* @__PURE__ */ new Map();
|
|
6974
7020
|
outboxBusy = /* @__PURE__ */ new Set();
|
|
6975
7021
|
approvalsAgain = false;
|
|
6976
7022
|
/** Unique to this run: a claim with this id is this process's own. */
|
|
@@ -7005,7 +7051,63 @@ var Watcher = class {
|
|
|
7005
7051
|
const value2 = got?.value;
|
|
7006
7052
|
this.stateEtag = got?.etag ?? null;
|
|
7007
7053
|
this.state = isRecord(value2) && value2.v === 1 ? { ...emptyWatchState(), ...value2 } : emptyWatchState();
|
|
7008
|
-
|
|
7054
|
+
delete this.state.peerPausedDay;
|
|
7055
|
+
for (const key of [...Object.keys(this.state.wakes), ...this.state.paused]) {
|
|
7056
|
+
const [peer, threadId] = key.split("/");
|
|
7057
|
+
if (peer && threadId && SAFE_ID.test(peer) && SAFE_ID.test(threadId)) {
|
|
7058
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
7059
|
+
record.ourPaused = this.state.paused.includes(key);
|
|
7060
|
+
}
|
|
7061
|
+
}
|
|
7062
|
+
this.syncNoticePeers();
|
|
7063
|
+
for (const record of Object.values(this.state.peerNotices))
|
|
7064
|
+
for (const slot of Object.values(record))
|
|
7065
|
+
slot.dispatchArrivals = "------";
|
|
7066
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
7067
|
+
const old = record;
|
|
7068
|
+
if (!record.episodes) {
|
|
7069
|
+
record.episodes = [emptyEpisode(), emptyEpisode()];
|
|
7070
|
+
for (const item of this.state.outbox) {
|
|
7071
|
+
if (item.kind !== "notify" || `${item.notice.peer}/${item.notice.threadId}` !== key || item.notice.kind !== "paused" && item.notice.kind !== "peer-paused" || item.id !== `${item.notice.kind}:${key}` || !item.notice.eventId)
|
|
7072
|
+
continue;
|
|
7073
|
+
this.queueEpisode(record, item.notice.kind, item.notice.eventId);
|
|
7074
|
+
}
|
|
7075
|
+
if (old.ourNoticePending)
|
|
7076
|
+
this.queueEpisode(record, "paused", record.ourMarkerId);
|
|
7077
|
+
if (old.peerNoticePending && old.peerNoticeId)
|
|
7078
|
+
this.queueEpisode(record, "peer-paused", old.peerNoticeId);
|
|
7079
|
+
}
|
|
7080
|
+
record.episodes = record.episodes.map((episode) => ({ ...emptyEpisode(), ...episode, dispatchArrivals: "------" }));
|
|
7081
|
+
delete old.ourNoticePending;
|
|
7082
|
+
delete old.peerNoticePending;
|
|
7083
|
+
delete old.peerNoticeId;
|
|
7084
|
+
}
|
|
7085
|
+
this.state.outbox = this.state.outbox.filter((item) => {
|
|
7086
|
+
if (item.kind === "inform")
|
|
7087
|
+
return true;
|
|
7088
|
+
const kind = needsKind(item.notice.kind);
|
|
7089
|
+
if (kind) {
|
|
7090
|
+
const slot2 = this.noticeSlot(item);
|
|
7091
|
+
if (!slot2 || item.id !== needsKey(item.notice.peer, kind))
|
|
7092
|
+
return false;
|
|
7093
|
+
this.rememberNotice(item);
|
|
7094
|
+
return true;
|
|
7095
|
+
}
|
|
7096
|
+
const key = `${item.notice.peer}/${item.notice.threadId}`;
|
|
7097
|
+
const record = this.state.threadNotices[key];
|
|
7098
|
+
const slot = record?.episodes.findIndex((episode) => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId) ?? -1;
|
|
7099
|
+
if (slot < 0 || item.id !== `${item.notice.kind}:${key}` && item.id !== episodeKey(key, slot))
|
|
7100
|
+
return false;
|
|
7101
|
+
item.id = episodeKey(key, slot);
|
|
7102
|
+
item.notice.repeats ??= record.episodes[slot].repeats;
|
|
7103
|
+
this.rememberNotice(item);
|
|
7104
|
+
return true;
|
|
7105
|
+
});
|
|
7106
|
+
for (const item of [...this.state.outbox]) {
|
|
7107
|
+
if ((item.tries ?? 0) >= (item.kind === "notify" ? MAX_NOTICE_ATTEMPTS : MAX_OUTBOX_TRIES))
|
|
7108
|
+
this.finishOutbox(item);
|
|
7109
|
+
}
|
|
7110
|
+
this.state.outbox = this.state.outbox.slice(-MAX_OUTBOX);
|
|
7009
7111
|
}
|
|
7010
7112
|
snapshot() {
|
|
7011
7113
|
return structuredClone(this.state);
|
|
@@ -7153,7 +7255,7 @@ var Watcher = class {
|
|
|
7153
7255
|
const { selfKv, log } = this.deps;
|
|
7154
7256
|
const mailbox = peer.theirHandleForMe;
|
|
7155
7257
|
const peerKv2 = await this.deps.kvFor(peer);
|
|
7156
|
-
const [keys] = await Promise.all([listAll(peerKv2, `${PREFIX}/to/${mailbox}/threads/`), this.syncPaused()]);
|
|
7258
|
+
const [keys] = await Promise.all([listAll(peerKv2, `${PREFIX}/to/${mailbox}/threads/`), this.syncPaused(), this.syncKnownThreads(peer)]);
|
|
7157
7259
|
const receipts = (await getJson(selfKv, k.receipts(peer.handle)))?.value ?? { v: 1, threads: {} };
|
|
7158
7260
|
receipts.threads ??= {};
|
|
7159
7261
|
const delivered = [];
|
|
@@ -7202,9 +7304,11 @@ var Watcher = class {
|
|
|
7202
7304
|
}
|
|
7203
7305
|
await this.mutate(() => {
|
|
7204
7306
|
if (this.deps.notifyOnly) {
|
|
7205
|
-
|
|
7206
|
-
|
|
7207
|
-
|
|
7307
|
+
this.syncNoticePeers();
|
|
7308
|
+
for (const item of [...delivered].sort((a, b) => a.messageId.localeCompare(b.messageId))) {
|
|
7309
|
+
if (item.intent && item.notice !== "thread-paused" && item.notice !== "thread-resumed") {
|
|
7310
|
+
this.queueNeeds(peer.handle, "message", item.threadId);
|
|
7311
|
+
}
|
|
7208
7312
|
}
|
|
7209
7313
|
}
|
|
7210
7314
|
for (const item of delivered) {
|
|
@@ -7229,20 +7333,50 @@ var Watcher = class {
|
|
|
7229
7333
|
entry.lastSeenAt = at;
|
|
7230
7334
|
}
|
|
7231
7335
|
for (const item of delivered) {
|
|
7232
|
-
if (item.notice !== "thread-paused"
|
|
7336
|
+
if (item.intent !== "inform" || item.notice !== "thread-paused" && item.notice !== "thread-resumed")
|
|
7233
7337
|
continue;
|
|
7234
|
-
const
|
|
7235
|
-
|
|
7236
|
-
|
|
7237
|
-
|
|
7238
|
-
|
|
7239
|
-
|
|
7240
|
-
}
|
|
7338
|
+
const record = this.state.threadNotices[`${peer.handle}/${item.threadId}`];
|
|
7339
|
+
if (!record || !ULID.test(item.messageId) || item.messageId <= record.lastPeerNoticeId)
|
|
7340
|
+
continue;
|
|
7341
|
+
let sentAt;
|
|
7342
|
+
try {
|
|
7343
|
+
sentAt = decodeTime(item.messageId);
|
|
7344
|
+
} catch {
|
|
7345
|
+
continue;
|
|
7346
|
+
}
|
|
7347
|
+
if (sentAt > at + MAX_PEER_CLOCK_SKEW_MS)
|
|
7348
|
+
continue;
|
|
7349
|
+
record.lastPeerNoticeId = item.messageId;
|
|
7350
|
+
const paused = item.notice === "thread-paused";
|
|
7351
|
+
if (paused && !record.peerPaused)
|
|
7352
|
+
this.queueEpisode(record, "peer-paused", item.messageId);
|
|
7353
|
+
record.peerPaused = paused;
|
|
7241
7354
|
}
|
|
7355
|
+
this.fillOutbox();
|
|
7242
7356
|
});
|
|
7243
7357
|
await commitReceipts(selfKv, /* @__PURE__ */ new Map([[peer.handle, delivered.map(({ threadId, messageId }) => ({ threadId, messageId }))]]));
|
|
7244
7358
|
await this.mirrorThreads(peer, peerKv2, keys);
|
|
7245
7359
|
}
|
|
7360
|
+
/** Only locally authored meta/messages/tasks, or a local wake/pause witness, establish a known thread. */
|
|
7361
|
+
async syncKnownThreads(peer) {
|
|
7362
|
+
const prefix = `${PREFIX}/to/${peer.handle}/threads/`;
|
|
7363
|
+
const keys = await listAll(this.deps.selfKv, prefix);
|
|
7364
|
+
const known = /* @__PURE__ */ new Set();
|
|
7365
|
+
for (const path18 of keys) {
|
|
7366
|
+
const match = /^([^/]+)\/(?:meta\.json|(?:msg|task)\/[^/]+\.json)$/.exec(path18.slice(prefix.length));
|
|
7367
|
+
if (match && SAFE_ID.test(match[1]))
|
|
7368
|
+
known.add(`${peer.handle}/${match[1]}`);
|
|
7369
|
+
}
|
|
7370
|
+
for (const key of Object.keys(this.state.wakes))
|
|
7371
|
+
if (key.startsWith(`${peer.handle}/`))
|
|
7372
|
+
known.add(key);
|
|
7373
|
+
const missing = [...known].filter((key) => !this.state.threadNotices[key]);
|
|
7374
|
+
if (missing.length)
|
|
7375
|
+
await this.mutate(() => {
|
|
7376
|
+
for (const key of missing)
|
|
7377
|
+
this.state.threadNotices[key] ??= threadNoticeState();
|
|
7378
|
+
});
|
|
7379
|
+
}
|
|
7246
7380
|
async mirrorCreate(key, value2) {
|
|
7247
7381
|
try {
|
|
7248
7382
|
await putCreate(this.deps.selfKv, key, value2);
|
|
@@ -7344,9 +7478,11 @@ var Watcher = class {
|
|
|
7344
7478
|
const at = this.now();
|
|
7345
7479
|
await this.mutate(() => {
|
|
7346
7480
|
this.state.approvalsSeen = [.../* @__PURE__ */ new Set([...this.state.approvalsSeen, ...handled])].sort();
|
|
7481
|
+
if (this.deps.notifyOnly)
|
|
7482
|
+
this.syncNoticePeers();
|
|
7347
7483
|
for (const { approval, peer, threadId, subject } of wakes) {
|
|
7348
7484
|
if (this.deps.notifyOnly) {
|
|
7349
|
-
this.
|
|
7485
|
+
this.queueNeeds(peer, "approval", threadId);
|
|
7350
7486
|
continue;
|
|
7351
7487
|
}
|
|
7352
7488
|
const key = `${peer}/${threadId}`;
|
|
@@ -7356,6 +7492,8 @@ var Watcher = class {
|
|
|
7356
7492
|
entry.approvals.push(approval);
|
|
7357
7493
|
entry.lastSeenAt = at;
|
|
7358
7494
|
}
|
|
7495
|
+
if (this.deps.notifyOnly)
|
|
7496
|
+
this.fillOutbox();
|
|
7359
7497
|
});
|
|
7360
7498
|
}
|
|
7361
7499
|
/** Reads one new approval and its request; `skip` says why it should not wake anyone. */
|
|
@@ -7388,13 +7526,15 @@ var Watcher = class {
|
|
|
7388
7526
|
if (this.deps.notifyOnly) {
|
|
7389
7527
|
if (Object.keys(this.state.pending).length)
|
|
7390
7528
|
await this.mutate(() => {
|
|
7529
|
+
this.syncNoticePeers();
|
|
7391
7530
|
for (const pending of Object.values(this.state.pending)) {
|
|
7392
|
-
|
|
7393
|
-
this.
|
|
7394
|
-
for (const
|
|
7395
|
-
this.
|
|
7531
|
+
for (const _message of pending.messages)
|
|
7532
|
+
this.queueNeeds(pending.peer, "message", pending.threadId);
|
|
7533
|
+
for (const _approval of pending.approvals ?? [])
|
|
7534
|
+
this.queueNeeds(pending.peer, "approval", pending.threadId);
|
|
7396
7535
|
}
|
|
7397
7536
|
this.state.pending = {};
|
|
7537
|
+
this.fillOutbox();
|
|
7398
7538
|
});
|
|
7399
7539
|
await this.runOutbox();
|
|
7400
7540
|
return;
|
|
@@ -7406,7 +7546,7 @@ var Watcher = class {
|
|
|
7406
7546
|
continue;
|
|
7407
7547
|
if (!force && wakeDueAt(pending, this.opts.debounceMs) > at)
|
|
7408
7548
|
continue;
|
|
7409
|
-
if (!force && (this.retryAt.get(key) ?? 0) > at)
|
|
7549
|
+
if (!force && !pending.approvals?.length && (this.retryAt.get(key) ?? 0) > at)
|
|
7410
7550
|
continue;
|
|
7411
7551
|
this.waking.add(key);
|
|
7412
7552
|
runs.push(this.wakeThread(key, pending, at).finally(() => this.waking.delete(key)));
|
|
@@ -7424,7 +7564,7 @@ var Watcher = class {
|
|
|
7424
7564
|
return;
|
|
7425
7565
|
await this.syncPaused();
|
|
7426
7566
|
let messages = [...pending.messages];
|
|
7427
|
-
|
|
7567
|
+
let approvals = [...pending.approvals ?? []];
|
|
7428
7568
|
if (messages.length) {
|
|
7429
7569
|
const used = this.recentWakes(key).length;
|
|
7430
7570
|
if (!this.pausedKeys.has(key) && used >= this.opts.wakeBudget)
|
|
@@ -7447,6 +7587,12 @@ var Watcher = class {
|
|
|
7447
7587
|
return;
|
|
7448
7588
|
}
|
|
7449
7589
|
}
|
|
7590
|
+
if ((this.state.wakeRetries[key]?.nextAttemptAt ?? 0) > at)
|
|
7591
|
+
messages = [];
|
|
7592
|
+
if ((this.state.approvalRetries[key]?.nextAttemptAt ?? 0) > at)
|
|
7593
|
+
approvals = [];
|
|
7594
|
+
if (!messages.length && !approvals.length)
|
|
7595
|
+
return;
|
|
7450
7596
|
const woken = messages.map((m) => m.messageId);
|
|
7451
7597
|
const approvedIds = approvals.map((a) => a.approvalId);
|
|
7452
7598
|
const oldestSentAt = messages[0]?.sentAt ?? "";
|
|
@@ -7457,6 +7603,7 @@ var Watcher = class {
|
|
|
7457
7603
|
}
|
|
7458
7604
|
try {
|
|
7459
7605
|
await this.mutate(() => {
|
|
7606
|
+
this.state.threadNotices[key] ??= threadNoticeState();
|
|
7460
7607
|
if (woken.length)
|
|
7461
7608
|
this.state.wakes[key] = [...this.recentWakes(key), this.now()];
|
|
7462
7609
|
});
|
|
@@ -7480,9 +7627,14 @@ var Watcher = class {
|
|
|
7480
7627
|
try {
|
|
7481
7628
|
await this.deps.wake({ peer: pending.peer, threadId: pending.threadId, subject: pending.subject, messages, approvals });
|
|
7482
7629
|
} catch (error) {
|
|
7483
|
-
|
|
7484
|
-
this.
|
|
7485
|
-
|
|
7630
|
+
let failures = 0;
|
|
7631
|
+
await this.mutate(() => {
|
|
7632
|
+
for (const retries of [...woken.length ? [this.state.wakeRetries] : [], ...approvedIds.length ? [this.state.approvalRetries] : []]) {
|
|
7633
|
+
const count = (retries[key]?.failures ?? 0) + 1;
|
|
7634
|
+
retries[key] = { failures: count, nextAttemptAt: this.now() + Math.min(15 * 6e4, WAKE_RETRY_MS * 2 ** (count - 1)) };
|
|
7635
|
+
failures = Math.max(failures, count);
|
|
7636
|
+
}
|
|
7637
|
+
});
|
|
7486
7638
|
this.deps.log("wake-error", { peer: pending.peer, threadId: pending.threadId, failures, ...errInfo(error) });
|
|
7487
7639
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
7488
7640
|
return;
|
|
@@ -7496,8 +7648,11 @@ var Watcher = class {
|
|
|
7496
7648
|
delete this.state.pending[key];
|
|
7497
7649
|
}
|
|
7498
7650
|
this.state.woken = [...this.state.woken, ...woken].slice(-WAKEN_MEMORY);
|
|
7651
|
+
if (woken.length)
|
|
7652
|
+
delete this.state.wakeRetries[key];
|
|
7653
|
+
if (approvedIds.length)
|
|
7654
|
+
delete this.state.approvalRetries[key];
|
|
7499
7655
|
});
|
|
7500
|
-
this.wakeFailures.delete(key);
|
|
7501
7656
|
this.retryAt.delete(key);
|
|
7502
7657
|
this.deps.log("woke", {
|
|
7503
7658
|
peer: pending.peer,
|
|
@@ -7511,8 +7666,9 @@ var Watcher = class {
|
|
|
7511
7666
|
async pauseThread(key, pending, used) {
|
|
7512
7667
|
const reason = `wake budget reached (${used} wakes today, UTC; limit ${this.opts.wakeBudget ?? WATCH_DEFAULTS.wakeBudget} per thread per day)`;
|
|
7513
7668
|
const at = new Date(this.now()).toISOString();
|
|
7669
|
+
const pauseId = await nextNoticeId(this.deps.selfKv, pending.peer, pending.threadId, this.now());
|
|
7514
7670
|
const wrote = await this.exclusive(async () => {
|
|
7515
|
-
const created = await writePauseMarker(this.deps.selfKv, { v: 1, peer: pending.peer, threadId: pending.threadId, by: "budget", at, reason });
|
|
7671
|
+
const created = await writePauseMarker(this.deps.selfKv, { v: 1, peer: pending.peer, threadId: pending.threadId, by: "budget", at, reason, informId: pauseId });
|
|
7516
7672
|
this.pausedKeys.add(key);
|
|
7517
7673
|
return created;
|
|
7518
7674
|
});
|
|
@@ -7524,8 +7680,10 @@ var Watcher = class {
|
|
|
7524
7680
|
await this.mutate(() => {
|
|
7525
7681
|
if (!this.state.paused.includes(key))
|
|
7526
7682
|
this.state.paused.push(key);
|
|
7527
|
-
|
|
7528
|
-
|
|
7683
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
7684
|
+
Object.assign(record, { ourPaused: true, ourMarkerId: pauseId, informPending: true, informText: text });
|
|
7685
|
+
this.queueEpisode(record, "paused", pauseId);
|
|
7686
|
+
this.fillOutbox();
|
|
7529
7687
|
});
|
|
7530
7688
|
this.deps.log("paused", { peer: pending.peer, threadId: pending.threadId, by: "budget", reason });
|
|
7531
7689
|
}
|
|
@@ -7541,29 +7699,239 @@ var Watcher = class {
|
|
|
7541
7699
|
return;
|
|
7542
7700
|
entry.messages = [...entry.messages, message].slice(-MAX_HELD_PER_THREAD);
|
|
7543
7701
|
}
|
|
7544
|
-
/**
|
|
7545
|
-
|
|
7546
|
-
if (
|
|
7547
|
-
|
|
7548
|
-
|
|
7549
|
-
|
|
7550
|
-
|
|
7551
|
-
|
|
7552
|
-
|
|
7553
|
-
|
|
7554
|
-
|
|
7555
|
-
|
|
7556
|
-
|
|
7557
|
-
|
|
7702
|
+
/** Two episode values per known thread. Additional transitions coalesce, without allocating another slot. */
|
|
7703
|
+
queueEpisode(record, kind, eventId) {
|
|
7704
|
+
if (!ULID.test(eventId) || record.episodes.some((episode) => episode.kind === kind && episode.eventId === eventId))
|
|
7705
|
+
return;
|
|
7706
|
+
const slot = record.episodes.find((episode) => !episode.kind);
|
|
7707
|
+
if (slot)
|
|
7708
|
+
Object.assign(slot, emptyEpisode(), { kind, eventId });
|
|
7709
|
+
else {
|
|
7710
|
+
const last = record.episodes[1];
|
|
7711
|
+
last.repeats = incrementCount(last.repeats);
|
|
7712
|
+
if (last.dispatchArrivals !== "------")
|
|
7713
|
+
last.dispatchArrivals = incrementCount(last.dispatchArrivals);
|
|
7714
|
+
}
|
|
7715
|
+
}
|
|
7716
|
+
/** Only the paired handles in our local configuration allocate needs-you state. */
|
|
7717
|
+
syncNoticePeers() {
|
|
7718
|
+
const paired = new Set(this.deps.loadPeers().filter((peer) => peer.status === "paired" && HANDLE.test(peer.handle)).map((peer) => peer.handle));
|
|
7719
|
+
for (const peer of Object.keys(this.state.peerNotices))
|
|
7720
|
+
if (!paired.has(peer))
|
|
7721
|
+
delete this.state.peerNotices[peer];
|
|
7722
|
+
if (this.deps.notifyOnly)
|
|
7723
|
+
for (const peer of paired) {
|
|
7724
|
+
if (!Object.hasOwn(this.state.peerNotices, peer))
|
|
7725
|
+
this.state.peerNotices[peer] = { message: emptyNeedsSlot(), approval: emptyNeedsSlot() };
|
|
7726
|
+
}
|
|
7727
|
+
for (const record of Object.values(this.state.peerNotices))
|
|
7728
|
+
for (const kind of ["message", "approval"]) {
|
|
7729
|
+
record[kind] = { ...emptyNeedsSlot(), ...record[kind] };
|
|
7558
7730
|
}
|
|
7559
|
-
|
|
7560
|
-
|
|
7731
|
+
}
|
|
7732
|
+
/** Bounded values only: a later delivery updates the same paired-peer slot. Called inside mutate. */
|
|
7733
|
+
queueNeeds(peer, kind, threadId) {
|
|
7734
|
+
const slot = this.state.peerNotices[peer]?.[kind];
|
|
7735
|
+
if (!slot || !ULID.test(threadId))
|
|
7736
|
+
return;
|
|
7737
|
+
try {
|
|
7738
|
+
decodeTime(threadId);
|
|
7739
|
+
} catch {
|
|
7740
|
+
return;
|
|
7741
|
+
}
|
|
7742
|
+
const at = new Date(this.now()).toISOString();
|
|
7743
|
+
if (slot.count === "000000") {
|
|
7744
|
+
slot.eventId = ulid(this.now());
|
|
7745
|
+
slot.firstSeenAt = at;
|
|
7746
|
+
}
|
|
7747
|
+
slot.threadId = threadId;
|
|
7748
|
+
slot.count = incrementCount(slot.count);
|
|
7749
|
+
if (slot.dispatchArrivals !== "------") {
|
|
7750
|
+
if (slot.dispatchArrivals === "000000")
|
|
7751
|
+
slot.arrivalFirstSeenAt = at;
|
|
7752
|
+
slot.dispatchArrivals = incrementCount(slot.dispatchArrivals);
|
|
7753
|
+
}
|
|
7754
|
+
}
|
|
7755
|
+
noticeSlot(item) {
|
|
7756
|
+
const kind = needsKind(item.notice.kind);
|
|
7757
|
+
if (kind) {
|
|
7758
|
+
const slot = this.state.peerNotices[item.notice.peer]?.[kind];
|
|
7759
|
+
return slot?.count !== "000000" && slot?.eventId === item.notice.eventId ? slot : void 0;
|
|
7760
|
+
}
|
|
7761
|
+
return this.state.threadNotices[`${item.notice.peer}/${item.notice.threadId}`]?.episodes.find((episode) => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId);
|
|
7762
|
+
}
|
|
7763
|
+
rememberNotice(item) {
|
|
7764
|
+
const episode = this.noticeSlot(item);
|
|
7765
|
+
if (episode)
|
|
7766
|
+
Object.assign(episode, { tries: item.tries ?? 0, nextAttemptAt: item.nextAttemptAt ?? 0, logged: item.logged ?? false });
|
|
7767
|
+
}
|
|
7768
|
+
/** Earliest rolling-hour retry under the global allowance and each paired peer's share. */
|
|
7769
|
+
noticeRateDeadline(peer, now = this.now()) {
|
|
7770
|
+
const recent = (this.state.noticeLog[peer] ?? []).filter((t) => now - t < 36e5);
|
|
7771
|
+
const global2 = this.state.noticeGlobalLog.filter((t) => now - t < 36e5);
|
|
7772
|
+
const peers = [.../* @__PURE__ */ new Set([...this.deps.loadPeers().filter((peer2) => peer2.status === "paired").map((peer2) => peer2.handle), peer])];
|
|
7773
|
+
const share = Math.min(NOTICE_RESERVE_PER_PEER, Math.floor(NOTICES_PER_HOUR / peers.length));
|
|
7774
|
+
const reservedForOthers = peers.filter((other) => other !== peer).reduce((reserved, other) => reserved + Math.max(0, share - (this.state.noticeLog[other] ?? []).filter((t) => now - t < 36e5).length), 0);
|
|
7775
|
+
if (recent.length < NOTICES_PER_PEER_PER_HOUR && global2.length + reservedForOthers < NOTICES_PER_HOUR)
|
|
7776
|
+
return 0;
|
|
7777
|
+
return Math.max(recent.length >= NOTICES_PER_PEER_PER_HOUR ? Math.min(...recent) + 36e5 : 0, global2.length ? Math.min(...global2) + 36e5 : now + 36e5);
|
|
7778
|
+
}
|
|
7779
|
+
/** -1 appends; a nonnegative index yields that place; undefined leaves the pending slot alone. */
|
|
7780
|
+
admissionIndex(item) {
|
|
7781
|
+
if (this.state.outbox.some((existing) => existing.id === item.id))
|
|
7782
|
+
return void 0;
|
|
7783
|
+
if (this.state.outbox.length < MAX_OUTBOX)
|
|
7784
|
+
return -1;
|
|
7785
|
+
const now = this.now();
|
|
7786
|
+
const eligible = item.kind === "inform" || (item.nextAttemptAt ?? 0) <= now && !this.noticeRateDeadline(item.notice.peer, now);
|
|
7787
|
+
if (!eligible)
|
|
7788
|
+
return void 0;
|
|
7789
|
+
const deferred = this.state.outbox.findIndex((existing) => existing.kind === "notify" && !this.outboxBusy.has(existing.id) && ((existing.nextAttemptAt ?? 0) > now || this.noticeRateDeadline(existing.notice.peer, now) > now));
|
|
7790
|
+
if (deferred >= 0)
|
|
7791
|
+
return deferred;
|
|
7792
|
+
if (item.kind === "notify") {
|
|
7793
|
+
const counts = /* @__PURE__ */ new Map();
|
|
7794
|
+
for (const existing of this.state.outbox)
|
|
7795
|
+
if (existing.kind === "notify")
|
|
7796
|
+
counts.set(existing.notice.peer, (counts.get(existing.notice.peer) ?? 0) + 1);
|
|
7797
|
+
const own = counts.get(item.notice.peer) ?? 0;
|
|
7798
|
+
const overrepresented = this.state.outbox.findIndex((existing) => existing.kind === "notify" && existing.notice.peer !== item.notice.peer && (counts.get(existing.notice.peer) ?? 0) > own + 1 && existing.notice.kind !== "paused" && !existing.tries && !this.outboxBusy.has(existing.id));
|
|
7799
|
+
if (overrepresented >= 0)
|
|
7800
|
+
return overrepresented;
|
|
7801
|
+
}
|
|
7802
|
+
if (item.kind === "inform" || item.notice.kind === "paused") {
|
|
7803
|
+
const unattempted = this.state.outbox.findIndex((existing) => existing.kind === "notify" && existing.notice.kind === "peer-paused" && !existing.tries && !this.outboxBusy.has(existing.id));
|
|
7804
|
+
if (unattempted >= 0)
|
|
7805
|
+
return unattempted;
|
|
7561
7806
|
}
|
|
7807
|
+
return void 0;
|
|
7808
|
+
}
|
|
7809
|
+
/** Queue places can yield; the two fixed episode records retain all retry metadata. */
|
|
7810
|
+
enqueue(item) {
|
|
7811
|
+
const index = this.admissionIndex(item);
|
|
7812
|
+
if (index === void 0)
|
|
7813
|
+
return;
|
|
7814
|
+
if (index >= 0)
|
|
7815
|
+
this.state.outbox.splice(index, 1);
|
|
7562
7816
|
this.state.outbox.push(item);
|
|
7563
|
-
|
|
7564
|
-
|
|
7565
|
-
|
|
7817
|
+
}
|
|
7818
|
+
*outboxCandidates() {
|
|
7819
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
7820
|
+
const [peer, threadId] = key.split("/");
|
|
7821
|
+
if (record.informPending)
|
|
7822
|
+
yield { id: `inform:${key}`, kind: "inform", peer, threadId, pauseId: record.ourMarkerId, text: record.informText };
|
|
7823
|
+
}
|
|
7824
|
+
for (const kind of ["paused", "peer-paused"]) {
|
|
7825
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
7826
|
+
const [peer, threadId] = key.split("/");
|
|
7827
|
+
for (const [slot, episode] of record.episodes.entries()) {
|
|
7828
|
+
if (episode.kind === kind)
|
|
7829
|
+
yield {
|
|
7830
|
+
id: episodeKey(key, slot),
|
|
7831
|
+
kind: "notify",
|
|
7832
|
+
tries: episode.tries,
|
|
7833
|
+
nextAttemptAt: episode.nextAttemptAt,
|
|
7834
|
+
logged: episode.logged,
|
|
7835
|
+
notice: {
|
|
7836
|
+
kind,
|
|
7837
|
+
peer,
|
|
7838
|
+
threadId,
|
|
7839
|
+
eventId: episode.eventId,
|
|
7840
|
+
repeats: episode.repeats,
|
|
7841
|
+
detail: kind === "paused" ? "thread paused on this side" : "the peer's watcher reported the thread paused"
|
|
7842
|
+
}
|
|
7843
|
+
};
|
|
7844
|
+
}
|
|
7845
|
+
}
|
|
7846
|
+
}
|
|
7847
|
+
for (const [peer, record] of Object.entries(this.state.peerNotices)) {
|
|
7848
|
+
for (const kind of ["approval", "message"]) {
|
|
7849
|
+
const slot = record[kind];
|
|
7850
|
+
if (slot.count !== "000000")
|
|
7851
|
+
yield {
|
|
7852
|
+
id: needsKey(peer, kind),
|
|
7853
|
+
kind: "notify",
|
|
7854
|
+
tries: slot.tries,
|
|
7855
|
+
nextAttemptAt: slot.nextAttemptAt,
|
|
7856
|
+
logged: slot.logged,
|
|
7857
|
+
notice: {
|
|
7858
|
+
kind: kind === "message" ? "new-message" : "new-approval",
|
|
7859
|
+
peer,
|
|
7860
|
+
threadId: slot.threadId,
|
|
7861
|
+
eventId: slot.eventId,
|
|
7862
|
+
count: slot.count,
|
|
7863
|
+
detail: ""
|
|
7864
|
+
}
|
|
7865
|
+
};
|
|
7866
|
+
}
|
|
7867
|
+
}
|
|
7868
|
+
}
|
|
7869
|
+
/** Admit local work first, allowing eligible peers to pass deferred entries in a full queue. */
|
|
7870
|
+
fillOutbox() {
|
|
7871
|
+
for (const item of this.fairOutbox([...this.outboxCandidates()]))
|
|
7872
|
+
this.enqueue(item);
|
|
7873
|
+
}
|
|
7874
|
+
fairOutbox(items) {
|
|
7875
|
+
const ordered = [];
|
|
7876
|
+
for (const priority of ["inform", "paused", "other"]) {
|
|
7877
|
+
const peers = /* @__PURE__ */ new Map();
|
|
7878
|
+
for (const item of items) {
|
|
7879
|
+
const group = item.kind === "inform" ? "inform" : item.notice.kind === "paused" ? "paused" : "other";
|
|
7880
|
+
if (group !== priority)
|
|
7881
|
+
continue;
|
|
7882
|
+
const peer = item.kind === "inform" ? item.peer : item.notice.peer;
|
|
7883
|
+
if (!peers.has(peer))
|
|
7884
|
+
peers.set(peer, []);
|
|
7885
|
+
peers.get(peer).push(item);
|
|
7886
|
+
}
|
|
7887
|
+
while (peers.size)
|
|
7888
|
+
for (const [peer, queue] of peers) {
|
|
7889
|
+
ordered.push(queue.shift());
|
|
7890
|
+
if (!queue.length)
|
|
7891
|
+
peers.delete(peer);
|
|
7892
|
+
}
|
|
7566
7893
|
}
|
|
7894
|
+
return ordered;
|
|
7895
|
+
}
|
|
7896
|
+
/** Completion matches both the fixed slot and episode ULID; another queued episode is independent. */
|
|
7897
|
+
finishOutbox(item, delivered = false) {
|
|
7898
|
+
const peer = item.kind === "inform" ? item.peer : item.notice.peer;
|
|
7899
|
+
const threadId = item.kind === "inform" ? item.threadId : item.notice.threadId;
|
|
7900
|
+
const record = this.state.threadNotices[`${peer}/${threadId}`];
|
|
7901
|
+
if (item.kind === "notify" && needsKind(item.notice.kind)) {
|
|
7902
|
+
const slot = this.noticeSlot(item);
|
|
7903
|
+
if (slot) {
|
|
7904
|
+
const arrivals = slot.dispatchArrivals === "------" ? "000000" : slot.dispatchArrivals;
|
|
7905
|
+
if (delivered && arrivals !== "000000")
|
|
7906
|
+
Object.assign(slot, emptyNeedsSlot(), {
|
|
7907
|
+
threadId: slot.threadId,
|
|
7908
|
+
eventId: ulid(this.now()),
|
|
7909
|
+
count: arrivals,
|
|
7910
|
+
firstSeenAt: slot.arrivalFirstSeenAt
|
|
7911
|
+
});
|
|
7912
|
+
else
|
|
7913
|
+
Object.assign(slot, emptyNeedsSlot());
|
|
7914
|
+
}
|
|
7915
|
+
} else if (record) {
|
|
7916
|
+
if (item.kind === "inform" && item.pauseId === record.ourMarkerId)
|
|
7917
|
+
record.informPending = false;
|
|
7918
|
+
if (item.kind === "notify") {
|
|
7919
|
+
record.lastNoticeSentAt = new Date(this.now()).toISOString();
|
|
7920
|
+
const slot = record.episodes.find((episode) => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId);
|
|
7921
|
+
if (slot) {
|
|
7922
|
+
const arrivedDuringDelivery = slot.dispatchArrivals === "------" ? 0 : Number(slot.dispatchArrivals);
|
|
7923
|
+
if (delivered && arrivedDuringDelivery > 0)
|
|
7924
|
+
Object.assign(slot, emptyEpisode(), {
|
|
7925
|
+
kind: slot.kind,
|
|
7926
|
+
eventId: slot.eventId,
|
|
7927
|
+
repeats: String(arrivedDuringDelivery - 1).padStart(6, "0")
|
|
7928
|
+
});
|
|
7929
|
+
else
|
|
7930
|
+
Object.assign(slot, emptyEpisode());
|
|
7931
|
+
}
|
|
7932
|
+
}
|
|
7933
|
+
}
|
|
7934
|
+
this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id);
|
|
7567
7935
|
}
|
|
7568
7936
|
/**
|
|
7569
7937
|
* Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
|
|
@@ -7591,7 +7959,10 @@ var Watcher = class {
|
|
|
7591
7959
|
const backlog = {};
|
|
7592
7960
|
await this.mutate(() => {
|
|
7593
7961
|
for (const key of resumed) {
|
|
7962
|
+
if (this.state.threadNotices[key])
|
|
7963
|
+
this.state.threadNotices[key].ourPaused = false;
|
|
7594
7964
|
delete this.state.wakes[key];
|
|
7965
|
+
delete this.state.wakeRetries[key];
|
|
7595
7966
|
const held = this.state.held[key];
|
|
7596
7967
|
if (held?.messages.length) {
|
|
7597
7968
|
backlog[key] = held.messages.length;
|
|
@@ -7603,6 +7974,10 @@ var Watcher = class {
|
|
|
7603
7974
|
}
|
|
7604
7975
|
delete this.state.held[key];
|
|
7605
7976
|
}
|
|
7977
|
+
for (const key of added) {
|
|
7978
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
7979
|
+
record.ourPaused = true;
|
|
7980
|
+
}
|
|
7606
7981
|
this.state.paused = [.../* @__PURE__ */ new Set([...this.state.paused.filter((key) => !resumed.includes(key)), ...added])].sort();
|
|
7607
7982
|
});
|
|
7608
7983
|
for (const key of resumed)
|
|
@@ -7616,64 +7991,112 @@ var Watcher = class {
|
|
|
7616
7991
|
/** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
|
|
7617
7992
|
async pruneState() {
|
|
7618
7993
|
const at = this.now();
|
|
7619
|
-
const
|
|
7620
|
-
|
|
7621
|
-
if (!stale.length)
|
|
7994
|
+
const stale = Object.keys(this.state.wakes).filter((key) => !this.recentWakes(key).length).concat(Object.keys(this.state.approvalRetries).filter((key) => !this.state.pending[key]?.approvals?.length)).concat(Object.keys(this.state.wakeRetries).filter((key) => !this.state.pending[key])).concat(Object.keys(this.state.noticeLog).filter((peer) => !this.state.noticeLog[peer].some((t) => at - t < 36e5)));
|
|
7995
|
+
if (!stale.length && !this.state.noticeGlobalLog.some((t) => at - t >= 36e5))
|
|
7622
7996
|
return;
|
|
7623
7997
|
await this.mutate(() => {
|
|
7998
|
+
for (const key of Object.keys(this.state.wakeRetries))
|
|
7999
|
+
if (!this.state.pending[key])
|
|
8000
|
+
delete this.state.wakeRetries[key];
|
|
7624
8001
|
for (const key of Object.keys(this.state.wakes))
|
|
7625
8002
|
if (!this.recentWakes(key).length)
|
|
7626
8003
|
delete this.state.wakes[key];
|
|
7627
|
-
for (const key of Object.keys(this.state.
|
|
7628
|
-
if (
|
|
7629
|
-
delete this.state.
|
|
8004
|
+
for (const key of Object.keys(this.state.approvalRetries))
|
|
8005
|
+
if (!this.state.pending[key]?.approvals?.length)
|
|
8006
|
+
delete this.state.approvalRetries[key];
|
|
8007
|
+
this.state.noticeGlobalLog = this.state.noticeGlobalLog.filter((t) => at - t < 36e5);
|
|
7630
8008
|
for (const peer of Object.keys(this.state.noticeLog))
|
|
7631
8009
|
if (!this.state.noticeLog[peer].some((t) => at - t < 36e5))
|
|
7632
8010
|
delete this.state.noticeLog[peer];
|
|
7633
8011
|
});
|
|
7634
8012
|
}
|
|
7635
|
-
/** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after
|
|
8013
|
+
/** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after 3 notification attempts or 8 inform failures. */
|
|
7636
8014
|
async runOutbox() {
|
|
7637
8015
|
const at = this.now();
|
|
8016
|
+
const refill = [...this.outboxCandidates()].some((item) => this.admissionIndex(item) !== void 0);
|
|
8017
|
+
if (refill)
|
|
8018
|
+
await this.mutate(() => this.fillOutbox());
|
|
7638
8019
|
const runs = [];
|
|
7639
|
-
|
|
8020
|
+
const items = this.fairOutbox([...this.state.outbox]);
|
|
8021
|
+
for (const item of items) {
|
|
7640
8022
|
if (this.outboxBusy.has(item.id) || (item.nextAttemptAt ?? 0) > at)
|
|
7641
8023
|
continue;
|
|
7642
8024
|
this.outboxBusy.add(item.id);
|
|
7643
8025
|
runs.push((async () => {
|
|
7644
8026
|
try {
|
|
7645
8027
|
if (item.kind === "inform")
|
|
7646
|
-
await this.deps.postInform?.(item.peer, item.threadId, item.text);
|
|
8028
|
+
await this.deps.postInform?.(item.peer, item.threadId, item.text, item.pauseId);
|
|
7647
8029
|
else {
|
|
7648
8030
|
const { notice } = item;
|
|
8031
|
+
let reserved = false;
|
|
8032
|
+
await this.mutate(() => {
|
|
8033
|
+
const stored = this.state.outbox.find((o) => o.id === item.id);
|
|
8034
|
+
if (stored?.kind !== "notify")
|
|
8035
|
+
return;
|
|
8036
|
+
const now = this.now();
|
|
8037
|
+
const recent = (this.state.noticeLog[notice.peer] ?? []).filter((t) => now - t < 36e5);
|
|
8038
|
+
if ((stored.tries ?? 0) >= MAX_NOTICE_ATTEMPTS) {
|
|
8039
|
+
this.finishOutbox(item);
|
|
8040
|
+
return;
|
|
8041
|
+
}
|
|
8042
|
+
const deadline = this.noticeRateDeadline(notice.peer, now);
|
|
8043
|
+
if (deadline) {
|
|
8044
|
+
stored.nextAttemptAt = deadline;
|
|
8045
|
+
this.rememberNotice(stored);
|
|
8046
|
+
this.deps.log("notice-rate-cap", { peer: notice.peer, id: item.id });
|
|
8047
|
+
return;
|
|
8048
|
+
}
|
|
8049
|
+
const episode = this.noticeSlot(stored);
|
|
8050
|
+
if (episode) {
|
|
8051
|
+
if ("repeats" in episode)
|
|
8052
|
+
item.notice.repeats = stored.notice.repeats = episode.repeats;
|
|
8053
|
+
else {
|
|
8054
|
+
item.notice.threadId = stored.notice.threadId = episode.threadId;
|
|
8055
|
+
item.notice.count = stored.notice.count = episode.count;
|
|
8056
|
+
episode.arrivalFirstSeenAt = EPOCH;
|
|
8057
|
+
}
|
|
8058
|
+
episode.dispatchArrivals = "000000";
|
|
8059
|
+
}
|
|
8060
|
+
this.state.noticeGlobalLog = [...this.state.noticeGlobalLog.filter((t) => now - t < 36e5), now];
|
|
8061
|
+
item.tries = stored.tries = (stored.tries ?? 0) + 1;
|
|
8062
|
+
this.rememberNotice(stored);
|
|
8063
|
+
this.state.noticeLog[notice.peer] = [...recent, now];
|
|
8064
|
+
reserved = true;
|
|
8065
|
+
});
|
|
8066
|
+
if (!reserved)
|
|
8067
|
+
return;
|
|
7649
8068
|
this.deps.log("notify", { kind: notice.kind, peer: notice.peer, threadId: notice.threadId, detail: notice.detail });
|
|
7650
8069
|
if (!item.logged && this.deps.notificationHome) {
|
|
7651
8070
|
appendNotice(this.deps.notificationHome, notice);
|
|
7652
8071
|
await this.mutate(() => {
|
|
7653
8072
|
const stored = this.state.outbox.find((o) => o.id === item.id);
|
|
7654
|
-
if (stored?.kind === "notify")
|
|
8073
|
+
if (stored?.kind === "notify") {
|
|
7655
8074
|
stored.logged = true;
|
|
8075
|
+
this.rememberNotice(stored);
|
|
8076
|
+
}
|
|
7656
8077
|
});
|
|
7657
8078
|
}
|
|
7658
8079
|
await this.deps.notify?.(notice);
|
|
7659
8080
|
}
|
|
7660
|
-
await this.mutate(() =>
|
|
7661
|
-
this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id);
|
|
7662
|
-
});
|
|
8081
|
+
await this.mutate(() => this.finishOutbox(item, true));
|
|
7663
8082
|
} catch (error) {
|
|
7664
|
-
const tries = (item.tries ?? 0) + 1;
|
|
8083
|
+
const tries = item.kind === "notify" ? item.tries ?? 1 : (item.tries ?? 0) + 1;
|
|
7665
8084
|
this.deps.log(item.kind === "inform" ? "inform-error" : "notify-error", { id: item.id, tries, ...errInfo(error) });
|
|
7666
|
-
if (tries >= MAX_OUTBOX_TRIES) {
|
|
8085
|
+
if (tries >= (item.kind === "notify" ? MAX_NOTICE_ATTEMPTS : MAX_OUTBOX_TRIES)) {
|
|
7667
8086
|
this.deps.log("outbox-dropped", { id: item.id, reason: "too-many-failures" });
|
|
7668
|
-
await this.mutate(() =>
|
|
7669
|
-
this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id);
|
|
7670
|
-
});
|
|
8087
|
+
await this.mutate(() => this.finishOutbox(item));
|
|
7671
8088
|
} else {
|
|
7672
8089
|
await this.mutate(() => {
|
|
7673
8090
|
const stored = this.state.outbox.find((o) => o.id === item.id);
|
|
7674
|
-
if (stored)
|
|
7675
|
-
|
|
7676
|
-
|
|
8091
|
+
if (!stored)
|
|
8092
|
+
return;
|
|
8093
|
+
stored.tries = tries;
|
|
8094
|
+
stored.nextAttemptAt = this.now() + Math.min(15 * 6e4, WAKE_RETRY_MS * 2 ** (tries - 1));
|
|
8095
|
+
if (stored.kind === "notify") {
|
|
8096
|
+
this.rememberNotice(stored);
|
|
8097
|
+
const episode = this.noticeSlot(stored);
|
|
8098
|
+
if (episode)
|
|
8099
|
+
episode.dispatchArrivals = "------";
|
|
7677
8100
|
}
|
|
7678
8101
|
});
|
|
7679
8102
|
}
|
|
@@ -7686,6 +8109,15 @@ var Watcher = class {
|
|
|
7686
8109
|
}
|
|
7687
8110
|
// ---------- run loop ----------
|
|
7688
8111
|
async pollOnce(via) {
|
|
8112
|
+
if (this.deps.notifyOnly) {
|
|
8113
|
+
const paired = this.deps.loadPeers().filter((peer) => peer.status === "paired" && HANDLE.test(peer.handle)).map((peer) => peer.handle);
|
|
8114
|
+
if (paired.some((peer) => !Object.hasOwn(this.state.peerNotices, peer)) || Object.keys(this.state.peerNotices).some((peer) => !paired.includes(peer))) {
|
|
8115
|
+
await this.mutate(() => {
|
|
8116
|
+
this.syncNoticePeers();
|
|
8117
|
+
this.state.outbox = this.state.outbox.filter((item) => item.kind !== "notify" || !needsKind(item.notice.kind) || paired.includes(item.notice.peer));
|
|
8118
|
+
});
|
|
8119
|
+
}
|
|
8120
|
+
}
|
|
7689
8121
|
await this.syncPaused();
|
|
7690
8122
|
await this.pruneState();
|
|
7691
8123
|
await Promise.all(this.deps.loadPeers().map((peer) => this.sync(peer, via)));
|
|
@@ -8523,8 +8955,8 @@ async function cmdWatch(args) {
|
|
|
8523
8955
|
loadPeers: () => Object.values(loadProfile().peers),
|
|
8524
8956
|
kvFor: (peer) => peerKv(node, loadProfile(), peer),
|
|
8525
8957
|
wake: createWaker(config, { run: spawnRunner, sessions: fileSessions() }),
|
|
8526
|
-
postInform: async (peer, threadId, text) => {
|
|
8527
|
-
await postPausedInform(node.kv, loadProfile(), peer, threadId, text);
|
|
8958
|
+
postInform: async (peer, threadId, text, pauseId) => {
|
|
8959
|
+
await postPausedInform(node.kv, loadProfile(), peer, threadId, text, pauseId);
|
|
8528
8960
|
},
|
|
8529
8961
|
notify: createNotifier(notifyConfig(args, config), { run: spawnRunner }),
|
|
8530
8962
|
holder: { profileId: watcherProfileId() },
|