@wakeshift/openclaw 0.0.0-placeholder → 1.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +56 -0
- package/dist/beacon.js +287 -0
- package/dist/control.js +150 -0
- package/dist/deliveries.js +507 -0
- package/dist/framing.js +200 -0
- package/dist/hooks.js +249 -0
- package/dist/index.js +287 -0
- package/dist/plugin.js +523 -0
- package/dist/reset.js +95 -0
- package/dist/store.js +202 -0
- package/dist/transport.js +474 -0
- package/dist/version.js +12 -0
- package/dist/wake.js +66 -0
- package/openclaw.plugin.json +42 -0
- package/package.json +45 -1
package/dist/control.js
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// Keeping OpenClaw's hooks.* set up for Wakeshift's wake (hooks.ts says
|
|
2
|
+
// what changes and why): turned on at the first connect, a connected
|
|
3
|
+
// conversation's session and agent allowed, and allowed no longer once it
|
|
4
|
+
// has gone. Every change is recorded in <state dir>/hooks.json before it's
|
|
5
|
+
// written, so the uninstaller can undo it (a crash between the two leaves
|
|
6
|
+
// the record naming one entry more than the config has, which undoing
|
|
7
|
+
// skips). Changes are made one at a time, through OpenClaw's own config
|
|
8
|
+
// writer (api.runtime.config.mutateConfigFile), which hot-reloads hooks.
|
|
9
|
+
import { randomBytes } from "node:crypto";
|
|
10
|
+
import { copyFileSync, readFileSync, rmSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { addScope, releaseScope, takeover } from "./hooks.js";
|
|
13
|
+
import { writeAtomic } from "./store.js";
|
|
14
|
+
const CHANGED = "OpenClaw's hooks settings changed meanwhile; call the app's tool again";
|
|
15
|
+
export class HooksControl {
|
|
16
|
+
o;
|
|
17
|
+
chain = Promise.resolve();
|
|
18
|
+
constructor(o) {
|
|
19
|
+
this.o = o;
|
|
20
|
+
}
|
|
21
|
+
get file() {
|
|
22
|
+
return join(this.o.dir, "hooks.json");
|
|
23
|
+
}
|
|
24
|
+
/** What the plugin changed so far (undefined: nothing yet). */
|
|
25
|
+
changes() {
|
|
26
|
+
try {
|
|
27
|
+
const c = JSON.parse(readFileSync(this.file, "utf8"));
|
|
28
|
+
return c && c.version === 1 && c.added ? c : undefined;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
save(c) {
|
|
35
|
+
writeAtomic(this.file, JSON.stringify(c, null, 2) + "\n");
|
|
36
|
+
if (this.o.restoreScript) {
|
|
37
|
+
try {
|
|
38
|
+
copyFileSync(this.o.restoreScript, join(this.o.dir, "restore.mjs"));
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
this.o.log.warn(`wakeshift: couldn't copy the uninstaller's restore script: ${e.message}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
on(fn) {
|
|
46
|
+
const p = this.chain.then(fn, fn);
|
|
47
|
+
this.chain = p.catch(() => { });
|
|
48
|
+
return p;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Makes sure OpenClaw's hooks let Wakeshift wake this session. Resolves
|
|
52
|
+
* to "" when they do, or to why not (said to the agent).
|
|
53
|
+
*/
|
|
54
|
+
allow(sessionKey, agentId) {
|
|
55
|
+
return this.on(async () => {
|
|
56
|
+
let ch = this.changes();
|
|
57
|
+
const cur = this.o.config.hooks();
|
|
58
|
+
if (!ch) {
|
|
59
|
+
const token = this.o.token ? this.o.token() : randomBytes(32).toString("hex");
|
|
60
|
+
const t = takeover(cur, token, { network: this.o.network(), allowNetwork: this.o.allowNetwork });
|
|
61
|
+
if (!t.ok)
|
|
62
|
+
return t.why;
|
|
63
|
+
// What's written, and recorded, is worked out from the config file as
|
|
64
|
+
// it is at the write (the snapshot can lag it): the record always says
|
|
65
|
+
// exactly what this write changed. Recorded before the file is.
|
|
66
|
+
let rec;
|
|
67
|
+
try {
|
|
68
|
+
await this.o.config.mutateHooks((now) => {
|
|
69
|
+
const again = takeover(now, token, { network: false, allowNetwork: true });
|
|
70
|
+
if (!again.ok || again.changes.owned !== t.changes.owned)
|
|
71
|
+
throw new Error(CHANGED);
|
|
72
|
+
rec = again.changes;
|
|
73
|
+
this.save(rec);
|
|
74
|
+
return again.next;
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
catch (e) {
|
|
78
|
+
rmSync(this.file, { force: true }); // nothing was turned on: no record of it
|
|
79
|
+
return `OpenClaw didn't save Wakeshift's hooks settings (${e.message})`;
|
|
80
|
+
}
|
|
81
|
+
ch = rec;
|
|
82
|
+
this.o.log.info("wakeshift: turned on OpenClaw's hooks for waking conversations (recorded in hooks.json)");
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
const why = stillUsable(cur, ch);
|
|
86
|
+
if (why)
|
|
87
|
+
return why;
|
|
88
|
+
}
|
|
89
|
+
// Nothing to add, as far as the snapshot shows: no write.
|
|
90
|
+
if (!addScope(this.o.config.hooks() ?? cur ?? {}, ch, sessionKey, agentId))
|
|
91
|
+
return "";
|
|
92
|
+
// Worked out, and recorded, from the config file as it is at the write,
|
|
93
|
+
// so the record says exactly what this write adds (never an entry the
|
|
94
|
+
// user added meanwhile, which an undo would then take out). A write
|
|
95
|
+
// that fails puts the record back as it was.
|
|
96
|
+
const before = ch;
|
|
97
|
+
try {
|
|
98
|
+
await this.o.config.mutateHooks((now) => {
|
|
99
|
+
const again = addScope(now ?? {}, before, sessionKey, agentId);
|
|
100
|
+
if (!again)
|
|
101
|
+
return now;
|
|
102
|
+
this.save(again.changes);
|
|
103
|
+
return again.next;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
catch (e) {
|
|
107
|
+
this.save(before);
|
|
108
|
+
return `OpenClaw didn't save this conversation's hooks settings (${e.message})`;
|
|
109
|
+
}
|
|
110
|
+
return "";
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
/** Allows no longer what the plugin added for conversations that have gone. */
|
|
114
|
+
release(inUse) {
|
|
115
|
+
return this.on(async () => {
|
|
116
|
+
const ch = this.changes();
|
|
117
|
+
if (!ch)
|
|
118
|
+
return;
|
|
119
|
+
// Worked out from the config file at the write, and recorded only once
|
|
120
|
+
// the write has landed (until then, the entries are still there and
|
|
121
|
+
// still the plugin's).
|
|
122
|
+
let rec;
|
|
123
|
+
try {
|
|
124
|
+
await this.o.config.mutateHooks((now) => {
|
|
125
|
+
const r = now ? releaseScope(now, ch, inUse) : null;
|
|
126
|
+
rec = r?.changes;
|
|
127
|
+
return r ? r.next : now;
|
|
128
|
+
});
|
|
129
|
+
if (rec)
|
|
130
|
+
this.save(rec);
|
|
131
|
+
}
|
|
132
|
+
catch (e) {
|
|
133
|
+
this.o.log.warn(`wakeshift: couldn't remove a gone conversation from OpenClaw's hooks settings: ${e.message}`);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/** Why hooks the plugin set up (or the user's, which it used) can't wake now, or "". */
|
|
139
|
+
export function stillUsable(cur, ch) {
|
|
140
|
+
if (cur?.enabled !== true) {
|
|
141
|
+
return ch.owned
|
|
142
|
+
? "OpenClaw's hooks were turned off since Wakeshift's plugin turned them on (hooks.enabled is false). Ask the user whether to turn them on again; only if they agree, set hooks.enabled to true"
|
|
143
|
+
: "OpenClaw's hooks are off here now (hooks.enabled is false)";
|
|
144
|
+
}
|
|
145
|
+
if (cur.allowRequestSessionKey !== true)
|
|
146
|
+
return "OpenClaw's hooks no longer let a caller name the session (hooks.allowRequestSessionKey is false)";
|
|
147
|
+
if (typeof cur.token !== "string" || !cur.token.trim())
|
|
148
|
+
return "OpenClaw's hooks have no token (hooks.token)";
|
|
149
|
+
return "";
|
|
150
|
+
}
|
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
// Delivery state: one durable record per delivery, and the one place that
|
|
2
|
+
// decides what happens to it.
|
|
3
|
+
//
|
|
4
|
+
// A record lives at <state dir>/deliveries/<sha256(conversation)>/<sha256(delivery ID)>.json
|
|
5
|
+
// (0600 in 0700 directories) and moves through
|
|
6
|
+
//
|
|
7
|
+
// taken → turn-started → acked
|
|
8
|
+
//
|
|
9
|
+
// (or `refused`: Wakeshift answered 400 for that ID alone, so it's not
|
|
10
|
+
// sent again; or `superseded`: a turn had it, but no stream that carries it
|
|
11
|
+
// is left to acknowledge it on, below).
|
|
12
|
+
//
|
|
13
|
+
// taken recorded on arrival, with the delivery; not yet handled
|
|
14
|
+
// by a turn that finished.
|
|
15
|
+
// turn-started a woken turn that had it finished (OpenClaw's /hooks/agent
|
|
16
|
+
// answered completion ok): only now may it be acknowledged.
|
|
17
|
+
// acked Wakeshift answered 2xx to its ack.
|
|
18
|
+
//
|
|
19
|
+
// (`enqueued`, from the plugin's earlier heartbeat wake, is read as taken.)
|
|
20
|
+
//
|
|
21
|
+
// Streams: a conversation can watch several apps, each on its own link
|
|
22
|
+
// (store.ts), so several streams feed one conversation's records. A record
|
|
23
|
+
// names the stream it last arrived on (`via`). A stream that was replaced by
|
|
24
|
+
// a newer link of its app (a new Wakeshift session superseding the old one,
|
|
25
|
+
// which cancels what the old one held), or that ended for good, is lost:
|
|
26
|
+
// what it brought still runs here (nothing else would bring it), but isn't
|
|
27
|
+
// acknowledged on any link, as none that's left carries it; it's marked
|
|
28
|
+
// superseded once a turn had it. If it arrives again on a live stream, it's
|
|
29
|
+
// that stream's, and acknowledged there.
|
|
30
|
+
//
|
|
31
|
+
// Invariants (tested in test/deliveries.test.ts, including random
|
|
32
|
+
// interleavings of redelivery, failures and restarts):
|
|
33
|
+
//
|
|
34
|
+
// 1. A delivery is in at most one wake request at a time: requests are
|
|
35
|
+
// made on the conversation's one chain, one after another, and a
|
|
36
|
+
// redelivery only opens a new window for the next try. A try after an
|
|
37
|
+
// outcome that isn't known (unreachable, busy, a timeout) sends the
|
|
38
|
+
// same request again, the same records with the same idempotency key
|
|
39
|
+
// (recorded as `unsure`), before anything newer, so OpenClaw replays a
|
|
40
|
+
// run it admitted instead of running it again; a try after a known
|
|
41
|
+
// one (a run that ended without ok, a refused request) uses a new key.
|
|
42
|
+
// 2. It's acknowledged only from `turn-started`, only on the stream it
|
|
43
|
+
// last arrived on and only while that stream is live, and only a 2xx
|
|
44
|
+
// moves it to `acked`; nothing is acknowledged after that.
|
|
45
|
+
// 3. A record isn't evicted while it isn't acked or refused (no capped
|
|
46
|
+
// sets): records are pruned only once acked or refused for ACKED_TTL,
|
|
47
|
+
// or after MAX_AGE whatever their state.
|
|
48
|
+
// 4. Every decision for a delivery reads and writes only its record (and
|
|
49
|
+
// for a publish, the records of its other copies), on the
|
|
50
|
+
// conversation's one chain, whichever of its streams brought it (the
|
|
51
|
+
// gateway is one process, and owns its state dir), so no two decisions
|
|
52
|
+
// for it interleave, and turns start one at a time per conversation.
|
|
53
|
+
// 5. A publish starts one turn: a copy of one that a turn already had (it
|
|
54
|
+
// reached the conversation through two of its app's sessions, the
|
|
55
|
+
// superseded one and its successor) is marked as had by that turn,
|
|
56
|
+
// without one of its own, and one wake request carries it once.
|
|
57
|
+
//
|
|
58
|
+
// The gaps (at least once, never lost): OpenClaw keeps its replay keys in
|
|
59
|
+
// memory, so a request in flight when the gateway restarts is sent again
|
|
60
|
+
// and may run a second time; so may a turn that finished after its
|
|
61
|
+
// request's answer was lost, once its key has aged out.
|
|
62
|
+
//
|
|
63
|
+
// Turns: a turn is a wake request (wake.ts). One that doesn't run is tried
|
|
64
|
+
// again with backoff for `wakeFor` from when the delivery arrived (or was
|
|
65
|
+
// redelivered); past that, the transport reconnects so Wakeshift hands it
|
|
66
|
+
// out again, which opens a new window. Acks: sent in chunks of at most
|
|
67
|
+
// 100, retried with backoff on their own timer; a 400 splits the chunk to
|
|
68
|
+
// find the bad ID (logged and marked refused), and the rest go on.
|
|
69
|
+
import { createHash } from "node:crypto";
|
|
70
|
+
import { chmodSync, mkdirSync, readdirSync, readFileSync, rmdirSync, rmSync } from "node:fs";
|
|
71
|
+
import { join } from "node:path";
|
|
72
|
+
import { prompt } from "./framing.js";
|
|
73
|
+
import { writeAtomic } from "./store.js";
|
|
74
|
+
export const ACKED_TTL = 24 * 3600 * 1000;
|
|
75
|
+
export const MAX_AGE = 7 * 24 * 3600 * 1000;
|
|
76
|
+
const hash = (s) => createHash("sha256").update(s).digest("hex").slice(0, 32);
|
|
77
|
+
/** The records on disk: one directory per conversation. */
|
|
78
|
+
export class Ledger {
|
|
79
|
+
dir;
|
|
80
|
+
constructor(dir) {
|
|
81
|
+
this.dir = dir;
|
|
82
|
+
}
|
|
83
|
+
convDir(conversation) {
|
|
84
|
+
return join(this.dir, hash(conversation));
|
|
85
|
+
}
|
|
86
|
+
path(conversation, id) {
|
|
87
|
+
return join(this.convDir(conversation), hash(id) + ".json");
|
|
88
|
+
}
|
|
89
|
+
get(conversation, id) {
|
|
90
|
+
try {
|
|
91
|
+
const r = JSON.parse(readFileSync(this.path(conversation, id), "utf8"));
|
|
92
|
+
return r.id === id && r.conversation === conversation ? r : undefined;
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
put(r) {
|
|
99
|
+
const d = this.convDir(r.conversation);
|
|
100
|
+
mkdirSync(d, { recursive: true, mode: 0o700 });
|
|
101
|
+
try {
|
|
102
|
+
chmodSync(this.dir, 0o700);
|
|
103
|
+
chmodSync(d, 0o700);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
/* best effort */
|
|
107
|
+
}
|
|
108
|
+
writeAtomic(this.path(r.conversation, r.id), JSON.stringify(r));
|
|
109
|
+
}
|
|
110
|
+
/** A conversation's records, oldest first (reads only its directory). */
|
|
111
|
+
of(conversation) {
|
|
112
|
+
return read(this.convDir(conversation))
|
|
113
|
+
.filter((r) => r.conversation === conversation)
|
|
114
|
+
.sort((a, b) => a.takenAt - b.takenAt);
|
|
115
|
+
}
|
|
116
|
+
all() {
|
|
117
|
+
return names(this.dir).flatMap((n) => (n.endsWith(".json") ? [] : read(join(this.dir, n))));
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Deletes records acked or refused over ACKED_TTL ago, and any over
|
|
121
|
+
* MAX_AGE, and directories left empty. Records in the old flat layout
|
|
122
|
+
* (<dir>/<hash>.json) are moved into their conversation's directory.
|
|
123
|
+
*/
|
|
124
|
+
prune(now = Date.now()) {
|
|
125
|
+
for (const n of names(this.dir)) {
|
|
126
|
+
if (!n.endsWith(".json"))
|
|
127
|
+
continue;
|
|
128
|
+
const p = join(this.dir, n);
|
|
129
|
+
try {
|
|
130
|
+
const r = JSON.parse(readFileSync(p, "utf8"));
|
|
131
|
+
if (r && typeof r.id === "string" && typeof r.conversation === "string" && !this.get(r.conversation, r.id))
|
|
132
|
+
this.put(r);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
/* unreadable: dropped */
|
|
136
|
+
}
|
|
137
|
+
rmSync(p, { force: true });
|
|
138
|
+
}
|
|
139
|
+
for (const n of names(this.dir)) {
|
|
140
|
+
const d = join(this.dir, n);
|
|
141
|
+
for (const r of read(d)) {
|
|
142
|
+
const done = finished(r) && now - (r.doneAt ?? r.takenAt) > ACKED_TTL;
|
|
143
|
+
if (done || now - r.takenAt > MAX_AGE)
|
|
144
|
+
rmSync(this.path(r.conversation, r.id), { force: true });
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
rmdirSync(d); // only if empty
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
/* not empty */
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function names(dir) {
|
|
156
|
+
try {
|
|
157
|
+
return readdirSync(dir);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
return [];
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
function read(dir) {
|
|
164
|
+
const out = [];
|
|
165
|
+
for (const n of names(dir)) {
|
|
166
|
+
if (!n.endsWith(".json"))
|
|
167
|
+
continue;
|
|
168
|
+
try {
|
|
169
|
+
out.push(JSON.parse(readFileSync(join(dir, n), "utf8")));
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
/* skip */
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return out;
|
|
176
|
+
}
|
|
177
|
+
/** Not yet handled by a turn that finished (enqueued: the earlier heartbeat wake's state). */
|
|
178
|
+
const pending = (r) => r.state === "taken" || r.state === "enqueued";
|
|
179
|
+
/** Done with: nothing more is sent for it. */
|
|
180
|
+
const finished = (r) => r.state === "acked" || r.state === "refused" || r.state === "superseded";
|
|
181
|
+
const viaOf = (r) => r.via ?? "";
|
|
182
|
+
/**
|
|
183
|
+
* Which of its app's publishes a delivery carries, the same in every
|
|
184
|
+
* session it reached ("" if it doesn't say): a topic copy has the
|
|
185
|
+
* publish's own ID as its event ID in every workspace, a direct copy names
|
|
186
|
+
* it in $emit (internal/appapi/fanout.go). Scoped by the app it's from, so
|
|
187
|
+
* two apps' IDs can't collide.
|
|
188
|
+
*/
|
|
189
|
+
export function publishKey(d) {
|
|
190
|
+
const ev = d?.event;
|
|
191
|
+
if (!ev || typeof ev !== "object")
|
|
192
|
+
return "";
|
|
193
|
+
const src = ev.source;
|
|
194
|
+
if (typeof src !== "string" || !src.startsWith("app:"))
|
|
195
|
+
return "";
|
|
196
|
+
const attrs = ev.attrs && typeof ev.attrs === "object" ? ev.attrs : {};
|
|
197
|
+
const id = attrs["$emit"] || ev.id;
|
|
198
|
+
return typeof id === "string" && id ? `${src}|${id}` : "";
|
|
199
|
+
}
|
|
200
|
+
/** The wake request's idempotency key: the same deliveries and attempt, the same key. */
|
|
201
|
+
export function idempotencyKey(conversation, ids, attempt, variant = "") {
|
|
202
|
+
return "wakeshift-" + createHash("sha256").update(JSON.stringify([conversation, [...ids].sort(), attempt, variant])).digest("hex").slice(0, 40);
|
|
203
|
+
}
|
|
204
|
+
/** One record per publish (the first), for a wake request's message. */
|
|
205
|
+
function once(rs) {
|
|
206
|
+
const seen = new Set();
|
|
207
|
+
return rs.filter((r) => {
|
|
208
|
+
const k = publishKey(r.delivery);
|
|
209
|
+
if (!k)
|
|
210
|
+
return true;
|
|
211
|
+
if (seen.has(k))
|
|
212
|
+
return false;
|
|
213
|
+
seen.add(k);
|
|
214
|
+
return true;
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
/** One conversation's deliveries, from every stream it has: every decision about them is made here, on one chain. */
|
|
218
|
+
export class Conversation {
|
|
219
|
+
o;
|
|
220
|
+
chain = Promise.resolve();
|
|
221
|
+
stopped = false;
|
|
222
|
+
turnTimer;
|
|
223
|
+
ackTimer;
|
|
224
|
+
redeliverTimer;
|
|
225
|
+
turnDelay = 0;
|
|
226
|
+
ackDelay = 0;
|
|
227
|
+
constructor(o) {
|
|
228
|
+
this.o = o;
|
|
229
|
+
if (o.after)
|
|
230
|
+
this.chain = o.after.catch(() => { });
|
|
231
|
+
}
|
|
232
|
+
now() {
|
|
233
|
+
return this.o.now ? this.o.now() : Date.now();
|
|
234
|
+
}
|
|
235
|
+
on(fn) {
|
|
236
|
+
const p = this.chain.then(() => (this.stopped ? undefined : fn()));
|
|
237
|
+
this.chain = p.then(() => { }, (e) => this.o.host.log.warn(`wakeshift (${this.o.sessionKey}): ${e.message}`));
|
|
238
|
+
return p;
|
|
239
|
+
}
|
|
240
|
+
/** Picks up what an earlier process left (turns, acks). */
|
|
241
|
+
start() {
|
|
242
|
+
void this.on(() => this.step());
|
|
243
|
+
}
|
|
244
|
+
stop() {
|
|
245
|
+
this.stopped = true;
|
|
246
|
+
clearTimeout(this.turnTimer);
|
|
247
|
+
clearTimeout(this.ackTimer);
|
|
248
|
+
clearTimeout(this.redeliverTimer);
|
|
249
|
+
}
|
|
250
|
+
/** Waits until everything queued on the chain so far is done. */
|
|
251
|
+
async settled() {
|
|
252
|
+
for (let c = this.chain;; c = this.chain) {
|
|
253
|
+
await c; // and whatever was queued on it meanwhile
|
|
254
|
+
if (c === this.chain)
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Records deliveries from the stream; resolves once recorded, with how
|
|
260
|
+
* many were new. A redelivery of one not yet handled opens a new turn
|
|
261
|
+
* window; one a turn handled, acked or refused is left alone.
|
|
262
|
+
*/
|
|
263
|
+
receive(ds, via = "") {
|
|
264
|
+
return this.on(async () => {
|
|
265
|
+
let fresh = 0;
|
|
266
|
+
const now = this.now();
|
|
267
|
+
// Arriving on a live stream makes it that stream's to acknowledge (a
|
|
268
|
+
// stream that was replaced may still hand over what it read).
|
|
269
|
+
const live = this.streamOf(via) === "live";
|
|
270
|
+
for (const d of ds) {
|
|
271
|
+
const r = this.o.ledger.get(this.o.conversation, d.delivery_id);
|
|
272
|
+
if (!r) {
|
|
273
|
+
this.o.ledger.put({ id: d.delivery_id, conversation: this.o.conversation, state: "taken", delivery: d, takenAt: now, via });
|
|
274
|
+
fresh++;
|
|
275
|
+
}
|
|
276
|
+
else if (pending(r)) {
|
|
277
|
+
this.o.ledger.put({ ...r, ...(live ? { via } : {}), windowFrom: now });
|
|
278
|
+
this.turnDelay = 0;
|
|
279
|
+
}
|
|
280
|
+
else if (r.state === "turn-started" && live) {
|
|
281
|
+
this.o.ledger.put({ ...r, via });
|
|
282
|
+
}
|
|
283
|
+
else if (r.state === "superseded" && live) {
|
|
284
|
+
// Had by a turn, and on a stream that can acknowledge it after all.
|
|
285
|
+
this.o.ledger.put({ ...r, via, state: "turn-started", doneAt: undefined });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
void this.on(() => this.step()); // recorded: the rest happens after, on the chain
|
|
289
|
+
return fresh;
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
streamOf(via) {
|
|
293
|
+
return this.o.stream ? this.o.stream(via) : "live";
|
|
294
|
+
}
|
|
295
|
+
// streamOf, asked once per stream for one pass over the records.
|
|
296
|
+
streams() {
|
|
297
|
+
const m = new Map();
|
|
298
|
+
return (via) => {
|
|
299
|
+
let s = m.get(via);
|
|
300
|
+
if (!s)
|
|
301
|
+
m.set(via, (s = this.streamOf(via)));
|
|
302
|
+
return s;
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
// Everything that can happen now, from the records alone.
|
|
306
|
+
async step() {
|
|
307
|
+
// After stop() nothing new starts (a step already past a wake request
|
|
308
|
+
// still records what came of it).
|
|
309
|
+
if (!this.stopped && this.o.ledger.of(this.o.conversation).some(pending))
|
|
310
|
+
await this.turn();
|
|
311
|
+
if (!this.stopped && this.o.ledger.of(this.o.conversation).some((r) => r.state === "turn-started"))
|
|
312
|
+
await this.acks();
|
|
313
|
+
}
|
|
314
|
+
// One wake request for everything not yet handled; or, after a request
|
|
315
|
+
// whose outcome isn't known, that same request again (the same records,
|
|
316
|
+
// the same key), so OpenClaw replays it rather than running it twice.
|
|
317
|
+
async turn() {
|
|
318
|
+
const { host } = this.o;
|
|
319
|
+
this.twins();
|
|
320
|
+
const all = this.o.ledger.of(this.o.conversation).filter(pending);
|
|
321
|
+
const unsure = all.find((r) => r.unsure)?.unsure;
|
|
322
|
+
const waiting = unsure ? all.filter((r) => r.unsure === unsure) : all;
|
|
323
|
+
if (waiting.length === 0)
|
|
324
|
+
return;
|
|
325
|
+
const attempt = Math.max(...waiting.map((w) => w.attempt ?? 0));
|
|
326
|
+
let req = {
|
|
327
|
+
sessionKey: this.o.sessionKey,
|
|
328
|
+
...(this.o.agentId ? { agentId: this.o.agentId } : {}),
|
|
329
|
+
message: prompt(once(waiting).map((w) => w.delivery)),
|
|
330
|
+
idempotencyKey: unsure ?? idempotencyKey(this.o.conversation, waiting.map((w) => w.id), attempt),
|
|
331
|
+
// A request sent again as is (unsure) keeps the route it had.
|
|
332
|
+
route: unsure ? waiting[0].unsureRoute : this.o.route?.(),
|
|
333
|
+
};
|
|
334
|
+
let r;
|
|
335
|
+
try {
|
|
336
|
+
r = await host.wake(req);
|
|
337
|
+
if (r.kind === "route") {
|
|
338
|
+
// The channel it last spoke on was refused (nothing ran): the reply
|
|
339
|
+
// stays in the session instead, a new request without the route.
|
|
340
|
+
host.log.info(`wakeshift (${this.o.sessionKey}): the reply's route was refused (${r.why}); waking without it`);
|
|
341
|
+
this.o.routeRefused?.();
|
|
342
|
+
req = { ...req, route: undefined, idempotencyKey: idempotencyKey(this.o.conversation, waiting.map((w) => w.id), attempt, "noroute") };
|
|
343
|
+
r = await host.wake(req);
|
|
344
|
+
if (r.kind === "route")
|
|
345
|
+
r = { kind: "broken", why: `OpenClaw refused the wake (400: ${r.why})` };
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
catch (e) {
|
|
349
|
+
r = { kind: "retry", fresh: false, why: e.message };
|
|
350
|
+
}
|
|
351
|
+
if (r.kind === "ran") {
|
|
352
|
+
const now = this.now();
|
|
353
|
+
for (const w of waiting)
|
|
354
|
+
this.o.ledger.put({ ...w, state: "turn-started", turnAt: now, unsure: undefined, unsureRoute: undefined });
|
|
355
|
+
this.turnDelay = 0;
|
|
356
|
+
this.o.problem?.("");
|
|
357
|
+
// What arrived while a request was sent again as is (unsure) wasn't
|
|
358
|
+
// in it: its own turn next (no timer or delivery would start one).
|
|
359
|
+
if (this.o.ledger.of(this.o.conversation).some(pending))
|
|
360
|
+
void this.on(() => this.step());
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
if (r.kind === "broken")
|
|
364
|
+
this.o.problem?.(r.why);
|
|
365
|
+
// A known outcome (a run that ended without ok, a refused request) needs
|
|
366
|
+
// a new key; an unknown one keeps it, with these records, so a run
|
|
367
|
+
// OpenClaw admitted is replayed, not run again.
|
|
368
|
+
const known = r.kind !== "retry" || r.fresh;
|
|
369
|
+
for (const w of waiting) {
|
|
370
|
+
this.o.ledger.put(known ? { ...w, attempt: attempt + 1, unsure: undefined, unsureRoute: undefined } : { ...w, unsure: req.idempotencyKey, unsureRoute: req.route });
|
|
371
|
+
}
|
|
372
|
+
host.log.info(`wakeshift (${this.o.sessionKey}): the wake didn't run (${r.why})`);
|
|
373
|
+
this.retryTurn(Math.min(...waiting.map((w) => w.windowFrom ?? w.takenAt)));
|
|
374
|
+
}
|
|
375
|
+
// A copy of a publish that a turn already had (the same publish through
|
|
376
|
+
// another session) is had by that turn: no turn of its own. Every record
|
|
377
|
+
// that isn't pending had a turn: refused and superseded are reached only
|
|
378
|
+
// from turn-started.
|
|
379
|
+
twins() {
|
|
380
|
+
const recs = this.o.ledger.of(this.o.conversation);
|
|
381
|
+
const had = new Set(recs.filter((r) => !pending(r)).map((r) => publishKey(r.delivery)).filter(Boolean));
|
|
382
|
+
if (had.size === 0)
|
|
383
|
+
return;
|
|
384
|
+
const now = this.now();
|
|
385
|
+
for (const r of recs) {
|
|
386
|
+
if (pending(r) && !r.unsure && had.has(publishKey(r.delivery))) {
|
|
387
|
+
this.o.ledger.put({ ...r, state: "turn-started", turnAt: now });
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
// Tries the turn again with backoff while the window is open; once it
|
|
392
|
+
// closes, has Wakeshift hand the delivery out again (a new window).
|
|
393
|
+
retryTurn(windowFrom) {
|
|
394
|
+
if (this.stopped || this.turnTimer)
|
|
395
|
+
return;
|
|
396
|
+
const base = this.o.wakeRetry ?? 5000;
|
|
397
|
+
this.turnDelay = this.turnDelay ? Math.min(this.turnDelay * 2, 60000) : base;
|
|
398
|
+
if (this.now() + this.turnDelay > windowFrom + (this.o.wakeFor ?? 300000)) {
|
|
399
|
+
this.o.host.log.warn(`wakeshift (${this.o.sessionKey}): no turn could run in time; Wakeshift will hand it out again`);
|
|
400
|
+
this.turnDelay = 0;
|
|
401
|
+
clearTimeout(this.redeliverTimer);
|
|
402
|
+
this.redeliverTimer = setTimeout(() => !this.stopped && void this.on(() => this.reopen()), this.o.redeliverAfter ?? 30000);
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
this.turnTimer = setTimeout(() => {
|
|
406
|
+
this.turnTimer = undefined;
|
|
407
|
+
void this.on(() => this.step());
|
|
408
|
+
}, this.turnDelay);
|
|
409
|
+
}
|
|
410
|
+
// A window closed: Wakeshift hands out again what a live stream carries;
|
|
411
|
+
// what no live stream carries (a replaced stream's) has its window opened
|
|
412
|
+
// here, as nothing else would bring it.
|
|
413
|
+
async reopen() {
|
|
414
|
+
const waiting = this.o.ledger.of(this.o.conversation).filter(pending);
|
|
415
|
+
const vias = new Set();
|
|
416
|
+
const now = this.now();
|
|
417
|
+
let local = false;
|
|
418
|
+
const streamOf = this.streams();
|
|
419
|
+
for (const r of waiting) {
|
|
420
|
+
if (streamOf(viaOf(r)) === "live")
|
|
421
|
+
vias.add(viaOf(r));
|
|
422
|
+
else {
|
|
423
|
+
this.o.ledger.put({ ...r, windowFrom: now });
|
|
424
|
+
local = true;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
if (vias.size > 0)
|
|
428
|
+
this.o.redeliver([...vias]);
|
|
429
|
+
if (local) {
|
|
430
|
+
this.turnDelay = 0;
|
|
431
|
+
await this.step();
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
// Acknowledges what a turn took, in chunks of 100, each on the stream it
|
|
435
|
+
// came from; what came from a lost stream is done with, unacknowledged.
|
|
436
|
+
async acks() {
|
|
437
|
+
const by = new Map();
|
|
438
|
+
const now = this.now();
|
|
439
|
+
const streamOf = this.streams();
|
|
440
|
+
for (const r of this.o.ledger.of(this.o.conversation)) {
|
|
441
|
+
if (r.state !== "turn-started")
|
|
442
|
+
continue;
|
|
443
|
+
const s = streamOf(viaOf(r));
|
|
444
|
+
if (s === "lost")
|
|
445
|
+
this.o.ledger.put({ ...r, state: "superseded", doneAt: now });
|
|
446
|
+
else if (s === "live")
|
|
447
|
+
by.set(viaOf(r), [...(by.get(viaOf(r)) ?? []), r.id]);
|
|
448
|
+
// waiting: acknowledged once its stream is back
|
|
449
|
+
}
|
|
450
|
+
let retry = false;
|
|
451
|
+
for (const [via, ids] of by) {
|
|
452
|
+
for (let i = 0; i < ids.length && !this.stopped; i += 100) {
|
|
453
|
+
const res = await this.ackChunk(ids.slice(i, i + 100), via);
|
|
454
|
+
if (res === "gone")
|
|
455
|
+
break; // the link no longer works: the transport ends it
|
|
456
|
+
if (res === "retry")
|
|
457
|
+
retry = true;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
if (retry)
|
|
461
|
+
this.retryAcks();
|
|
462
|
+
else
|
|
463
|
+
this.ackDelay = 0;
|
|
464
|
+
}
|
|
465
|
+
// One chunk: 2xx marks them acked; a 400 splits it to find the bad ID.
|
|
466
|
+
async ackChunk(ids, via) {
|
|
467
|
+
const status = await this.o.ack(ids, via);
|
|
468
|
+
if (status >= 200 && status < 300) {
|
|
469
|
+
const now = this.now();
|
|
470
|
+
for (const id of ids) {
|
|
471
|
+
const r = this.o.ledger.get(this.o.conversation, id);
|
|
472
|
+
if (r && r.state === "turn-started")
|
|
473
|
+
this.o.ledger.put({ ...r, state: "acked", doneAt: now });
|
|
474
|
+
}
|
|
475
|
+
return "ok";
|
|
476
|
+
}
|
|
477
|
+
if (status === 400) {
|
|
478
|
+
if (ids.length === 1) {
|
|
479
|
+
this.o.host.log.warn(`wakeshift (${this.o.sessionKey}): Wakeshift refused the ack for ${ids[0]}; not sending it again`);
|
|
480
|
+
const r = this.o.ledger.get(this.o.conversation, ids[0]);
|
|
481
|
+
if (r)
|
|
482
|
+
this.o.ledger.put({ ...r, state: "refused", doneAt: this.now() });
|
|
483
|
+
return "ok";
|
|
484
|
+
}
|
|
485
|
+
const half = Math.ceil(ids.length / 2);
|
|
486
|
+
const a = await this.ackChunk(ids.slice(0, half), via);
|
|
487
|
+
const b = await this.ackChunk(ids.slice(half), via);
|
|
488
|
+
return a === "gone" || b === "gone" ? "gone" : a === "retry" || b === "retry" ? "retry" : "ok";
|
|
489
|
+
}
|
|
490
|
+
// Gone only as the transport counts it (401, 410; a 403 or 404 that's
|
|
491
|
+
// Wakeshift's link_revoked comes back as 401): another 403 or 404 is retried.
|
|
492
|
+
if (status === 401 || status === 410)
|
|
493
|
+
return "gone";
|
|
494
|
+
this.o.host.log.info(`wakeshift (${this.o.sessionKey}): ack: ${status ? `Wakeshift answered ${status}` : "Wakeshift couldn't be reached"}; trying again`);
|
|
495
|
+
return "retry";
|
|
496
|
+
}
|
|
497
|
+
retryAcks() {
|
|
498
|
+
if (this.stopped || this.ackTimer)
|
|
499
|
+
return;
|
|
500
|
+
const base = this.o.ackRetry ?? 1000;
|
|
501
|
+
this.ackDelay = this.ackDelay ? Math.min(this.ackDelay * 2, 30000) : base;
|
|
502
|
+
this.ackTimer = setTimeout(() => {
|
|
503
|
+
this.ackTimer = undefined;
|
|
504
|
+
void this.on(() => this.acks());
|
|
505
|
+
}, this.ackDelay);
|
|
506
|
+
}
|
|
507
|
+
}
|