@saadnvd1/agent-os 0.3.514 → 0.3.517
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -1
- package/app/api/devices/me/push/route.test.ts +209 -0
- package/app/api/devices/me/push/route.ts +115 -0
- package/app/api/devices/me/route.ts +12 -0
- package/app/api/devices/route.ts +2 -0
- package/app/api/push/test/route.test.ts +112 -0
- package/app/api/push/test/route.ts +38 -0
- package/components/Devices/DeviceRow.tsx +52 -3
- package/components/Devices/DevicesPanel.tsx +13 -0
- package/data/devices/queries.ts +30 -2
- package/lib/away/run.ts +15 -3
- package/lib/chat/registry.ts +2 -0
- package/lib/chat/runner.ts +3 -0
- package/lib/connect/control-api.ts +38 -0
- package/lib/connect/serve.ts +107 -31
- package/lib/connect/supervise.ts +110 -11
- package/lib/db/migrations.ts +59 -0
- package/lib/done/index.ts +5 -1
- package/lib/load/alert.ts +5 -1
- package/lib/notify/copy.ts +190 -0
- package/lib/notify/index.ts +46 -7
- package/lib/notify/limiter.ts +6 -1
- package/lib/notify/push/encode.ts +42 -0
- package/lib/notify/push/events.ts +72 -0
- package/lib/notify/push/index.ts +183 -0
- package/lib/notify/push/outbox.ts +238 -0
- package/lib/notify/push/policy.ts +33 -0
- package/lib/notify/push/presence.ts +45 -0
- package/lib/notify/push/relay.ts +50 -0
- package/lib/notify/push/sender.ts +271 -0
- package/lib/notify/push/testing.ts +30 -0
- package/lib/notify/push/waiting.ts +29 -0
- package/lib/notify/push-types.ts +79 -0
- package/lib/notify/redact.ts +68 -0
- package/lib/orchestrator/asks.ts +5 -1
- package/lib/orchestrator/review.ts +3 -0
- package/lib/program-status/tap.ts +11 -1
- package/lib/schedules/start.ts +11 -2
- package/lib/security/demo.ts +1 -0
- package/lib/security/device-me.ts +12 -0
- package/lib/security/devices.ts +213 -5
- package/lib/security/net.ts +47 -11
- package/lib/tasks/finish.ts +2 -0
- package/lib/tasks/session.ts +26 -2
- package/package.json +1 -1
- package/server.ts +3 -0
package/data/devices/queries.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
|
2
|
-
import type {
|
|
2
|
+
import type { ListedDevice } from "@/lib/security/devices";
|
|
3
3
|
import type { NetworkState } from "@/lib/security/network-state";
|
|
4
4
|
import type { PairingOffer } from "@/lib/security/pair-qr";
|
|
5
5
|
import { deviceKeys } from "./keys";
|
|
@@ -12,7 +12,9 @@ async function json<T>(res: Response): Promise<T> {
|
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
export interface DevicesResponse {
|
|
15
|
-
devices:
|
|
15
|
+
devices: ListedDevice[];
|
|
16
|
+
// Whether this machine can reach the push relay at all.
|
|
17
|
+
push: { linked: boolean; unavailable: string | null };
|
|
16
18
|
current: { deviceId: string | null; via: string | null };
|
|
17
19
|
}
|
|
18
20
|
|
|
@@ -128,3 +130,29 @@ export function useSetCanApprove() {
|
|
|
128
130
|
queryClient.invalidateQueries({ queryKey: deviceKeys.list() }),
|
|
129
131
|
});
|
|
130
132
|
}
|
|
133
|
+
|
|
134
|
+
export interface PushTestResponse {
|
|
135
|
+
results: {
|
|
136
|
+
deviceId: string;
|
|
137
|
+
status: "sent" | "gone" | "failed";
|
|
138
|
+
error: string | null;
|
|
139
|
+
}[];
|
|
140
|
+
unavailable: string | null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function useSendTestPush() {
|
|
144
|
+
const queryClient = useQueryClient();
|
|
145
|
+
return useMutation({
|
|
146
|
+
mutationFn: async (deviceId: string) =>
|
|
147
|
+
json<PushTestResponse>(
|
|
148
|
+
await fetch("/api/push/test", {
|
|
149
|
+
method: "POST",
|
|
150
|
+
headers: { "Content-Type": "application/json" },
|
|
151
|
+
body: JSON.stringify({ deviceId }),
|
|
152
|
+
})
|
|
153
|
+
),
|
|
154
|
+
// A dead token is forgotten, and the last error changes.
|
|
155
|
+
onSettled: () =>
|
|
156
|
+
queryClient.invalidateQueries({ queryKey: deviceKeys.list() }),
|
|
157
|
+
});
|
|
158
|
+
}
|
package/lib/away/run.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { review } from "../orchestrator/review";
|
|
|
19
19
|
import { repoOf } from "../orchestrator/repo";
|
|
20
20
|
import { clip } from "../orchestrator/digest";
|
|
21
21
|
import { notifyPhone } from "../notify";
|
|
22
|
+
import { takeFoldedPushes } from "../notify/push";
|
|
22
23
|
import { getProject } from "../projects";
|
|
23
24
|
import { loadProjectConfig } from "../project-config";
|
|
24
25
|
import { run } from "../tasks/gh";
|
|
@@ -194,7 +195,11 @@ export function turnAwayOff(
|
|
|
194
195
|
// no report), then the summary. A second later, so the window takes
|
|
195
196
|
// everything up to now.
|
|
196
197
|
notifyBatch(ended, now + 1000);
|
|
197
|
-
notifyPhone(PHONE_SOURCE, phoneSummary(report)
|
|
198
|
+
notifyPhone(PHONE_SOURCE, phoneSummary(report), {
|
|
199
|
+
category: "digest",
|
|
200
|
+
key: `digest:end:${ended.id}`,
|
|
201
|
+
collapse: "digest",
|
|
202
|
+
});
|
|
198
203
|
return { ...ended, report: JSON.stringify(report) };
|
|
199
204
|
}
|
|
200
205
|
|
|
@@ -516,7 +521,9 @@ export function notifyBatch(away: AwayRow, now = Date.now()): string | null {
|
|
|
516
521
|
}[];
|
|
517
522
|
const deploys = held.filter((h) => h.kind === "deploy_failed");
|
|
518
523
|
const notes = held.filter((h) => h.kind === "notify");
|
|
519
|
-
|
|
524
|
+
// What push held for this digest (ready for review, CI, agent messages).
|
|
525
|
+
const pushed = takeFoldedPushes(now);
|
|
526
|
+
if (!asks.length && !held.length && !pushed) return null;
|
|
520
527
|
const lines = [
|
|
521
528
|
...deploys.map(
|
|
522
529
|
(d) => `- ${d.workspace}: deploy failed: ${clip(d.detail ?? "", 120)}`
|
|
@@ -534,6 +541,7 @@ export function notifyBatch(away: AwayRow, now = Date.now()): string | null {
|
|
|
534
541
|
"note from an orchestrator",
|
|
535
542
|
"notes from orchestrators"
|
|
536
543
|
),
|
|
544
|
+
...count(pushed, "other update", "other updates"),
|
|
537
545
|
];
|
|
538
546
|
const head = `While you're away: ${parts.length > 1 ? `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}` : parts[0]}`;
|
|
539
547
|
const shown = lines.slice(0, PHONE_LINES);
|
|
@@ -541,7 +549,11 @@ export function notifyBatch(away: AwayRow, now = Date.now()): string | null {
|
|
|
541
549
|
const text = [head, ...shown, ...(more ? [`- and ${more} more`] : [])].join(
|
|
542
550
|
"\n"
|
|
543
551
|
);
|
|
544
|
-
notifyPhone(PHONE_SOURCE, text
|
|
552
|
+
notifyPhone(PHONE_SOURCE, text, {
|
|
553
|
+
category: "digest",
|
|
554
|
+
key: `digest:${away.id}:${now}`,
|
|
555
|
+
collapse: "digest",
|
|
556
|
+
});
|
|
545
557
|
return text;
|
|
546
558
|
}
|
|
547
559
|
|
package/lib/chat/registry.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { noteWaiting } from "../notify/push/waiting";
|
|
1
2
|
import { db, type Session } from "../db";
|
|
2
3
|
import type { WorkerClient } from "./worker/client";
|
|
3
4
|
import { notifyStatusChanged } from "../status/hub";
|
|
@@ -90,6 +91,7 @@ export function emit(sessionId: string, m: ChatServerMessage): void {
|
|
|
90
91
|
// Items come from the worker, which writes them a moment after saying so.
|
|
91
92
|
if (m.type === "state" || m.type === "compacting" || m.type === "item")
|
|
92
93
|
markStatusDirty(sessionId, EMIT_SETTLE_MS);
|
|
94
|
+
if (m.type === "state") noteWaiting(sessionId, m.state === "waiting");
|
|
93
95
|
registry.listeners.get(sessionId)?.forEach((fn) => fn(m));
|
|
94
96
|
// Idle, running, waiting and back, and a finished turn (unread): the
|
|
95
97
|
// sidebar hears at once.
|
package/lib/chat/runner.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* events to whoever is watching. Restarting the server leaves turns running.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import { pushPlanReady } from "../notify/push/events";
|
|
8
9
|
import { handoffAfterPlan } from "../models/plan-step";
|
|
9
10
|
import { randomUUID } from "crypto";
|
|
10
11
|
import os from "os";
|
|
@@ -152,6 +153,8 @@ function onWorkerEvent(sessionId: string, live: Live, e: WorkerEvent): void {
|
|
|
152
153
|
if (e.type === "item" && e.item.kind === "task")
|
|
153
154
|
retireIfStale(sessionId, live);
|
|
154
155
|
if (e.type === "item") {
|
|
156
|
+
if (e.item.kind === "plan" && !e.item.carried)
|
|
157
|
+
pushPlanReady(sessionId, e.item.id);
|
|
155
158
|
if ("streaming" in e.item && e.item.streaming)
|
|
156
159
|
live.streaming.set(e.item.id, e.item);
|
|
157
160
|
else live.streaming.delete(e.item.id);
|
|
@@ -226,3 +226,41 @@ export async function heartbeat(
|
|
|
226
226
|
);
|
|
227
227
|
return (await res.json()) as { enabled: boolean };
|
|
228
228
|
}
|
|
229
|
+
|
|
230
|
+
export interface PushRequestBody {
|
|
231
|
+
v: 1;
|
|
232
|
+
devices: { token: string; env: "production" | "sandbox" }[];
|
|
233
|
+
push_type: "alert" | "background";
|
|
234
|
+
priority: 10 | 5;
|
|
235
|
+
level: "passive" | "active" | "time-sensitive";
|
|
236
|
+
collapse_id?: string;
|
|
237
|
+
thread_id?: string;
|
|
238
|
+
payload: Record<string, unknown>;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export interface PushResult {
|
|
242
|
+
status: "sent" | "gone" | "retry" | "rejected";
|
|
243
|
+
apns_status?: number;
|
|
244
|
+
reason?: string;
|
|
245
|
+
prune?: boolean;
|
|
246
|
+
retry_after?: number;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** One APNs send per device, answered in device order. Signed. */
|
|
250
|
+
export async function pushRelay(
|
|
251
|
+
config: ConnectConfig,
|
|
252
|
+
privateKey: string,
|
|
253
|
+
body: PushRequestBody
|
|
254
|
+
): Promise<PushResult[]> {
|
|
255
|
+
const res = await signed(config, privateKey)(
|
|
256
|
+
"POST",
|
|
257
|
+
`/api/v1/machines/${config.machineId}/push`,
|
|
258
|
+
body
|
|
259
|
+
);
|
|
260
|
+
const r = (await res.json().catch(() => null)) as {
|
|
261
|
+
results?: PushResult[];
|
|
262
|
+
} | null;
|
|
263
|
+
if (!Array.isArray(r?.results))
|
|
264
|
+
throw new Error("the Connect service sent no push results");
|
|
265
|
+
return r.results;
|
|
266
|
+
}
|
package/lib/connect/serve.ts
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
* - takes only the machine's own Host (cloudflared pins it) and strips
|
|
9
9
|
* X-Forwarded-Host, Forwarded and X-Original-Host before any handler.
|
|
10
10
|
*
|
|
11
|
+
* cloudflared starts only once that listener is bound and a connection to
|
|
12
|
+
* the port is seen to arrive on it, and stops if the listener goes: it must
|
|
13
|
+
* never publish whatever else holds the port.
|
|
14
|
+
*
|
|
11
15
|
* Connect follows connect.json while running: `agent-os disconnect` or the
|
|
12
16
|
* switch in Devices stops cloudflared and closes every tunnel connection at
|
|
13
17
|
* once. Nothing here may stop AgentOS from starting.
|
|
@@ -15,6 +19,7 @@
|
|
|
15
19
|
|
|
16
20
|
import fs from "fs";
|
|
17
21
|
import http from "http";
|
|
22
|
+
import net from "net";
|
|
18
23
|
import path from "path";
|
|
19
24
|
import type { IncomingMessage, ServerResponse } from "http";
|
|
20
25
|
import type { Socket } from "net";
|
|
@@ -36,7 +41,13 @@ import {
|
|
|
36
41
|
} from "./control-api";
|
|
37
42
|
import { appVersion, startHeartbeat } from "./heartbeat";
|
|
38
43
|
import { ensureCloudflared } from "./cloudflared";
|
|
39
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
reapTunnel,
|
|
46
|
+
superviseTunnel,
|
|
47
|
+
type ReapDeps,
|
|
48
|
+
type Spawn,
|
|
49
|
+
type TunnelState,
|
|
50
|
+
} from "./supervise";
|
|
40
51
|
|
|
41
52
|
export interface Handlers {
|
|
42
53
|
onRequest: (req: IncomingMessage, res: ServerResponse) => void;
|
|
@@ -101,12 +112,51 @@ export interface ConnectDeps {
|
|
|
101
112
|
delays?: Parameters<typeof superviseTunnel>[0]["delays"];
|
|
102
113
|
heartbeat?: typeof sendHeartbeat;
|
|
103
114
|
heartbeatMs?: number;
|
|
115
|
+
reap?: ReapDeps;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Whether connections to 127.0.0.1:<port> reach `server`: one is made and
|
|
120
|
+
* must show up on it. Never rejects.
|
|
121
|
+
*/
|
|
122
|
+
export function listenerIsOurs(
|
|
123
|
+
server: http.Server,
|
|
124
|
+
port: number,
|
|
125
|
+
timeoutMs = 3000
|
|
126
|
+
): Promise<boolean> {
|
|
127
|
+
return new Promise((resolve) => {
|
|
128
|
+
const arrived = new Set<number>();
|
|
129
|
+
let probePort: number | undefined;
|
|
130
|
+
let done = false;
|
|
131
|
+
const finish = (ok: boolean) => {
|
|
132
|
+
if (done) return;
|
|
133
|
+
done = true;
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
server.removeListener("connection", onConnection);
|
|
136
|
+
client.destroy();
|
|
137
|
+
resolve(ok);
|
|
138
|
+
};
|
|
139
|
+
const onConnection = (s: Socket) => {
|
|
140
|
+
if (s.remotePort === undefined) return;
|
|
141
|
+
arrived.add(s.remotePort);
|
|
142
|
+
if (s.remotePort === probePort) finish(true);
|
|
143
|
+
};
|
|
144
|
+
server.prependListener("connection", onConnection);
|
|
145
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
146
|
+
timer.unref?.();
|
|
147
|
+
const client = net.connect(port, "127.0.0.1", () => {
|
|
148
|
+
probePort = client.localPort;
|
|
149
|
+
// The server may have seen it before this callback ran.
|
|
150
|
+
if (probePort !== undefined && arrived.has(probePort)) finish(true);
|
|
151
|
+
});
|
|
152
|
+
client.on("error", () => finish(false));
|
|
153
|
+
});
|
|
104
154
|
}
|
|
105
155
|
|
|
106
156
|
interface Running {
|
|
107
157
|
server: http.Server;
|
|
108
158
|
sockets: Set<Socket>;
|
|
109
|
-
supervisor: ReturnType<typeof superviseTunnel
|
|
159
|
+
supervisor: ReturnType<typeof superviseTunnel> | null;
|
|
110
160
|
heartbeat: ReturnType<typeof startHeartbeat>;
|
|
111
161
|
release: () => void;
|
|
112
162
|
}
|
|
@@ -131,12 +181,12 @@ export function startConnect(
|
|
|
131
181
|
let bindError: string | undefined;
|
|
132
182
|
g.__agentosConnect = () => {
|
|
133
183
|
if (!running) return base;
|
|
134
|
-
const s: TunnelState = running.supervisor
|
|
184
|
+
const s: TunnelState = running.supervisor?.state() ?? "connecting";
|
|
135
185
|
return {
|
|
136
186
|
...base,
|
|
137
187
|
hostname: hostname || base.hostname,
|
|
138
188
|
state: bindError ? "error" : s === "stopped" ? "off" : s,
|
|
139
|
-
reason: bindError ?? running.supervisor
|
|
189
|
+
reason: bindError ?? running.supervisor?.reason(),
|
|
140
190
|
};
|
|
141
191
|
};
|
|
142
192
|
|
|
@@ -153,7 +203,7 @@ export function startConnect(
|
|
|
153
203
|
if (!running) return;
|
|
154
204
|
bindError = undefined;
|
|
155
205
|
running.heartbeat.stop();
|
|
156
|
-
running.supervisor
|
|
206
|
+
running.supervisor?.stop();
|
|
157
207
|
running.sockets.forEach((s) => s.destroy());
|
|
158
208
|
running.server.close();
|
|
159
209
|
running.release();
|
|
@@ -200,19 +250,50 @@ export function startConnect(
|
|
|
200
250
|
log(`connect: ${base.reason}`);
|
|
201
251
|
return;
|
|
202
252
|
}
|
|
253
|
+
// The lock is ours, so a cloudflared in the pid file belongs to an
|
|
254
|
+
// AgentOS that died: it goes before anything else happens.
|
|
255
|
+
const reaped = reapTunnel(dir, undefined, deps.reap).catch(() => false);
|
|
203
256
|
const server = tunnelListener(() => hostname, handlers);
|
|
204
257
|
const sockets = new Set<Socket>();
|
|
205
258
|
server.on("connection", (s: Socket) => {
|
|
206
259
|
sockets.add(s);
|
|
207
260
|
s.once("close", () => sockets.delete(s));
|
|
208
261
|
});
|
|
262
|
+
const ensure =
|
|
263
|
+
deps.ensureBinary ?? ((d: string) => ensureCloudflared({ dir: d }));
|
|
264
|
+
const ask = deps.requestTunnel ?? requestTunnel;
|
|
265
|
+
const startTunnel = () =>
|
|
266
|
+
superviseTunnel({
|
|
267
|
+
dir,
|
|
268
|
+
log,
|
|
269
|
+
spawn: deps.spawn,
|
|
270
|
+
delays: deps.delays,
|
|
271
|
+
prepare: async () => {
|
|
272
|
+
const bin = await ensure(dir);
|
|
273
|
+
// Fresh every start; the token lives only in this call and the child's env.
|
|
274
|
+
const tunnel = await ask(config, machineKey, tunnelPort);
|
|
275
|
+
if (tunnel.hostname !== hostname) {
|
|
276
|
+
hostname = tunnel.hostname;
|
|
277
|
+
saveConnect({ ...config, hostname }, dir);
|
|
278
|
+
log(`connect: reachable at https://${hostname}`);
|
|
279
|
+
}
|
|
280
|
+
return { bin, token: tunnel.token };
|
|
281
|
+
},
|
|
282
|
+
});
|
|
283
|
+
const stopTunnel = () => {
|
|
284
|
+
if (running?.server !== server || !running.supervisor) return;
|
|
285
|
+
running.supervisor.stop();
|
|
286
|
+
running.supervisor = null;
|
|
287
|
+
};
|
|
209
288
|
// Something else may hold the port for a while (an AgentOS shutting
|
|
210
|
-
// down, say): keep trying with backoff rather than giving up
|
|
289
|
+
// down, say): keep trying with backoff rather than giving up, with
|
|
290
|
+
// cloudflared stopped meanwhile.
|
|
211
291
|
const bindDelays = deps.delays ?? { min: 1000, max: 60_000 };
|
|
212
292
|
let bindDelay = bindDelays.min;
|
|
213
|
-
|
|
293
|
+
const rebind = (why: string) => {
|
|
214
294
|
if (running?.server !== server) return;
|
|
215
|
-
|
|
295
|
+
stopTunnel();
|
|
296
|
+
bindError = `can't listen on 127.0.0.1:${tunnelPort}: ${why}`;
|
|
216
297
|
log(`connect: ${bindError}; trying again`);
|
|
217
298
|
const t = setTimeout(() => {
|
|
218
299
|
if (running?.server !== server) return;
|
|
@@ -221,30 +302,19 @@ export function startConnect(
|
|
|
221
302
|
}, bindDelay);
|
|
222
303
|
t.unref?.();
|
|
223
304
|
bindDelay = Math.min(bindDelay * 2, bindDelays.max);
|
|
224
|
-
}
|
|
225
|
-
server.on("
|
|
305
|
+
};
|
|
306
|
+
server.on("error", (err) => rebind(err.message));
|
|
307
|
+
server.on("close", () => stopTunnel());
|
|
308
|
+
server.on("listening", async () => {
|
|
309
|
+
const [ours] = await Promise.all([
|
|
310
|
+
listenerIsOurs(server, tunnelPort),
|
|
311
|
+
reaped,
|
|
312
|
+
]);
|
|
313
|
+
if (running?.server !== server || !server.listening) return;
|
|
314
|
+
if (!ours) return rebind("connections there don't reach AgentOS");
|
|
226
315
|
bindError = undefined;
|
|
227
316
|
bindDelay = bindDelays.min;
|
|
228
|
-
|
|
229
|
-
const ensure =
|
|
230
|
-
deps.ensureBinary ?? ((d: string) => ensureCloudflared({ dir: d }));
|
|
231
|
-
const ask = deps.requestTunnel ?? requestTunnel;
|
|
232
|
-
const supervisor = superviseTunnel({
|
|
233
|
-
dir,
|
|
234
|
-
log,
|
|
235
|
-
spawn: deps.spawn,
|
|
236
|
-
delays: deps.delays,
|
|
237
|
-
prepare: async () => {
|
|
238
|
-
const bin = await ensure(dir);
|
|
239
|
-
// Fresh every start; the token lives only in this call and the child's env.
|
|
240
|
-
const tunnel = await ask(config, machineKey, tunnelPort);
|
|
241
|
-
if (tunnel.hostname !== hostname) {
|
|
242
|
-
hostname = tunnel.hostname;
|
|
243
|
-
saveConnect({ ...config, hostname }, dir);
|
|
244
|
-
log(`connect: reachable at https://${hostname}`);
|
|
245
|
-
}
|
|
246
|
-
return { bin, token: tunnel.token };
|
|
247
|
-
},
|
|
317
|
+
if (!running.supervisor) running.supervisor = startTunnel();
|
|
248
318
|
});
|
|
249
319
|
const beat = deps.heartbeat ?? sendHeartbeat;
|
|
250
320
|
const version = appVersion();
|
|
@@ -261,7 +331,13 @@ export function startConnect(
|
|
|
261
331
|
down("machine no longer linked");
|
|
262
332
|
},
|
|
263
333
|
});
|
|
264
|
-
running = {
|
|
334
|
+
running = {
|
|
335
|
+
server,
|
|
336
|
+
sockets,
|
|
337
|
+
supervisor: null,
|
|
338
|
+
heartbeat,
|
|
339
|
+
release: lock.release,
|
|
340
|
+
};
|
|
265
341
|
server.listen(tunnelPort, "127.0.0.1");
|
|
266
342
|
};
|
|
267
343
|
|
package/lib/connect/supervise.ts
CHANGED
|
@@ -3,13 +3,24 @@
|
|
|
3
3
|
* the Connect service for a fresh connector token, which goes to the child
|
|
4
4
|
* in TUNNEL_TOKEN only: never argv, never a file, never a log line. The
|
|
5
5
|
* child exits → it's started again with backoff; stop() ends it.
|
|
6
|
+
*
|
|
7
|
+
* cloudflared must never outlive AgentOS: it would go on publishing
|
|
8
|
+
* whatever next holds the tunnel port. So it runs in a process group of its
|
|
9
|
+
* own under a small shell that kills it the moment AgentOS goes, however it
|
|
10
|
+
* goes (SIGKILL and out-of-memory included), and a copy a crash left behind
|
|
11
|
+
* anyway is killed before the next one starts.
|
|
6
12
|
*/
|
|
7
13
|
|
|
8
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
execFile,
|
|
16
|
+
spawn as nodeSpawn,
|
|
17
|
+
type ChildProcess,
|
|
18
|
+
type SpawnOptions,
|
|
19
|
+
} from "child_process";
|
|
9
20
|
import fs from "fs";
|
|
10
21
|
import path from "path";
|
|
11
22
|
import readline from "readline";
|
|
12
|
-
import type { Readable } from "stream";
|
|
23
|
+
import type { Readable, Writable } from "stream";
|
|
13
24
|
import { ApiError, clean } from "./control-api";
|
|
14
25
|
|
|
15
26
|
export type TunnelState = "connecting" | "up" | "denied" | "error" | "stopped";
|
|
@@ -21,6 +32,7 @@ export interface Prepared {
|
|
|
21
32
|
|
|
22
33
|
interface Child {
|
|
23
34
|
pid?: number;
|
|
35
|
+
stdin?: Writable | null;
|
|
24
36
|
stdout: Readable | null;
|
|
25
37
|
stderr: Readable | null;
|
|
26
38
|
kill(signal?: NodeJS.Signals): boolean;
|
|
@@ -53,7 +65,48 @@ const DELAYS = {
|
|
|
53
65
|
// cloudflared can be chatty while the network flaps.
|
|
54
66
|
const LOG_LINES_PER_MINUTE = 20;
|
|
55
67
|
|
|
56
|
-
|
|
68
|
+
// $1 is cloudflared, the rest its arguments. Its stdin is a pipe only
|
|
69
|
+
// AgentOS holds the other end of: when AgentOS ends, the read sees EOF and
|
|
70
|
+
// cloudflared is killed outright (a SIGTERM would let it drain, still
|
|
71
|
+
// forwarding, for half a minute). The shell exits with cloudflared's code.
|
|
72
|
+
const WATCHDOG = `exec 3<&0 </dev/null
|
|
73
|
+
"$@" 3<&- &
|
|
74
|
+
child=$!
|
|
75
|
+
trap 'kill -TERM "$child" 2>/dev/null' TERM INT HUP
|
|
76
|
+
{ read -r _ <&3; kill -KILL "$child" 2>/dev/null; } &
|
|
77
|
+
watch=$!
|
|
78
|
+
exec 3<&-
|
|
79
|
+
while :; do
|
|
80
|
+
wait "$child"; code=$?
|
|
81
|
+
kill -0 "$child" 2>/dev/null || break
|
|
82
|
+
done
|
|
83
|
+
kill "$watch" 2>/dev/null
|
|
84
|
+
exit "$code"`;
|
|
85
|
+
|
|
86
|
+
/** cloudflared under the watchdog, its group killed as one. */
|
|
87
|
+
export const spawnTunnel: Spawn = (bin, args, opts) => {
|
|
88
|
+
if (process.platform === "win32")
|
|
89
|
+
return (nodeSpawn as unknown as Spawn)(bin, args, opts);
|
|
90
|
+
const proc: ChildProcess = nodeSpawn(
|
|
91
|
+
"/bin/sh",
|
|
92
|
+
["-c", WATCHDOG, "agentos-cloudflared", bin, ...args],
|
|
93
|
+
{
|
|
94
|
+
env: opts.env,
|
|
95
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
96
|
+
detached: true,
|
|
97
|
+
} as SpawnOptions
|
|
98
|
+
);
|
|
99
|
+
proc.stdin?.on("error", () => {});
|
|
100
|
+
return Object.assign(proc, {
|
|
101
|
+
kill: (signal: NodeJS.Signals = "SIGTERM") => {
|
|
102
|
+
try {
|
|
103
|
+
if (proc.pid) return process.kill(-proc.pid, signal);
|
|
104
|
+
} catch {}
|
|
105
|
+
return proc.kill(signal);
|
|
106
|
+
},
|
|
107
|
+
}) as unknown as Child;
|
|
108
|
+
};
|
|
109
|
+
|
|
57
110
|
const commandOf = (pid: number) =>
|
|
58
111
|
new Promise<string>((resolve) =>
|
|
59
112
|
execFile(
|
|
@@ -64,25 +117,60 @@ const commandOf = (pid: number) =>
|
|
|
64
117
|
)
|
|
65
118
|
);
|
|
66
119
|
|
|
67
|
-
|
|
120
|
+
const signal = (pid: number, sig: NodeJS.Signals | 0) => {
|
|
121
|
+
try {
|
|
122
|
+
process.kill(pid, sig);
|
|
123
|
+
return true;
|
|
124
|
+
} catch {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export interface ReapDeps {
|
|
130
|
+
commandOf?: (pid: number) => Promise<string>;
|
|
131
|
+
/** process.kill: false when there was nothing to signal. */
|
|
132
|
+
kill?: (pid: number, sig: NodeJS.Signals | 0) => boolean;
|
|
133
|
+
waitMs?: number;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Kills a cloudflared an AgentOS that died left running, named by the pid
|
|
138
|
+
* file, and only if that pid is still one of ours (pids get reused). Call
|
|
139
|
+
* it only while holding connect.lock: then no live AgentOS owns it.
|
|
140
|
+
*/
|
|
141
|
+
export async function reapTunnel(
|
|
142
|
+
dir: string,
|
|
143
|
+
binDir = path.join(dir, "bin"),
|
|
144
|
+
deps: ReapDeps = {}
|
|
145
|
+
) {
|
|
146
|
+
const pidFile = path.join(dir, "cloudflared.pid");
|
|
147
|
+
const kill = deps.kill ?? signal;
|
|
68
148
|
let pid = 0;
|
|
69
149
|
try {
|
|
70
150
|
pid = Number(fs.readFileSync(pidFile, "utf8")) || 0;
|
|
71
151
|
} catch {
|
|
72
|
-
return;
|
|
152
|
+
return false;
|
|
73
153
|
}
|
|
154
|
+
let reaped = false;
|
|
74
155
|
if (pid > 1) {
|
|
75
|
-
const command = await commandOf(pid);
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
156
|
+
const command = await (deps.commandOf ?? commandOf)(pid);
|
|
157
|
+
if (command.includes(binDir + path.sep)) {
|
|
158
|
+
// Its group (the watchdog and cloudflared), or a bare cloudflared an
|
|
159
|
+
// older AgentOS started outside one.
|
|
160
|
+
reaped = kill(-pid, "SIGKILL");
|
|
161
|
+
reaped = kill(pid, "SIGKILL") || reaped;
|
|
162
|
+
const end = Date.now() + (deps.waitMs ?? 2000);
|
|
163
|
+
while (kill(pid, 0) && Date.now() < end)
|
|
164
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
165
|
+
}
|
|
79
166
|
}
|
|
80
167
|
fs.rmSync(pidFile, { force: true });
|
|
168
|
+
return reaped;
|
|
81
169
|
}
|
|
82
170
|
|
|
83
171
|
export function superviseTunnel(opts: SuperviseOptions) {
|
|
84
172
|
const log = opts.log ?? (() => {});
|
|
85
|
-
const spawn = opts.spawn ??
|
|
173
|
+
const spawn = opts.spawn ?? spawnTunnel;
|
|
86
174
|
const delays = opts.delays ?? DELAYS;
|
|
87
175
|
const pidFile = path.join(opts.dir, "cloudflared.pid");
|
|
88
176
|
const home = path.join(opts.dir, "cloudflared-home");
|
|
@@ -145,7 +233,7 @@ export function superviseTunnel(opts: SuperviseOptions) {
|
|
|
145
233
|
|
|
146
234
|
const run = async ({ bin, token }: Prepared) => {
|
|
147
235
|
const redact = (line: string) => clean(line.split(token).join("[token]"));
|
|
148
|
-
await
|
|
236
|
+
await reapTunnel(opts.dir, path.dirname(bin));
|
|
149
237
|
if (stopped) return;
|
|
150
238
|
fs.mkdirSync(home, { recursive: true, mode: 0o700 });
|
|
151
239
|
// Its own config and home, so no config.yml elsewhere on the machine
|
|
@@ -256,8 +344,18 @@ export function superviseTunnel(opts: SuperviseOptions) {
|
|
|
256
344
|
proc.on("exit", (code) => end(`cloudflared exited (${code ?? "signal"})`));
|
|
257
345
|
};
|
|
258
346
|
|
|
347
|
+
// A stop that never ran (the process exiting, or a signal nobody else
|
|
348
|
+
// handles) still takes cloudflared's whole group down first.
|
|
259
349
|
const killNow = () => child?.kill("SIGTERM");
|
|
350
|
+
const SIGNALS = ["SIGTERM", "SIGINT", "SIGHUP"] as const;
|
|
351
|
+
const onSignal = (sig: NodeJS.Signals) => {
|
|
352
|
+
killNow();
|
|
353
|
+
for (const s of SIGNALS) process.removeListener(s, onSignal);
|
|
354
|
+
// Nobody else handles it: end the way the signal would have.
|
|
355
|
+
if (process.listenerCount(sig) === 0) process.kill(process.pid, sig);
|
|
356
|
+
};
|
|
260
357
|
process.once("exit", killNow);
|
|
358
|
+
for (const s of SIGNALS) process.on(s, onSignal);
|
|
261
359
|
tick();
|
|
262
360
|
|
|
263
361
|
return {
|
|
@@ -268,6 +366,7 @@ export function superviseTunnel(opts: SuperviseOptions) {
|
|
|
268
366
|
stopped = true;
|
|
269
367
|
state = "stopped";
|
|
270
368
|
process.removeListener("exit", killNow);
|
|
369
|
+
for (const s of SIGNALS) process.removeListener(s, onSignal);
|
|
271
370
|
if (timer) clearTimeout(timer);
|
|
272
371
|
const proc = child;
|
|
273
372
|
if (!proc) return;
|
package/lib/db/migrations.ts
CHANGED
|
@@ -1630,6 +1630,65 @@ const migrations: Migration[] = [
|
|
|
1630
1630
|
);
|
|
1631
1631
|
},
|
|
1632
1632
|
},
|
|
1633
|
+
{
|
|
1634
|
+
id: 71,
|
|
1635
|
+
name: "devices_push",
|
|
1636
|
+
up: (db) => {
|
|
1637
|
+
// Push belongs to a paired device, so revoking the device clears it.
|
|
1638
|
+
// push_categories is a JSON array; NULL means the defaults.
|
|
1639
|
+
// Each column checked, so a run cut off part way can be run again.
|
|
1640
|
+
const have = new Set(
|
|
1641
|
+
(
|
|
1642
|
+
db.prepare(`PRAGMA table_info(devices)`).all() as { name: string }[]
|
|
1643
|
+
).map((c) => c.name)
|
|
1644
|
+
);
|
|
1645
|
+
for (const col of [
|
|
1646
|
+
"push_token",
|
|
1647
|
+
"push_platform",
|
|
1648
|
+
"push_environment",
|
|
1649
|
+
"push_categories",
|
|
1650
|
+
"push_updated_at",
|
|
1651
|
+
"push_last_error",
|
|
1652
|
+
])
|
|
1653
|
+
if (!have.has(col))
|
|
1654
|
+
db.exec(`ALTER TABLE devices ADD COLUMN ${col} TEXT`);
|
|
1655
|
+
db.exec(
|
|
1656
|
+
`CREATE INDEX IF NOT EXISTS idx_devices_push_token ON devices(push_token)
|
|
1657
|
+
WHERE push_token IS NOT NULL`
|
|
1658
|
+
);
|
|
1659
|
+
},
|
|
1660
|
+
},
|
|
1661
|
+
{
|
|
1662
|
+
id: 72,
|
|
1663
|
+
name: "push_outbox",
|
|
1664
|
+
up: (db) => {
|
|
1665
|
+
// One row per (event, device). device_id is '' for a row folded into
|
|
1666
|
+
// the Away digest. state: pending, sending, sent, failed, gone,
|
|
1667
|
+
// capped, superseded, folded, digested.
|
|
1668
|
+
db.exec(`
|
|
1669
|
+
CREATE TABLE IF NOT EXISTS push_outbox (
|
|
1670
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1671
|
+
event_key TEXT NOT NULL,
|
|
1672
|
+
device_id TEXT NOT NULL,
|
|
1673
|
+
category TEXT NOT NULL,
|
|
1674
|
+
level TEXT NOT NULL,
|
|
1675
|
+
payload TEXT NOT NULL,
|
|
1676
|
+
collapse_id TEXT,
|
|
1677
|
+
state TEXT NOT NULL,
|
|
1678
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
1679
|
+
next_attempt_at INTEGER NOT NULL,
|
|
1680
|
+
created_at INTEGER NOT NULL,
|
|
1681
|
+
sent_at INTEGER,
|
|
1682
|
+
error TEXT,
|
|
1683
|
+
fallback_source TEXT,
|
|
1684
|
+
fallback_text TEXT,
|
|
1685
|
+
UNIQUE(event_key, device_id)
|
|
1686
|
+
);
|
|
1687
|
+
CREATE INDEX IF NOT EXISTS idx_push_outbox_due ON push_outbox(state, next_attempt_at);
|
|
1688
|
+
CREATE INDEX IF NOT EXISTS idx_push_outbox_device ON push_outbox(device_id, sent_at);
|
|
1689
|
+
`);
|
|
1690
|
+
},
|
|
1691
|
+
},
|
|
1633
1692
|
];
|
|
1634
1693
|
|
|
1635
1694
|
// `upTo`: stop after this id (tests that start from an older database).
|
package/lib/done/index.ts
CHANGED
|
@@ -26,6 +26,7 @@ import { releasePorts } from "../ports";
|
|
|
26
26
|
import { dropSessionDatabase } from "../project-config/database";
|
|
27
27
|
import { doneOnPeer, peerSessionLink } from "../hosts/peer-actions";
|
|
28
28
|
import { captureMergedTask } from "../session-logs";
|
|
29
|
+
import { pushMerged } from "../notify/push/events";
|
|
29
30
|
|
|
30
31
|
export interface DoneOptions {
|
|
31
32
|
// Who asked: the orchestrator's gate failures count toward Saad, as its
|
|
@@ -103,7 +104,10 @@ function mark(s: Session, status: "merged" | "done", pr: TaskPR | null) {
|
|
|
103
104
|
: `UPDATE sessions SET task_status = 'done' WHERE id = ?`
|
|
104
105
|
).run(s.id);
|
|
105
106
|
syncTaskCardInBackground(s, status, pr);
|
|
106
|
-
if (status === "merged")
|
|
107
|
+
if (status === "merged") {
|
|
108
|
+
pushMerged(s.id);
|
|
109
|
+
void captureMergedTask(s.id, { head: pr?.head });
|
|
110
|
+
}
|
|
107
111
|
}
|
|
108
112
|
|
|
109
113
|
export async function doneSession(
|
package/lib/load/alert.ts
CHANGED
|
@@ -30,7 +30,11 @@ const BACK_OFF =
|
|
|
30
30
|
"Ask your running tasks to use targeted tests and hold off on full suites, type checks and builds until it clears. Nothing has been stopped.";
|
|
31
31
|
|
|
32
32
|
export function raiseAlert(text: string): void {
|
|
33
|
-
notifyPhone("load", text
|
|
33
|
+
notifyPhone("load", text, {
|
|
34
|
+
category: "system",
|
|
35
|
+
key: `system:load:${Date.now()}`,
|
|
36
|
+
collapse: "system:load",
|
|
37
|
+
});
|
|
34
38
|
for (const orch of listOrchestrators()) {
|
|
35
39
|
if (orch.archived_at) continue;
|
|
36
40
|
sendMessage({
|