@bom0792/herdgate 0.9.31 → 0.9.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEPLOY.md +85 -126
- package/DESIGN.md +9 -4
- package/README.md +12 -2
- package/bin/herdgate.mjs +6 -1
- package/dist-readme.md +3 -1
- package/package.json +34 -8
- package/plugin/server/e2e.d.mts +2 -2
- package/plugin/server/e2e.mjs +51 -32
- package/plugin/server/index.mjs +128 -32
- package/plugin/server/pi-receipts.d.mts +28 -0
- package/plugin/server/pi-receipts.mjs +77 -0
- package/plugin/server/pi-worker.d.mts +47 -0
- package/plugin/server/pi-worker.mjs +127 -0
- package/scripts/build-host-bundle.sh +14 -8
- package/scripts/build-release.sh +47 -0
- package/scripts/deploy-vps.sh +5 -78
- package/src/cloud/gateway.ts +163 -134
- package/src/cloud/main.ts +72 -15
- package/src/cloud/pairing.ts +7 -2
- package/src/cloud/push.ts +54 -0
- package/src/cloud/sshd.ts +76 -38
- package/src/cloud/util.ts +5 -5
- package/src/web/app.js +88 -61
- package/src/web/e2e.js +51 -32
- package/src/web/sw.js +1 -1
- package/templates/cloud/.dockerignore +7 -0
- package/templates/cloud/Caddyfile.example +3 -0
- package/templates/cloud/Dockerfile +10 -0
- package/templates/cloud/README.md +74 -0
- package/templates/cloud/compose.env.example +18 -0
- package/templates/cloud/compose.yml +41 -0
- package/templates/cloud/gitignore.example +4 -0
- package/templates/cloud/herdgate-cloud-edge.service +18 -0
- package/templates/cloud/herdgate-cloud.service +26 -0
- package/tsconfig.cloud.json +11 -0
- package/tsconfig.json +18 -0
- package/src/web/herdgate-host.zip +0 -0
package/plugin/server/index.mjs
CHANGED
|
@@ -7,12 +7,15 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
+
import { randomUUID } from "node:crypto";
|
|
10
11
|
import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
|
|
11
12
|
import { join, dirname } from "node:path";
|
|
12
13
|
import { hostname, homedir } from "node:os";
|
|
13
14
|
import { fileURLToPath } from "node:url";
|
|
14
15
|
import { dirs, readConfig, writeConfig, readState, writeState, herdrSnapshot, summarize, herdrPanes, herdrRead } from "./common.mjs";
|
|
15
16
|
import { createSshBridge } from "./ssh.mjs";
|
|
17
|
+
import { findReceipt, journalBase as receiptJournalBase, publicReceipt } from "./pi-receipts.mjs";
|
|
18
|
+
import { JOB_ID, MAX_PENDING, MAX_PROMPT, appendJob, jobsFile, publicJob, queueDepth, readHeartbeat, readJobs, workerBase } from "./pi-worker.mjs";
|
|
16
19
|
import { registerPublisher } from "./publisher-tui-lifecycle.mjs";
|
|
17
20
|
import { HostChannel, b64d, b64e, fingerprint, generateKeyPair } from "./e2e.mjs";
|
|
18
21
|
|
|
@@ -196,45 +199,48 @@ async function publisherLoop() {
|
|
|
196
199
|
// A device that has not completed the handshake keeps working in the clear —
|
|
197
200
|
// the plaintext path stays the compat fallback.
|
|
198
201
|
const e2eKeyPath = D.stateDir ? join(D.stateDir, "e2e-host.json") : "";
|
|
199
|
-
let
|
|
200
|
-
const e2eSelfKey = async () => {
|
|
201
|
-
if (e2eSelf) return e2eSelf;
|
|
202
|
+
let e2eIdentity;
|
|
203
|
+
const e2eSelfKey = () => e2eIdentity ??= (async () => {
|
|
202
204
|
try {
|
|
203
205
|
const j = JSON.parse(readFileSync(e2eKeyPath, "utf8"));
|
|
204
|
-
|
|
205
|
-
return e2eSelf;
|
|
206
|
-
} catch {
|
|
207
|
-
/* mint a fresh identity below */
|
|
208
|
-
}
|
|
209
|
-
e2eSelf = await generateKeyPair();
|
|
210
|
-
try {
|
|
211
|
-
mkdirSync(dirname(e2eKeyPath), { recursive: true });
|
|
212
|
-
writeFileSync(e2eKeyPath, JSON.stringify({ jwk: e2eSelf.jwk, pub: b64e(e2eSelf.pub) }), { mode: 0o600 });
|
|
206
|
+
return { jwk: j.jwk, pub: b64d(j.pub) };
|
|
213
207
|
} catch (e) {
|
|
214
|
-
|
|
208
|
+
if (e.code !== "ENOENT") throw new Error("cannot load existing E2E host identity");
|
|
215
209
|
}
|
|
216
|
-
|
|
217
|
-
|
|
210
|
+
const key = await generateKeyPair();
|
|
211
|
+
mkdirSync(dirname(e2eKeyPath), { recursive: true, mode: 0o700 });
|
|
212
|
+
writeFileSync(e2eKeyPath, JSON.stringify({ jwk: key.jwk, pub: b64e(key.pub) }), { mode: 0o600, flag: "wx" });
|
|
213
|
+
return key;
|
|
214
|
+
})();
|
|
218
215
|
const e2eSessions = new Map();
|
|
216
|
+
const acceptingFinish = new WeakSet();
|
|
219
217
|
/** Per-device last-call times for the subprocess-backed term.* ops (1/s each). */
|
|
220
218
|
const termThrottle = new Map();
|
|
221
219
|
/** null = not an e2e payload; {reply} = handshake answer to send; {inner} = decrypted device payload. */
|
|
222
|
-
const e2eOnDevice = async (device, payload) => {
|
|
220
|
+
const e2eOnDevice = async (ws, device, payload) => {
|
|
223
221
|
if (payload && typeof payload === "object" && typeof payload.op === "string" && payload.op.startsWith("e2e")) log(`e2e: op ${payload.op} from ${device}`);
|
|
224
222
|
if (!payload || typeof payload !== "object") return null;
|
|
225
223
|
if (payload.op === "e2e.init") {
|
|
226
224
|
const ch = new HostChannel(await e2eSelfKey());
|
|
225
|
+
if (wsRef !== ws || ws.readyState !== 1 || !devices.some(d => d.id === device)) return { reply: null };
|
|
227
226
|
e2eSessions.set(device, ch);
|
|
228
|
-
|
|
227
|
+
const reply = await ch.acceptInit(payload);
|
|
228
|
+
return e2eSessions.get(device) === ch ? { reply: { op: "e2e.reply", ...reply } } : { reply: null };
|
|
229
229
|
}
|
|
230
230
|
if (payload.op === "e2e.finish") {
|
|
231
231
|
const ch = e2eSessions.get(device);
|
|
232
|
-
if (!ch) return { reply:
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
232
|
+
if (!ch || ch.ready() || acceptingFinish.has(ch)) return { reply: null };
|
|
233
|
+
acceptingFinish.add(ch);
|
|
234
|
+
try {
|
|
235
|
+
const fp = await fingerprint((await e2eSelfKey()).pub);
|
|
236
|
+
await ch.acceptFinish(payload);
|
|
237
|
+
if (e2eSessions.get(device) === ch && wsRef === ws && ws.readyState === 1) {
|
|
238
|
+
// Queue the ack before allowing the first ciphertext to be sent.
|
|
239
|
+
ws.send(JSON.stringify({ v: 1, t: "frame", device, payload: { op: "e2e.ready" } }));
|
|
240
|
+
log(`e2e: session up with ${device} (host fp ${fp})`);
|
|
241
|
+
}
|
|
242
|
+
return { reply: null };
|
|
243
|
+
} finally { acceptingFinish.delete(ch); }
|
|
238
244
|
}
|
|
239
245
|
if (payload.op === "e2e") {
|
|
240
246
|
const ch = e2eSessions.get(device);
|
|
@@ -244,6 +250,7 @@ async function publisherLoop() {
|
|
|
244
250
|
} catch {
|
|
245
251
|
// A decrypt failure means the counters can never resync: drop the session and
|
|
246
252
|
// make the device renegotiate.
|
|
253
|
+
if (e2eSessions.get(device) !== ch) return { reply: null };
|
|
247
254
|
e2eSessions.delete(device);
|
|
248
255
|
log(`e2e: dropped session with ${device} (decrypt failed — renegotiate)`);
|
|
249
256
|
return { reply: { op: "error", code: "e2e", message: "session dropped — renegotiate" } };
|
|
@@ -251,10 +258,15 @@ async function publisherLoop() {
|
|
|
251
258
|
}
|
|
252
259
|
return null;
|
|
253
260
|
};
|
|
254
|
-
/** Encrypt for this device when a session exists; passthrough
|
|
255
|
-
const
|
|
261
|
+
/** Encrypt for this device when a session exists; passthrough for legacy clients. */
|
|
262
|
+
const sendPayload = async (ws, device, payload) => {
|
|
263
|
+
if (wsRef !== ws || ws.readyState !== 1 || !devices.some(d => d.id === device)) return;
|
|
256
264
|
const ch = e2eSessions.get(device);
|
|
257
|
-
|
|
265
|
+
if (acceptingFinish.has(ch)) return;
|
|
266
|
+
const wire = ch?.ready() ? await ch.send(payload) : payload;
|
|
267
|
+
if (wsRef === ws && ws.readyState === 1 && e2eSessions.get(device) === ch) {
|
|
268
|
+
ws.send(JSON.stringify({ v: 1, t: "frame", device, payload: wire }));
|
|
269
|
+
}
|
|
258
270
|
};
|
|
259
271
|
|
|
260
272
|
const ssh = createSshBridge({
|
|
@@ -278,7 +290,7 @@ async function publisherLoop() {
|
|
|
278
290
|
const payload = await snapshotPayload();
|
|
279
291
|
if (!payload) return;
|
|
280
292
|
for (const d of devices) {
|
|
281
|
-
ws
|
|
293
|
+
await sendPayload(ws, d.id, payload);
|
|
282
294
|
}
|
|
283
295
|
};
|
|
284
296
|
|
|
@@ -339,7 +351,10 @@ async function publisherLoop() {
|
|
|
339
351
|
}
|
|
340
352
|
if (m.t === "room.update") {
|
|
341
353
|
const next = m.devices ?? [];
|
|
342
|
-
for (const d of devices) if (!next.some((n) => n.id === d.id && n.scope === d.scope))
|
|
354
|
+
for (const d of devices) if (!next.some((n) => n.id === d.id && n.scope === d.scope)) {
|
|
355
|
+
ssh.dropDevice(d.id);
|
|
356
|
+
e2eSessions.delete(d.id);
|
|
357
|
+
}
|
|
343
358
|
devices = next;
|
|
344
359
|
void publish(ws);
|
|
345
360
|
return;
|
|
@@ -352,7 +367,7 @@ async function publisherLoop() {
|
|
|
352
367
|
ssh.handle(m.device, m.payload);
|
|
353
368
|
return;
|
|
354
369
|
}
|
|
355
|
-
const unwrapped = await e2eOnDevice(m.device, m.payload);
|
|
370
|
+
const unwrapped = await e2eOnDevice(ws, m.device, m.payload);
|
|
356
371
|
if (unwrapped) {
|
|
357
372
|
if (unwrapped.reply !== null && unwrapped.reply !== undefined) {
|
|
358
373
|
ws.send(JSON.stringify({ v: 1, t: "frame", device: m.device, payload: unwrapped.reply }));
|
|
@@ -361,10 +376,90 @@ async function publisherLoop() {
|
|
|
361
376
|
}
|
|
362
377
|
const payload = unwrapped?.inner ?? m.payload;
|
|
363
378
|
const op = payload && typeof payload === "object" ? payload.op : undefined;
|
|
364
|
-
const replyTo =
|
|
365
|
-
ws.send(JSON.stringify({ v: 1, t: "frame", device: m.device, payload: await e2eToDevice(m.device, out) }));
|
|
366
|
-
};
|
|
379
|
+
const replyTo = (out) => sendPayload(ws, m.device, out).catch(e => log(`payload send failed: ${e.message}`));
|
|
367
380
|
if (op === "ping") return void replyTo({ op: "pong", n: payload.n ?? null, at: new Date().toISOString() });
|
|
381
|
+
if (op === "receipt.get") {
|
|
382
|
+
// Experimental (opt-in HERDGATE_PI_RECEIPTS=1): answer from the local Pi
|
|
383
|
+
// receipt journal written by the experiments/pi-receipt observer.
|
|
384
|
+
// Metadata only — the observer itself never opens a socket.
|
|
385
|
+
const id = typeof payload.id === "string" ? payload.id : "";
|
|
386
|
+
const enabled = process.env.HERDGATE_PI_RECEIPTS === "1";
|
|
387
|
+
const entry = enabled ? findReceipt(receiptJournalBase(), id) : null;
|
|
388
|
+
log(`receipts: ${m.device} queried ${id || "?"} -> ${entry ? entry.status : "unknown"}`);
|
|
389
|
+
return void replyTo({ op: "receipt.result", id: id || null,
|
|
390
|
+
...publicReceipt(entry), ...(enabled ? {} : { reason: "pi_receipts_disabled" }) });
|
|
391
|
+
}
|
|
392
|
+
if (op === "worker.health") {
|
|
393
|
+
// Experimental (opt-in HERDGATE_PI_WORKER=1): liveness for the owner — queue
|
|
394
|
+
// depth plus the last heartbeat written by the scheduler/worker experiment.
|
|
395
|
+
// Metadata only; read scope is enough; missing heartbeat is reported as null.
|
|
396
|
+
const enabled = process.env.HERDGATE_PI_WORKER === "1";
|
|
397
|
+
let queued = null;
|
|
398
|
+
let running = null;
|
|
399
|
+
let beat = null;
|
|
400
|
+
if (enabled) {
|
|
401
|
+
try { const depth = queueDepth(readJobs(jobsFile(workerBase()))); queued = depth.queued; running = depth.running; }
|
|
402
|
+
catch { /* unreadable queue -> null, fail closed */ }
|
|
403
|
+
beat = readHeartbeat(workerBase());
|
|
404
|
+
}
|
|
405
|
+
const age = beat ? Math.max(0, Math.round((Date.now() - beat.ts) / 1000)) : null;
|
|
406
|
+
log(`worker: ${m.device} health -> queued ${queued ?? "?"} running ${running ?? "?"} heartbeat ${beat ? `${age}s ago` : "none"}`);
|
|
407
|
+
return void replyTo({ op: "worker.health.result", enabled,
|
|
408
|
+
...(enabled
|
|
409
|
+
? { queued, running, heartbeat: beat ? { ts: beat.ts, source: beat.source, ageSeconds: age } : null }
|
|
410
|
+
: { reason: "pi_worker_disabled" }) });
|
|
411
|
+
}
|
|
412
|
+
if (op === "worker.submit" || op === "worker.status") {
|
|
413
|
+
// Experimental (opt-in HERDGATE_PI_WORKER=1): admission to the local Pi worker
|
|
414
|
+
// queue (plugin/server/pi-worker.mjs). The publisher never runs Pi itself — a
|
|
415
|
+
// separately started experiments/pi-worker drains the queue and journals receipts.
|
|
416
|
+
// worker.submit carries a prompt, so it is a control operation (act scope only).
|
|
417
|
+
const enabled = process.env.HERDGATE_PI_WORKER === "1";
|
|
418
|
+
const file = jobsFile(workerBase());
|
|
419
|
+
const id = typeof payload.id === "string" && JOB_ID.test(payload.id) ? payload.id : "";
|
|
420
|
+
if (op === "worker.status") {
|
|
421
|
+
let job = null;
|
|
422
|
+
if (enabled && id) {
|
|
423
|
+
try { job = readJobs(file).get(id) ?? null; } catch { job = null; }
|
|
424
|
+
}
|
|
425
|
+
log(`worker: ${m.device} status ${id || "?"} -> ${job ? job.state : "unknown"}`);
|
|
426
|
+
return void replyTo({ op: "worker.status.result", id: id || null, ...publicJob(job),
|
|
427
|
+
...(enabled ? {} : { reason: "pi_worker_disabled" }) });
|
|
428
|
+
}
|
|
429
|
+
if (!enabled) {
|
|
430
|
+
return void replyTo({ op: "error", code: "pi_worker_disabled", message: "the Pi worker queue is not enabled on this host" });
|
|
431
|
+
}
|
|
432
|
+
if (devices.find(d => d.id === m.device)?.scope !== "act") {
|
|
433
|
+
return void replyTo({ op: "error", code: "requires-act", message: "worker.submit needs an act device" });
|
|
434
|
+
}
|
|
435
|
+
const prompt = typeof payload.prompt === "string" ? payload.prompt : "";
|
|
436
|
+
if (!prompt.trim() || prompt.length > MAX_PROMPT) {
|
|
437
|
+
return void replyTo({ op: "error", code: "bad-prompt", message: `worker.submit needs a prompt of 1..${MAX_PROMPT} chars` });
|
|
438
|
+
}
|
|
439
|
+
let jobs;
|
|
440
|
+
try { jobs = readJobs(file); }
|
|
441
|
+
catch (e) {
|
|
442
|
+
return void replyTo({ op: "error", code: "worker-queue", message: `queue unreadable: ${e instanceof Error ? e.message : String(e)}` });
|
|
443
|
+
}
|
|
444
|
+
const existing = id ? jobs.get(id) : undefined;
|
|
445
|
+
const settled = !existing && id ? findReceipt(receiptJournalBase(), id) : null;
|
|
446
|
+
if (existing || settled) {
|
|
447
|
+
const state = existing ? existing.state : String(settled?.status);
|
|
448
|
+
log(`worker: ${m.device} resubmitted ${id} -> ${state} (duplicate)`);
|
|
449
|
+
return void replyTo({ op: "worker.submitted", id, state, duplicate: true });
|
|
450
|
+
}
|
|
451
|
+
const pending = [...jobs.values()].filter(j => j.state === "queued" || j.state === "running").length;
|
|
452
|
+
if (pending >= MAX_PENDING) {
|
|
453
|
+
return void replyTo({ op: "error", code: "queue-full", message: `worker queue is full (${MAX_PENDING} pending)` });
|
|
454
|
+
}
|
|
455
|
+
const jobId = id || randomUUID();
|
|
456
|
+
try { appendJob(file, { v: 1, id: jobId, prompt, ts: Date.now(), state: "queued" }); }
|
|
457
|
+
catch (e) {
|
|
458
|
+
return void replyTo({ op: "error", code: "worker-queue", message: `queue append failed: ${e instanceof Error ? e.message : String(e)}` });
|
|
459
|
+
}
|
|
460
|
+
log(`worker: ${m.device} submitted ${jobId} (${prompt.length} chars)`);
|
|
461
|
+
return void replyTo({ op: "worker.submitted", id: jobId, state: "queued", duplicate: false });
|
|
462
|
+
}
|
|
368
463
|
if (op === "view") {
|
|
369
464
|
// Answer this device now, even if the view has not changed since the last push.
|
|
370
465
|
lastSig = "";
|
|
@@ -431,6 +526,7 @@ async function publisherLoop() {
|
|
|
431
526
|
if (timer) clearInterval(timer);
|
|
432
527
|
timer = null;
|
|
433
528
|
ssh.closeAll();
|
|
529
|
+
e2eSessions.clear();
|
|
434
530
|
devices = [];
|
|
435
531
|
const wait = Math.min(30_000, backoff) + Math.random() * 500;
|
|
436
532
|
backoff *= 2;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export interface JournalEntry {
|
|
2
|
+
v?: number;
|
|
3
|
+
id: string;
|
|
4
|
+
status: string;
|
|
5
|
+
reason?: string;
|
|
6
|
+
ts?: number;
|
|
7
|
+
sessionId?: string;
|
|
8
|
+
paneId?: string;
|
|
9
|
+
requestEntryId?: string;
|
|
10
|
+
responseEntryId?: string;
|
|
11
|
+
runtime?: string;
|
|
12
|
+
[key: string]: unknown;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface PublicReceipt {
|
|
16
|
+
id?: string;
|
|
17
|
+
status: string;
|
|
18
|
+
reason?: string;
|
|
19
|
+
requestEntryId?: string;
|
|
20
|
+
responseEntryId?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function journalBase(env?: Record<string, string | undefined>): string;
|
|
24
|
+
export function scopeFor(cwd: string): string;
|
|
25
|
+
export function validEntry(entry: unknown): JournalEntry | null;
|
|
26
|
+
export function readJournal(dir: string): JournalEntry[];
|
|
27
|
+
export function findReceipt(base: string, id: string): JournalEntry | null;
|
|
28
|
+
export function publicReceipt(entry: JournalEntry | null): PublicReceipt;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// Shared reader for the experimental Pi receipt journal (see experiments/pi-receipt).
|
|
2
|
+
// The observer writes; the publisher may answer receipt.get frames from paired
|
|
3
|
+
// devices when HERDGATE_PI_RECEIPTS=1. Read-only, bounded, no network access here.
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
import { homedir } from 'node:os';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { readFileSync, readdirSync } from 'node:fs';
|
|
8
|
+
|
|
9
|
+
export const RECEIPT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
10
|
+
const RECEIPTS = 'receipts.jsonl';
|
|
11
|
+
|
|
12
|
+
/** Base directory holding one hashed sub-directory per Pi workspace. */
|
|
13
|
+
export function journalBase(env = process.env) {
|
|
14
|
+
const override = env.HERDGATE_PI_RECEIPT_DIR;
|
|
15
|
+
if (override) return override;
|
|
16
|
+
return join(env.XDG_STATE_HOME || join(homedir(), '.local', 'state'), 'herdgate', 'pi-receipts');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function scopeFor(cwd) {
|
|
20
|
+
return createHash('sha256').update(cwd).digest('hex').slice(0, 16);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function validEntry(entry) {
|
|
24
|
+
return validEntryShape(entry) ? entry : null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function validEntryShape(entry) {
|
|
28
|
+
return Boolean(entry && entry.v === 1 && RECEIPT_ID.test(entry.id ?? '')
|
|
29
|
+
&& ['replied', 'failed', 'canceled', 'unknown'].includes(entry.status)
|
|
30
|
+
&& typeof entry.sessionId === 'string' && typeof entry.ts === 'number'
|
|
31
|
+
&& ['reason', 'requestEntryId', 'responseEntryId', 'paneId'].every(key => entry[key] === undefined || typeof entry[key] === 'string'));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Parse a journal file; tolerate a torn tail and skip corrupt lines. */
|
|
35
|
+
export function readJournal(file) {
|
|
36
|
+
let raw;
|
|
37
|
+
try { raw = readFileSync(file, 'utf8'); }
|
|
38
|
+
catch (error) { if (error.code === 'ENOENT') return []; throw error; }
|
|
39
|
+
const lines = raw.split('\n');
|
|
40
|
+
lines.pop(); // the empty tail; a torn final line without \n drops with it
|
|
41
|
+
const entries = [];
|
|
42
|
+
for (const line of lines) {
|
|
43
|
+
try {
|
|
44
|
+
const entry = JSON.parse(line);
|
|
45
|
+
if (validEntry(entry)) entries.push(entry);
|
|
46
|
+
} catch { /* skip */ }
|
|
47
|
+
}
|
|
48
|
+
return entries;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Find the newest receipt for a request ID across workspace journals.
|
|
53
|
+
* Lookup by exact UUID only — there is no enumeration. Returns null when the
|
|
54
|
+
* journals are unreadable or the ID is unknown; callers fail closed.
|
|
55
|
+
*/
|
|
56
|
+
export function findReceipt(base, id) {
|
|
57
|
+
if (!RECEIPT_ID.test(id ?? '')) return null;
|
|
58
|
+
try {
|
|
59
|
+
let found = null;
|
|
60
|
+
for (const name of readdirSync(base)) {
|
|
61
|
+
if (!/^[0-9a-f]{16}$/.test(name)) continue;
|
|
62
|
+
for (const entry of readJournal(join(base, name, RECEIPTS))) {
|
|
63
|
+
if (entry.id === id && (!found || entry.ts >= found.ts)) found = entry;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return found;
|
|
67
|
+
} catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The public answer for a device: journal internals stay on the host. */
|
|
73
|
+
export function publicReceipt(entry) {
|
|
74
|
+
if (!entry) return { status: 'unknown', reason: 'not_retained_in_runtime' };
|
|
75
|
+
const { v, ts, sessionId, paneId, ...rest } = entry;
|
|
76
|
+
return rest;
|
|
77
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export type JobState = 'queued' | 'running' | 'replied' | 'failed' | 'canceled' | 'unknown';
|
|
2
|
+
|
|
3
|
+
export interface JobRecord {
|
|
4
|
+
v: 1;
|
|
5
|
+
id: string;
|
|
6
|
+
prompt: string;
|
|
7
|
+
ts: number;
|
|
8
|
+
state: JobState;
|
|
9
|
+
sessionId?: string;
|
|
10
|
+
reason?: string;
|
|
11
|
+
requestEntryId?: string;
|
|
12
|
+
responseEntryId?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface PublicJob {
|
|
16
|
+
id?: string;
|
|
17
|
+
state: JobState;
|
|
18
|
+
ts?: number;
|
|
19
|
+
reason?: string;
|
|
20
|
+
sessionId?: string;
|
|
21
|
+
requestEntryId?: string;
|
|
22
|
+
responseEntryId?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const JOB_ID: RegExp;
|
|
26
|
+
export const MAX_PROMPT: number;
|
|
27
|
+
export const MAX_PENDING: number;
|
|
28
|
+
/** Oldest Pi release the experiments are verified against (floor, not a pin). */
|
|
29
|
+
export const MIN_PI_VERSION: string;
|
|
30
|
+
export function versionAtLeast(version: string, min?: string): boolean;
|
|
31
|
+
export function workerBase(env?: Record<string, string | undefined>): string;
|
|
32
|
+
export function jobsFile(base: string): string;
|
|
33
|
+
export function validJob(job: unknown): JobRecord | null;
|
|
34
|
+
export function readJobs(file: string): Map<string, JobRecord>;
|
|
35
|
+
export function appendJob(file: string, job: JobRecord): void;
|
|
36
|
+
export function publicJob(job: JobRecord | null): PublicJob;
|
|
37
|
+
export function queueDepth(jobs: Map<string, JobRecord>): { queued: number; running: number };
|
|
38
|
+
export function scheduledRunId(scheduleId: string, bucket: number): string;
|
|
39
|
+
export interface Heartbeat {
|
|
40
|
+
v: 1;
|
|
41
|
+
ts: number;
|
|
42
|
+
source: string;
|
|
43
|
+
[key: string]: unknown;
|
|
44
|
+
}
|
|
45
|
+
export function heartbeatPath(base: string): string;
|
|
46
|
+
export function writeHeartbeat(base: string, entry: Heartbeat): void;
|
|
47
|
+
export function readHeartbeat(base: string): Heartbeat | null;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Shared format for the experimental Pi worker queue (see experiments/pi-worker).
|
|
2
|
+
// The publisher appends jobs when HERDGATE_PI_WORKER=1; the separately started
|
|
3
|
+
// experiment worker consumes them and journals receipts via the shared journal.
|
|
4
|
+
// This module only owns the on-disk queue format — no network, no Pi SDK here.
|
|
5
|
+
import { createHash } from 'node:crypto';
|
|
6
|
+
import { homedir } from 'node:os';
|
|
7
|
+
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { dirname, join } from 'node:path';
|
|
9
|
+
|
|
10
|
+
export const JOB_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
11
|
+
export const MAX_PROMPT = 16 * 1024; // admission bound for one prompt
|
|
12
|
+
export const MAX_PENDING = 100; // queued + running jobs admitted at once
|
|
13
|
+
export const JOB_STATES = new Set(['queued', 'running', 'replied', 'failed', 'canceled', 'unknown']);
|
|
14
|
+
const JOBS = 'jobs.jsonl';
|
|
15
|
+
|
|
16
|
+
/** Oldest Pi release the experiments are verified against. Newer Pi releases run
|
|
17
|
+
* without code changes; only a Pi older than this is refused. */
|
|
18
|
+
export const MIN_PI_VERSION = '1.5.2';
|
|
19
|
+
|
|
20
|
+
/** Loose semver floor: numeric triple comparison, 'v' prefix and pre-release
|
|
21
|
+
* suffixes ignored — except a pre-release of the exact min triple does not pass. */
|
|
22
|
+
export function versionAtLeast(version, min = MIN_PI_VERSION) {
|
|
23
|
+
const parse = value => String(value ?? '').trim().replace(/^v/, '').split('-')[0].split('.').map(Number);
|
|
24
|
+
const actual = parse(version);
|
|
25
|
+
const floor = parse(min);
|
|
26
|
+
for (let i = 0; i < 3; i++) {
|
|
27
|
+
const a = actual[i] ?? 0;
|
|
28
|
+
const b = floor[i] ?? 0;
|
|
29
|
+
if (!Number.isInteger(a) || !Number.isInteger(b)) return false;
|
|
30
|
+
if (a !== b) return a > b;
|
|
31
|
+
}
|
|
32
|
+
if (String(version).includes('-') && !String(min).includes('-')) return false; // 1.5.2-rc1 is not >= 1.5.2
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Directory holding jobs.jsonl and worker state. One worker per host (prototype). */
|
|
37
|
+
export function workerBase(env = process.env) {
|
|
38
|
+
if (env.HERDGATE_PI_WORKER_DIR) return env.HERDGATE_PI_WORKER_DIR;
|
|
39
|
+
return join(env.XDG_STATE_HOME || join(homedir(), '.local', 'state'), 'herdgate', 'pi-worker');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function jobsFile(base) {
|
|
43
|
+
return join(base, JOBS);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function validJob(job) {
|
|
47
|
+
return validJobShape(job) ? job : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function validJobShape(job) {
|
|
51
|
+
return Boolean(job && job.v === 1 && JOB_ID.test(job.id ?? '')
|
|
52
|
+
&& JOB_STATES.has(job.state)
|
|
53
|
+
&& typeof job.prompt === 'string' && job.prompt.length <= MAX_PROMPT
|
|
54
|
+
&& typeof job.ts === 'number'
|
|
55
|
+
&& ['sessionId', 'reason', 'requestEntryId', 'responseEntryId'].every(key =>
|
|
56
|
+
job[key] === undefined || typeof job[key] === 'string'));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Newest record per job id; corrupt lines and a torn tail are skipped. */
|
|
60
|
+
export function readJobs(file) {
|
|
61
|
+
let raw;
|
|
62
|
+
try { raw = readFileSync(file, 'utf8'); }
|
|
63
|
+
catch (error) { if (error.code === 'ENOENT') return new Map(); throw error; }
|
|
64
|
+
const lines = raw.split('\n');
|
|
65
|
+
lines.pop();
|
|
66
|
+
const jobs = new Map();
|
|
67
|
+
for (const line of lines) {
|
|
68
|
+
try {
|
|
69
|
+
const job = JSON.parse(line);
|
|
70
|
+
if (validJob(job)) jobs.set(job.id, job);
|
|
71
|
+
} catch { /* skip */ }
|
|
72
|
+
}
|
|
73
|
+
return jobs;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function appendJob(file, job) {
|
|
77
|
+
if (!validJob(job)) throw new Error('invalid job record');
|
|
78
|
+
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
79
|
+
appendFileSync(file, `${JSON.stringify(job)}\n`, { mode: 0o600 });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The public answer for a device: the prompt never leaves the host. */
|
|
83
|
+
export function publicJob(job) {
|
|
84
|
+
if (!job) return { state: 'unknown', reason: 'not_admitted' };
|
|
85
|
+
const { v, prompt, ...rest } = job;
|
|
86
|
+
return rest;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function queueDepth(jobs) {
|
|
90
|
+
let queued = 0;
|
|
91
|
+
let running = 0;
|
|
92
|
+
for (const job of jobs.values()) {
|
|
93
|
+
if (job.state === 'queued') queued += 1;
|
|
94
|
+
else if (job.state === 'running') running += 1;
|
|
95
|
+
}
|
|
96
|
+
return { queued, running };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Deterministic UUID for one scheduled run: same (schedule, bucket) -> same id, so a
|
|
100
|
+
* restarted scheduler re-admits the same job instead of duplicating the run. */
|
|
101
|
+
export function scheduledRunId(scheduleId, bucket) {
|
|
102
|
+
const digest = createHash('sha256').update(`${scheduleId}|${bucket}`).digest('hex');
|
|
103
|
+
const variant = (parseInt(digest[16], 16) & 0x3) | 0x8;
|
|
104
|
+
return [digest.slice(0, 8), digest.slice(8, 12), `4${digest.slice(13, 16)}`,
|
|
105
|
+
`${variant.toString(16)}${digest.slice(17, 20)}`, digest.slice(20, 32)].join('-');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function heartbeatPath(base) {
|
|
109
|
+
return join(base, 'heartbeat.json');
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** One heartbeat record per tick; the publisher serves it via worker.health. */
|
|
113
|
+
export function writeHeartbeat(base, entry) {
|
|
114
|
+
if (!entry || entry.v !== 1 || typeof entry.ts !== 'number' || typeof entry.source !== 'string') {
|
|
115
|
+
throw new Error('invalid heartbeat');
|
|
116
|
+
}
|
|
117
|
+
mkdirSync(base, { recursive: true, mode: 0o700 });
|
|
118
|
+
writeFileSync(heartbeatPath(base), JSON.stringify(entry), { mode: 0o600 });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function readHeartbeat(base) {
|
|
122
|
+
try {
|
|
123
|
+
const entry = JSON.parse(readFileSync(heartbeatPath(base), 'utf8'));
|
|
124
|
+
if (entry && entry.v === 1 && typeof entry.ts === 'number' && typeof entry.source === 'string') return entry;
|
|
125
|
+
} catch { /* missing or corrupt -> no heartbeat */ }
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
2
|
+
# The default still puts the bundle in the PWA's web root. For staging, pass a ZIP path.
|
|
3
3
|
set -euo pipefail
|
|
4
4
|
DIR=$(cd "$(dirname "$0")/.." && pwd)
|
|
5
|
+
[[ $# -le 1 ]] || { echo "usage: build-host-bundle.sh [output.zip]" >&2; exit 2; }
|
|
6
|
+
DEST=${1:-"$DIR/src/web/herdgate-host.zip"}
|
|
5
7
|
OUT=$(mktemp -d)
|
|
6
8
|
trap 'rm -rf "$OUT"' EXIT
|
|
7
|
-
mkdir -p "$OUT/
|
|
8
|
-
cp
|
|
9
|
-
cp
|
|
10
|
-
cp "$DIR/
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
mkdir -p "$OUT/host/mod/herdgate" "$OUT/host/plugin/server" "$OUT/host/windows" "$OUT/host/templates"
|
|
10
|
+
cp "$DIR/app-mod/herdgate/"{main.js,mod.json,herdgate.svg} "$OUT/host/mod/herdgate/"
|
|
11
|
+
cp "$DIR/plugin/herdr-plugin.toml" "$OUT/host/plugin/"
|
|
12
|
+
cp "$DIR/plugin/server/"*.mjs "$OUT/host/plugin/server/"
|
|
13
|
+
cp "$DIR/windows/"*.mjs "$OUT/host/windows/"
|
|
14
|
+
cp "$DIR/templates/herdgate-via.mjs" "$OUT/host/templates/"
|
|
15
|
+
cp "$DIR/dist-readme.md" "$OUT/host/README.md"
|
|
16
|
+
( cd "$OUT/host" && zip -qr "$OUT/host.zip" mod plugin windows templates README.md )
|
|
17
|
+
mkdir -p "$(dirname "$DEST")"
|
|
18
|
+
mv -- "$OUT/host.zip" "$DEST"
|
|
19
|
+
printf '%s\n' "$DEST"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Build a fresh, deployable release set; never use local config, installed dependencies or live state.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
|
5
|
+
cd "$ROOT"
|
|
6
|
+
[[ $# -le 1 ]] || { echo 'usage: build-release.sh [output-directory]' >&2; exit 2; }
|
|
7
|
+
VERSION=$(node -p 'require("./package.json").version')
|
|
8
|
+
DEST=${1:-"$ROOT/dist/$VERSION"}
|
|
9
|
+
[[ "$DEST" = /* ]] || DEST="$ROOT/$DEST"
|
|
10
|
+
if [[ -e "$DEST" || -L "$DEST" ]]; then
|
|
11
|
+
echo "release output already exists: $DEST" >&2
|
|
12
|
+
exit 1
|
|
13
|
+
fi
|
|
14
|
+
mkdir -p "$(dirname "$DEST")"
|
|
15
|
+
STAGE=$(mktemp -d "$(dirname "$DEST")/.herdgate-release.XXXXXX")
|
|
16
|
+
trap 'rm -rf "$STAGE"' EXIT
|
|
17
|
+
mkdir "$STAGE/release"
|
|
18
|
+
OUT="$STAGE/release"
|
|
19
|
+
NAME="herdgate-cloud-$VERSION"
|
|
20
|
+
CLOUD="$STAGE/$NAME"
|
|
21
|
+
mkdir -p "$CLOUD/runtime/web/icons" "$CLOUD/scripts"
|
|
22
|
+
|
|
23
|
+
npm run build:cloud -- --outDir "$CLOUD/runtime"
|
|
24
|
+
cp src/web/{index.html,app.js,app.css,app.webmanifest,sw.js,e2e.js} "$CLOUD/runtime/web/"
|
|
25
|
+
cp src/web/icons/{icon.svg,icon-192.png,icon-512.png} "$CLOUD/runtime/web/icons/"
|
|
26
|
+
cp templates/cloud/{Dockerfile,compose.yml,compose.env.example,.dockerignore,herdgate-cloud.service,herdgate-cloud-edge.service,Caddyfile.example,README.md} "$CLOUD/"
|
|
27
|
+
cp templates/cloud/gitignore.example "$CLOUD/.gitignore"
|
|
28
|
+
cp scripts/patch-ssh2.mjs "$CLOUD/scripts/"
|
|
29
|
+
cp package-lock.json "$CLOUD/"
|
|
30
|
+
# Use the same versions and lock as npm, without the host installer or Pi postinstall.
|
|
31
|
+
node --input-type=module - "$CLOUD/package.json" <<'NODE'
|
|
32
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
33
|
+
const { name, version, type, engines, dependencies } = JSON.parse(readFileSync('package.json', 'utf8'));
|
|
34
|
+
const scripts = { start: 'node runtime/cloud/main.js', 'ssh-tcp-edge': 'node runtime/cloud/ssh-tcp-edge.js' };
|
|
35
|
+
writeFileSync(process.argv[2], JSON.stringify({ name, version, type, engines, private: true, scripts, dependencies }, null, 2) + '\n');
|
|
36
|
+
NODE
|
|
37
|
+
|
|
38
|
+
HOST="herdgate-host-$VERSION.zip"
|
|
39
|
+
bash scripts/build-host-bundle.sh "$OUT/$HOST"
|
|
40
|
+
cp "$OUT/$HOST" "$CLOUD/runtime/web/herdgate-host.zip"
|
|
41
|
+
tar -czf "$OUT/$NAME.tar.gz" -C "$STAGE" "$NAME"
|
|
42
|
+
npm pack --ignore-scripts --pack-destination "$OUT"
|
|
43
|
+
( cd "$OUT" && sha256sum -- *.tar.gz *.tgz *.zip > SHA256SUMS )
|
|
44
|
+
# Move the complete set into place; don't nest or replace an existing release.
|
|
45
|
+
mkdir -- "$DEST"
|
|
46
|
+
mv -- "$OUT/"* "$DEST/"
|
|
47
|
+
printf 'Herdgate %s: %s\n' "$VERSION" "$DEST"
|