@bom0792/herdgate 0.9.38 → 0.9.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -166,6 +166,21 @@ One persistent shell with marker framing replaces exec-per-command; phone keystr
166
166
  bounded, coalescing, in-order-acked queue that resyncs instead of replaying; file polls are
167
167
  checksum-stamped (`unchanged` costs one word).
168
168
 
169
+ ### Latency budget (measured 22/11)
170
+
171
+ Interactive typing (mobile TUI, remote herdr attach) rides the SSH bridge **direct**
172
+ to the host — no CDN hop, feels local. The control/PWA leg goes through the tunnel
173
+ edge (~130ms RTT hairpin from the same LAN; a phone on 4G sees similar). To keep
174
+ that leg smooth the publisher now:
175
+
176
+ - pushes view snapshots every `HERDGATE_PUBLISH_MS` (default 30s; the home unit sets
177
+ 5s — a settle is visible on the phone within seconds, not half a minute);
178
+ - replays the last snapshot for a `view` op inside `HERDGATE_SNAPSHOT_FRESH_MS`
179
+ (default 2s) instead of spawning `herdr api snapshot` again — a view op costs ~0
180
+ server-side work when the cache is warm;
181
+ - coalesces concurrent snapshot demands into one spawn (single-flight), and a cached
182
+ view no longer resets the change-sig (so it cannot re-wake away phones for nothing).
183
+
169
184
  ## The SSH leg (the Herdr phone app)
170
185
 
171
186
  The phone app adds machines over SSH. Herdgate terminates SSH at the Gate; use the Gate's reachable SSH
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bom0792/herdgate",
3
- "version": "0.9.38",
3
+ "version": "0.9.40",
4
4
  "description": "Cloud relay hub for herdr-core: every machine dials out, phones and browsers follow (PWA + SSH leg, E2E payloads).",
5
5
  "type": "module",
6
6
  "engines": {
@@ -1,6 +1,6 @@
1
1
  id = "dev.herdgate.host"
2
2
  name = "herdgate"
3
- version = "0.1.0"
3
+ version = "0.9.38"
4
4
  min_herdr_version = "0.9.0"
5
5
  description = "Publish this Herdr session to your own devices through a herdgate cloud relay (read-only by default)."
6
6
  platforms = ["linux", "macos"]
@@ -262,6 +262,15 @@ async function publisherLoop() {
262
262
  let lastSig = "";
263
263
  let devices = [];
264
264
  let timer = null;
265
+ // Push cadence and view-request freshness. The 30s default is gentle on a
266
+ // battery-powered host; a plugged-in one sets HERDGATE_PUBLISH_MS=5000 so a
267
+ // phone sees a settle within seconds instead of half a minute. A `view` op
268
+ // inside the freshness window replays the last snapshot instead of spawning
269
+ // `herdr api snapshot` again (bimodal view latency: 4ms cached vs 150ms cold).
270
+ const PUBLISH_MS = Math.max(2_000, Number.parseInt(process.env.HERDGATE_PUBLISH_MS ?? "30000", 10) || 30000);
271
+ const SNAPSHOT_FRESH_MS = Math.max(0, Number.parseInt(process.env.HERDGATE_SNAPSHOT_FRESH_MS ?? "2000", 10) || 0);
272
+ let snapshotInflight = null;
273
+ let snapshotCached = null; // { payload, at }
265
274
 
266
275
  const snapshotPayload = async () => {
267
276
  let snap;
@@ -280,7 +289,18 @@ async function publisherLoop() {
280
289
  writeState(D.statePath, { lastPublish: { at: new Date().toISOString(), summary } });
281
290
  }
282
291
  if (changed) wakeAway(true);
283
- return { op: "view", view: { v: 1, source: "herdr", takenAt: new Date().toISOString(), summary } };
292
+ const payload = { op: "view", view: { v: 1, source: "herdr", takenAt: new Date().toISOString(), summary } };
293
+ snapshotCached = { payload, at: Date.now() };
294
+ return payload;
295
+ };
296
+
297
+ /** Single-flight: concurrent demands (interval tick + a device's view op)
298
+ * share one `herdr api snapshot` spawn instead of stampeding. */
299
+ const snapshotOnce = () => {
300
+ if (!snapshotInflight) {
301
+ snapshotInflight = snapshotPayload().finally(() => { snapshotInflight = null; });
302
+ }
303
+ return snapshotInflight;
284
304
  };
285
305
 
286
306
  let wsRef = null;
@@ -380,7 +400,7 @@ async function publisherLoop() {
380
400
  });
381
401
  const publish = async (ws) => {
382
402
  if (ws.readyState !== 1 || devices.length === 0) return;
383
- const payload = await snapshotPayload();
403
+ const payload = await snapshotOnce();
384
404
  if (!payload) return;
385
405
  for (const d of devices) {
386
406
  await sendPayload(ws, d.id, payload);
@@ -469,7 +489,7 @@ async function publisherLoop() {
469
489
  // Advertise honestly: `act` only when the owner's switch allows it.
470
490
  let allowAct = false;
471
491
  try { allowAct = (JSON.parse(readFileSync(SETTINGS, "utf8")) ?? {}).allowAct === true; } catch { /* no settings yet */ }
472
- ws.send(JSON.stringify({ v: 1, t: "hello", roomToken: token, info: { name: NAME, version: "0.1.0", capabilities: ["dev.herdgate.host", allowAct ? "act" : "read-only"] } }));
492
+ ws.send(JSON.stringify({ v: 1, t: "hello", roomToken: token, info: { name: NAME, version: "0.9.38", capabilities: ["dev.herdgate.host", allowAct ? "act" : "read-only"] } }));
473
493
  };
474
494
  ws.onmessage = (ev) => {
475
495
  void (async () => {
@@ -485,7 +505,7 @@ async function publisherLoop() {
485
505
  backoff = 1_000;
486
506
  devices = m.devices ?? [];
487
507
  lifecycle?.setConnected(true);
488
- log(`connected to ${url}${loopbackPort ? ` via loopback 127.0.0.1:${loopbackPort}` : ""} (room ${m.roomId}, ${devices.length} device(s))`);
508
+ log(`connected to ${url}${loopbackPort ? ` via loopback 127.0.0.1:${loopbackPort}` : ""} (room ${m.roomId}, ${devices.length} device(s), publish ${PUBLISH_MS}ms)`);
489
509
  if (process.env.HERDGATE_PI_RECEIPTS === "1" && !receiptWatch) {
490
510
  // Settles push to subscribed devices only. Parent wakes no longer ride
491
511
  // this watcher — the child-cooperative design (arm/marker/observer)
@@ -501,7 +521,7 @@ async function publisherLoop() {
501
521
  for (const d of devices) flushWatches(d.id);
502
522
  void publish(ws);
503
523
  if (timer) clearInterval(timer);
504
- timer = setInterval(() => void publish(ws), 30_000);
524
+ timer = setInterval(() => void publish(ws), PUBLISH_MS);
505
525
  timer.unref?.();
506
526
  return;
507
527
  }
@@ -814,7 +834,13 @@ async function publisherLoop() {
814
834
  }
815
835
  }
816
836
  if (op === "view") {
817
- // Answer this device now, even if the view has not changed since the last push.
837
+ // Answer this device now. Inside the freshness window replay the last
838
+ // snapshot (no spawn, no lastSig reset — so the next full tick cannot
839
+ // re-detect a "change" and wake away phones for nothing).
840
+ if (snapshotCached && Date.now() - snapshotCached.at < SNAPSHOT_FRESH_MS) {
841
+ for (const d of devices) await sendPayload(ws, d.id, snapshotCached.payload);
842
+ return;
843
+ }
818
844
  lastSig = "";
819
845
  return void publish(ws);
820
846
  }
@@ -9,7 +9,7 @@
9
9
  import { execFileSync } from 'node:child_process';
10
10
  import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
11
11
  import { dirname, join } from 'node:path';
12
- import { readSpawns } from '../../experiments/pi-spawn/registry.mjs';
12
+ import { readSpawns } from './spawn-registry.mjs';
13
13
  import { findReceipt } from './pi-receipts.mjs';
14
14
 
15
15
  const note = (line) => console.error(`herdgate pi-wake: ${line}`);
@@ -65,6 +65,13 @@ async function launch({ action, bin, hc, mobile, mobileScope, base, state }) {
65
65
  }
66
66
  const env = {
67
67
  ...process.env,
68
+ // The TUI publisher is an interactive-machine default (a laptop you are
69
+ // looking at), not a battery-sipping box: push views every 5s unless the
70
+ // operator set an explicit cadence. The plugin/systemd default stays 30s.
71
+ HERDGATE_PUBLISH_MS: process.env.HERDGATE_PUBLISH_MS ?? "5000",
72
+ // Match the freshness window to the cadence so a view op almost always
73
+ // replays the last snapshot instead of forcing a fresh spawn.
74
+ HERDGATE_SNAPSHOT_FRESH_MS: process.env.HERDGATE_SNAPSHOT_FRESH_MS ?? process.env.HERDGATE_PUBLISH_MS ?? "5000",
68
75
  HERDGATE_MOBILE_HC_BIN: mobileHcBin,
69
76
  HERDGATE_MOBILE_TUI_DEVICE: mobile,
70
77
  HERDGATE_MOBILE_TUI_SCOPE: mobileScope,
@@ -0,0 +1,23 @@
1
+ export const MAX_SPAWNS: number;
2
+ export const COMPACT_AT: number;
3
+ export const SPAWN_NAME: RegExp;
4
+
5
+ export interface SpawnRecord {
6
+ id: string;
7
+ sessionId: string;
8
+ paneId: string;
9
+ name: string;
10
+ parentSessionId?: string;
11
+ cwd?: string;
12
+ ts: number;
13
+ status?: string;
14
+ restoredAt?: number;
15
+ [key: string]: unknown;
16
+ }
17
+
18
+ export function spawnsFile(base: string): string;
19
+ export function readSpawns(file: string): SpawnRecord[];
20
+ export function recordSpawn(base: string, spawn: SpawnRecord): SpawnRecord & { status: string };
21
+ export function updateSpawn(base: string, id: string, patch: Partial<SpawnRecord>): SpawnRecord | null;
22
+ export function findSpawn(base: string, ref: string): SpawnRecord | null;
23
+ export function spawnBase(): string;
@@ -0,0 +1,216 @@
1
+ // Parent→child spawn registry for Herdgate-controlled Pi agents (experiment).
2
+ // One id serves the whole chain: receipt request id == Pi --session-id, so a
3
+ // controller knows the child's session identity the moment it spawns and can
4
+ // reopen it later with the same flag. Stores only ids/panes/status — no briefs.
5
+ //
6
+ // State hardening (miko-misa/herdr-portfwd state.rs lessons, deep/portfwd.md):
7
+ // - a damaged file never reads as empty: the reader throws with the FILE NAMED
8
+ // (the StateError::Damaged refuse); callers warn and continue on the lines
9
+ // that did parse instead of silently inventing a smaller registry;
10
+ // - writes are atomic through an O_EXCL temp with an unguessable name + rename
11
+ // (a symlink planted at a guessable temp path must not be written through);
12
+ // - records sort by a stable key before every write, so the same set of records
13
+ // serializes to byte-identical files across processes (diffable by a human);
14
+ // - spawnBase() refuses a base dir owned by another uid or writable by
15
+ // group/others, naming the dir: registry records decide which panes get
16
+ // closed, and a dir some other uid can write into is a takeover.
17
+ import { randomBytes } from 'node:crypto';
18
+ import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
19
+ import { dirname, join } from 'node:path';
20
+ import { journalBase, scopeFor } from './pi-receipts.mjs';
21
+
22
+ export const MAX_SPAWNS = 200;
23
+ export const COMPACT_AT = 300;
24
+ export const SPAWN_NAME = /^[a-z][a-z0-9_-]{0,31}$/;
25
+ const PANE = /^[\w:.-]{1,32}$/;
26
+ // Pi session ids are v7 (SDK-generated); receipt request ids are v4. Both are one-id
27
+ // chain material, so the registry accepts either shape.
28
+ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[47][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
29
+
30
+ export function spawnsFileFor(base, cwd) {
31
+ return join(base, scopeFor(cwd), 'spawns.jsonl');
32
+ }
33
+
34
+ export function spawnsFile(base) {
35
+ return spawnsFileFor(base, process.cwd());
36
+ }
37
+
38
+ function valid(entry) {
39
+ // Controller entries carry the pane they run in; self-registered ones (an agent
40
+ // declaring its own parent, e.g. a worker job session) have no pane until restored.
41
+ const paneOk = entry?.self === true
42
+ ? (entry.paneId === undefined || PANE.test(entry.paneId))
43
+ : PANE.test(entry?.paneId ?? '');
44
+ return Boolean(entry && UUID.test(entry.id ?? '') && typeof entry.sessionId === 'string' && entry.sessionId
45
+ && paneOk && SPAWN_NAME.test(entry.name ?? '')
46
+ && (entry.parentSessionId === undefined || typeof entry.parentSessionId === 'string')
47
+ && (entry.cwd === undefined || typeof entry.cwd === 'string')
48
+ && (entry.sessionFile === undefined || typeof entry.sessionFile === 'string')
49
+ && Number.isFinite(entry.ts));
50
+ }
51
+
52
+ /** A damaged registry file. The message names the file — a corrupt line must
53
+ * never read as "no children" — and `.entries` carries the lines that DID
54
+ * parse, so a caller can warn and keep going on the valid portion. */
55
+ export class DamagedSpawnsError extends Error {
56
+ constructor(file, bad, entries) {
57
+ super(`spawns file ${file} is damaged: ${bad} unreadable line(s) — refusing to read it as if it were fine`);
58
+ this.name = 'DamagedSpawnsError';
59
+ this.file = file;
60
+ this.bad = bad;
61
+ this.entries = entries;
62
+ }
63
+ }
64
+
65
+ /** Read one registry file. Missing file → [] (a fresh workspace is not damage).
66
+ * Any line that fails to parse or pass the schema → DamagedSpawnsError naming
67
+ * the file (the entwurf crash-don't-warn rule for malformed records). */
68
+ export function readSpawns(file) {
69
+ let raw;
70
+ try { raw = readFileSync(file, 'utf8'); }
71
+ catch (error) {
72
+ if (error?.code === 'ENOENT') return [];
73
+ throw error;
74
+ }
75
+ const entries = [];
76
+ let bad = 0;
77
+ for (const line of raw.split('\n')) {
78
+ if (!line) continue;
79
+ let parsed = null;
80
+ try { parsed = JSON.parse(line); } catch { parsed = null; }
81
+ if (valid(parsed)) entries.push(parsed); else bad++;
82
+ }
83
+ if (bad > 0) throw new DamagedSpawnsError(file, bad, entries);
84
+ return entries;
85
+ }
86
+
87
+ /** Warn-and-continue read for callers that must survive a damaged scope: prints
88
+ * the named error once, returns the valid records. Writers use this so one
89
+ * corrupt line compacts away on the next rewrite instead of blocking work. */
90
+ export function readSpawnsTolerant(file, warn = console.error) {
91
+ try {
92
+ return readSpawns(file);
93
+ } catch (error) {
94
+ if (error instanceof DamagedSpawnsError) {
95
+ warn(`[spawn-registry] ${error.message}; continuing with ${error.entries.length} valid record(s)`);
96
+ return error.entries;
97
+ }
98
+ throw error;
99
+ }
100
+ }
101
+
102
+ // Byte-stable output (state.rs:471): dedupe by id happens at the call sites,
103
+ // the write sorts by (ts, id) so the same records always produce the same
104
+ // bytes regardless of which process or insertion order produced them.
105
+ const byStableKey = (a, b) =>
106
+ a.ts !== b.ts ? (a.ts < b.ts ? -1 : 1) : (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
107
+
108
+ let tempSeq = 0;
109
+ function write(file, entries) {
110
+ mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
111
+ const body = [...entries].sort(byStableKey).map(entry => JSON.stringify(entry)).join('\n') + '\n';
112
+ // O_EXCL ('wx') + an unguessable name: a symlink somebody planted at a
113
+ // guessable temp path cannot be written through (state.rs:700-718). The
114
+ // rename then replaces the registry atomically — no window where it is gone.
115
+ const tmp = `${file}.tmp-${process.pid}-${tempSeq++}-${randomBytes(8).toString('hex')}`;
116
+ const fd = openSync(tmp, 'wx', 0o600);
117
+ try {
118
+ writeFileSync(fd, body);
119
+ } catch (error) {
120
+ try { unlinkSync(tmp); } catch { /* best effort cleanup */ }
121
+ throw error;
122
+ } finally {
123
+ try { closeSync(fd); } catch { /* already closed on the error path */ }
124
+ }
125
+ try {
126
+ renameSync(tmp, file);
127
+ } catch (error) {
128
+ try { unlinkSync(tmp); } catch { /* best effort cleanup */ }
129
+ throw error;
130
+ }
131
+ }
132
+
133
+ /** Record (or supersede) one spawn. The same id updates in place. */
134
+ export function recordSpawn(base, spawn) {
135
+ if (!valid(spawn)) throw new Error('invalid_spawn');
136
+ const file = spawnsFile(base);
137
+ let entries = readSpawnsTolerant(file).filter(entry => entry.id !== spawn.id);
138
+ entries.push({ status: 'running', ...spawn });
139
+ if (entries.length > COMPACT_AT) entries = entries.slice(-MAX_SPAWNS);
140
+ write(file, entries);
141
+ return { status: 'running', ...spawn };
142
+ }
143
+
144
+ /** Patch one record by id (restore/retire); returns the updated record or null. */
145
+ export function updateSpawn(base, id, patch) {
146
+ const file = spawnsFile(base);
147
+ const entries = readSpawnsTolerant(file);
148
+ let updated = null;
149
+ for (let i = 0; i < entries.length; i++) {
150
+ if (entries[i].id !== id) continue;
151
+ entries[i] = { ...entries[i], ...patch };
152
+ updated = entries[i];
153
+ }
154
+ if (!updated) return null;
155
+ write(file, entries);
156
+ return updated;
157
+ }
158
+
159
+ /** Find one record by id or name. A name can be shadowed by an orphan record
160
+ * (the 22/11 bug: a dead pane's newer record hid the live one), so when the
161
+ * caller passes the pane ids herdr still lists, a LIVE record wins over a
162
+ * newer orphan; ties and the no-liveness case resolve to the newest ts. An
163
+ * exact id is unique and never tie-broken. */
164
+ export function findSpawn(base, ref, livePaneIds = null) {
165
+ const entries = readSpawnsTolerant(spawnsFile(base));
166
+ const byId = entries.find(entry => entry.id === ref);
167
+ if (byId) return byId;
168
+ const named = entries.filter(entry => entry.name === ref);
169
+ if (!named.length) return null;
170
+ const live = livePaneIds ? new Set(livePaneIds) : null;
171
+ const liveNamed = live ? named.filter(entry => entry.paneId && live.has(entry.paneId)) : [];
172
+ const pool = liveNamed.length ? liveNamed : named;
173
+ return pool.reduce((best, entry) => (best && best.ts >= entry.ts ? best : entry), null);
174
+ }
175
+
176
+ /** Record an agent declaring its OWN parent (no controller pane involved). */
177
+ export function recordSelf(base, self) {
178
+ if (!self?.sessionId || !UUID.test(self.sessionId) || typeof self.parentSessionId !== 'string'
179
+ || !self.parentSessionId || !SPAWN_NAME.test(self.name ?? '')) throw new Error('invalid_self');
180
+ // One-id chain: a self-registered child serves as its own receipt request id.
181
+ // Scoped to the workspace the child runs in, so a controller sitting there sees it.
182
+ const spawn = { status: 'self', ...self, id: self.id ?? self.sessionId, self: true };
183
+ if (!valid(spawn)) throw new Error('invalid_self');
184
+ const file = spawnsFileFor(base, self.cwd ?? process.cwd());
185
+ let entries = readSpawnsTolerant(file).filter(entry => entry.id !== spawn.id);
186
+ entries.push(spawn);
187
+ if (entries.length > COMPACT_AT) entries = entries.slice(-MAX_SPAWNS);
188
+ write(file, entries);
189
+ return spawn;
190
+ }
191
+
192
+ export function spawnBase() {
193
+ const base = journalBase(); // same override + scope rules as the receipt journal
194
+ // state.rs:613-650: refuse a state dir another uid owns or that OTHERS can
195
+ // write into — the dir is NAMED in the error, never hidden behind a generic
196
+ // failure. Group-writable is the lenient direction the original teaches
197
+ // (state.rs:655): a dir we own gets tightened in place; one that cannot be
198
+ // tightened is refused exactly like a foreign dir. Without a POSIX uid/mode
199
+ // model (non-POSIX platforms) there is nothing equivalent to check.
200
+ if (typeof process.geteuid !== 'function') return base;
201
+ mkdirSync(base, { recursive: true, mode: 0o700 });
202
+ const meta = statSync(base);
203
+ const me = process.geteuid();
204
+ if (meta.uid !== me)
205
+ throw new Error(`spawn dir ${base} belongs to uid ${meta.uid}, not uid ${me} — refusing to use it`);
206
+ const mode = meta.mode & 0o777;
207
+ if (mode & 0o002)
208
+ throw new Error(`spawn dir ${base} has mode ${mode.toString(8).padStart(4, '0')} — others can write in it; refusing to use it`);
209
+ if (mode & 0o020) {
210
+ chmodSync(base, mode & ~0o020); // auto-tighten our own dir (state.rs:655)
211
+ const after = statSync(base).mode & 0o777;
212
+ if (after & 0o022)
213
+ throw new Error(`spawn dir ${base} has mode ${after.toString(8).padStart(4, '0')} — group/others can write in it and it cannot be tightened; refusing to use it`);
214
+ }
215
+ return base;
216
+ }