niahere 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/chat/engine.ts +3 -0
- package/src/core/engine-guard.ts +29 -5
- package/src/core/runner.ts +3 -0
- package/src/db/models/active_engine.ts +49 -0
package/package.json
CHANGED
package/src/chat/engine.ts
CHANGED
|
@@ -291,6 +291,9 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
291
291
|
|
|
292
292
|
try {
|
|
293
293
|
for await (const ev of sess.send(userMessage, attachments)) {
|
|
294
|
+
// Keep the lease alive while the turn runs, so a slow reply is
|
|
295
|
+
// never mistaken for a crashed one.
|
|
296
|
+
void ignore(ActiveEngine.throttledTouch(room), "touch active-engine");
|
|
294
297
|
switch (ev.type) {
|
|
295
298
|
case "session": {
|
|
296
299
|
if (!sessionId || ev.backendSessionId !== sessionId) {
|
package/src/core/engine-guard.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { ActiveEngine } from "../db/models";
|
|
10
|
+
import { isStale, type ActiveEngine as ActiveEngineRow } from "../db/models/active_engine";
|
|
10
11
|
import { withDb } from "../db/with-db";
|
|
11
12
|
import { DIM, RESET, ICON_WARN } from "../utils/cli";
|
|
12
13
|
|
|
@@ -38,21 +39,41 @@ export function withDefaultWait(opts: GuardOptions, defaultWaitMinutes: number):
|
|
|
38
39
|
interface ActiveSummary {
|
|
39
40
|
count: number;
|
|
40
41
|
rooms: string[];
|
|
42
|
+
/** Rows nothing has pinged lately — ignored, but worth saying out loud. */
|
|
43
|
+
stale: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* A row only counts as work if something is still pinging it. Without this a
|
|
48
|
+
* crash — or a test pointed at the wrong database — leaves a row that blocks
|
|
49
|
+
* stop, restart and update indefinitely, including the restart whose startup
|
|
50
|
+
* would have cleared it.
|
|
51
|
+
*/
|
|
52
|
+
export function partitionEngines(
|
|
53
|
+
engines: ActiveEngineRow[],
|
|
54
|
+
now: number = Date.now(),
|
|
55
|
+
): { live: ActiveEngineRow[]; stale: ActiveEngineRow[] } {
|
|
56
|
+
const live: ActiveEngineRow[] = [];
|
|
57
|
+
const stale: ActiveEngineRow[] = [];
|
|
58
|
+
for (const e of engines) (isStale(e.lastPing, now) ? stale : live).push(e);
|
|
59
|
+
return { live, stale };
|
|
41
60
|
}
|
|
42
61
|
|
|
43
62
|
async function getActiveEngines(): Promise<ActiveSummary> {
|
|
44
63
|
let count = 0;
|
|
45
64
|
let rooms: string[] = [];
|
|
65
|
+
let stale = 0;
|
|
46
66
|
try {
|
|
47
67
|
await withDb(async () => {
|
|
48
|
-
const
|
|
49
|
-
count =
|
|
50
|
-
rooms =
|
|
68
|
+
const partitioned = partitionEngines(await ActiveEngine.list());
|
|
69
|
+
count = partitioned.live.length;
|
|
70
|
+
rooms = partitioned.live.map((e) => `${e.room} (${e.channel})`);
|
|
71
|
+
stale = partitioned.stale.length;
|
|
51
72
|
});
|
|
52
73
|
} catch {
|
|
53
74
|
// DB unreachable — no engines to worry about
|
|
54
75
|
}
|
|
55
|
-
return { count, rooms };
|
|
76
|
+
return { count, rooms, stale };
|
|
56
77
|
}
|
|
57
78
|
|
|
58
79
|
/**
|
|
@@ -62,7 +83,10 @@ async function getActiveEngines(): Promise<ActiveSummary> {
|
|
|
62
83
|
export async function guardActiveEngines(action: string, opts: GuardOptions): Promise<boolean> {
|
|
63
84
|
if (opts.force) return true;
|
|
64
85
|
|
|
65
|
-
const { count, rooms } = await getActiveEngines();
|
|
86
|
+
const { count, rooms, stale } = await getActiveEngines();
|
|
87
|
+
if (stale > 0) {
|
|
88
|
+
console.log(`${DIM}ignoring ${stale} stale engine row${stale > 1 ? "s" : ""} (no heartbeat)${RESET}`);
|
|
89
|
+
}
|
|
66
90
|
if (count === 0) return true;
|
|
67
91
|
|
|
68
92
|
// Active engines found
|
package/src/core/runner.ts
CHANGED
|
@@ -60,6 +60,9 @@ async function consumeBackendRun(
|
|
|
60
60
|
|
|
61
61
|
try {
|
|
62
62
|
for await (const ev of session.send(prompt)) {
|
|
63
|
+
// Keep the lease alive while the turn runs. Throttled, so a long job
|
|
64
|
+
// stays live without one write per event.
|
|
65
|
+
if (activeRoom) void ignore(ActiveEngine.throttledTouch(activeRoom), "touch active-engine");
|
|
63
66
|
if (ev.type === "thinking") onActivity?.(ev.delta);
|
|
64
67
|
else if (ev.type === "tool") onActivity?.(ev.summary ?? ev.name);
|
|
65
68
|
else if (ev.type === "result") {
|
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
import { getSql } from "../connection";
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* How often a running turn re-stamps its row. Called from the event loops
|
|
5
|
+
* rather than a timer on purpose: a timer outlives the work it describes, so a
|
|
6
|
+
* process that died mid-turn would keep its row looking alive. A loop that
|
|
7
|
+
* stops simply stops pinging.
|
|
8
|
+
*/
|
|
9
|
+
export const PING_INTERVAL_MS = 30_000;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Silence longer than this means the turn is gone, not slow — several missed
|
|
13
|
+
* pings, not one. Nothing read this column for a long time, so a row left
|
|
14
|
+
* behind by a crash (or a test aimed at the wrong database) counted as live
|
|
15
|
+
* work forever, and blocked the very restart that clears it.
|
|
16
|
+
*/
|
|
17
|
+
export const STALE_AFTER_MS = 3 * 60_000;
|
|
18
|
+
|
|
3
19
|
export interface ActiveEngine {
|
|
4
20
|
room: string;
|
|
5
21
|
channel: string;
|
|
@@ -21,12 +37,45 @@ export async function ping(room: string): Promise<void> {
|
|
|
21
37
|
await sql`UPDATE active_engines SET last_ping = NOW() WHERE room = ${room}`;
|
|
22
38
|
}
|
|
23
39
|
|
|
40
|
+
/** An unreadable timestamp counts as live: `--force` is the escape hatch, and
|
|
41
|
+
* killing real work is the worse mistake. */
|
|
42
|
+
export function isStale(lastPing: string, now: number = Date.now()): boolean {
|
|
43
|
+
const t = Date.parse(lastPing);
|
|
44
|
+
return Number.isFinite(t) ? now - t > STALE_AFTER_MS : false;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const lastTouch = new Map<string, number>();
|
|
48
|
+
|
|
49
|
+
export interface TouchDeps {
|
|
50
|
+
now?: number;
|
|
51
|
+
seen?: Map<string, number>;
|
|
52
|
+
ping?: (room: string) => Promise<void>;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Re-stamp a running turn, at most once per interval. Safe to call from a hot
|
|
56
|
+
* loop — the chat path fires per token. */
|
|
57
|
+
export async function throttledTouch(room: string, deps: TouchDeps = {}): Promise<void> {
|
|
58
|
+
const now = deps.now ?? Date.now();
|
|
59
|
+
const seen = deps.seen ?? lastTouch;
|
|
60
|
+
const previous = seen.get(room);
|
|
61
|
+
if (previous !== undefined && now - previous < PING_INTERVAL_MS) return;
|
|
62
|
+
seen.set(room, now);
|
|
63
|
+
await (deps.ping ?? ping)(room);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Forget a room's throttle state so a later turn pings immediately. */
|
|
67
|
+
export function forgetTouch(room: string): void {
|
|
68
|
+
lastTouch.delete(room);
|
|
69
|
+
}
|
|
70
|
+
|
|
24
71
|
export async function unregister(room: string): Promise<void> {
|
|
72
|
+
forgetTouch(room);
|
|
25
73
|
const sql = getSql();
|
|
26
74
|
await sql`DELETE FROM active_engines WHERE room = ${room}`;
|
|
27
75
|
}
|
|
28
76
|
|
|
29
77
|
export async function clearAll(): Promise<void> {
|
|
78
|
+
lastTouch.clear();
|
|
30
79
|
const sql = getSql();
|
|
31
80
|
await sql`DELETE FROM active_engines`;
|
|
32
81
|
}
|