niahere 0.4.6 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agent/backends/claude-normalize.ts +5 -6
- package/src/agent/backends/claude.ts +6 -4
- package/src/agent/backends/codex-normalize.ts +19 -3
- package/src/agent/backends/codex.ts +100 -35
- package/src/agent/chain.ts +60 -0
- package/src/agent/failure.ts +90 -0
- package/src/agent/health.ts +41 -0
- package/src/agent/index.ts +3 -1
- package/src/agent/mcp-endpoint.ts +5 -3
- package/src/agent/models.ts +59 -0
- package/src/agent/registry.ts +54 -29
- package/src/agent/types.ts +9 -4
- package/src/channels/common/chat-session.ts +60 -0
- package/src/channels/phone/consult.ts +2 -1
- package/src/channels/phone/index.ts +5 -3
- package/src/channels/phone/relay.ts +15 -5
- package/src/channels/slack.ts +24 -34
- package/src/channels/sms.ts +16 -36
- package/src/channels/telegram.ts +22 -34
- package/src/channels/twilio/media-cache.ts +4 -3
- package/src/channels/twilio/rest.ts +0 -31
- package/src/channels/twilio/server.ts +0 -5
- package/src/channels/twilio/shared.ts +36 -0
- package/src/channels/whatsapp.ts +24 -58
- package/src/chat/engine.ts +83 -34
- package/src/chat/gap-marker.ts +63 -0
- package/src/chat/repl.ts +5 -5
- package/src/cli/config.ts +71 -0
- package/src/cli/index.ts +23 -288
- package/src/cli/job.ts +1 -2
- package/src/cli/logs.ts +55 -0
- package/src/cli/run.ts +74 -0
- package/src/cli/skills.ts +13 -0
- package/src/cli/status.ts +0 -1
- package/src/cli/test.ts +32 -0
- package/src/cli/update.ts +79 -0
- package/src/commands/backup.ts +1 -2
- package/src/commands/init.ts +1 -2
- package/src/commands/validate.ts +17 -13
- package/src/core/alive.ts +6 -5
- package/src/core/consolidator.ts +78 -31
- package/src/core/daemon.ts +3 -2
- package/src/core/finalizer.ts +9 -5
- package/src/core/runner.ts +64 -47
- package/src/core/scheduler.ts +19 -4
- package/src/core/skills.ts +0 -4
- package/src/db/migrations/017_sessions_consolidated_count.ts +9 -0
- package/src/db/models/active_engine.ts +0 -7
- package/src/db/models/job.ts +3 -2
- package/src/db/models/message.ts +10 -0
- package/src/db/models/session.ts +12 -0
- package/src/mcp/gate.ts +46 -0
- package/src/mcp/server.ts +2 -1
- package/src/mcp/tools/misc.ts +2 -1
- package/src/mcp/tools/send.ts +5 -4
- package/src/types/config.ts +4 -4
- package/src/utils/config.ts +6 -8
- package/src/utils/errors.ts +26 -0
- package/src/utils/retry.ts +0 -45
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "niahere",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "A personal AI assistant daemon — chat, scheduled jobs, persona system, extensible via skills.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"typecheck": "tsc --noEmit",
|
|
14
14
|
"check:cycles": "madge --circular --extensions ts src/",
|
|
15
15
|
"seed": "bun run src/db/seed.ts",
|
|
16
|
-
"release": "npm version patch && npm publish && git push"
|
|
16
|
+
"release": "npm version patch && npm publish && git push --follow-tags"
|
|
17
17
|
},
|
|
18
18
|
"keywords": [
|
|
19
19
|
"ai",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AgentEvent, Normalizer } from "../types";
|
|
2
2
|
import { truncate } from "../../utils/format-activity";
|
|
3
|
-
import {
|
|
3
|
+
import { isRetryable, scopeOf, parseFailure } from "../failure";
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Pure reducer: Claude Agent SDK messages → normalized `AgentEvent`s.
|
|
@@ -120,14 +120,13 @@ export class SdkNormalizer implements Normalizer {
|
|
|
120
120
|
};
|
|
121
121
|
}
|
|
122
122
|
const raw = (msg.errors?.join(", ") as string) || "unknown error";
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
// - providerDown: blank/"unknown error" → the provider is down → failover.
|
|
123
|
+
// A transient error carries no scope yet — it only becomes one once the
|
|
124
|
+
// session has burned its retries.
|
|
126
125
|
return {
|
|
127
126
|
type: "error",
|
|
128
127
|
message: raw,
|
|
129
|
-
retryable:
|
|
130
|
-
|
|
128
|
+
retryable: isRetryable(raw),
|
|
129
|
+
failover: scopeOf(parseFailure(raw), "provider"),
|
|
131
130
|
terminalReason: msg.terminal_reason,
|
|
132
131
|
};
|
|
133
132
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
2
|
+
import { asError } from "../../utils/errors";
|
|
2
3
|
import { randomUUID } from "crypto";
|
|
3
4
|
import { existsSync } from "fs";
|
|
4
5
|
import { join } from "path";
|
|
@@ -126,7 +127,7 @@ class ClaudeSession implements AgentSession {
|
|
|
126
127
|
res = await this.iterator!.next();
|
|
127
128
|
} catch (err) {
|
|
128
129
|
if (this.aborted) throw new Error(this.aborted);
|
|
129
|
-
throw
|
|
130
|
+
throw asError(err);
|
|
130
131
|
}
|
|
131
132
|
if (this.aborted) throw new Error(this.aborted);
|
|
132
133
|
if (res.done) {
|
|
@@ -151,11 +152,12 @@ class ClaudeSession implements AgentSession {
|
|
|
151
152
|
retry = true;
|
|
152
153
|
break;
|
|
153
154
|
}
|
|
154
|
-
// A retryable error that survived
|
|
155
|
-
//
|
|
155
|
+
// A retryable error that survived every retry is a capacity problem
|
|
156
|
+
// with THIS model, not proof the provider is gone — scope it to the
|
|
157
|
+
// model so a second model on the same provider still gets its turn.
|
|
156
158
|
const out =
|
|
157
159
|
ev.type === "error" && ev.retryable && this.retryCount >= MAX_SEND_RETRIES
|
|
158
|
-
? { ...ev,
|
|
160
|
+
? { ...ev, failover: ev.failover ?? ("model" as const) }
|
|
159
161
|
: ev;
|
|
160
162
|
yield out;
|
|
161
163
|
if (out.type === "result" || out.type === "error") {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AgentEvent, Normalizer } from "../types";
|
|
2
2
|
import { truncate } from "../../utils/format-activity";
|
|
3
|
+
import { scopeOf, parseFailure } from "../failure";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Pure reducer: Codex `codex exec --json` JSONL events → normalized `AgentEvent`s.
|
|
@@ -7,13 +8,15 @@ import { truncate } from "../../utils/format-activity";
|
|
|
7
8
|
* Codex is batch (no token streaming): the assistant message arrives whole in a
|
|
8
9
|
* single `item.completed`/`agent_message`, and `turn.completed` carries token
|
|
9
10
|
* usage. So `text` is emitted once (full), then `result` on `turn.completed`.
|
|
10
|
-
* No I/O — the session that drives it owns process lifecycle.
|
|
11
|
-
*
|
|
12
|
-
* items are
|
|
11
|
+
* No I/O — the session that drives it owns process lifecycle. A failed turn
|
|
12
|
+
* arrives as a top-level `error` and/or `turn.failed` carrying the upstream
|
|
13
|
+
* message; `error` *items* are non-fatal warnings (service tier, model metadata,
|
|
14
|
+
* skill budget) and are dropped.
|
|
13
15
|
*/
|
|
14
16
|
export class CodexNormalizer implements Normalizer {
|
|
15
17
|
private threadId = "";
|
|
16
18
|
private agentText = "";
|
|
19
|
+
private failed = false;
|
|
17
20
|
|
|
18
21
|
get backendSessionId(): string {
|
|
19
22
|
return this.threadId;
|
|
@@ -28,6 +31,10 @@ export class CodexNormalizer implements Normalizer {
|
|
|
28
31
|
case "item.started":
|
|
29
32
|
case "item.completed":
|
|
30
33
|
return this.consumeItem(e.type === "item.completed", e.item);
|
|
34
|
+
case "error":
|
|
35
|
+
return this.fail(typeof e.message === "string" ? e.message : "");
|
|
36
|
+
case "turn.failed":
|
|
37
|
+
return this.fail(typeof e.error?.message === "string" ? e.error.message : "");
|
|
31
38
|
case "turn.completed":
|
|
32
39
|
return [
|
|
33
40
|
{
|
|
@@ -47,6 +54,15 @@ export class CodexNormalizer implements Normalizer {
|
|
|
47
54
|
}
|
|
48
55
|
}
|
|
49
56
|
|
|
57
|
+
/** Codex reports the same failure twice (`error` then `turn.failed`); only the
|
|
58
|
+
* first ends the turn. */
|
|
59
|
+
private fail(raw: string): AgentEvent[] {
|
|
60
|
+
if (this.failed) return [];
|
|
61
|
+
this.failed = true;
|
|
62
|
+
const failure = parseFailure(raw);
|
|
63
|
+
return [{ type: "error", message: failure.message, retryable: false, failover: scopeOf(failure) }];
|
|
64
|
+
}
|
|
65
|
+
|
|
50
66
|
private consumeItem(completed: boolean, item: any): AgentEvent[] {
|
|
51
67
|
if (!item) return [];
|
|
52
68
|
switch (item.type) {
|
|
@@ -6,7 +6,8 @@ import type { Attachment } from "../../types/attachment";
|
|
|
6
6
|
import type { McpSourceContext } from "../../mcp";
|
|
7
7
|
import { CodexNormalizer } from "./codex-normalize";
|
|
8
8
|
import { mintRun, revokeRun } from "../mcp-endpoint";
|
|
9
|
-
import {
|
|
9
|
+
import { scopeOf, parseFailure } from "../failure";
|
|
10
|
+
import { ignore } from "../../utils/errors";
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Resolve the codex binary's absolute path. The daemon runs under launchd with a
|
|
@@ -42,21 +43,30 @@ export interface CliProc {
|
|
|
42
43
|
}
|
|
43
44
|
export type SpawnFn = (args: string[], opts: { cwd: string; env: Record<string, string> }) => CliProc;
|
|
44
45
|
|
|
45
|
-
//
|
|
46
|
-
// authenticates via its own
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
53
|
-
"
|
|
46
|
+
// An allowlist, so a newly-added secret is excluded by default rather than
|
|
47
|
+
// leaking to a third-party subprocess. Codex authenticates via its own
|
|
48
|
+
// ~/.codex login, so it needs no credential of ours.
|
|
49
|
+
const ENV_ALLOW = new Set([
|
|
50
|
+
"PATH",
|
|
51
|
+
"HOME",
|
|
52
|
+
"USER",
|
|
53
|
+
"LOGNAME",
|
|
54
|
+
"SHELL",
|
|
55
|
+
"TERM",
|
|
56
|
+
"TMPDIR",
|
|
57
|
+
"TZ",
|
|
58
|
+
"LANG",
|
|
59
|
+
"XDG_CONFIG_HOME",
|
|
60
|
+
"XDG_CACHE_HOME",
|
|
61
|
+
"XDG_DATA_HOME",
|
|
54
62
|
]);
|
|
63
|
+
const ENV_ALLOW_PREFIX = ["LC_", "CODEX_"];
|
|
55
64
|
|
|
56
|
-
function
|
|
65
|
+
function subprocessEnv(extra: Record<string, string>): Record<string, string> {
|
|
57
66
|
const env: Record<string, string> = {};
|
|
58
67
|
for (const [k, v] of Object.entries(process.env)) {
|
|
59
|
-
if (
|
|
68
|
+
if (v == null) continue;
|
|
69
|
+
if (ENV_ALLOW.has(k) || ENV_ALLOW_PREFIX.some((p) => k.startsWith(p))) env[k] = v;
|
|
60
70
|
}
|
|
61
71
|
return { ...env, ...extra };
|
|
62
72
|
}
|
|
@@ -76,8 +86,9 @@ function defaultSpawn(args: string[], opts: { cwd: string; env: Record<string, s
|
|
|
76
86
|
};
|
|
77
87
|
}
|
|
78
88
|
|
|
79
|
-
|
|
80
|
-
|
|
89
|
+
/** Takes the reader rather than the stream so the caller can cancel a read that
|
|
90
|
+
* would otherwise never resolve. */
|
|
91
|
+
async function* readLines(reader: ReadableStreamDefaultReader<Uint8Array>): AsyncGenerator<string> {
|
|
81
92
|
const decoder = new TextDecoder();
|
|
82
93
|
let buf = "";
|
|
83
94
|
while (true) {
|
|
@@ -93,28 +104,53 @@ async function* readLines(stream: ReadableStream<Uint8Array>): AsyncGenerator<st
|
|
|
93
104
|
if (buf.trim()) yield buf;
|
|
94
105
|
}
|
|
95
106
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
}
|
|
105
|
-
|
|
107
|
+
const IDLE = Symbol("idle");
|
|
108
|
+
|
|
109
|
+
/** A cancellable deadline, used to race a read that may never resolve. */
|
|
110
|
+
function idleAfter(ms: number): { promise: Promise<typeof IDLE>; cancel: () => void } {
|
|
111
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
112
|
+
const promise = new Promise<typeof IDLE>((resolve) => {
|
|
113
|
+
timer = setTimeout(() => resolve(IDLE), ms);
|
|
114
|
+
});
|
|
115
|
+
return { promise, cancel: () => clearTimeout(timer) };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const STDERR_CAP = 16_000;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Consume stderr to EOF, keeping only the tail. Must run alongside the process:
|
|
122
|
+
* an undrained pipe blocks the child once its buffer fills.
|
|
123
|
+
*/
|
|
124
|
+
function drainStderr(stream: ReadableStream<Uint8Array>): Promise<string> {
|
|
125
|
+
return (async () => {
|
|
126
|
+
let out = "";
|
|
127
|
+
const decoder = new TextDecoder();
|
|
128
|
+
const reader = stream.getReader();
|
|
129
|
+
while (true) {
|
|
130
|
+
const { value, done } = await reader.read();
|
|
131
|
+
if (done) break;
|
|
132
|
+
out += decoder.decode(value, { stream: true });
|
|
133
|
+
if (out.length > STDERR_CAP) out = out.slice(-STDERR_CAP);
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
})().catch(() => "");
|
|
106
137
|
}
|
|
107
138
|
|
|
139
|
+
/** A codex that emits nothing for this long is wedged, not working. */
|
|
140
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 10 * 60 * 1000;
|
|
141
|
+
|
|
108
142
|
export class CodexBackend implements AgentBackend {
|
|
109
143
|
readonly name = "codex" as const;
|
|
110
144
|
private spawnFn: SpawnFn;
|
|
145
|
+
private idleTimeoutMs: number;
|
|
111
146
|
|
|
112
|
-
constructor(deps?: { spawnFn?: SpawnFn }) {
|
|
147
|
+
constructor(deps?: { spawnFn?: SpawnFn; idleTimeoutMs?: number }) {
|
|
113
148
|
this.spawnFn = deps?.spawnFn ?? defaultSpawn;
|
|
149
|
+
this.idleTimeoutMs = deps?.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
114
150
|
}
|
|
115
151
|
|
|
116
152
|
async openSession(ctx: AgentSessionContext): Promise<AgentSession> {
|
|
117
|
-
return new CodexSession(ctx, this.spawnFn);
|
|
153
|
+
return new CodexSession(ctx, this.spawnFn, this.idleTimeoutMs);
|
|
118
154
|
}
|
|
119
155
|
|
|
120
156
|
async canResume(): Promise<boolean> {
|
|
@@ -127,10 +163,12 @@ class CodexSession implements AgentSession {
|
|
|
127
163
|
private _sessionId: string | null = null;
|
|
128
164
|
private aborted: string | null = null;
|
|
129
165
|
private proc: CliProc | null = null;
|
|
166
|
+
private idledOut = false;
|
|
130
167
|
|
|
131
168
|
constructor(
|
|
132
169
|
private ctx: AgentSessionContext,
|
|
133
170
|
private spawnFn: SpawnFn,
|
|
171
|
+
private idleTimeoutMs: number,
|
|
134
172
|
) {}
|
|
135
173
|
|
|
136
174
|
get backendSessionId(): string | null {
|
|
@@ -157,15 +195,32 @@ class CodexSession implements AgentSession {
|
|
|
157
195
|
];
|
|
158
196
|
if (this.ctx.model && this.ctx.model !== "default") args.push("-m", this.ctx.model);
|
|
159
197
|
|
|
160
|
-
const proc = this.spawnFn(args, { cwd: this.ctx.cwd, env:
|
|
198
|
+
const proc = this.spawnFn(args, { cwd: this.ctx.cwd, env: subprocessEnv({ NIA_MCP_TOKEN: token }) });
|
|
161
199
|
this.proc = proc;
|
|
162
200
|
|
|
201
|
+
// Started now, not after exit: an undrained pipe blocks the child.
|
|
202
|
+
const stderr = drainStderr(proc.stderr);
|
|
203
|
+
|
|
163
204
|
const normalizer = new CodexNormalizer();
|
|
164
|
-
|
|
205
|
+
const stdout = proc.stdout.getReader();
|
|
206
|
+
const lines = readLines(stdout)[Symbol.asyncIterator]();
|
|
207
|
+
let sawTerminal = false;
|
|
165
208
|
try {
|
|
166
|
-
|
|
209
|
+
while (true) {
|
|
210
|
+
// Race the read rather than trusting kill() to close the pipe — a child
|
|
211
|
+
// that ignores the signal would otherwise hang the run forever.
|
|
212
|
+
const idle = idleAfter(this.idleTimeoutMs);
|
|
213
|
+
const next = await Promise.race([lines.next(), idle.promise]);
|
|
214
|
+
idle.cancel();
|
|
215
|
+
if (next === IDLE) {
|
|
216
|
+
this.idledOut = true;
|
|
217
|
+
proc.kill();
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
if (next.done) break;
|
|
221
|
+
|
|
167
222
|
if (this.aborted) throw new Error(this.aborted);
|
|
168
|
-
const trimmed =
|
|
223
|
+
const trimmed = next.value.trim();
|
|
169
224
|
if (!trimmed) continue;
|
|
170
225
|
let parsed: unknown;
|
|
171
226
|
try {
|
|
@@ -177,22 +232,32 @@ class CodexSession implements AgentSession {
|
|
|
177
232
|
if (ev.type === "session" || ev.type === "result") {
|
|
178
233
|
this._sessionId = ev.backendSessionId || this._sessionId;
|
|
179
234
|
}
|
|
180
|
-
if (ev.type === "result")
|
|
235
|
+
if (ev.type === "result" || ev.type === "error") sawTerminal = true;
|
|
181
236
|
yield ev;
|
|
182
237
|
}
|
|
183
238
|
}
|
|
239
|
+
if (this.idledOut) {
|
|
240
|
+
yield {
|
|
241
|
+
type: "error",
|
|
242
|
+
message: `codex produced no output for ${Math.round(this.idleTimeoutMs / 1000)}s`,
|
|
243
|
+
retryable: false,
|
|
244
|
+
failover: "provider",
|
|
245
|
+
};
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
184
248
|
const exit = await proc.exited;
|
|
185
249
|
if (this.aborted) throw new Error(this.aborted);
|
|
186
|
-
if (exit !== 0 && !
|
|
187
|
-
const
|
|
250
|
+
if (exit !== 0 && !sawTerminal) {
|
|
251
|
+
const text = await stderr;
|
|
188
252
|
yield {
|
|
189
253
|
type: "error",
|
|
190
|
-
message:
|
|
254
|
+
message: text.trim() || `codex exited ${exit}`,
|
|
191
255
|
retryable: false,
|
|
192
|
-
|
|
256
|
+
failover: scopeOf(parseFailure(text), "provider"),
|
|
193
257
|
};
|
|
194
258
|
}
|
|
195
259
|
} finally {
|
|
260
|
+
await ignore(stdout.cancel(), "cancel codex stdout reader");
|
|
196
261
|
revokeRun(token);
|
|
197
262
|
this.proc = null;
|
|
198
263
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { AgentBackend, FailoverScope } from "./types";
|
|
2
|
+
import { providerHealth, type ProviderHealth } from "./health";
|
|
3
|
+
|
|
4
|
+
/** One attempt: a backend paired with the model to run it on. */
|
|
5
|
+
export interface ChainEntry {
|
|
6
|
+
backend: AgentBackend;
|
|
7
|
+
/** Absent → let the backend pick its own default. */
|
|
8
|
+
model?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** `provider:model` for logs. */
|
|
12
|
+
export function describeEntry(entry: ChainEntry): string {
|
|
13
|
+
return `${entry.backend.name}:${entry.model ?? "default"}`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Walks the resolved chain. A model-scoped failure advances one entry — possibly
|
|
18
|
+
* the same provider on another model; a provider-scoped one writes that provider
|
|
19
|
+
* off for the rest of the run.
|
|
20
|
+
*/
|
|
21
|
+
export class ChainCursor {
|
|
22
|
+
private index: number;
|
|
23
|
+
private readonly down = new Set<string>();
|
|
24
|
+
|
|
25
|
+
constructor(
|
|
26
|
+
private readonly chain: ChainEntry[],
|
|
27
|
+
private readonly health: ProviderHealth = providerHealth,
|
|
28
|
+
) {
|
|
29
|
+
// Skip a provider that recently failed, but never strand the run: if every
|
|
30
|
+
// provider is cooling down, start at the head and let it try.
|
|
31
|
+
const usable = chain.findIndex((e) => !health.isDown(e.backend.name));
|
|
32
|
+
this.index = usable === -1 ? 0 : usable;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
get current(): ChainEntry | undefined {
|
|
36
|
+
return this.chain[this.index];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
get atHead(): boolean {
|
|
40
|
+
return this.index === 0;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Position in the chain, for deciding whether a fresh cursor would differ. */
|
|
44
|
+
get entry(): ChainEntry | undefined {
|
|
45
|
+
return this.chain[this.index];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Record the failure and move on. Undefined when nothing usable is left. */
|
|
49
|
+
advance(scope: FailoverScope): ChainEntry | undefined {
|
|
50
|
+
if (scope === "provider") {
|
|
51
|
+
const name = this.chain[this.index]!.backend.name;
|
|
52
|
+
this.down.add(name);
|
|
53
|
+
this.health.markDown(name);
|
|
54
|
+
}
|
|
55
|
+
const next = this.chain.findIndex((e, i) => i > this.index && !this.down.has(e.backend.name));
|
|
56
|
+
if (next === -1) return undefined;
|
|
57
|
+
this.index = next;
|
|
58
|
+
return this.chain[next];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { FailoverScope } from "./types";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Failure classification shared by every backend adapter.
|
|
5
|
+
*
|
|
6
|
+
* Prefer the structured signal. Agents talk about logins, credentials and API
|
|
7
|
+
* keys all day, and the codex binary bundles an AWS SDK that mentions them too,
|
|
8
|
+
* so prose is only consulted when there is no HTTP status to go on — and the
|
|
9
|
+
* patterns that read prose are anchored to failure wording rather than to the
|
|
10
|
+
* bare nouns.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const RETRYABLE = [/\b500\b/i, /internal server error/i, /overloaded/i, /529/, /rate limit/i];
|
|
14
|
+
|
|
15
|
+
const MODEL_SCOPED = [
|
|
16
|
+
/model .*not (found|supported|available|allowed)/i,
|
|
17
|
+
/(unknown|invalid|unsupported) model/i,
|
|
18
|
+
/model .*(does not exist|is not supported)/i,
|
|
19
|
+
/context ?window ?exceeded/i,
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
const PROVIDER_SCOPED = [
|
|
23
|
+
/not logged in/i,
|
|
24
|
+
/failed to (authenticate|authorize)/i,
|
|
25
|
+
/(authentication|authorization) (failed|required|error)/i,
|
|
26
|
+
/\b(authorizationfailed|authorizationrequired|noauthorizationsupport)\b/i,
|
|
27
|
+
/failed to (load|refresh)\b.{0,40}\b(credential|token|auth)/i,
|
|
28
|
+
/(session|token|refresh token|credentials) (has |have )?expired/i,
|
|
29
|
+
/\b(tokenexpired|tokenrefreshfailed|refreshtokenfailed|tokenexchangefailed)\b/i,
|
|
30
|
+
/invalid (api[ -]?key|credentials|token)/i,
|
|
31
|
+
/\b(401 unauthorized|403 forbidden)\b/i,
|
|
32
|
+
/\bstatus:? (401|403)\b/i,
|
|
33
|
+
/(usage limit reached|quota exceeded)/i,
|
|
34
|
+
/\b(usagelimitreached|quotaexceeded)\b/i,
|
|
35
|
+
/\b(econnrefused|enotfound|etimedout|econnreset)\b/i,
|
|
36
|
+
/connection (refused|reset|timed out)/i,
|
|
37
|
+
/network (error|unreachable)/i,
|
|
38
|
+
/failed to refresh available models/i,
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
export interface Failure {
|
|
42
|
+
message: string;
|
|
43
|
+
/** HTTP status, when the backend reported a structured error. */
|
|
44
|
+
status?: number;
|
|
45
|
+
/** Provider's own error type, e.g. `invalid_request_error`. */
|
|
46
|
+
type?: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Unwrap a JSON error envelope; plain text passes through. */
|
|
50
|
+
export function parseFailure(raw: string | null | undefined): Failure {
|
|
51
|
+
const text = (raw ?? "").trim();
|
|
52
|
+
try {
|
|
53
|
+
const parsed = JSON.parse(text);
|
|
54
|
+
const inner = parsed?.error?.message ?? parsed?.message;
|
|
55
|
+
return {
|
|
56
|
+
message: typeof inner === "string" && inner.trim() ? inner : text,
|
|
57
|
+
status: typeof parsed?.status === "number" ? parsed.status : undefined,
|
|
58
|
+
type: typeof parsed?.error?.type === "string" ? parsed.error.type : undefined,
|
|
59
|
+
};
|
|
60
|
+
} catch {
|
|
61
|
+
return { message: text };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Transient enough for the backend to retry in place. */
|
|
66
|
+
export function isRetryable(text: string | null | undefined): boolean {
|
|
67
|
+
const t = text?.trim();
|
|
68
|
+
return !!t && RETRYABLE.some((p) => p.test(t));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function scopeOfStatus(status: number, message: string): FailoverScope | undefined {
|
|
72
|
+
if (status === 404) return "model";
|
|
73
|
+
if (status === 401 || status === 403 || status === 408 || status === 429 || status >= 500) return "provider";
|
|
74
|
+
if (MODEL_SCOPED.some((p) => p.test(message))) return "model";
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* How far the chain should skip. `blank` is what an empty or opaque message
|
|
80
|
+
* implies: a run that reported nothing points at the provider, a structured
|
|
81
|
+
* error that merely lacks a message does not.
|
|
82
|
+
*/
|
|
83
|
+
export function scopeOf(failure: Failure, blank?: FailoverScope): FailoverScope | undefined {
|
|
84
|
+
const t = failure.message.trim();
|
|
85
|
+
if (!t || t.toLowerCase() === "unknown error") return blank;
|
|
86
|
+
if (failure.status !== undefined) return scopeOfStatus(failure.status, t);
|
|
87
|
+
if (MODEL_SCOPED.some((p) => p.test(t))) return "model";
|
|
88
|
+
if (PROVIDER_SCOPED.some((p) => p.test(t))) return "provider";
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-wide provider cooldown. Without it every job and every chat turn
|
|
3
|
+
* re-discovers the same outage from scratch, paying the full retry ladder before
|
|
4
|
+
* failing over; with it a provider that just failed is skipped until it has had
|
|
5
|
+
* time to recover, and picked up again once the cooldown lapses.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const DEFAULT_COOLDOWN_MS = 5 * 60 * 1000;
|
|
9
|
+
|
|
10
|
+
export interface ProviderHealth {
|
|
11
|
+
markDown(provider: string): void;
|
|
12
|
+
isDown(provider: string): boolean;
|
|
13
|
+
/** Test seam. */
|
|
14
|
+
clear(): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function createProviderHealth(
|
|
18
|
+
cooldownMs: number = DEFAULT_COOLDOWN_MS,
|
|
19
|
+
now: () => number = Date.now,
|
|
20
|
+
): ProviderHealth {
|
|
21
|
+
const downUntil = new Map<string, number>();
|
|
22
|
+
return {
|
|
23
|
+
markDown(provider) {
|
|
24
|
+
downUntil.set(provider, now() + cooldownMs);
|
|
25
|
+
},
|
|
26
|
+
isDown(provider) {
|
|
27
|
+
const until = downUntil.get(provider);
|
|
28
|
+
if (until === undefined) return false;
|
|
29
|
+
if (now() > until) {
|
|
30
|
+
downUntil.delete(provider);
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
return true;
|
|
34
|
+
},
|
|
35
|
+
clear() {
|
|
36
|
+
downUntil.clear();
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export const providerHealth = createProviderHealth();
|
package/src/agent/index.ts
CHANGED
|
@@ -8,5 +8,7 @@ export type {
|
|
|
8
8
|
Normalizer,
|
|
9
9
|
} from "./types";
|
|
10
10
|
export { isResultEvent } from "./types";
|
|
11
|
-
export {
|
|
11
|
+
export type { FailoverScope } from "./types";
|
|
12
|
+
export { getBackend, setBackend, setBackendChain, resolveChain, buildChain } from "./registry";
|
|
13
|
+
export { ChainCursor, describeEntry, type ChainEntry } from "./chain";
|
|
12
14
|
export { resolveSdkModel } from "./backends/claude";
|
|
@@ -4,6 +4,8 @@ import { randomBytes, randomUUID } from "crypto";
|
|
|
4
4
|
import type { NiaTool } from "../mcp/tools/types";
|
|
5
5
|
import type { McpSourceContext } from "../mcp";
|
|
6
6
|
import { log } from "../utils/log";
|
|
7
|
+
import { ignore } from "../utils/errors";
|
|
8
|
+
import { gateSideEffects } from "../mcp/gate";
|
|
7
9
|
|
|
8
10
|
/**
|
|
9
11
|
* Loopback MCP endpoint — how out-of-process CLI backends (Codex/Gemini) reach
|
|
@@ -33,7 +35,7 @@ let endpointTools: NiaTool[] = [];
|
|
|
33
35
|
/** Build a per-run MCP server whose tool closures bake in the frozen context. */
|
|
34
36
|
function buildRunServer(ctx: McpSourceContext, tools: NiaTool[]): McpServer {
|
|
35
37
|
const mcp = new McpServer({ name: "nia", version: "0.1.0" });
|
|
36
|
-
for (const t of tools) {
|
|
38
|
+
for (const t of gateSideEffects(tools)) {
|
|
37
39
|
mcp.registerTool(t.name, { description: t.description, inputSchema: t.schema }, async (args: unknown) => ({
|
|
38
40
|
content: [{ type: "text" as const, text: await t.handler(args, ctx) }],
|
|
39
41
|
}));
|
|
@@ -92,8 +94,8 @@ export function revokeRun(token: string): void {
|
|
|
92
94
|
const entry = runs.get(token);
|
|
93
95
|
if (!entry) return;
|
|
94
96
|
runs.delete(token);
|
|
95
|
-
entry.transport.close()
|
|
96
|
-
entry.server.close()
|
|
97
|
+
void ignore(entry.transport.close(), "close run transport");
|
|
98
|
+
void ignore(entry.server.close(), "close run server");
|
|
97
99
|
}
|
|
98
100
|
|
|
99
101
|
/** Test/diagnostic: number of live runs. */
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { BackendName } from "../types/config";
|
|
2
|
+
|
|
3
|
+
/** Model → provider resolution. Config names models, never backends, so one
|
|
4
|
+
* provider's model id can never reach another. */
|
|
5
|
+
|
|
6
|
+
export type ProviderName = BackendName;
|
|
7
|
+
|
|
8
|
+
export interface ModelRef {
|
|
9
|
+
provider: ProviderName;
|
|
10
|
+
/** Absent → let the provider pick. */
|
|
11
|
+
model?: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Order of the implicit cross-provider tail. */
|
|
15
|
+
export const PROVIDER_ORDER: readonly ProviderName[] = ["claude", "codex", "gemini"];
|
|
16
|
+
|
|
17
|
+
/** Selects a provider without naming a model. */
|
|
18
|
+
const BARE_PROVIDERS: Record<string, ProviderName> = {
|
|
19
|
+
default: "claude",
|
|
20
|
+
claude: "claude",
|
|
21
|
+
codex: "codex",
|
|
22
|
+
gemini: "gemini",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** Short names the provider understands directly. */
|
|
26
|
+
const ALIASES: Record<string, ProviderName> = {
|
|
27
|
+
sonnet: "claude",
|
|
28
|
+
opus: "claude",
|
|
29
|
+
opusplan: "claude",
|
|
30
|
+
haiku: "claude",
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const PREFIXES: [RegExp, ProviderName][] = [
|
|
34
|
+
[/^claude[-.]/, "claude"],
|
|
35
|
+
[/^(gpt|o[134]|codex)[-.]/, "codex"],
|
|
36
|
+
[/^gemini[-.]/, "gemini"],
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
/** Unrecognized names are assumed to be Claude's — a rejected model is a
|
|
40
|
+
* model-scoped failure, so a typo degrades to the next entry. */
|
|
41
|
+
export function resolveModel(name: string): ModelRef {
|
|
42
|
+
const model = name.trim();
|
|
43
|
+
const key = model.toLowerCase();
|
|
44
|
+
|
|
45
|
+
const bare = BARE_PROVIDERS[key];
|
|
46
|
+
if (bare) return { provider: bare };
|
|
47
|
+
|
|
48
|
+
const alias = ALIASES[key];
|
|
49
|
+
if (alias) return { provider: alias, model };
|
|
50
|
+
|
|
51
|
+
for (const [pattern, provider] of PREFIXES) {
|
|
52
|
+
if (pattern.test(key)) return { provider, model };
|
|
53
|
+
}
|
|
54
|
+
return { provider: "claude", model };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function providerDefault(provider: ProviderName): ModelRef {
|
|
58
|
+
return { provider };
|
|
59
|
+
}
|