dsh-daruma 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/lib/client/FailoverNoticeRow.d.ts +24 -0
- package/lib/client/FailoverNoticeRow.js +63 -0
- package/lib/client/FailoverNoticeRow.js.map +1 -0
- package/lib/client/failover-notice.d.ts +111 -0
- package/lib/client/failover-notice.js +147 -0
- package/lib/client/failover-notice.js.map +1 -0
- package/lib/client/health.d.ts +7 -5
- package/lib/client/health.js +7 -5
- package/lib/client/health.js.map +1 -1
- package/lib/client/index.js +38 -0
- package/lib/client/index.js.map +1 -1
- package/lib/client/locales.d.ts +8 -0
- package/lib/client/locales.js +8 -0
- package/lib/client/locales.js.map +1 -1
- package/lib/client.js +251 -12
- package/lib/client.js.map +1 -1
- package/lib/config.d.ts +3 -0
- package/lib/config.js +10 -2
- package/lib/config.js.map +1 -1
- package/lib/engine.d.ts +8 -0
- package/lib/engine.js +14 -1
- package/lib/engine.js.map +1 -1
- package/lib/failover-events.d.ts +43 -1
- package/lib/failover-events.js +15 -4
- package/lib/failover-events.js.map +1 -1
- package/lib/failover-log.d.ts +67 -0
- package/lib/failover-log.js +49 -0
- package/lib/failover-log.js.map +1 -0
- package/lib/index.d.ts +3 -2
- package/lib/index.js +93 -1
- package/lib/index.js.map +1 -1
- package/package.json +3 -3
package/lib/failover-events.d.ts
CHANGED
|
@@ -52,5 +52,47 @@ export interface FailoverEventInput {
|
|
|
52
52
|
/** Scope give-up budget. */
|
|
53
53
|
readonly giveUpBudget: number;
|
|
54
54
|
}
|
|
55
|
-
/** Assemble the durable failover event from decision-site facts.
|
|
55
|
+
/** Assemble the durable failover event from decision-site facts.
|
|
56
|
+
*
|
|
57
|
+
* Absent optional fields are omitted (not set to `undefined`): the host's
|
|
58
|
+
* session append validates data with a lossless-JSON snapshotter that rejects
|
|
59
|
+
* own properties whose value is `undefined`, so the event must carry only
|
|
60
|
+
* keys that actually have values.
|
|
61
|
+
*/
|
|
56
62
|
export declare function buildDarumaFailoverEvent(input: FailoverEventInput): DarumaFailoverEvent;
|
|
63
|
+
/** Why recovery gave up (mirrors the core engine's GIVE_UP reasons). */
|
|
64
|
+
export type DarumaGiveUpReason = 'give-up-budget-exhausted' | 'no-routable-fallback';
|
|
65
|
+
/** Durable session event appended when recovery gives up entirely. */
|
|
66
|
+
export interface DarumaGiveUpEvent {
|
|
67
|
+
/** Discriminator-compatible shape with {@link DarumaFailoverEvent}. */
|
|
68
|
+
readonly kind: 'give-up';
|
|
69
|
+
/** Channel id the final failure occurred on. */
|
|
70
|
+
readonly from: string;
|
|
71
|
+
/** {@link DarumaGiveUpReason}. */
|
|
72
|
+
readonly reason: DarumaGiveUpReason;
|
|
73
|
+
/** Epoch ms of the give-up decision. */
|
|
74
|
+
readonly at: number;
|
|
75
|
+
/** Agent whose recovery gave up (its scope id). */
|
|
76
|
+
readonly agentId: string;
|
|
77
|
+
/** Turn containing the final failed request (1-based). */
|
|
78
|
+
readonly turn: number;
|
|
79
|
+
/** Step containing the final failed request attempt (1-based). */
|
|
80
|
+
readonly step: number;
|
|
81
|
+
/** Scope failover count at the give-up decision. */
|
|
82
|
+
readonly failoverCount: number;
|
|
83
|
+
/** Scope give-up budget the count counted against. */
|
|
84
|
+
readonly giveUpBudget: number;
|
|
85
|
+
}
|
|
86
|
+
/** Event data required from the give-up decision site. */
|
|
87
|
+
export interface GiveUpEventInput {
|
|
88
|
+
readonly from: string;
|
|
89
|
+
readonly reason: DarumaGiveUpReason;
|
|
90
|
+
readonly at: number;
|
|
91
|
+
readonly agentId: string;
|
|
92
|
+
readonly turn: number;
|
|
93
|
+
readonly step: number;
|
|
94
|
+
readonly failoverCount: number;
|
|
95
|
+
readonly giveUpBudget: number;
|
|
96
|
+
}
|
|
97
|
+
/** Assemble the durable give-up event from decision-site facts. */
|
|
98
|
+
export declare function buildDarumaGiveUpEvent(input: GiveUpEventInput): DarumaGiveUpEvent;
|
package/lib/failover-events.js
CHANGED
|
@@ -16,9 +16,16 @@ function clip(value, limit) {
|
|
|
16
16
|
return value;
|
|
17
17
|
return `${value.slice(0, limit)}…`;
|
|
18
18
|
}
|
|
19
|
-
/** Assemble the durable failover event from decision-site facts.
|
|
19
|
+
/** Assemble the durable failover event from decision-site facts.
|
|
20
|
+
*
|
|
21
|
+
* Absent optional fields are omitted (not set to `undefined`): the host's
|
|
22
|
+
* session append validates data with a lossless-JSON snapshotter that rejects
|
|
23
|
+
* own properties whose value is `undefined`, so the event must carry only
|
|
24
|
+
* keys that actually have values.
|
|
25
|
+
*/
|
|
20
26
|
export function buildDarumaFailoverEvent(input) {
|
|
21
27
|
const { failure } = input;
|
|
28
|
+
const message = clip(failure.message, FAILOVER_MESSAGE_LIMIT);
|
|
22
29
|
return {
|
|
23
30
|
from: input.from,
|
|
24
31
|
to: input.to,
|
|
@@ -27,11 +34,15 @@ export function buildDarumaFailoverEvent(input) {
|
|
|
27
34
|
agentId: input.agentId,
|
|
28
35
|
turn: input.turn,
|
|
29
36
|
step: input.step,
|
|
30
|
-
status: failure.status,
|
|
31
|
-
requestId: failure.requestId,
|
|
32
|
-
message
|
|
37
|
+
...(failure.status !== undefined ? { status: failure.status } : {}),
|
|
38
|
+
...(failure.requestId !== undefined ? { requestId: failure.requestId } : {}),
|
|
39
|
+
...(message !== undefined ? { message } : {}),
|
|
33
40
|
failoverCount: input.failoverCount,
|
|
34
41
|
giveUpBudget: input.giveUpBudget,
|
|
35
42
|
};
|
|
36
43
|
}
|
|
44
|
+
/** Assemble the durable give-up event from decision-site facts. */
|
|
45
|
+
export function buildDarumaGiveUpEvent(input) {
|
|
46
|
+
return { kind: 'give-up', ...input };
|
|
47
|
+
}
|
|
37
48
|
//# sourceMappingURL=failover-events.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"failover-events.js","sourceRoot":"","sources":["../src/failover-events.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAgCH,mEAAmE;AACnE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAkBzC,SAAS,IAAI,CAAC,KAAyB,EAAE,KAAa;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,KAAK,CAAA;IACvC,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAA;AACpC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"failover-events.js","sourceRoot":"","sources":["../src/failover-events.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAgCH,mEAAmE;AACnE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAkBzC,SAAS,IAAI,CAAC,KAAyB,EAAE,KAAa;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,KAAK,CAAA;IACvC,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAA;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAyB;IAChE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;IACzB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAA;IAC7D,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,YAAY,EAAE,KAAK,CAAC,YAAY;KACjC,CAAA;AACH,CAAC;AAuCD,mEAAmE;AACnE,MAAM,UAAU,sBAAsB,CAAC,KAAuB;IAC5D,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,CAAA;AACtC,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only JSONL log of daruma's recovery decisions.
|
|
3
|
+
*
|
|
4
|
+
* The harness does not yet persist out-of-repo session events, and the web
|
|
5
|
+
* server's stdout (where the ctx.logger.warn lines go) is not captured to
|
|
6
|
+
* disk — so without this file a channel switch leaves **no auditable trace**.
|
|
7
|
+
* Every failover / give-up / boot is therefore appended here as one JSON
|
|
8
|
+
* line, giving post-hoc audits (like the 2026-09-08 one) a durable record.
|
|
9
|
+
*
|
|
10
|
+
* Failure semantics: logging must never break recovery. Every I/O error is
|
|
11
|
+
* swallowed; the worst case is a missing or truncated line, never a lost
|
|
12
|
+
* failover.
|
|
13
|
+
*/
|
|
14
|
+
/** Rotate when the log exceeds this size (kept to one `.1` generation). */
|
|
15
|
+
export declare const FAILOVER_LOG_ROTATE_BYTES: number;
|
|
16
|
+
/** One JSONL record. `kind` discriminates the payload shape. */
|
|
17
|
+
export type FailoverLogRecord = {
|
|
18
|
+
readonly kind: 'failover';
|
|
19
|
+
/** Epoch ms of the decision. */
|
|
20
|
+
readonly t: number;
|
|
21
|
+
readonly agentId: string;
|
|
22
|
+
/** Channel the request failed on. */
|
|
23
|
+
readonly from: string;
|
|
24
|
+
/** Channel recovery switched to. */
|
|
25
|
+
readonly to: string;
|
|
26
|
+
/** Stable failure code, e.g. `RATE_LIMIT`. */
|
|
27
|
+
readonly reason: string;
|
|
28
|
+
/** HTTP status of the failed request, when available. */
|
|
29
|
+
readonly status?: number;
|
|
30
|
+
/** Turn / step of the failed request. */
|
|
31
|
+
readonly turn?: number;
|
|
32
|
+
readonly step?: number;
|
|
33
|
+
/** Scope failover count AFTER this decision. */
|
|
34
|
+
readonly failoverCount?: number;
|
|
35
|
+
/** Scope give-up budget the count counts against. */
|
|
36
|
+
readonly giveUpBudget?: number;
|
|
37
|
+
} | {
|
|
38
|
+
readonly kind: 'give-up';
|
|
39
|
+
readonly t: number;
|
|
40
|
+
readonly agentId: string;
|
|
41
|
+
readonly from: string;
|
|
42
|
+
/** `give-up-budget-exhausted` or `no-routable-fallback`. */
|
|
43
|
+
readonly reason: string;
|
|
44
|
+
readonly status?: number;
|
|
45
|
+
readonly turn?: number;
|
|
46
|
+
readonly step?: number;
|
|
47
|
+
readonly failoverCount?: number;
|
|
48
|
+
readonly giveUpBudget?: number;
|
|
49
|
+
} | {
|
|
50
|
+
readonly kind: 'boot';
|
|
51
|
+
readonly t: number;
|
|
52
|
+
/** Plugin process id, to correlate with server restarts. */
|
|
53
|
+
readonly pid: number;
|
|
54
|
+
/** Configured chain channel ids, priority order. */
|
|
55
|
+
readonly channels: readonly string[];
|
|
56
|
+
/** Configured budgets, for interpreting later records. */
|
|
57
|
+
readonly failureBudget: number;
|
|
58
|
+
readonly cooldownMs: number;
|
|
59
|
+
readonly giveUpBudget: number;
|
|
60
|
+
};
|
|
61
|
+
export declare class JsonlFailoverLogStore {
|
|
62
|
+
private readonly file;
|
|
63
|
+
constructor(file: string);
|
|
64
|
+
/** Append one record; rotate first when over budget. Never throws. */
|
|
65
|
+
append(record: FailoverLogRecord): void;
|
|
66
|
+
private rotateIfNeeded;
|
|
67
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only JSONL log of daruma's recovery decisions.
|
|
3
|
+
*
|
|
4
|
+
* The harness does not yet persist out-of-repo session events, and the web
|
|
5
|
+
* server's stdout (where the ctx.logger.warn lines go) is not captured to
|
|
6
|
+
* disk — so without this file a channel switch leaves **no auditable trace**.
|
|
7
|
+
* Every failover / give-up / boot is therefore appended here as one JSON
|
|
8
|
+
* line, giving post-hoc audits (like the 2026-09-08 one) a durable record.
|
|
9
|
+
*
|
|
10
|
+
* Failure semantics: logging must never break recovery. Every I/O error is
|
|
11
|
+
* swallowed; the worst case is a missing or truncated line, never a lost
|
|
12
|
+
* failover.
|
|
13
|
+
*/
|
|
14
|
+
import { appendFileSync, renameSync, statSync } from 'node:fs';
|
|
15
|
+
import { mkdirSync } from 'node:fs';
|
|
16
|
+
/** Rotate when the log exceeds this size (kept to one `.1` generation). */
|
|
17
|
+
export const FAILOVER_LOG_ROTATE_BYTES = 2 * 1024 * 1024;
|
|
18
|
+
export class JsonlFailoverLogStore {
|
|
19
|
+
file;
|
|
20
|
+
constructor(file) {
|
|
21
|
+
this.file = file;
|
|
22
|
+
}
|
|
23
|
+
/** Append one record; rotate first when over budget. Never throws. */
|
|
24
|
+
append(record) {
|
|
25
|
+
try {
|
|
26
|
+
this.rotateIfNeeded();
|
|
27
|
+
mkdirSync(dirnameOf(this.file), { recursive: true });
|
|
28
|
+
appendFileSync(this.file, `${JSON.stringify(record)}\n`, 'utf8');
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// A lost log line must never take down recovery.
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
rotateIfNeeded() {
|
|
35
|
+
try {
|
|
36
|
+
if (statSync(this.file).size < FAILOVER_LOG_ROTATE_BYTES)
|
|
37
|
+
return;
|
|
38
|
+
renameSync(this.file, `${this.file}.1`);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// Missing file, or Windows lock on rename: keep appending (no rotation).
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function dirnameOf(file) {
|
|
46
|
+
const index = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\'));
|
|
47
|
+
return index <= 0 ? '.' : file.slice(0, index);
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=failover-log.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failover-log.js","sourceRoot":"","sources":["../src/failover-log.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAEnC,2EAA2E;AAC3E,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAmDxD,MAAM,OAAO,qBAAqB;IACH;IAA7B,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,sEAAsE;IACtE,MAAM,CAAC,MAAyB;QAC9B,IAAI,CAAC;YACH,IAAI,CAAC,cAAc,EAAE,CAAA;YACrB,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACpD,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,iDAAiD;QACnD,CAAC;IACH,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC;YACH,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,yBAAyB;gBAAE,OAAM;YAChE,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;IACH,CAAC;CACF;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;IACrE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;AAChD,CAAC"}
|
package/lib/index.d.ts
CHANGED
|
@@ -18,13 +18,14 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import type { Context } from '@deepseek-ai/cordis';
|
|
20
20
|
import { type PluginConfig } from './config.ts';
|
|
21
|
-
import { type DarumaFailoverEvent } from './failover-events.ts';
|
|
21
|
+
import { type DarumaFailoverEvent, type DarumaGiveUpEvent } from './failover-events.ts';
|
|
22
22
|
export declare const name = "dsh-daruma";
|
|
23
23
|
export declare const inject: readonly ["agents", "settings", "llm"];
|
|
24
|
-
export type { DarumaFailoverEvent };
|
|
24
|
+
export type { DarumaFailoverEvent, DarumaGiveUpEvent };
|
|
25
25
|
declare module '@deepseek-ai/dsh-session/types' {
|
|
26
26
|
interface SessionEventMap {
|
|
27
27
|
'daruma/failover': DarumaFailoverEvent;
|
|
28
|
+
'daruma/give-up': DarumaGiveUpEvent;
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
31
|
export declare function apply(ctx: Context, rawConfig?: PluginConfig): void;
|
package/lib/index.js
CHANGED
|
@@ -20,7 +20,8 @@ import { channelIdOf, channelIdOfConfig, toCallConfig, toFailureSignal } from ".
|
|
|
20
20
|
import { resolveConfig } from "./config.js";
|
|
21
21
|
import { RecoveryEngine } from "./engine.js";
|
|
22
22
|
import { JsonFileChannelHealthStore } from "./store.js";
|
|
23
|
-
import {
|
|
23
|
+
import { JsonlFailoverLogStore } from "./failover-log.js";
|
|
24
|
+
import { buildDarumaFailoverEvent, buildDarumaGiveUpEvent } from "./failover-events.js";
|
|
24
25
|
import { mountStatus } from "./status.js";
|
|
25
26
|
import { mountRpc } from "./rpc.js";
|
|
26
27
|
import { modelId } from 'daruma-core';
|
|
@@ -30,11 +31,27 @@ export function apply(ctx, rawConfig = {}) {
|
|
|
30
31
|
const config = resolveConfig(rawConfig);
|
|
31
32
|
const store = new JsonFileChannelHealthStore(config.stateFile);
|
|
32
33
|
const engine = new RecoveryEngine(config, store);
|
|
34
|
+
const failoverLog = new JsonlFailoverLogStore(config.logFile);
|
|
33
35
|
const status = mountStatus(ctx);
|
|
36
|
+
// Boot record: one line per plugin start, so audits can align restart
|
|
37
|
+
// boundaries with the failover/give-up lines that follow.
|
|
38
|
+
failoverLog.append({
|
|
39
|
+
kind: 'boot',
|
|
40
|
+
t: Date.now(),
|
|
41
|
+
pid: process.pid,
|
|
42
|
+
channels: config.channels.map((channel) => channel.id),
|
|
43
|
+
failureBudget: config.failureBudget,
|
|
44
|
+
cooldownMs: config.cooldownMs,
|
|
45
|
+
giveUpBudget: config.giveUpBudget,
|
|
46
|
+
});
|
|
34
47
|
// Per-agent channel currently in use (tracked from the last request).
|
|
35
48
|
const currentChannel = new Map();
|
|
36
49
|
// Per-agent failover target armed on the previous failed request.
|
|
37
50
|
const pending = new Map();
|
|
51
|
+
// Agents whose latest request-attempt failed and has not been followed by a
|
|
52
|
+
// fresh request yet. `agent/pre-step` infers "the last request succeeded"
|
|
53
|
+
// only when the agent is absent here (see onSuccess wiring below).
|
|
54
|
+
const failedSinceRequest = new Set();
|
|
38
55
|
/** The user-chosen backup channel, if set and healthy. */
|
|
39
56
|
const backupChannel = () => {
|
|
40
57
|
const backup = status.getBackup();
|
|
@@ -43,6 +60,8 @@ export function apply(ctx, rawConfig = {}) {
|
|
|
43
60
|
return { id: channelIdOf(backup.provider, backup.model), provider: backup.provider, model: modelId(backup.model) };
|
|
44
61
|
};
|
|
45
62
|
ctx.on('agent/request', async (payload, next) => {
|
|
63
|
+
// A new request attempt supersedes any recorded failure attribution.
|
|
64
|
+
failedSinceRequest.delete(payload.agent.id);
|
|
46
65
|
const current = await next();
|
|
47
66
|
const armed = pending.get(payload.agent.id);
|
|
48
67
|
if (armed) {
|
|
@@ -56,6 +75,7 @@ export function apply(ctx, rawConfig = {}) {
|
|
|
56
75
|
return current;
|
|
57
76
|
});
|
|
58
77
|
ctx.on('agent/request-error', async (payload, next) => {
|
|
78
|
+
failedSinceRequest.add(payload.agent.id);
|
|
59
79
|
const channel = currentChannel.get(payload.agent.id) ?? channelIdOf(payload.provider, '');
|
|
60
80
|
const signal = toFailureSignal(payload.failure, channel, Date.now());
|
|
61
81
|
const plan = engine.onFailure(signal, backupChannel(), payload.agent.id);
|
|
@@ -74,18 +94,90 @@ export function apply(ctx, rawConfig = {}) {
|
|
|
74
94
|
giveUpBudget: engine.giveUpBudget,
|
|
75
95
|
});
|
|
76
96
|
payload.agent.session.append('daruma/failover', event);
|
|
97
|
+
failoverLog.append({
|
|
98
|
+
kind: 'failover',
|
|
99
|
+
t: event.at,
|
|
100
|
+
agentId: payload.agent.id,
|
|
101
|
+
from: channel,
|
|
102
|
+
to: plan.verdict.target.id,
|
|
103
|
+
reason: signal.code,
|
|
104
|
+
status: payload.failure.status,
|
|
105
|
+
turn: payload.turn,
|
|
106
|
+
step: payload.step,
|
|
107
|
+
failoverCount: plan.failoverCount,
|
|
108
|
+
giveUpBudget: engine.giveUpBudget,
|
|
109
|
+
});
|
|
77
110
|
ctx.logger.warn(`dsh-daruma: failover ${channel} -> ${plan.verdict.target.id} (${signal.code}) agent=${payload.agent.id} turn=${payload.turn} step=${payload.step}`);
|
|
78
111
|
return { kind: 'retry' };
|
|
79
112
|
}
|
|
80
113
|
if (plan.verdict.kind === 'GIVE_UP') {
|
|
114
|
+
const giveUpEvent = buildDarumaGiveUpEvent({
|
|
115
|
+
from: channel,
|
|
116
|
+
reason: plan.verdict.reason === 'no-routable-fallback' ? 'no-routable-fallback' : 'give-up-budget-exhausted',
|
|
117
|
+
at: Date.now(),
|
|
118
|
+
agentId: payload.agent.id,
|
|
119
|
+
turn: payload.turn,
|
|
120
|
+
step: payload.step,
|
|
121
|
+
failoverCount: plan.failoverCount,
|
|
122
|
+
giveUpBudget: engine.giveUpBudget,
|
|
123
|
+
});
|
|
124
|
+
payload.agent.session.append('daruma/give-up', giveUpEvent);
|
|
125
|
+
failoverLog.append({
|
|
126
|
+
kind: 'give-up',
|
|
127
|
+
t: giveUpEvent.at,
|
|
128
|
+
agentId: payload.agent.id,
|
|
129
|
+
from: channel,
|
|
130
|
+
reason: giveUpEvent.reason,
|
|
131
|
+
status: payload.failure.status,
|
|
132
|
+
turn: payload.turn,
|
|
133
|
+
step: payload.step,
|
|
134
|
+
failoverCount: plan.failoverCount,
|
|
135
|
+
giveUpBudget: engine.giveUpBudget,
|
|
136
|
+
});
|
|
81
137
|
ctx.logger.error(`dsh-daruma: giving up (${plan.verdict.reason})`);
|
|
82
138
|
}
|
|
83
139
|
// RETRY_NOW / GIVE_UP: delegate downstream (retry may still own it).
|
|
84
140
|
return next();
|
|
85
141
|
});
|
|
142
|
+
// The host exposes no request-success event. Two adjacent host events prove
|
|
143
|
+
// "the previous model request completed without erroring":
|
|
144
|
+
// 1. `agent/pre-step` — the loop proposes the next step;
|
|
145
|
+
// 2. `agent/turn-stopping` — the turn is about to close.
|
|
146
|
+
// At either point, `currentChannel` still names the channel of the last
|
|
147
|
+
// request, so it earns a success record — unless a failure is still
|
|
148
|
+
// un-attributed (`failedSinceRequest`), which gates the false-positive
|
|
149
|
+
// where the turn died on an error and never sent another request.
|
|
150
|
+
// The turn-stopping arm is what makes single-step tasks (headless one-shot
|
|
151
|
+
// prompts) reset health: their only pre-step fires before any request.
|
|
152
|
+
// Never blocks the host loop: errors are logged and swallowed; the
|
|
153
|
+
// pre-step waterfall always calls through.
|
|
154
|
+
const recordInferredSuccess = (agentId) => {
|
|
155
|
+
const channel = currentChannel.get(agentId);
|
|
156
|
+
if (channel !== undefined && !failedSinceRequest.has(agentId)) {
|
|
157
|
+
engine.onSuccess(channel);
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
ctx.on('agent/pre-step', async (payload, next) => {
|
|
161
|
+
try {
|
|
162
|
+
recordInferredSuccess(payload.agent.id);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
ctx.logger.warn(`dsh-daruma: success recording failed: ${String(error)}`);
|
|
166
|
+
}
|
|
167
|
+
return next();
|
|
168
|
+
});
|
|
169
|
+
ctx.on('agent/turn-stopping', ({ agent }) => {
|
|
170
|
+
try {
|
|
171
|
+
recordInferredSuccess(agent.id);
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
ctx.logger.warn(`dsh-daruma: success recording failed: ${String(error)}`);
|
|
175
|
+
}
|
|
176
|
+
});
|
|
86
177
|
ctx.on('agent/disposed', ({ agent }) => {
|
|
87
178
|
currentChannel.delete(agent.id);
|
|
88
179
|
pending.delete(agent.id);
|
|
180
|
+
failedSinceRequest.delete(agent.id);
|
|
89
181
|
engine.clearScope(agent.id);
|
|
90
182
|
});
|
|
91
183
|
mountRpc(ctx, { engine, currentChannel, status, getLlm: () => ctx.llm, getSettings: () => ctx.settings });
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC5F,OAAO,EAAE,aAAa,EAAqB,MAAM,aAAa,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AACvD,OAAO,EAAE,wBAAwB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC5F,OAAO,EAAE,aAAa,EAAqB,MAAM,aAAa,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AACzD,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAoD,MAAM,sBAAsB,CAAA;AACzI,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,OAAO,EAAgC,MAAM,aAAa,CAAA;AAEnE,MAAM,CAAC,MAAM,IAAI,GAAG,YAAY,CAAA;AAChC,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAU,CAAA;AAW5D,MAAM,UAAU,KAAK,CAAC,GAAY,EAAE,YAA0B,EAAE;IAC9D,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,0BAA0B,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAC9D,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;IAE/B,sEAAsE;IACtE,0DAA0D;IAC1D,WAAW,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM;QACZ,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE;QACb,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;KAClC,CAAC,CAAA;IAEF,sEAAsE;IACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAqB,CAAA;IACnD,kEAAkE;IAClE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAA;IAC1C,4EAA4E;IAC5E,0EAA0E;IAC1E,mEAAmE;IACnE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAA;IAE5C,0DAA0D;IAC1D,MAAM,aAAa,GAAG,GAAwB,EAAE;QAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAA;QACjC,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,SAAS,CAAA;QAC1C,OAAO,EAAE,EAAE,EAAE,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;IACpH,CAAC,CAAA;IAED,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC9C,qEAAqE;QACrE,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAkB,MAAM,IAAI,EAAE,CAAA;QAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC3C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAChC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;YAC5C,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;YAChE,GAAG,CAAC,MAAM,CAAC,IAAI,CACb,yBAAyB,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,OAAO,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,EAAE,CACrG,CAAA;YACD,OAAO,OAAO,CAAA;QAChB,CAAC;QACD,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;QAChE,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,EAAE,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAA+B,EAAE;QACjF,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QACzF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAExE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,wBAAwB,CAAC;gBACrC,IAAI,EAAE,OAAO;gBACb,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBAC1B,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;gBACd,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBACzB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAA;YACF,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;YACtD,WAAW,CAAC,MAAM,CAAC;gBACjB,IAAI,EAAE,UAAU;gBAChB,CAAC,EAAE,KAAK,CAAC,EAAE;gBACX,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBACzB,IAAI,EAAE,OAAO;gBACb,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBAC1B,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;gBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAA;YACF,GAAG,CAAC,MAAM,CAAC,IAAI,CACb,wBAAwB,OAAO,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,IAAI,WAAW,OAAO,CAAC,KAAK,CAAC,EAAE,SAAS,OAAO,CAAC,IAAI,SAAS,OAAO,CAAC,IAAI,EAAE,CACpJ,CAAA;YACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,sBAAsB,CAAC;gBACzC,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,sBAAsB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,0BAA0B;gBAC5G,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;gBACd,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBACzB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAA;YACF,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAA;YAC3D,WAAW,CAAC,MAAM,CAAC;gBACjB,IAAI,EAAE,SAAS;gBACf,CAAC,EAAE,WAAW,CAAC,EAAE;gBACjB,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBACzB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;gBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAA;YACF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAA;QACpE,CAAC;QAED,qEAAqE;QACrE,OAAO,IAAI,EAAE,CAAA;IACf,CAAC,CAAC,CAAA;IAEF,4EAA4E;IAC5E,2DAA2D;IAC3D,2DAA2D;IAC3D,2DAA2D;IAC3D,wEAAwE;IACxE,oEAAoE;IACpE,uEAAuE;IACvE,kEAAkE;IAClE,2EAA2E;IAC3E,uEAAuE;IACvE,mEAAmE;IACnE,2CAA2C;IAC3C,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAQ,EAAE;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC3C,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9D,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;QAC3B,CAAC;IACH,CAAC,CAAA;IAED,GAAG,CAAC,EAAE,CAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC/C,IAAI,CAAC;YACH,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC3E,CAAC;QACD,OAAO,IAAI,EAAE,CAAA;IACf,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QAC1C,IAAI,CAAC;YACH,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC3E,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACrC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACxB,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACnC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;AAC3G,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
{
|
|
1
|
+
{
|
|
2
2
|
"name": "dsh-daruma",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "Daruma resilience plugin for DeepSeek Harness
|
|
3
|
+
"version": "0.1.4",
|
|
4
|
+
"description": "Daruma resilience plugin for DeepSeek Harness 鈥?detects model-request failures and fails over to another channel to keep long tasks alive.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "SISCHOI",
|