@ziggs-ai/agent-sdk 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/dist/AgentHost.d.ts +12 -20
- package/dist/AgentHost.js +68 -142
- package/dist/agent/Agent.js +4 -0
- package/dist/agent/ClaudeSdkAgent.js +13 -4
- package/dist/agent/claudeSdkProtocol.d.ts +8 -7
- package/dist/agent/claudeSdkProtocol.js +8 -7
- package/dist/agent/claudeSdkTools.d.ts +4 -2
- package/dist/agent/claudeSdkTools.js +9 -4
- package/dist/agent/toolManagerToSdkMcp.js +1 -0
- package/dist/index.d.ts +5 -4
- package/dist/index.js +4 -2
- package/dist/ingress/normalizeIncoming.d.ts +23 -8
- package/dist/ingress/normalizeIncoming.js +21 -32
- package/dist/runtime/AgentMachine.d.ts +5 -2
- package/dist/runtime/AgentMachine.js +8 -2
- package/dist/runtime/PromptBuilder.js +2 -2
- package/dist/runtime/runMechanical.d.ts +3 -1
- package/dist/runtime/runMechanical.js +2 -2
- package/dist/runtime/runTurn.d.ts +3 -1
- package/dist/runtime/runTurn.js +27 -9
- package/dist/runtime/wakeWriteKeys.d.ts +25 -0
- package/dist/runtime/wakeWriteKeys.js +134 -0
- package/dist/server/ConnectionPool.d.ts +31 -9
- package/dist/server/ConnectionPool.js +21 -24
- package/dist/server/ZiggsEffectHandler.js +17 -12
- package/dist/server/createHealthServer.js +5 -5
- package/dist/server/inboxLoop.d.ts +50 -0
- package/dist/server/{InboxCatchUp.js → inboxLoop.js} +64 -81
- package/dist/server/runLauncher.d.ts +1 -1
- package/dist/server/runLauncher.js +20 -7
- package/dist/server/staleBacklogGate.d.ts +53 -0
- package/dist/server/staleBacklogGate.js +120 -0
- package/dist/server/tasks/TaskService.d.ts +2 -1
- package/dist/server/tasks/TaskService.js +5 -2
- package/dist/server/tools/tier1/protocolRunner.d.ts +2 -1
- package/dist/server/tools/tier1/protocolRunner.js +43 -9
- package/dist/server/tools/tier1/protocolTools.js +1 -1
- package/dist/server/tools/tier2/contextTools.js +1 -1
- package/dist/shared/operatorKeyClaims.d.ts +22 -0
- package/dist/shared/operatorKeyClaims.js +35 -0
- package/dist/types.d.ts +40 -0
- package/package.json +2 -2
- package/dist/server/InboxCatchUp.d.ts +0 -24
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ConnectionPool — multi-agent fleet manager.
|
|
3
|
+
*
|
|
4
|
+
* Holds N `AgentHostOptions` configs but only constructs/wakes an
|
|
5
|
+
* `AgentHost` (one WS connection) when needed. Idle hosts are slept after
|
|
6
|
+
* `idleTimeoutMs`. At most `maxActive` hosts are connected at a time; LRU
|
|
7
|
+
* eviction drops the oldest if the cap is reached. With `startControl`, a
|
|
8
|
+
* separate "launcher" socket receives backend-driven wake hints so the pool
|
|
9
|
+
* spins up the right host on demand.
|
|
10
|
+
*
|
|
11
|
+
* This is how a single Node process serves a fleet of agents without
|
|
12
|
+
* holding 1000 open sockets — typical config: 1000+ registered, ~50 live.
|
|
13
|
+
*/
|
|
14
|
+
import { type InboxOperatorAgentEntry } from '@ziggs-ai/api-client';
|
|
1
15
|
import { AgentHost, type AgentHostOptions } from '../AgentHost.js';
|
|
2
16
|
interface ConnectionManagerMeta {
|
|
3
17
|
domain?: string;
|
|
@@ -13,6 +27,11 @@ interface QueryFilter {
|
|
|
13
27
|
export interface PoolOptions {
|
|
14
28
|
maxActive?: number;
|
|
15
29
|
idleTimeoutMs?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Gate on the operator-poll lazy start (ZIG-910): return false to leave an
|
|
32
|
+
* actionable agent's host down this pass. Explicit startAgent()s are not gated.
|
|
33
|
+
*/
|
|
34
|
+
shouldLazyStart?: (entry: InboxOperatorAgentEntry) => boolean;
|
|
16
35
|
}
|
|
17
36
|
export interface SendToOptions {
|
|
18
37
|
timeoutMs?: number;
|
|
@@ -25,17 +44,17 @@ export declare class ConnectionPool {
|
|
|
25
44
|
private readonly manager;
|
|
26
45
|
private readonly configs;
|
|
27
46
|
private readonly responseListeners;
|
|
28
|
-
constructor({ maxActive, idleTimeoutMs }?: PoolOptions);
|
|
47
|
+
constructor({ maxActive, idleTimeoutMs, shouldLazyStart }?: PoolOptions);
|
|
29
48
|
/**
|
|
30
|
-
* Register N agent configs. Each will be lazily
|
|
49
|
+
* Register N agent configs. Each will be lazily started (a fresh
|
|
31
50
|
* `AgentHost` constructed + connected) the first time it's referenced.
|
|
32
51
|
*/
|
|
33
52
|
register(configs: AgentHostOptions[], metaArr?: ConnectionManagerMeta[]): void;
|
|
34
|
-
|
|
53
|
+
startAgent(agentId: string): Promise<AgentHost>;
|
|
35
54
|
sleep(agentId: string): Promise<void>;
|
|
36
55
|
disconnectAll(): Promise<void>;
|
|
37
56
|
/**
|
|
38
|
-
*
|
|
57
|
+
* Start the named agent, deliver a message, and wait for its first outbound
|
|
39
58
|
* `send-message` effect — i.e. the agent's reply. Returns the reply text.
|
|
40
59
|
*
|
|
41
60
|
* The effectTap injected in `register` drains a per-agent listener queue;
|
|
@@ -44,7 +63,7 @@ export declare class ConnectionPool {
|
|
|
44
63
|
*/
|
|
45
64
|
requestFrom(agentId: string, text: string, metadata?: Record<string, unknown>, { timeoutMs }?: SendToOptions): Promise<string>;
|
|
46
65
|
/**
|
|
47
|
-
*
|
|
66
|
+
* Start the named agent and deliver one wire frame to it. Returns once the
|
|
48
67
|
* host has finished processing.
|
|
49
68
|
*
|
|
50
69
|
* Note: does not return the agent's response. For request/response
|
|
@@ -52,11 +71,14 @@ export declare class ConnectionPool {
|
|
|
52
71
|
*/
|
|
53
72
|
sendTo(agentId: string, text: string, metadata?: Record<string, unknown>, { timeoutMs }?: SendToOptions): Promise<null>;
|
|
54
73
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
74
|
+
* Start the fleet lazy-start loop (delivery law, phase 4): the manager
|
|
75
|
+
* long-polls `GET /inbox/operator` with this operator key and starts the host
|
|
76
|
+
* for any owned agent whose slice has actionable content. Each started host
|
|
77
|
+
* then consumes via its own per-agent inbox loop — this is ordinary lazy
|
|
78
|
+
* process management, not a pushed signal. `wsUrl` is accepted for
|
|
79
|
+
* back-compat but unused (the poll is HTTP).
|
|
58
80
|
*/
|
|
59
|
-
startControl({
|
|
81
|
+
startControl({ operatorKey }?: ControlOptions): void;
|
|
60
82
|
stopControl(): void;
|
|
61
83
|
query(filter: QueryFilter): string[];
|
|
62
84
|
list(): string[];
|
|
@@ -19,11 +19,11 @@ export class ConnectionPool {
|
|
|
19
19
|
configs = new Map();
|
|
20
20
|
// Listeners enqueued by requestFrom — drained on outbound send-message effects.
|
|
21
21
|
responseListeners = new Map();
|
|
22
|
-
constructor({ maxActive = 50, idleTimeoutMs = 60_000 } = {}) {
|
|
23
|
-
this.manager = new ConnectionManager({ maxActive, idleTimeoutMs });
|
|
22
|
+
constructor({ maxActive = 50, idleTimeoutMs = 60_000, shouldLazyStart } = {}) {
|
|
23
|
+
this.manager = new ConnectionManager({ maxActive, idleTimeoutMs, shouldLazyStart });
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
|
-
* Register N agent configs. Each will be lazily
|
|
26
|
+
* Register N agent configs. Each will be lazily started (a fresh
|
|
27
27
|
* `AgentHost` constructed + connected) the first time it's referenced.
|
|
28
28
|
*/
|
|
29
29
|
register(configs, metaArr = []) {
|
|
@@ -56,7 +56,7 @@ export class ConnectionPool {
|
|
|
56
56
|
},
|
|
57
57
|
});
|
|
58
58
|
await host.connectAsync();
|
|
59
|
-
runtimeLog.debug('ConnectionPool', `
|
|
59
|
+
runtimeLog.debug('ConnectionPool', `started "${agentId}" (active: ${this.manager.listActive().length}/${this.maxActive})`);
|
|
60
60
|
return host;
|
|
61
61
|
}, async (host) => {
|
|
62
62
|
host.disconnect();
|
|
@@ -64,8 +64,8 @@ export class ConnectionPool {
|
|
|
64
64
|
}, metaByAgentId.get(agentId));
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
|
-
async
|
|
68
|
-
return await this.manager.
|
|
67
|
+
async startAgent(agentId) {
|
|
68
|
+
return await this.manager.startAgent(agentId);
|
|
69
69
|
}
|
|
70
70
|
async sleep(agentId) {
|
|
71
71
|
return this.manager.sleep(agentId);
|
|
@@ -74,7 +74,7 @@ export class ConnectionPool {
|
|
|
74
74
|
return this.manager.sleepAll();
|
|
75
75
|
}
|
|
76
76
|
/**
|
|
77
|
-
*
|
|
77
|
+
* Start the named agent, deliver a message, and wait for its first outbound
|
|
78
78
|
* `send-message` effect — i.e. the agent's reply. Returns the reply text.
|
|
79
79
|
*
|
|
80
80
|
* The effectTap injected in `register` drains a per-agent listener queue;
|
|
@@ -82,7 +82,7 @@ export class ConnectionPool {
|
|
|
82
82
|
* does not reply within `timeoutMs` (default 60 s for LLM agents).
|
|
83
83
|
*/
|
|
84
84
|
async requestFrom(agentId, text, metadata = {}, { timeoutMs = 60_000 } = {}) {
|
|
85
|
-
const host = await this.
|
|
85
|
+
const host = await this.startAgent(agentId);
|
|
86
86
|
return new Promise((resolve, reject) => {
|
|
87
87
|
let settled = false;
|
|
88
88
|
const timer = setTimeout(() => {
|
|
@@ -123,39 +123,36 @@ export class ConnectionPool {
|
|
|
123
123
|
});
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
|
-
*
|
|
126
|
+
* Start the named agent and deliver one wire frame to it. Returns once the
|
|
127
127
|
* host has finished processing.
|
|
128
128
|
*
|
|
129
129
|
* Note: does not return the agent's response. For request/response
|
|
130
130
|
* semantics use `requestFrom`.
|
|
131
131
|
*/
|
|
132
132
|
async sendTo(agentId, text, metadata = {}, { timeoutMs = 30_000 } = {}) {
|
|
133
|
-
const host = await this.
|
|
133
|
+
const host = await this.startAgent(agentId);
|
|
134
134
|
const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`sendTo timeout for "${agentId}" after ${timeoutMs}ms`)), timeoutMs));
|
|
135
135
|
await Promise.race([host.handleMessage(text, metadata), timeoutPromise]);
|
|
136
136
|
this.manager.touch(agentId);
|
|
137
137
|
return null;
|
|
138
138
|
}
|
|
139
139
|
/**
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
140
|
+
* Start the fleet lazy-start loop (delivery law, phase 4): the manager
|
|
141
|
+
* long-polls `GET /inbox/operator` with this operator key and starts the host
|
|
142
|
+
* for any owned agent whose slice has actionable content. Each started host
|
|
143
|
+
* then consumes via its own per-agent inbox loop — this is ordinary lazy
|
|
144
|
+
* process management, not a pushed signal. `wsUrl` is accepted for
|
|
145
|
+
* back-compat but unused (the poll is HTTP).
|
|
143
146
|
*/
|
|
144
|
-
startControl({
|
|
145
|
-
if (!
|
|
146
|
-
runtimeLog.warn('ConnectionPool', 'startControl:
|
|
147
|
+
startControl({ operatorKey } = {}) {
|
|
148
|
+
if (!operatorKey) {
|
|
149
|
+
runtimeLog.warn('ConnectionPool', 'startControl: operatorKey is required — skipping');
|
|
147
150
|
return;
|
|
148
151
|
}
|
|
149
|
-
|
|
150
|
-
// private field is the smallest way to add it later. (Cleaner fix lives
|
|
151
|
-
// in api-client: expose a `setControl` setter.)
|
|
152
|
-
this.manager['_controlOpts'] = { wsUrl, operatorKey };
|
|
153
|
-
this.manager.start();
|
|
152
|
+
this.manager.start(operatorKey);
|
|
154
153
|
}
|
|
155
154
|
stopControl() {
|
|
156
|
-
|
|
157
|
-
m['_controlHandle']?.close?.();
|
|
158
|
-
m['_controlHandle'] = null;
|
|
155
|
+
void this.manager.stop();
|
|
159
156
|
}
|
|
160
157
|
query(filter) { return this.manager.query(filter); }
|
|
161
158
|
list() { return this.manager.list(); }
|
|
@@ -69,7 +69,7 @@ function previewToolArgs(args) {
|
|
|
69
69
|
* Chat timeline: `Message.tsx` parses `entryType: artifact`, `content_type: operation` with
|
|
70
70
|
* `operation_*:{json}` where json includes `state` and `tool`.
|
|
71
71
|
*/
|
|
72
|
-
async function emitToolOperationArtifact(client, chatId, phase, payload) {
|
|
72
|
+
async function emitToolOperationArtifact(client, chatId, phase, payload, idempotencyKey) {
|
|
73
73
|
const prefix = phase === 'started' ? 'operation_started' : phase === 'completed' ? 'operation_completed' : 'operation_error';
|
|
74
74
|
const text = `${prefix}:${JSON.stringify(payload)}`;
|
|
75
75
|
try {
|
|
@@ -78,6 +78,7 @@ async function emitToolOperationArtifact(client, chatId, phase, payload) {
|
|
|
78
78
|
text,
|
|
79
79
|
content_type: 'operation',
|
|
80
80
|
visibility: 'agent-private',
|
|
81
|
+
idempotencyKey,
|
|
81
82
|
});
|
|
82
83
|
}
|
|
83
84
|
catch (e) {
|
|
@@ -108,7 +109,7 @@ export async function executeToolWithSideEffects(deps, eff, effect, roster) {
|
|
|
108
109
|
state: 'started',
|
|
109
110
|
tool: effect.name,
|
|
110
111
|
...(argsPreview ? { args: argsPreview } : {}),
|
|
111
|
-
});
|
|
112
|
+
}, effect.wakeKeys?.artifactKey());
|
|
112
113
|
try {
|
|
113
114
|
const lastUser = roster.users.find((u) => u.userId || u.id);
|
|
114
115
|
// Accumulate token usage from any nested LLM call the tool makes (llmTool),
|
|
@@ -141,6 +142,7 @@ export async function executeToolWithSideEffects(deps, eff, effect, roster) {
|
|
|
141
142
|
senderType: effect.senderType ?? (lastUser ? 'USER' : undefined),
|
|
142
143
|
onBehalfOfAgreementId: effect.onBehalfOfAgreementId,
|
|
143
144
|
memory: effect.memory,
|
|
145
|
+
wakeKeys: effect.wakeKeys,
|
|
144
146
|
reportUsage,
|
|
145
147
|
turnScratch: {},
|
|
146
148
|
});
|
|
@@ -148,7 +150,7 @@ export async function executeToolWithSideEffects(deps, eff, effect, roster) {
|
|
|
148
150
|
state: 'completed',
|
|
149
151
|
tool: effect.name,
|
|
150
152
|
result: clipForOperationArtifact(result),
|
|
151
|
-
});
|
|
153
|
+
}, effect.wakeKeys?.artifactKey());
|
|
152
154
|
sendTelemetry(deps.telemetryClient, buildToolIngestPayload({
|
|
153
155
|
sessionId: effect.sessionId,
|
|
154
156
|
tool: effect.name,
|
|
@@ -166,7 +168,7 @@ export async function executeToolWithSideEffects(deps, eff, effect, roster) {
|
|
|
166
168
|
state: 'failed',
|
|
167
169
|
tool: effect.name,
|
|
168
170
|
error: errMsg,
|
|
169
|
-
});
|
|
171
|
+
}, effect.wakeKeys?.artifactKey());
|
|
170
172
|
sendTelemetry(deps.telemetryClient, buildToolIngestPayload({
|
|
171
173
|
sessionId: effect.sessionId,
|
|
172
174
|
tool: effect.name,
|
|
@@ -242,13 +244,13 @@ export function createZiggsEffectHandler(deps) {
|
|
|
242
244
|
let usage;
|
|
243
245
|
if (deps.llm.chatMessagesStream) {
|
|
244
246
|
const streamed = await deps.llm.chatMessagesStream(messages, (chunk) => {
|
|
245
|
-
// Live-
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
//
|
|
247
|
+
// Live-token display is best-effort and optional. Hosted agents wire
|
|
248
|
+
// no chunkSender (delivery law, phase 4: no agent socket, no live
|
|
249
|
+
// chunk transport), so partials are simply dropped — `text`
|
|
250
|
+
// accumulates independently and the final send-message (below, in
|
|
251
|
+
// runTurn) carries the full reply as one whole HTTP message. Any
|
|
252
|
+
// chunk-send failure is swallowed so a display hiccup never aborts
|
|
253
|
+
// the stream and loses the reply.
|
|
252
254
|
try {
|
|
253
255
|
if (chunk && chunk.trim())
|
|
254
256
|
deps.chunkSender?.(sessionId, receiverId, chunk, messageId);
|
|
@@ -288,7 +290,9 @@ export function createZiggsEffectHandler(deps) {
|
|
|
288
290
|
// visible to this agent across turns, not to other chat parties.
|
|
289
291
|
try {
|
|
290
292
|
if (effect.entry.kind === 'thought' && typeof effect.entry.text === 'string') {
|
|
291
|
-
await deps.artifactsClient.recordThought(effect.sessionId, effect.entry.text
|
|
293
|
+
await deps.artifactsClient.recordThought(effect.sessionId, effect.entry.text, {
|
|
294
|
+
idempotencyKey: effect.idempotencyKey,
|
|
295
|
+
});
|
|
292
296
|
}
|
|
293
297
|
else {
|
|
294
298
|
await deps.artifactsClient.write({
|
|
@@ -296,6 +300,7 @@ export function createZiggsEffectHandler(deps) {
|
|
|
296
300
|
text: JSON.stringify(effect.entry),
|
|
297
301
|
content_type: String(effect.entry.kind ?? 'event'),
|
|
298
302
|
visibility: 'agent-private',
|
|
303
|
+
idempotencyKey: effect.idempotencyKey,
|
|
299
304
|
});
|
|
300
305
|
}
|
|
301
306
|
}
|
|
@@ -19,11 +19,11 @@ function readJsonBody(req) {
|
|
|
19
19
|
export function createHealthServer({ port = Number(process.env.PORT) || 8080, label = 'agents', pool, } = {}) {
|
|
20
20
|
const server = createServer(async (req, res) => {
|
|
21
21
|
const url = req.url ?? '/';
|
|
22
|
-
const
|
|
23
|
-
if (req.method === 'POST' &&
|
|
24
|
-
const agentId = decodeURIComponent(
|
|
22
|
+
const startMatch = url.match(/^\/fleet-wake\/([^/?]+)/);
|
|
23
|
+
if (req.method === 'POST' && startMatch && pool) {
|
|
24
|
+
const agentId = decodeURIComponent(startMatch[1]);
|
|
25
25
|
try {
|
|
26
|
-
await pool.
|
|
26
|
+
await pool.startAgent(agentId);
|
|
27
27
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
28
28
|
res.end(JSON.stringify({ ok: true, agentId, active: pool.listActive().includes(agentId) }));
|
|
29
29
|
}
|
|
@@ -45,7 +45,7 @@ export function createHealthServer({ port = Number(process.env.PORT) || 8080, la
|
|
|
45
45
|
res.end(JSON.stringify({ ok: false, error: 'agentId, chatId, text, senderId required' }));
|
|
46
46
|
return;
|
|
47
47
|
}
|
|
48
|
-
const host = (await pool.
|
|
48
|
+
const host = (await pool.startAgent(agentId));
|
|
49
49
|
await host.handleMessage(text, {
|
|
50
50
|
chatId,
|
|
51
51
|
entryType: 'message',
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type Creds } from '@ziggs-ai/api-client';
|
|
2
|
+
import { SeenMessages } from './SeenMessages.js';
|
|
3
|
+
/**
|
|
4
|
+
* Per-agent inbox long-poll loop (delivery law, phase 4). This is the ONLY way
|
|
5
|
+
* a hosted agent consumes: `GET /inbox?wait=` → grant-fenced reads of the delta
|
|
6
|
+
* → deliver into the host's lane/tick machinery → `POST /inbox/ack`. No socket,
|
|
7
|
+
* no push, no separate catch-up channel — a crash-replay is just the next
|
|
8
|
+
* iteration (the cursor is server-side), and writes carry idempotency keys so
|
|
9
|
+
* re-processing a redelivered window is wasteful, never harmful. Act-before-ack
|
|
10
|
+
* is the one ordering rule.
|
|
11
|
+
*/
|
|
12
|
+
export interface InboxLoopDeps {
|
|
13
|
+
creds: Creds;
|
|
14
|
+
ownAgentId: string;
|
|
15
|
+
baseUrl?: string;
|
|
16
|
+
seen: SeenMessages;
|
|
17
|
+
onMessage: (text: string, metadata: Record<string, unknown>) => Promise<void>;
|
|
18
|
+
onResourceEvent: (event: Record<string, unknown>) => Promise<void>;
|
|
19
|
+
label?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface InboxPassResult {
|
|
22
|
+
scopes: number;
|
|
23
|
+
messagesDelivered: number;
|
|
24
|
+
resourceEvents: number;
|
|
25
|
+
acked: number;
|
|
26
|
+
truncated: boolean;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* One long-poll pass: hold up to `waitSeconds` for new inbox content, then read
|
|
30
|
+
* the deltas, deliver them into the host, and ack. Returns what it delivered.
|
|
31
|
+
* Reused each iteration of {@link runInboxLoop}; also usable as a single catch-up
|
|
32
|
+
* read (`waitSeconds: 0`) at bootstrap.
|
|
33
|
+
*/
|
|
34
|
+
export declare function pollInboxOnce(deps: InboxLoopDeps, waitSeconds?: number): Promise<InboxPassResult>;
|
|
35
|
+
export interface InboxLoopHandle {
|
|
36
|
+
/** Stop the loop after the current pass; resolves when it has fully exited. */
|
|
37
|
+
stop(): Promise<void>;
|
|
38
|
+
/** True while the loop is running. */
|
|
39
|
+
readonly running: boolean;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Run {@link pollInboxOnce} forever (until stopped). Each iteration long-polls
|
|
43
|
+
* for `waitSeconds`; a passing envelope returns instantly, an empty one holds.
|
|
44
|
+
* Errors retry with exponential backoff + jitter and never exit the loop —
|
|
45
|
+
* there is nothing to resynchronize (the cursor is server-side), so recovery is
|
|
46
|
+
* just the next iteration.
|
|
47
|
+
*/
|
|
48
|
+
export declare function runInboxLoop(deps: InboxLoopDeps, opts?: {
|
|
49
|
+
waitSeconds?: number;
|
|
50
|
+
}): InboxLoopHandle;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ContextReadClient, GrantsClient, InboxClient, getChatsForAgreement,
|
|
1
|
+
import { ContextReadClient, GrantsClient, InboxClient, getChatsForAgreement, } from '@ziggs-ai/api-client';
|
|
2
2
|
import { runtimeLog } from '../shared/runtimeLog.js';
|
|
3
3
|
const READ_PAGE = 100;
|
|
4
4
|
function log(label, msg) {
|
|
@@ -7,47 +7,7 @@ function log(label, msg) {
|
|
|
7
7
|
function warn(label, msg) {
|
|
8
8
|
runtimeLog.warn(label, msg);
|
|
9
9
|
}
|
|
10
|
-
|
|
11
|
-
/** ZIG-832 P4: is this agreement finished (so its backlog must not be re-run)? */
|
|
12
|
-
async function isTerminalAgreement(agreementId, creds) {
|
|
13
|
-
try {
|
|
14
|
-
const res = (await getAgreementStatus(agreementId, creds));
|
|
15
|
-
const status = res?.status ?? res?.agreement?.status;
|
|
16
|
-
return status != null && TERMINAL_AGREEMENT_STATUSES.has(status);
|
|
17
|
-
}
|
|
18
|
-
catch {
|
|
19
|
-
// On lookup failure, don't skip — fall through to normal delivery.
|
|
20
|
-
return false;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
/** Map Mongo message row → wire metadata for normalizeIncomingEvent. */
|
|
24
|
-
function rowToMetadata(row, chatId) {
|
|
25
|
-
const senderRaw = row.sender;
|
|
26
|
-
const senderId = String(senderRaw?.agentId ?? senderRaw?.id ?? '');
|
|
27
|
-
const receiver = row.receiver;
|
|
28
|
-
const senderType = senderRaw?.type ??
|
|
29
|
-
(receiver?.type === 'agent' ? 'agent' : 'user');
|
|
30
|
-
return {
|
|
31
|
-
chatId,
|
|
32
|
-
chat_id: chatId,
|
|
33
|
-
messageId: row.messageId,
|
|
34
|
-
sender: { id: senderId, type: senderType },
|
|
35
|
-
senderId,
|
|
36
|
-
senderType: String(senderType).toUpperCase(),
|
|
37
|
-
receiver: receiver ?? null,
|
|
38
|
-
receiverId: receiver?.id ?? null,
|
|
39
|
-
entryType: row.entryType ?? 'message',
|
|
40
|
-
content_type: row.content_type ?? row.contentType ?? 'text',
|
|
41
|
-
contentType: row.content_type ?? row.contentType ?? 'text',
|
|
42
|
-
task: row.task ?? null,
|
|
43
|
-
agreement: row.agreement ?? null,
|
|
44
|
-
operation: row.operation ?? null,
|
|
45
|
-
agreementId: row.agreementId ?? null,
|
|
46
|
-
timestamp: row.timestamp,
|
|
47
|
-
text: row.text,
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
async function chatIdsForScope(scope, creds, baseUrl, grants) {
|
|
10
|
+
async function chatIdsForScope(scope, creds, grants) {
|
|
51
11
|
const { kind, id } = scope.scope;
|
|
52
12
|
switch (kind) {
|
|
53
13
|
case 'chat':
|
|
@@ -57,8 +17,8 @@ async function chatIdsForScope(scope, creds, baseUrl, grants) {
|
|
|
57
17
|
return [...new Set(links.map((l) => l.chatId).filter(Boolean))];
|
|
58
18
|
}
|
|
59
19
|
case 'org': {
|
|
60
|
-
// All pages of live chat grants — an org-scope
|
|
61
|
-
//
|
|
20
|
+
// All pages of live chat grants — an org-scope read must not miss chats
|
|
21
|
+
// hidden behind pagination (GET /grants defaults to one page).
|
|
62
22
|
const items = await grants.listAllGrants({
|
|
63
23
|
scopeKind: 'chat',
|
|
64
24
|
health: 'active',
|
|
@@ -71,7 +31,7 @@ async function chatIdsForScope(scope, creds, baseUrl, grants) {
|
|
|
71
31
|
}
|
|
72
32
|
async function deliverMessagesForScope(deps, scope, chatIds, reader, countCap) {
|
|
73
33
|
let delivered = 0;
|
|
74
|
-
const label = deps.label ?? '
|
|
34
|
+
const label = deps.label ?? 'inboxLoop';
|
|
75
35
|
const expected = scope.newMessages;
|
|
76
36
|
let remaining = expected >= countCap ? Number.POSITIVE_INFINITY : expected;
|
|
77
37
|
for (const chatId of chatIds) {
|
|
@@ -95,8 +55,7 @@ async function deliverMessagesForScope(deps, scope, chatIds, reader, countCap) {
|
|
|
95
55
|
const text = typeof row.text === 'string' ? row.text : '';
|
|
96
56
|
if (!text.trim())
|
|
97
57
|
continue;
|
|
98
|
-
|
|
99
|
-
await deps.onMessage(text, metadata);
|
|
58
|
+
await deps.onMessage(text, { ...row, chatId });
|
|
100
59
|
delivered++;
|
|
101
60
|
if (remaining !== Number.POSITIVE_INFINITY)
|
|
102
61
|
remaining--;
|
|
@@ -113,49 +72,31 @@ async function deliverMessagesForScope(deps, scope, chatIds, reader, countCap) {
|
|
|
113
72
|
return delivered;
|
|
114
73
|
}
|
|
115
74
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
75
|
+
* One long-poll pass: hold up to `waitSeconds` for new inbox content, then read
|
|
76
|
+
* the deltas, deliver them into the host, and ack. Returns what it delivered.
|
|
77
|
+
* Reused each iteration of {@link runInboxLoop}; also usable as a single catch-up
|
|
78
|
+
* read (`waitSeconds: 0`) at bootstrap.
|
|
119
79
|
*/
|
|
120
|
-
export async function
|
|
121
|
-
const label = deps.label ?? '
|
|
80
|
+
export async function pollInboxOnce(deps, waitSeconds = 0) {
|
|
81
|
+
const label = deps.label ?? 'inboxLoop';
|
|
122
82
|
const inbox = new InboxClient(deps.creds.operatorKey, deps.creds.agentId, deps.baseUrl);
|
|
123
83
|
const reader = new ContextReadClient(deps.creds.operatorKey, deps.creds.agentId, deps.baseUrl);
|
|
124
84
|
const grants = new GrantsClient(deps.creds.operatorKey, deps.creds.agentId, deps.baseUrl);
|
|
125
|
-
const envelope = await inbox.getInbox();
|
|
85
|
+
const envelope = await inbox.getInbox(waitSeconds > 0 ? { waitSeconds } : {});
|
|
126
86
|
const countCap = envelope.countCap;
|
|
127
87
|
let messagesDelivered = 0;
|
|
128
88
|
let resourceEvents = 0;
|
|
129
89
|
const toAck = [];
|
|
130
90
|
if (envelope.truncatedScopes > 0) {
|
|
131
|
-
// Informational only — does NOT block acking the scopes
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
// acking here meant a busy long-lived agent (reach > cap) would never ack
|
|
137
|
-
// anything and re-deliver its entire backlog on every reconnect forever.
|
|
91
|
+
// Informational only — does NOT block acking the scopes returned below.
|
|
92
|
+
// truncatedScopes just means the caller's total reach exceeded the server's
|
|
93
|
+
// per-call scope cap; each returned scope is still an independently accurate,
|
|
94
|
+
// safe-to-ack unit of news. Never acking here would make a busy agent
|
|
95
|
+
// (reach > cap) re-deliver its whole backlog every pass forever.
|
|
138
96
|
warn(label, `inbox truncatedScopes=${envelope.truncatedScopes} (reach exceeds per-call cap — examined scopes are still acked normally)`);
|
|
139
97
|
}
|
|
140
98
|
for (const scope of envelope.scopes) {
|
|
141
|
-
|
|
142
|
-
// re-deliver an already-finished agreement's backlog, re-running a completed
|
|
143
|
-
// pipeline and burning tokens. If this is an agreement scope that has reached
|
|
144
|
-
// a terminal status, ack it (so it drops out of future catch-ups) and skip
|
|
145
|
-
// re-delivery. Only agreement scopes — chat scopes may carry genuinely new
|
|
146
|
-
// conversation, so those are never skipped on status. (The deeper fix, acking
|
|
147
|
-
// the watermark on the live-push path so nothing re-delivers, is tracked
|
|
148
|
-
// separately.)
|
|
149
|
-
if (scope.scope.kind === 'agreement' &&
|
|
150
|
-
(scope.newMessages > 0 || scope.newArtifacts > 0) &&
|
|
151
|
-
(await isTerminalAgreement(scope.scope.id, deps.creds))) {
|
|
152
|
-
if (scope.latestAt) {
|
|
153
|
-
toAck.push({ kind: scope.scope.kind, id: scope.scope.id, upTo: scope.latestAt });
|
|
154
|
-
}
|
|
155
|
-
log(label, `scope agreement:${scope.scope.id} terminal — acked, not re-delivered`);
|
|
156
|
-
continue;
|
|
157
|
-
}
|
|
158
|
-
const chatIds = await chatIdsForScope(scope, deps.creds, deps.baseUrl, grants);
|
|
99
|
+
const chatIds = await chatIdsForScope(scope, deps.creds, grants);
|
|
159
100
|
if (!chatIds.length && (scope.newMessages > 0 || scope.newArtifacts > 0)) {
|
|
160
101
|
warn(label, `no chat ids for scope ${scope.scope.kind}:${scope.scope.id} (${scope.newMessages} msgs)`);
|
|
161
102
|
}
|
|
@@ -201,9 +142,6 @@ export async function runInboxCatchUp(deps) {
|
|
|
201
142
|
acked = toAck.length;
|
|
202
143
|
log(label, `acked ${acked} scope(s)`);
|
|
203
144
|
}
|
|
204
|
-
if (messagesDelivered > 0 || resourceEvents > 0) {
|
|
205
|
-
log(label, `catch-up complete: ${messagesDelivered} message(s), ${resourceEvents} resource hint(s)`);
|
|
206
|
-
}
|
|
207
145
|
return {
|
|
208
146
|
scopes: envelope.scopes.length,
|
|
209
147
|
messagesDelivered,
|
|
@@ -212,3 +150,48 @@ export async function runInboxCatchUp(deps) {
|
|
|
212
150
|
truncated: envelope.truncatedScopes > 0,
|
|
213
151
|
};
|
|
214
152
|
}
|
|
153
|
+
const DEFAULT_WAIT_SECONDS = 25;
|
|
154
|
+
const ERROR_BACKOFF_BASE_MS = 1_000;
|
|
155
|
+
const ERROR_BACKOFF_MAX_MS = 30_000;
|
|
156
|
+
function sleep(ms) {
|
|
157
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Run {@link pollInboxOnce} forever (until stopped). Each iteration long-polls
|
|
161
|
+
* for `waitSeconds`; a passing envelope returns instantly, an empty one holds.
|
|
162
|
+
* Errors retry with exponential backoff + jitter and never exit the loop —
|
|
163
|
+
* there is nothing to resynchronize (the cursor is server-side), so recovery is
|
|
164
|
+
* just the next iteration.
|
|
165
|
+
*/
|
|
166
|
+
export function runInboxLoop(deps, opts = {}) {
|
|
167
|
+
const label = deps.label ?? 'inboxLoop';
|
|
168
|
+
const waitSeconds = opts.waitSeconds ?? DEFAULT_WAIT_SECONDS;
|
|
169
|
+
let running = true;
|
|
170
|
+
let consecutiveErrors = 0;
|
|
171
|
+
const done = (async () => {
|
|
172
|
+
while (running) {
|
|
173
|
+
try {
|
|
174
|
+
await pollInboxOnce(deps, waitSeconds);
|
|
175
|
+
consecutiveErrors = 0;
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
if (!running)
|
|
179
|
+
break;
|
|
180
|
+
consecutiveErrors++;
|
|
181
|
+
const backoff = Math.min(ERROR_BACKOFF_MAX_MS, ERROR_BACKOFF_BASE_MS * 2 ** (consecutiveErrors - 1));
|
|
182
|
+
const jitter = backoff * (0.5 + (consecutiveErrors % 7) / 14); // deterministic-ish, no Math.random
|
|
183
|
+
warn(label, `inbox loop pass failed (#${consecutiveErrors}): ${err?.message ?? err} — retrying in ~${Math.round(jitter)}ms`);
|
|
184
|
+
await sleep(jitter);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
})();
|
|
188
|
+
return {
|
|
189
|
+
get running() {
|
|
190
|
+
return running;
|
|
191
|
+
},
|
|
192
|
+
async stop() {
|
|
193
|
+
running = false;
|
|
194
|
+
await done.catch(() => { });
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|
|
@@ -18,7 +18,7 @@ interface FleetOptions extends SingleOptions {
|
|
|
18
18
|
wsUrl?: string;
|
|
19
19
|
agentMeta?: Record<string, unknown>[];
|
|
20
20
|
primary?: AgentConfig | ((pool: ConnectionPool) => AgentConfig);
|
|
21
|
-
|
|
21
|
+
preStart?: string[];
|
|
22
22
|
poolOptions?: {
|
|
23
23
|
maxActive?: number;
|
|
24
24
|
idleTimeoutMs?: number;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { ConnectionPool } from './ConnectionPool.js';
|
|
2
2
|
import { AgentHost } from '../AgentHost.js';
|
|
3
3
|
import { createHealthServer } from './createHealthServer.js';
|
|
4
|
+
import { createStaleBacklogGate, resolveReplayPolicy } from './staleBacklogGate.js';
|
|
5
|
+
import { boundKeyFleetWarning } from '../shared/operatorKeyClaims.js';
|
|
4
6
|
import { runtimeLog } from '../shared/runtimeLog.js';
|
|
5
7
|
export async function runLauncher(configOrConfigs, opts = {}) {
|
|
6
8
|
if (Array.isArray(configOrConfigs))
|
|
@@ -34,8 +36,14 @@ async function runSingle(config, { operatorKey, label = 'agent', port = process.
|
|
|
34
36
|
});
|
|
35
37
|
return { agent, healthServer: server ?? undefined };
|
|
36
38
|
}
|
|
37
|
-
async function runFleet(configs, { operatorKey, wsUrl = process.env.WS_URL, port = process.env.PORT || 8080, healthServer = true, label = 'launcher', agentMeta, primary: primaryConfig,
|
|
38
|
-
|
|
39
|
+
async function runFleet(configs, { operatorKey, wsUrl = process.env.WS_URL, port = process.env.PORT || 8080, healthServer = true, label = 'launcher', agentMeta, primary: primaryConfig, preStart = [], poolOptions, onShutdown, } = {}) {
|
|
40
|
+
// ZIG-910: don't resurrect stale backlogs at bootstrap. Env-driven default
|
|
41
|
+
// (FLEET_REPLAY_MAX_AGE / FLEET_REPLAY_STALE); an explicit
|
|
42
|
+
// poolOptions.shouldLazyStart wins.
|
|
43
|
+
const pool = new ConnectionPool({
|
|
44
|
+
shouldLazyStart: createStaleBacklogGate({ ...resolveReplayPolicy(), label }),
|
|
45
|
+
...poolOptions,
|
|
46
|
+
});
|
|
39
47
|
const effectiveKey = operatorKey ?? process.env.ZIGGS_OPERATOR_KEY;
|
|
40
48
|
const decoratedConfigs = effectiveKey
|
|
41
49
|
? configs.map(c => (c.operatorKey ? c : { ...c, operatorKey: effectiveKey }))
|
|
@@ -50,10 +58,15 @@ async function runFleet(configs, { operatorKey, wsUrl = process.env.WS_URL, port
|
|
|
50
58
|
}
|
|
51
59
|
const controlKey = effectiveKey ?? configs.find(c => c.operatorKey)?.operatorKey ?? resolvedPrimaryConfig?.operatorKey;
|
|
52
60
|
if (controlKey) {
|
|
61
|
+
// ZIG-911: an agent-bound key half-works in a confusing way — say the big
|
|
62
|
+
// thing once at startup instead of letting N-1 per-agent loops 403.
|
|
63
|
+
const boundWarning = boundKeyFleetWarning(controlKey, pool.size);
|
|
64
|
+
if (boundWarning)
|
|
65
|
+
runtimeLog.warn(label, boundWarning);
|
|
53
66
|
pool.startControl({ wsUrl, operatorKey: controlKey });
|
|
54
67
|
}
|
|
55
68
|
else {
|
|
56
|
-
runtimeLog.warn(label, 'No operatorKey provided — skipping
|
|
69
|
+
runtimeLog.warn(label, 'No operatorKey provided — skipping fleet inbox poll (no agents will be lazily started)');
|
|
57
70
|
}
|
|
58
71
|
let primary = null;
|
|
59
72
|
if (resolvedPrimaryConfig) {
|
|
@@ -68,10 +81,10 @@ async function runFleet(configs, { operatorKey, wsUrl = process.env.WS_URL, port
|
|
|
68
81
|
runtimeLog.warn(label, `Primary agent "${resolvedPrimaryConfig.name || resolvedPrimaryConfig.ownAgentId || 'primary'}" failed to connect at startup: ${err.message} — will retry`);
|
|
69
82
|
}
|
|
70
83
|
}
|
|
71
|
-
if (
|
|
72
|
-
await Promise.all(
|
|
73
|
-
.then(() => runtimeLog.debug(label, `Pre-
|
|
74
|
-
.catch(err => runtimeLog.warn(label, `Could not pre-
|
|
84
|
+
if (preStart.length > 0) {
|
|
85
|
+
await Promise.all(preStart.map(id => pool.startAgent(id)
|
|
86
|
+
.then(() => runtimeLog.debug(label, `Pre-started "${id}"`))
|
|
87
|
+
.catch(err => runtimeLog.warn(label, `Could not pre-start "${id}": ${err.message}`))));
|
|
75
88
|
}
|
|
76
89
|
runtimeLog.info(label, `Ready — ${pool.size} agent(s) registered, ${pool.listActive().length} connected`);
|
|
77
90
|
process.on('SIGTERM', async () => {
|