@blindmarket/sdk 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +141 -40
- package/dist/executor/WorkerRuntime.d.ts +80 -18
- package/dist/executor/WorkerRuntime.js +306 -101
- package/dist/index.d.ts +101 -22
- package/dist/index.js +158 -14
- package/dist/tools/helpers.js +33 -11
- package/dist/types.d.ts +96 -9
- package/dist/worker/Worker.d.ts +5 -3
- package/dist/worker/Worker.js +5 -3
- package/package.json +9 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ethers } from 'ethers';
|
|
2
|
-
import { BlindMarket } from '../index.js';
|
|
2
|
+
import { BlindMarket, ApiError } from '../index.js';
|
|
3
3
|
import { eciesDecrypt, aesDecrypt, derivePublicKey } from '../crypto/index.js';
|
|
4
4
|
/**
|
|
5
5
|
* Chains this runtime's CODE can sign submitEvidence on. What it registers as
|
|
@@ -22,8 +22,7 @@ function settlementChain(taskId, reported) {
|
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* The RPC this runtime signs on for `chain`. `rpcUrls` is per chain; the
|
|
25
|
-
* single `rpcUrl` is the 0G RPC
|
|
26
|
-
* in for another chain: a Base submitEvidence sent through it is rejected by
|
|
25
|
+
* single `rpcUrl` is the 0G RPC and never stands in for another chain: a Base submitEvidence sent through it is rejected by
|
|
27
26
|
* ethers' chainId pin — after the handler has already run.
|
|
28
27
|
*/
|
|
29
28
|
function rpcFor(config, chain) {
|
|
@@ -34,8 +33,39 @@ const DEFAULTS = {
|
|
|
34
33
|
browseIntervalMs: 15_000,
|
|
35
34
|
watchIntervalMs: 5_000,
|
|
36
35
|
maxConcurrentTasks: 3,
|
|
37
|
-
|
|
36
|
+
wrapTimeoutMs: 600_000,
|
|
37
|
+
assignmentPendingTimeoutMs: 180_000,
|
|
38
38
|
};
|
|
39
|
+
/** First back-off after an /accept the backend released or that never confirmed; doubles per failure. */
|
|
40
|
+
const RELEASED_BACKOFF_MS = 30_000;
|
|
41
|
+
const MAX_BACKOFF_MS = 3_600_000;
|
|
42
|
+
const MAX_WRAP_BACKOFF_MS = 86_400_000;
|
|
43
|
+
/** Re-accept rounds for a task that may still be held for this executor before it is dropped. */
|
|
44
|
+
const MAX_REACCEPT_ROUNDS = 6;
|
|
45
|
+
/**
|
|
46
|
+
* NEEDS_WRAP messages after which waiting is pointless: the platform cannot
|
|
47
|
+
* re-wrap (custody key rotated — only the poster's own client still can), or
|
|
48
|
+
* this executor has no public key on record. Newer backends say so in
|
|
49
|
+
* `error.reason` (CUSTODY_ROTATED / NO_PUBLIC_KEY); the message match covers
|
|
50
|
+
* backends that send only the NEEDS_WRAP code.
|
|
51
|
+
*/
|
|
52
|
+
const WRAP_IMPOSSIBLE_REASONS = new Set(['CUSTODY_ROTATED', 'NO_PUBLIC_KEY']);
|
|
53
|
+
const WRAP_IMPOSSIBLE = /rotated custody key|cannot re-wrap|no public key/i;
|
|
54
|
+
/** /accept was abandoned; `held` = the backend may still hold the task for this executor. */
|
|
55
|
+
class AcceptAbandoned extends Error {
|
|
56
|
+
cause;
|
|
57
|
+
held;
|
|
58
|
+
constructor(cause, held) {
|
|
59
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
60
|
+
this.cause = cause;
|
|
61
|
+
this.held = held;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
65
|
+
/** base, 2·base, 4·base … capped. `n` is 1 for the first failure. */
|
|
66
|
+
function backoff(base, n, cap) {
|
|
67
|
+
return Math.min(base * 2 ** Math.max(0, n - 1), cap);
|
|
68
|
+
}
|
|
39
69
|
// ── WorkerRuntime ───────────────────────────────────────────────────────────
|
|
40
70
|
export class WorkerRuntime {
|
|
41
71
|
bb;
|
|
@@ -45,8 +75,9 @@ export class WorkerRuntime {
|
|
|
45
75
|
running = false;
|
|
46
76
|
paused = false;
|
|
47
77
|
browseTimer;
|
|
48
|
-
watchTimers = new Map();
|
|
49
78
|
executions = new Map();
|
|
79
|
+
retries = new Map();
|
|
80
|
+
retryTimers = new Set();
|
|
50
81
|
listeners = new Set();
|
|
51
82
|
constructor(config) {
|
|
52
83
|
this.config = { ...DEFAULTS, ...config };
|
|
@@ -63,10 +94,9 @@ export class WorkerRuntime {
|
|
|
63
94
|
return [...this.executions.values()];
|
|
64
95
|
}
|
|
65
96
|
/**
|
|
66
|
-
* The chains this runtime declares to the backend
|
|
67
|
-
* for AND it has an RPC for.
|
|
68
|
-
*
|
|
69
|
-
* them until the poster's deadline.
|
|
97
|
+
* The chains this runtime declares to the backend and claims tasks on:
|
|
98
|
+
* those its code can sign for AND it has an RPC for. The backend stores the
|
|
99
|
+
* list but does not filter by it — browse() and executeTask() enforce it.
|
|
70
100
|
*/
|
|
71
101
|
get declaredChains() {
|
|
72
102
|
return SETTLEMENT_CHAINS.filter((chain) => !!rpcFor(this.config, chain));
|
|
@@ -92,40 +122,62 @@ export class WorkerRuntime {
|
|
|
92
122
|
async start() {
|
|
93
123
|
if (this.running)
|
|
94
124
|
return this.profile;
|
|
125
|
+
// Refuse, before any request, a configuration that can only strand tasks.
|
|
126
|
+
const key = this.config.privateKey ?? this.config.existingPrivateKey;
|
|
127
|
+
if (!key) {
|
|
128
|
+
throw new Error('[WorkerRuntime] no executor key. Pass `privateKey`: the private key of the wallet that owns `apiKey` ' +
|
|
129
|
+
'(or `existingPrivateKey` to restore without re-registering). The backend assigns accepted tasks on-chain to ' +
|
|
130
|
+
"the API key's owner, and only that wallet can sign submitEvidence — a keyless runtime would overwrite the " +
|
|
131
|
+
"owner's registered public key and strand every task it accepted. To only look at tasks, call " +
|
|
132
|
+
'BlindMarket.browseA2ATasks() directly.');
|
|
133
|
+
}
|
|
134
|
+
if (this.declaredChains.length === 0) {
|
|
135
|
+
throw new Error('[WorkerRuntime] no RPC configured. Set `rpcUrl` (0G) and/or `rpcUrls.base` to the network the backend at ' +
|
|
136
|
+
'`apiBase` settles on. There is no default: submitEvidence is signed on this RPC after the task is already ' +
|
|
137
|
+
'assigned, so a guessed network strands it.');
|
|
138
|
+
}
|
|
95
139
|
// "No tasks arrive" looks the same as "no work available", so say which
|
|
96
|
-
// chains this runtime
|
|
140
|
+
// chains this runtime claims tasks on, and what is missing.
|
|
97
141
|
const undeclared = SETTLEMENT_CHAINS.filter((c) => !this.declaredChains.includes(c));
|
|
98
142
|
if (undeclared.length > 0) {
|
|
99
143
|
console.warn(`[WorkerRuntime] declaring chains: ${this.declaredChains.join(', ') || 'none'}. ` +
|
|
100
|
-
`No RPC for ${undeclared.join(', ')} —
|
|
144
|
+
`No RPC for ${undeclared.join(', ')} — tasks on ${undeclared.length > 1 ? 'those chains' : 'that chain'} are skipped; ` +
|
|
145
|
+
`set ${undeclared.map((c) => `rpcUrls.${c}`).join(', ')} to claim them ` +
|
|
101
146
|
`(production posts new tasks on Base).`);
|
|
102
147
|
}
|
|
103
148
|
// 1. Register or restore executor
|
|
104
|
-
if (this.config.
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
};
|
|
110
|
-
// Fetch existing profile — backend identifies by API key
|
|
111
|
-
const result = await this.bb.getExecutorProfile();
|
|
112
|
-
this.profile = await this.declareSupportedChains(result.agent);
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
149
|
+
if (this.config.privateKey) {
|
|
150
|
+
// The API key owner's own key: createAgent() checks it against the API
|
|
151
|
+
// key's owner BEFORE registering (no side effects on a mismatch), then
|
|
152
|
+
// registers its uncompressed public key. Registration is an upsert that
|
|
153
|
+
// keeps reputation, so this is safe on every start.
|
|
115
154
|
const result = await this.bb.createAgent({
|
|
155
|
+
privateKey: this.config.privateKey,
|
|
116
156
|
displayName: this.config.displayName,
|
|
117
157
|
capabilities: this.config.capabilities,
|
|
118
158
|
minReward: this.config.minReward,
|
|
119
159
|
preferredCapabilities: this.config.preferredCapabilities,
|
|
120
160
|
supportedChains: this.declaredChains,
|
|
121
161
|
});
|
|
122
|
-
this.wallet =
|
|
123
|
-
address: result.wallet.address,
|
|
124
|
-
privateKey: result.wallet.privateKey,
|
|
125
|
-
publicKey: result.wallet.publicKey,
|
|
126
|
-
};
|
|
162
|
+
this.wallet = result.wallet;
|
|
127
163
|
this.profile = result.executor;
|
|
128
164
|
}
|
|
165
|
+
else {
|
|
166
|
+
const signer = new ethers.Wallet(this.config.existingPrivateKey);
|
|
167
|
+
if (this.config.existingAddress && this.config.existingAddress.toLowerCase() !== signer.address.toLowerCase()) {
|
|
168
|
+
throw new Error(`[WorkerRuntime] existingAddress ${this.config.existingAddress} is not existingPrivateKey's address (${signer.address})`);
|
|
169
|
+
}
|
|
170
|
+
// Fetch existing profile — backend identifies by API key
|
|
171
|
+
const result = await this.bb.getExecutorProfile();
|
|
172
|
+
// Checked before declareSupportedChains() can register anything: tasks
|
|
173
|
+
// are assigned to the profile's address, which must be this signer.
|
|
174
|
+
if (result.agent.address.toLowerCase() !== signer.address.toLowerCase()) {
|
|
175
|
+
throw new Error(`[WorkerRuntime] the API key resolves to executor ${result.agent.address} but existingPrivateKey belongs to ` +
|
|
176
|
+
`${signer.address}; it could accept tasks but never sign submitEvidence. Use the owner wallet's key, or an API key minted by this wallet.`);
|
|
177
|
+
}
|
|
178
|
+
this.wallet = { address: signer.address, privateKey: signer.privateKey, publicKey: signer.signingKey.publicKey };
|
|
179
|
+
this.profile = await this.declareSupportedChains(result.agent);
|
|
180
|
+
}
|
|
129
181
|
this.emit({ type: 'registered', profile: this.profile });
|
|
130
182
|
// 2. Start the browse loop
|
|
131
183
|
this.running = true;
|
|
@@ -139,9 +191,11 @@ export class WorkerRuntime {
|
|
|
139
191
|
* chain this runtime has no RPC for — it would be offered, accept and
|
|
140
192
|
* strand those tasks. A stored list that is a SUBSET of what the runtime
|
|
141
193
|
* can settle is left alone: an operator who registered ['base'] through
|
|
142
|
-
* the MCP or PATCH meant it. The
|
|
143
|
-
*
|
|
144
|
-
*
|
|
194
|
+
* the MCP or PATCH meant it. The stored list is a declaration only — the
|
|
195
|
+
* backend does not filter offers by it; this runtime's own browse filter
|
|
196
|
+
* uses `declaredChains`, not the stored list — and a restore never
|
|
197
|
+
* registers otherwise, so an executor first registered by an older SDK
|
|
198
|
+
* would keep its old list.
|
|
145
199
|
*
|
|
146
200
|
* A /profile response with no `supportedChains` key comes from a backend
|
|
147
201
|
* that predates the field. That backend would drop the field anyway, and
|
|
@@ -206,9 +260,9 @@ export class WorkerRuntime {
|
|
|
206
260
|
clearInterval(this.browseTimer);
|
|
207
261
|
this.browseTimer = undefined;
|
|
208
262
|
}
|
|
209
|
-
for (const t of this.
|
|
210
|
-
|
|
211
|
-
this.
|
|
263
|
+
for (const t of this.retryTimers)
|
|
264
|
+
clearTimeout(t);
|
|
265
|
+
this.retryTimers.clear();
|
|
212
266
|
this.emit({ type: 'stopped' });
|
|
213
267
|
}
|
|
214
268
|
pause() {
|
|
@@ -235,75 +289,241 @@ export class WorkerRuntime {
|
|
|
235
289
|
if (this.paused)
|
|
236
290
|
return;
|
|
237
291
|
try {
|
|
238
|
-
|
|
239
|
-
const inFlight = active.filter(e => e.status === 'bidding' || e.status === 'working').length;
|
|
240
|
-
if (inFlight >= this.config.maxConcurrentTasks)
|
|
292
|
+
if (this.inFlight() >= this.config.maxConcurrentTasks)
|
|
241
293
|
return;
|
|
242
294
|
const result = await this.bb.browseA2ATasks({
|
|
243
295
|
capabilities: this.config.capabilities,
|
|
244
296
|
});
|
|
245
297
|
this.emit({ type: 'browse_done', found: result.tasks.length });
|
|
246
|
-
|
|
247
|
-
|
|
298
|
+
// GET /a2a/tasks serves { meta, state } entries — id and status live on
|
|
299
|
+
// `state`. Nobody "assigns" a task: the executor claims it by calling
|
|
300
|
+
// /accept itself (which also assigns it on-chain), so there is no
|
|
301
|
+
// 'assigned' status to wait for.
|
|
302
|
+
const listed = new Set();
|
|
303
|
+
for (const entry of result.tasks) {
|
|
304
|
+
const state = entry.state;
|
|
305
|
+
const taskId = state?.taskId ?? entry.meta?.taskId;
|
|
306
|
+
if (!taskId)
|
|
248
307
|
continue;
|
|
249
|
-
|
|
308
|
+
listed.add(taskId);
|
|
309
|
+
if (state.status !== 'open')
|
|
250
310
|
continue;
|
|
251
|
-
|
|
311
|
+
// An accept assigns on-chain and cannot be released, so never claim a
|
|
312
|
+
// task browse already says is on a chain this runtime did not declare.
|
|
313
|
+
// The backend does not filter by the registered supportedChains, so
|
|
314
|
+
// this filter (and the post-accept check in executeTask, which covers
|
|
315
|
+
// rows with no chain) is the only thing keeping such tasks out.
|
|
316
|
+
if (entry.meta?.chain && !this.declaredChains.includes(entry.meta.chain))
|
|
252
317
|
continue;
|
|
253
|
-
this.
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
318
|
+
this.claim(taskId, state, entry.meta);
|
|
319
|
+
}
|
|
320
|
+
// Tasks that may still be held for this executor are not in the open
|
|
321
|
+
// listing: re-try their /accept here once their back-off has passed.
|
|
322
|
+
for (const [taskId, retry] of this.retries) {
|
|
323
|
+
if (retry.reaccept)
|
|
324
|
+
this.claim(taskId, retry.reaccept.state, retry.reaccept.meta);
|
|
325
|
+
// Anything else that left the listing is gone (taken, cancelled,
|
|
326
|
+
// expired): drop its bookkeeping so the map stays bounded.
|
|
327
|
+
else if (!listed.has(taskId))
|
|
328
|
+
this.retries.delete(taskId);
|
|
263
329
|
}
|
|
264
330
|
}
|
|
265
331
|
catch (err) {
|
|
266
332
|
this.emit({ type: 'error', error: `Browse failed: ${err}` });
|
|
267
333
|
}
|
|
268
334
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
335
|
+
/** Executions holding a concurrency slot. A task waiting for a wrap or backing off holds none. */
|
|
336
|
+
inFlight() {
|
|
337
|
+
let n = 0;
|
|
338
|
+
for (const e of this.executions.values()) {
|
|
339
|
+
if (e.status === 'bidding' || e.status === 'assigned' || e.status === 'working')
|
|
340
|
+
n++;
|
|
341
|
+
}
|
|
342
|
+
return n;
|
|
343
|
+
}
|
|
344
|
+
/** Start executing `taskId` if it is not running, not backing off, and a slot is free. */
|
|
345
|
+
claim(taskId, state, meta) {
|
|
346
|
+
if (this.executions.has(taskId))
|
|
347
|
+
return false;
|
|
348
|
+
const retry = this.retries.get(taskId);
|
|
349
|
+
if (retry && retry.notBefore > Date.now())
|
|
350
|
+
return false;
|
|
351
|
+
if (this.inFlight() >= this.config.maxConcurrentTasks)
|
|
352
|
+
return false;
|
|
353
|
+
this.executions.set(taskId, { taskId, status: 'bidding', task: state, startedAt: Date.now() });
|
|
354
|
+
// Announce a task once, not on every re-try.
|
|
355
|
+
if (!retry)
|
|
356
|
+
this.emit({ type: 'task_found', taskId, task: state });
|
|
357
|
+
void this.executeTask(taskId, state, meta);
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
retryState(taskId) {
|
|
361
|
+
let retry = this.retries.get(taskId);
|
|
362
|
+
if (!retry) {
|
|
363
|
+
retry = { notBefore: 0, failures: 0, wrapTimeouts: 0, bidded: false };
|
|
364
|
+
this.retries.set(taskId, retry);
|
|
365
|
+
}
|
|
366
|
+
return retry;
|
|
367
|
+
}
|
|
368
|
+
// ── Accept ──────────────────────────────────────────────────────────────
|
|
369
|
+
/**
|
|
370
|
+
* POST /accept, re-trying while the backend says the claim is still ours:
|
|
371
|
+
* 503 ASSIGNMENT_PENDING means the assign tx is broadcast but unconfirmed
|
|
372
|
+
* and the task stays `accepted` for this executor until a retry confirms it
|
|
373
|
+
* (or the backend's sweep releases it). A 503 SETTLEMENT_FAILED seen AFTER a
|
|
374
|
+
* pending answer is the idempotent re-check failing, not a release, so it is
|
|
375
|
+
* re-tried too. Bounded by assignmentPendingTimeoutMs, then AcceptAbandoned
|
|
376
|
+
* with `held: true`. Every other error is thrown as it came.
|
|
377
|
+
*/
|
|
378
|
+
async acceptUntilAssigned(taskId) {
|
|
379
|
+
const giveUpAt = Date.now() + this.config.assignmentPendingTimeoutMs;
|
|
380
|
+
let delay = Math.min(2_000, this.config.watchIntervalMs);
|
|
381
|
+
let pending = false;
|
|
382
|
+
for (;;) {
|
|
277
383
|
try {
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
if (
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
384
|
+
return await this.bb.acceptTask(taskId);
|
|
385
|
+
}
|
|
386
|
+
catch (err) {
|
|
387
|
+
const code = err instanceof ApiError ? err.code : undefined;
|
|
388
|
+
if (code === 'ASSIGNMENT_PENDING')
|
|
389
|
+
pending = true;
|
|
390
|
+
else if (!(pending && code === 'SETTLEMENT_FAILED'))
|
|
391
|
+
throw err;
|
|
392
|
+
if (!this.running || Date.now() + delay > giveUpAt)
|
|
393
|
+
throw new AcceptAbandoned(err, true);
|
|
394
|
+
await sleep(delay);
|
|
395
|
+
delay = Math.min(delay * 2, 15_000);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* /accept did not hand over the task. Release the slot and decide when (if
|
|
401
|
+
* ever) the task is touched again. Returns the message for `task_failed`.
|
|
402
|
+
*/
|
|
403
|
+
async onAcceptFailed(taskId, a2a, meta, err) {
|
|
404
|
+
this.executions.delete(taskId);
|
|
405
|
+
const now = Date.now();
|
|
406
|
+
const cause = err instanceof AcceptAbandoned ? err.cause : err;
|
|
407
|
+
const api = cause instanceof ApiError ? cause : undefined;
|
|
408
|
+
// 403 NEEDS_WRAP: the brief key is not wrapped to our pubkey. Bid (the
|
|
409
|
+
// poster wraps to bidders) and wait WITHOUT a slot: the task is re-tried
|
|
410
|
+
// every watchIntervalMs until wrapTimeoutMs, then backed off.
|
|
411
|
+
if (api?.code === 'NEEDS_WRAP') {
|
|
412
|
+
const retry = this.retryState(taskId);
|
|
413
|
+
if (!retry.bidded) {
|
|
414
|
+
try {
|
|
415
|
+
await this.bb.bidOnTask(taskId);
|
|
416
|
+
retry.bidded = true;
|
|
417
|
+
this.emit({ type: 'task_bidded', taskId });
|
|
418
|
+
}
|
|
419
|
+
catch (bidErr) {
|
|
420
|
+
retry.failures++;
|
|
421
|
+
retry.notBefore = now + backoff(RELEASED_BACKOFF_MS, retry.failures, MAX_BACKOFF_MS);
|
|
422
|
+
return `not accepted: NEEDS_WRAP, and the bid failed: ${bidErr}`;
|
|
287
423
|
}
|
|
288
424
|
}
|
|
289
|
-
|
|
290
|
-
|
|
425
|
+
const reason = api.body?.error?.reason;
|
|
426
|
+
const impossible = typeof reason === 'string'
|
|
427
|
+
? WRAP_IMPOSSIBLE_REASONS.has(reason)
|
|
428
|
+
: WRAP_IMPOSSIBLE.test(api.message);
|
|
429
|
+
retry.wrapSince ??= now;
|
|
430
|
+
if (!impossible && now - retry.wrapSince < this.config.wrapTimeoutMs) {
|
|
431
|
+
retry.notBefore = now + this.config.watchIntervalMs;
|
|
432
|
+
this.scheduleRetry(taskId, a2a, meta, this.config.watchIntervalMs);
|
|
433
|
+
return 'not accepted yet: NEEDS_WRAP — bid placed, waiting for the poster to wrap the brief key';
|
|
291
434
|
}
|
|
435
|
+
// Timed out, or the backend says the platform can never wrap it: only
|
|
436
|
+
// the poster's own client still can, so look again much later.
|
|
437
|
+
retry.wrapTimeouts++;
|
|
438
|
+
retry.wrapSince = undefined;
|
|
439
|
+
retry.bidded = false; // bid again when the back-off ends
|
|
440
|
+
const wait = backoff(this.config.wrapTimeoutMs, retry.wrapTimeouts, MAX_WRAP_BACKOFF_MS);
|
|
441
|
+
retry.notBefore = now + wait;
|
|
442
|
+
return impossible
|
|
443
|
+
? `not accepted: ${api.message} Skipping for ${Math.round(wait / 60_000)} min.`
|
|
444
|
+
: `not accepted: no wrapped key after ${Math.round(this.config.wrapTimeoutMs / 1000)}s; skipping for ${Math.round(wait / 60_000)} min`;
|
|
445
|
+
}
|
|
446
|
+
// Lost the race / offer held by another agent / gone: nothing was claimed.
|
|
447
|
+
// A still-open task (OFFER_HELD, ACCEPT_LOCKED) is re-tried by a later browse.
|
|
448
|
+
if (api?.status === 409) {
|
|
449
|
+
this.retries.delete(taskId);
|
|
450
|
+
return `not accepted: ${api.code ?? api.message}`;
|
|
451
|
+
}
|
|
452
|
+
// Refused for a reason a retry cannot change (SELF_ACCEPT, IS_VERIFIER,
|
|
453
|
+
// NOT_REGISTERED, NOT_TARGET_EXECUTOR, 404 …): nothing was claimed. Keep
|
|
454
|
+
// the task out of the way for as long as it stays listed.
|
|
455
|
+
if (api && api.status < 500 && api.status !== 408 && api.status !== 429) {
|
|
456
|
+
const retry = this.retryState(taskId);
|
|
457
|
+
retry.reaccept = undefined;
|
|
458
|
+
retry.notBefore = now + MAX_WRAP_BACKOFF_MS;
|
|
459
|
+
return `not accepted: ${api.code ?? api.message}`;
|
|
460
|
+
}
|
|
461
|
+
// The backend released the task back to open (503 REWRAP_FAILED, or a 503
|
|
462
|
+
// SETTLEMENT_FAILED on a fresh accept): a later browse may claim it again,
|
|
463
|
+
// after a per-task back-off so a persistent failure does not hot-loop.
|
|
464
|
+
const retry = this.retryState(taskId);
|
|
465
|
+
retry.failures++;
|
|
466
|
+
// The backend says "released" in the message only when it did re-open the
|
|
467
|
+
// task (its release is compare-and-set and can fail; SETTLEMENT_FAILED from
|
|
468
|
+
// the idempotent re-check never releases). There is no separate code.
|
|
469
|
+
const released = !(err instanceof AcceptAbandoned) &&
|
|
470
|
+
(api?.code === 'REWRAP_FAILED' || api?.code === 'SETTLEMENT_FAILED') &&
|
|
471
|
+
/released/i.test(api.message);
|
|
472
|
+
if (released) {
|
|
473
|
+
retry.reaccept = undefined;
|
|
474
|
+
retry.notBefore = now + backoff(RELEASED_BACKOFF_MS, retry.failures, MAX_BACKOFF_MS);
|
|
475
|
+
return `not accepted: ${api.code} — the backend released the task; re-trying after a back-off`;
|
|
476
|
+
}
|
|
477
|
+
// ASSIGNMENT_PENDING that never confirmed, a REWRAP/SETTLEMENT failure the
|
|
478
|
+
// backend did not release, an unknown 5xx, a rate limit or a network error: the task MAY be held for (or already assigned to) this executor,
|
|
479
|
+
// and then it is not in the open listing. /accept is idempotent for the
|
|
480
|
+
// recorded executor, so browse re-tries it directly, with back-off, a
|
|
481
|
+
// bounded number of times.
|
|
482
|
+
const rounds = (retry.reaccept?.rounds ?? 0) + 1;
|
|
483
|
+
if (rounds > MAX_REACCEPT_ROUNDS) {
|
|
484
|
+
this.retries.delete(taskId);
|
|
485
|
+
return `accept never confirmed after ${MAX_REACCEPT_ROUNDS} rounds (${api?.code ?? cause}); giving up. If the assignment landed on-chain the task is in getExecutions().`;
|
|
486
|
+
}
|
|
487
|
+
retry.reaccept = { state: a2a, meta, rounds };
|
|
488
|
+
retry.notBefore = now + backoff(RELEASED_BACKOFF_MS, retry.failures, MAX_BACKOFF_MS);
|
|
489
|
+
return `accept not confirmed (${api?.code ?? cause}); the task may still be held for this executor — re-trying /accept after a back-off`;
|
|
490
|
+
}
|
|
491
|
+
/** Re-try one task sooner than the next browse tick (NEEDS_WRAP wait). */
|
|
492
|
+
scheduleRetry(taskId, a2a, meta, ms) {
|
|
493
|
+
if (!this.running)
|
|
494
|
+
return;
|
|
495
|
+
const timer = setTimeout(() => {
|
|
496
|
+
this.retryTimers.delete(timer);
|
|
497
|
+
// No free slot / paused: the next browse picks it up instead.
|
|
498
|
+
if (this.running && !this.paused)
|
|
499
|
+
this.claim(taskId, a2a, meta);
|
|
292
500
|
}, ms);
|
|
293
|
-
this.
|
|
501
|
+
this.retryTimers.add(timer);
|
|
294
502
|
}
|
|
295
503
|
// ── Task execution ──────────────────────────────────────────────────────
|
|
296
|
-
async executeTask(taskId, a2a) {
|
|
504
|
+
async executeTask(taskId, a2a, meta) {
|
|
297
505
|
const exec = this.executions.get(taskId);
|
|
298
506
|
if (!exec)
|
|
299
507
|
return;
|
|
300
508
|
try {
|
|
301
|
-
exec.status = 'assigned';
|
|
302
509
|
// Accept task — get the rootHash + this executor's ECIES-wrapped AES
|
|
303
510
|
// key. wrappedKey is a single hex string (this caller's slice), not a
|
|
304
511
|
// Record — see acceptTask()'s doc comment in ../index.ts.
|
|
305
|
-
|
|
512
|
+
let acceptResult;
|
|
513
|
+
try {
|
|
514
|
+
acceptResult = await this.acceptUntilAssigned(taskId);
|
|
515
|
+
}
|
|
516
|
+
catch (err) {
|
|
517
|
+
// Never leave a task that was not handed over in `executions`: it
|
|
518
|
+
// would be skipped by every later browse and never executed.
|
|
519
|
+
const error = await this.onAcceptFailed(taskId, a2a, meta, err);
|
|
520
|
+
this.emit({ type: 'task_failed', taskId, error });
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
this.retries.delete(taskId);
|
|
524
|
+
exec.status = 'assigned';
|
|
306
525
|
exec.task = a2a;
|
|
526
|
+
this.emit({ type: 'task_assigned', taskId });
|
|
307
527
|
this.emit({ type: 'task_accepted', taskId });
|
|
308
528
|
// Fail before running the handler if this runtime can't settle the task:
|
|
309
529
|
// an unknown chain, or one it has no RPC for (the send would fail only
|
|
@@ -338,38 +558,23 @@ export class WorkerRuntime {
|
|
|
338
558
|
const result = await this.config.executeTask({
|
|
339
559
|
taskId,
|
|
340
560
|
task: a2a,
|
|
561
|
+
meta,
|
|
341
562
|
instructions,
|
|
342
563
|
});
|
|
343
564
|
exec.status = 'submitted';
|
|
344
565
|
this.emit({ type: 'task_executed', taskId });
|
|
345
|
-
// Submit
|
|
346
|
-
//
|
|
347
|
-
// unsigned tx
|
|
348
|
-
|
|
349
|
-
//
|
|
350
|
-
//
|
|
351
|
-
//
|
|
352
|
-
//
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
// broadcast onto 0G. The tx now also carries chainId, so a wrong RPC
|
|
358
|
-
// fails loudly at ethers instead of landing on the wrong network.
|
|
359
|
-
const chain = settlementChain(taskId, submitResult.chain);
|
|
360
|
-
// rpcUrls per chain; rpcUrl is the 0G RPC only (rpcFor). The chainId
|
|
361
|
-
// pin still rejects a genuine mismatch at ethers before anything is
|
|
362
|
-
// broadcast.
|
|
363
|
-
const rpc = rpcFor(this.config, chain);
|
|
364
|
-
if (!rpc) {
|
|
365
|
-
throw new Error(`task ${taskId} is escrowed on ${chain} but no RPC is configured for it — set rpcUrls.${chain} in the WorkerRuntime config`);
|
|
366
|
-
}
|
|
367
|
-
const provider = new ethers.JsonRpcProvider(rpc);
|
|
368
|
-
const signer = new ethers.Wallet(this.wallet.privateKey, provider);
|
|
369
|
-
const tx = await signer.sendTransaction(submitResult.unsignedSubmitEvidence);
|
|
370
|
-
await tx.wait();
|
|
371
|
-
}
|
|
372
|
-
const finalizeResult = await this.bb.finalize(taskId);
|
|
566
|
+
// Submit → sign + broadcast submitEvidence on the chain /submit names
|
|
567
|
+
// (the contract's onlyWorker means the backend can only hand back an
|
|
568
|
+
// unsigned tx) → finalize, healing a stranded 'submitted' state via
|
|
569
|
+
// /rebroadcast. Without the broadcast the on-chain task never leaves
|
|
570
|
+
// 'Assigned' and the task only LOOKS complete. deliverResult() signs
|
|
571
|
+
// only on a chain named here: rpcUrls per chain, rpcUrl for 0G only, so
|
|
572
|
+
// an unknown chain refuses before anything is sent, and the chainId pin
|
|
573
|
+
// still rejects a genuine mismatch at ethers.
|
|
574
|
+
const { submitTxHash: _submitTxHash, ...finalizeResult } = await this.bb.deliverResult(taskId, result, {
|
|
575
|
+
privateKey: this.wallet.privateKey,
|
|
576
|
+
rpcUrls: Object.fromEntries(SETTLEMENT_CHAINS.map((c) => [c, rpcFor(this.config, c)])),
|
|
577
|
+
});
|
|
373
578
|
exec.status = 'completed';
|
|
374
579
|
this.emit({ type: 'task_submitted', taskId, result });
|
|
375
580
|
this.emit({ type: 'task_finalized', taskId, finalize: finalizeResult });
|