@ixo/decision-oracle 0.2.28 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/db/schema.d.ts +17 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +10 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/lib/api-lane/accept.d.ts.map +1 -1
- package/dist/lib/api-lane/accept.js +27 -10
- package/dist/lib/api-lane/accept.js.map +1 -1
- package/dist/lib/api-lane/notify.d.ts +9 -0
- package/dist/lib/api-lane/notify.d.ts.map +1 -1
- package/dist/lib/api-lane/notify.js +10 -3
- package/dist/lib/api-lane/notify.js.map +1 -1
- package/dist/lib/api-lane/record.d.ts.map +1 -1
- package/dist/lib/api-lane/record.js +6 -3
- package/dist/lib/api-lane/record.js.map +1 -1
- package/dist/lib/engine/client.d.ts +14 -2
- package/dist/lib/engine/client.d.ts.map +1 -1
- package/dist/lib/engine/client.js +90 -11
- package/dist/lib/engine/client.js.map +1 -1
- package/dist/lib/ixo/grant-chain.js +6 -6
- package/dist/lib/ixo/grant-chain.js.map +1 -1
- package/dist/lib/openhands/why.d.ts.map +1 -1
- package/dist/lib/openhands/why.js +7 -2
- package/dist/lib/openhands/why.js.map +1 -1
- package/dist/lib/work-order/index.d.ts +4 -2
- package/dist/lib/work-order/index.d.ts.map +1 -1
- package/dist/lib/work-order/index.js +19 -4
- package/dist/lib/work-order/index.js.map +1 -1
- package/dist/middleware/api/keys.d.ts.map +1 -1
- package/dist/middleware/api/keys.js +10 -2
- package/dist/middleware/api/keys.js.map +1 -1
- package/dist/middleware/api/rate-limit.js +3 -4
- package/dist/middleware/api/rate-limit.js.map +1 -1
- package/dist/middleware/api/wait.d.ts +17 -2
- package/dist/middleware/api/wait.d.ts.map +1 -1
- package/dist/middleware/api/wait.js +80 -19
- package/dist/middleware/api/wait.js.map +1 -1
- package/dist/middleware/ucan-auth.d.ts.map +1 -1
- package/dist/middleware/ucan-auth.js +7 -2
- package/dist/middleware/ucan-auth.js.map +1 -1
- package/dist/routes/v1/api/claims.d.ts +15 -3
- package/dist/routes/v1/api/claims.d.ts.map +1 -1
- package/dist/routes/v1/api/claims.js +85 -14
- package/dist/routes/v1/api/claims.js.map +1 -1
- package/dist/workflows/claim-ingest.d.ts +19 -125
- package/dist/workflows/claim-ingest.d.ts.map +1 -1
- package/dist/workflows/claim-ingest.js +49 -765
- package/dist/workflows/claim-ingest.js.map +1 -1
- package/dist/workflows/evaluate.d.ts +193 -0
- package/dist/workflows/evaluate.d.ts.map +1 -0
- package/dist/workflows/evaluate.js +980 -0
- package/dist/workflows/evaluate.js.map +1 -0
- package/docs/api.md +24 -4
- package/docs/guide.md +19 -1
- package/migrations/0029_inline_lease.sql +1 -0
- package/migrations/meta/0029_snapshot.json +3021 -0
- package/migrations/meta/_journal.json +7 -0
- package/package.json +4 -4
|
@@ -4,10 +4,16 @@
|
|
|
4
4
|
* each step checkpointed and auto-retried, so a crash/redeploy/transient failure never re-does
|
|
5
5
|
* work or drops a claim.
|
|
6
6
|
*
|
|
7
|
-
* THE WORKFLOW OWNS ITS LEDGER ROW — one writer
|
|
7
|
+
* THE WORKFLOW OWNS ITS LEDGER ROW for its whole lifecycle — the one other writer is the API
|
|
8
|
+
* lane's inline run (`evaluateInline`, workflows/evaluate.ts), the same `evaluate` body run by the
|
|
9
|
+
* accepting request for a caller that waits; it takes the row only from `queued` (writing its lease,
|
|
10
|
+
* `inlineUntil`, in the same statement), and the steps below find what it wrote:
|
|
8
11
|
* step "record-queued" → upsert the `claim_ingests` row as `queued` (durable: the row exists
|
|
9
|
-
* iff the instance started, so nothing can ever strand a row)
|
|
10
|
-
*
|
|
12
|
+
* iff the instance started, so nothing can ever strand a row) — never
|
|
13
|
+
* over a row already `processing` / `processed`, which an inline run
|
|
14
|
+
* (or an earlier run of this instance) wrote and a rerun must not undo
|
|
15
|
+
* step "evaluate" → (workflows/evaluate.ts) a recorded decision is returned as is; else
|
|
16
|
+
* mark the row `processing` (so a later attempt knows one ran), then
|
|
11
17
|
* ONE pull (chain metadata + body + attachment descriptors), the facts
|
|
12
18
|
* lane over the same in-memory claim (`buildFacts`: read the engine's
|
|
13
19
|
* rubric manifest, run the producer, freeze the FactSet), and `POST /v1/decisions` on
|
|
@@ -28,7 +34,11 @@
|
|
|
28
34
|
* ingest — the decision is recorded, and an operator rerun re-enters
|
|
29
35
|
* delivery idempotently. A `pending_review` decision waits for the
|
|
30
36
|
* review lane (routes/v1/reviews.ts → ReviewComplete) to act instead
|
|
31
|
-
* step "mark-processed" → row → `processed`
|
|
37
|
+
* step "mark-processed" → row → `processed` — right after the decision (and the act), so the
|
|
38
|
+
* ledger reads `processed` beside it without waiting on a receiver;
|
|
39
|
+
* AFTER the notify for a `pending_review`, whose record reads its
|
|
40
|
+
* tasks off the `review_required` event the notify writes
|
|
41
|
+
* step "notify-review" → the source's notify lane (fail-soft, its own delivery ledger)
|
|
32
42
|
* terminal catch → row → `failed` (+ the message), instance shows Errored in the dashboard
|
|
33
43
|
* The cron (cron/) only discovers claims and creates instances; it never writes this table.
|
|
34
44
|
*
|
|
@@ -85,68 +95,27 @@
|
|
|
85
95
|
import { WorkflowEntrypoint } from "cloudflare:workers";
|
|
86
96
|
import { NonRetryableError } from "cloudflare:workflows";
|
|
87
97
|
import * as Sentry from "@sentry/cloudflare";
|
|
88
|
-
import {
|
|
98
|
+
import { eq, ne, notInArray } from "drizzle-orm";
|
|
89
99
|
import { getDb } from "../db/client.js";
|
|
90
|
-
import {
|
|
91
|
-
import { claimErrorCode
|
|
92
|
-
import { archiveAnswer } from "../lib/archive.js";
|
|
93
|
-
import { createEngineClient } from "../lib/engine/client.js";
|
|
94
|
-
import { EngineError, isManifestUnusable, } from "../lib/engine/types.js";
|
|
95
|
-
import { AppError } from "../lib/errors.js";
|
|
96
|
-
import { buildFacts, factDeps } from "../lib/facts/builder.js";
|
|
100
|
+
import { claimIngests, decisions } from "../db/schema.js";
|
|
101
|
+
import { claimErrorCode } from "../lib/api-lane/record.js";
|
|
97
102
|
import { factCacheDelete } from "../lib/facts/fact-cache.js";
|
|
98
|
-
import {
|
|
99
|
-
import { finalVerdictUpdate, registryVerdictToken, voidUndecidedUpdate, } from "../lib/facts/stock/unique-store.js";
|
|
103
|
+
import { voidUndecidedUpdate } from "../lib/facts/stock/unique-store.js";
|
|
100
104
|
import { depsFromEnv } from "../lib/ixo/index.js";
|
|
101
105
|
import { getClaimById } from "../lib/ixo/blocksync/index.js";
|
|
102
|
-
import {
|
|
103
|
-
import {
|
|
104
|
-
import { BILLING_PAUSES, CHAIN_CLAIM_ID, companyOf, planAllowsReviewBilling, releaseChainClaim, reserveChainClaim, resolvePayer, } from "../lib/payments/review-billing.js";
|
|
105
|
-
import { settleClaim } from "../lib/payments/settle.js";
|
|
106
|
-
import { payoutRequestFor } from "../lib/payouts.js";
|
|
107
|
-
import { reviewerNoteEnabled, writeReviewerNote } from "../lib/review/reviewer-note.js";
|
|
106
|
+
import { scopedStep, withMetrics } from "../lib/metrics.js";
|
|
107
|
+
import { releaseChainClaim } from "../lib/payments/review-billing.js";
|
|
108
108
|
import { reportAlert, reportError } from "../lib/sentry.js";
|
|
109
|
-
import { getOracle } from "../oracle.js";
|
|
110
109
|
import { isChain, sourceFor } from "./claim-source.js";
|
|
111
|
-
import {
|
|
110
|
+
import { ClaimEvaluator, UNBILLABLE } from "./evaluate.js";
|
|
111
|
+
import { actStep, RECEIPT_MISSING, stepError } from "./steps.js";
|
|
112
|
+
export { GRANT_RETRY_DELAY, PAYER_BILLING } from "./evaluate.js";
|
|
112
113
|
// Light garbage guard for manually-created instances, NOT a format check: real chain claimIds are
|
|
113
114
|
// CIDs *and* Matrix event ids (`$…`), so anything printable-ASCII of sane length is acceptable —
|
|
114
115
|
// the chain lookup is the actual validation (unknown id → fail-closed 404).
|
|
115
116
|
const CLAIM_ID_RE = /^[\x21-\x7e]{1,512}$/;
|
|
116
|
-
/**
|
|
117
|
-
* How a review-billing trouble is told apart in the ledger's `error` (a run that gave up while its
|
|
118
|
-
* collection was paused for payment): the resume cron restarts exactly those.
|
|
119
|
-
*/
|
|
120
|
-
export const PAYER_BILLING = "payer-billing:";
|
|
121
|
-
/** `evaluate`'s result for a claim a per-review collection cannot bill (see the header). */
|
|
122
|
-
const UNBILLABLE = "unbillable";
|
|
123
|
-
const isBillingPause = (reason) => BILLING_PAUSES.includes(reason ?? "");
|
|
124
|
-
/** The facts lane's deterministic negatives → the `decisions.status` they are recorded as (schema.ts). */
|
|
125
|
-
const BUILD_FAILURE_STATUS = {
|
|
126
|
-
"rubric-unusable": "unusable",
|
|
127
|
-
"invalid-facts": "invalid_facts",
|
|
128
|
-
};
|
|
129
|
-
/** The engine's refusals of the claim's content → the `decisions.status` they are recorded as. */
|
|
130
|
-
const CONTENT_REFUSAL_STATUS = {
|
|
131
|
-
CONTENT_MISMATCH: "content_mismatch",
|
|
132
|
-
PAYLOAD_TOO_LARGE: "payload_too_large",
|
|
133
|
-
};
|
|
134
|
-
/**
|
|
135
|
-
* `403 NO_EVALUATE_GRANT` is retry-ONCE (docs/ENGINE-CONTRACT.md): the grant may have landed on
|
|
136
|
-
* chain seconds ago — the portal grants and registers in one flow — so wait about a block and ask
|
|
137
|
-
* again before calling it a real "not hired". In-step on purpose: the FactSet may not leave the
|
|
138
|
-
* step, so a step-level sleep would cost a full rebuild (connectors, AI charges). An object so a
|
|
139
|
-
* test can shorten it.
|
|
140
|
-
*/
|
|
141
|
-
export const GRANT_RETRY_DELAY = { ms: 30_000 };
|
|
142
117
|
/** The evaluate step's retries (below) — its last attempt is the one that freezes outages (P0-7). */
|
|
143
118
|
const EVALUATE_RETRY_LIMIT = 8;
|
|
144
|
-
const resultOf = (row) => ({
|
|
145
|
-
status: row.status,
|
|
146
|
-
verdict: row.verdict,
|
|
147
|
-
resultDigest: row.resultDigest,
|
|
148
|
-
receiptCid: row.receiptCid,
|
|
149
|
-
});
|
|
150
119
|
export class ClaimIngest extends WorkflowEntrypoint {
|
|
151
120
|
async run(event, step) {
|
|
152
121
|
// Params may be absent when an operator hand-creates an instance whose id IS a claim CID; for
|
|
@@ -225,7 +194,9 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
225
194
|
async (ctx) => {
|
|
226
195
|
const finalAttempt = ctx !== undefined && ctx.attempt > (ctx.config?.retries?.limit ?? EVALUATE_RETRY_LIMIT);
|
|
227
196
|
try {
|
|
228
|
-
return await this.evaluate(deps, source, claimId,
|
|
197
|
+
return await new ClaimEvaluator(this.env).evaluate(deps, source, claimId, {
|
|
198
|
+
finalAttempt,
|
|
199
|
+
});
|
|
229
200
|
}
|
|
230
201
|
catch (err) {
|
|
231
202
|
throw stepError(err);
|
|
@@ -244,6 +215,20 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
244
215
|
workflow: "claim-ingest",
|
|
245
216
|
});
|
|
246
217
|
}
|
|
218
|
+
// `processed` as soon as the decision (and its act) is in: nothing after it can fail the
|
|
219
|
+
// ingest — the notify is fail-soft — so the ledger never makes a waiting caller sit through a
|
|
220
|
+
// receiver's webhook round trip to read `processed` beside its decision. Except a claim that
|
|
221
|
+
// waits for review: its record reads the tasks off the `review_required` event the notify
|
|
222
|
+
// writes, so `processed` follows it there and no answer shows placeholder tasks.
|
|
223
|
+
const markProcessed = () => step.do("mark-processed", async () => {
|
|
224
|
+
await this.recordStatus(claimId, event, {
|
|
225
|
+
status: "processed",
|
|
226
|
+
collectionId: collectionId,
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
const reviewFirst = result.status === "pending_review";
|
|
230
|
+
if (!reviewFirst)
|
|
231
|
+
await markProcessed();
|
|
247
232
|
// The source's notify lane (chain: the Matrix review card) — its own step, NEVER inside
|
|
248
233
|
// `evaluate`: a step retry there would re-send. Self-contained and fail-soft like `act`
|
|
249
234
|
// above: the delivery has its own ledger (`matrix_deliveries`), and a dead receiver never
|
|
@@ -258,12 +243,8 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
258
243
|
deps.logger.error("notify-review.failed", { claimId, detail: String(err) });
|
|
259
244
|
reportError(err, { lane: "notify", step: "notify-review", claimId, collectionId });
|
|
260
245
|
}
|
|
261
|
-
|
|
262
|
-
await
|
|
263
|
-
status: "processed",
|
|
264
|
-
collectionId: collectionId,
|
|
265
|
-
});
|
|
266
|
-
});
|
|
246
|
+
if (reviewFirst)
|
|
247
|
+
await markProcessed();
|
|
267
248
|
return { claimId, ok: true, ...result };
|
|
268
249
|
}
|
|
269
250
|
catch (err) {
|
|
@@ -318,700 +299,6 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
318
299
|
throw err;
|
|
319
300
|
}
|
|
320
301
|
}
|
|
321
|
-
/**
|
|
322
|
-
* The evaluate step's body: pull → facts → decide → the `decisions` row. Reads the DB, not step
|
|
323
|
-
* results, so an operator rerun converges — and a retry that finds the row is done.
|
|
324
|
-
*/
|
|
325
|
-
async evaluate(deps, source, claimId, finalAttempt = false) {
|
|
326
|
-
const db = getDb(this.env);
|
|
327
|
-
// In this order: what the ledger says about this step's history, then the `processing` mark
|
|
328
|
-
// for the NEXT attempt — written before anything is pulled or spent, so a crash anywhere below
|
|
329
|
-
// leaves it behind. (Not one `db.batch`: Drizzle's D1 batch throws on a relational read that
|
|
330
|
-
// finds no row.) A `processed` row is never re-marked: it has its decision (the early return
|
|
331
|
-
// below), and an orphaned attempt must not walk it back (see `recordStatus`).
|
|
332
|
-
const ingest = await db.query.claimIngests.findFirst({
|
|
333
|
-
where: (t, { eq: eqOp }) => eqOp(t.claimId, claimId),
|
|
334
|
-
});
|
|
335
|
-
await db
|
|
336
|
-
.update(claimIngests)
|
|
337
|
-
.set({ status: "processing" })
|
|
338
|
-
.where(and(eq(claimIngests.claimId, claimId), ne(claimIngests.status, "processed")));
|
|
339
|
-
const existing = await db.query.decisions.findFirst({
|
|
340
|
-
where: (t, { eq: eqOp }) => eqOp(t.claimId, claimId),
|
|
341
|
-
});
|
|
342
|
-
if (existing)
|
|
343
|
-
return resultOf(existing);
|
|
344
|
-
const engine = createEngineClient({
|
|
345
|
-
config: deps.config,
|
|
346
|
-
logger: deps.logger,
|
|
347
|
-
trace: deps.trace,
|
|
348
|
-
cache: deps.cache,
|
|
349
|
-
});
|
|
350
|
-
// `processing` already set ⇒ a previous attempt of this step ran and died before its row
|
|
351
|
-
// landed — possibly AFTER the engine decided. Decided-once on the engine means a resend would
|
|
352
|
-
// only replay that stored row, so one read recovers it and the claim is never stranded: the
|
|
353
|
-
// engine is NOT asked to decide again below. A 404 means the engine never got that far: the
|
|
354
|
-
// normal path. A first attempt (`queued`) never pays the read.
|
|
355
|
-
if (ingest?.status === "processing") {
|
|
356
|
-
const recovered = await engine.getDecision(claimId);
|
|
357
|
-
if (recovered) {
|
|
358
|
-
deps.logger.info("evaluate.recovered", {
|
|
359
|
-
claimId,
|
|
360
|
-
collectionId: ingest.collectionId,
|
|
361
|
-
status: recovered.decision.status,
|
|
362
|
-
resultDigest: recovered.resultDigest,
|
|
363
|
-
receiptCid: recovered.receipt?.cid ?? null,
|
|
364
|
-
});
|
|
365
|
-
if (!recovered.receipt && recovered.decision.status !== "pending_review") {
|
|
366
|
-
// Since CA10 the GET mints a missing receipt or answers 503, so this is a contract
|
|
367
|
-
// breach. Never recorded receipt-less (the receipt is the one credential act and the
|
|
368
|
-
// payout spend), and never re-decided either: that would re-run the handlers' AI spend.
|
|
369
|
-
// `requireReceipt` retries the step, whose next GET recovers it with its receipt.
|
|
370
|
-
deps.logger.warn("evaluate.recovered-without-receipt", {
|
|
371
|
-
claimId,
|
|
372
|
-
collectionId: ingest.collectionId,
|
|
373
|
-
status: recovered.decision.status,
|
|
374
|
-
});
|
|
375
|
-
}
|
|
376
|
-
requireReceipt(recovered, "decision");
|
|
377
|
-
return this.recordRecovered(deps, source, db, engine, claimId, ingest.collectionId, recovered);
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
// Out of engine credits, every decide call 402s until the credits cron resumes the scope, so an
|
|
381
|
-
// attempt here would pull, build and PAY for its AI calls up to eight times only to be refused.
|
|
382
|
-
// One projected read instead, AFTER the two ledger reads above (a decided or recovered claim
|
|
383
|
-
// returns regardless): the step retries on its own backoff and spends nothing meanwhile.
|
|
384
|
-
// The chain lane off (config.ts `CHAIN_LANE`): a chain run started before the switch, still
|
|
385
|
-
// retrying, or restarted by hand must not pull, spend AI or ask the engine to decide on the
|
|
386
|
-
// owner's credits. Retried like the pause below, never recorded `skipped` — the scan would then
|
|
387
|
-
// count the claim as seen for good — so the run carries on once the lane is back on (within the
|
|
388
|
-
// step's ~4 h of retries; past them it ends `failed`, rerun like any failed claim).
|
|
389
|
-
if (isChain(source) && !deps.config.CHAIN_LANE) {
|
|
390
|
-
throw AppError.unavailable(`chain lane off (CHAIN_LANE=false) — not pulling ${claimId}`);
|
|
391
|
-
}
|
|
392
|
-
const paused = ingest?.collectionId ? await source.pausedReason(ingest.collectionId) : null;
|
|
393
|
-
if (paused === "engine-credits") {
|
|
394
|
-
throw AppError.unavailable(`${source.lane === "chain" ? "collection" : "source"} paused for engine credits — not pulling ${claimId}`);
|
|
395
|
-
}
|
|
396
|
-
// The same for a review nobody can pay for yet: the payer's fix, resumed by the payer cron.
|
|
397
|
-
if (isBillingPause(paused)) {
|
|
398
|
-
throw AppError.unavailable(`${PAYER_BILLING} collection paused ${paused} — not pulling ${claimId}`);
|
|
399
|
-
}
|
|
400
|
-
const billing = isChain(source) && ingest?.collectionId
|
|
401
|
-
? await this.billingFor(deps, source, db, claimId, ingest.collectionId)
|
|
402
|
-
: null;
|
|
403
|
-
if (billing === UNBILLABLE) {
|
|
404
|
-
await db
|
|
405
|
-
.update(claimIngests)
|
|
406
|
-
.set({ status: "skipped", error: "unbillable-claim-id" })
|
|
407
|
-
.where(eq(claimIngests.claimId, claimId));
|
|
408
|
-
deps.logger.warn("reserve.unbillable-claim-id", {
|
|
409
|
-
claimId,
|
|
410
|
-
collectionId: ingest?.collectionId,
|
|
411
|
-
});
|
|
412
|
-
return { status: UNBILLABLE, verdict: null, resultDigest: null, receiptCid: null };
|
|
413
|
-
}
|
|
414
|
-
return this.decideFresh(deps, source, db, engine, claimId, billing, finalAttempt);
|
|
415
|
-
}
|
|
416
|
-
/**
|
|
417
|
-
* Who pays for this chain claim's review (header): null when its collection pays by its
|
|
418
|
-
* evaluation leg (or the oracle no longer bills per review); `reserved` when a charge already
|
|
419
|
-
* holds it (a retry); else the payer, resolved fresh — never a guess: the engine's or the chain's
|
|
420
|
-
* trouble retries the step, and no usable delegation pauses the collection `payer-delegation`.
|
|
421
|
-
*/
|
|
422
|
-
async billingFor(deps, source, db, claimId, collectionId) {
|
|
423
|
-
const watched = await deps.getWatched(collectionId);
|
|
424
|
-
if (!watched?.billingSourceId)
|
|
425
|
-
return null;
|
|
426
|
-
// Registered to be paid per review, it is never decided unpaid: a plan IXO no longer grants
|
|
427
|
-
// pauses it (re-register on the escrow gate), and a plan the engine cannot tell retries.
|
|
428
|
-
if (!(await planAllowsReviewBilling(deps))) {
|
|
429
|
-
await this.pausePayer(deps, db, collectionId, "oracle-plan", "plan does not allow it");
|
|
430
|
-
throw AppError.unavailable(`${PAYER_BILLING} the plan does not allow per-review billing`);
|
|
431
|
-
}
|
|
432
|
-
if (!CHAIN_CLAIM_ID.test(claimId))
|
|
433
|
-
return UNBILLABLE;
|
|
434
|
-
const charge = await db.query.apiCharges.findFirst({
|
|
435
|
-
columns: { status: true },
|
|
436
|
-
where: eq(apiCharges.claimId, claimId),
|
|
437
|
-
});
|
|
438
|
-
if (charge && ["reserved", "settled", "paid"].includes(charge.status))
|
|
439
|
-
return "reserved";
|
|
440
|
-
const billingDeps = { ...deps, db };
|
|
441
|
-
const payer = await resolvePayer(billingDeps, await companyOf(billingDeps, collectionId), await source.protocolDid(collectionId));
|
|
442
|
-
if (!payer.ok) {
|
|
443
|
-
await this.pausePayer(deps, db, collectionId, "payer-delegation", payer.detail);
|
|
444
|
-
if (payer.detail.startsWith("no published price")) {
|
|
445
|
-
// The operator's config, not the payer's: every claim of the protocol stops here.
|
|
446
|
-
await reportAlert(deps, "payments.review.no-price", { collectionId });
|
|
447
|
-
}
|
|
448
|
-
throw AppError.unavailable(`${PAYER_BILLING} no one pays for collection ${collectionId}`);
|
|
449
|
-
}
|
|
450
|
-
// The organisation now pays from another source (one deleted, one added): follow it.
|
|
451
|
-
if (payer.source.sourceId !== watched.billingSourceId) {
|
|
452
|
-
await db
|
|
453
|
-
.update(watchedCollections)
|
|
454
|
-
.set({ billingSourceId: payer.source.sourceId })
|
|
455
|
-
.where(eq(watchedCollections.collectionId, collectionId));
|
|
456
|
-
}
|
|
457
|
-
return payer;
|
|
458
|
-
}
|
|
459
|
-
/**
|
|
460
|
-
* A billing pause. The payer's two: the owner's notice, never a page (a customer's condition).
|
|
461
|
-
* `oracle-plan`: IXO's to fix — paged, no notice to the owner or the payer.
|
|
462
|
-
*/
|
|
463
|
-
async pausePayer(deps, db, collectionId, reason, detail) {
|
|
464
|
-
if (reason === "oracle-plan") {
|
|
465
|
-
await db
|
|
466
|
-
.update(watchedCollections)
|
|
467
|
-
.set({ status: "paused", pausedReason: reason })
|
|
468
|
-
.where(eq(watchedCollections.collectionId, collectionId));
|
|
469
|
-
await reportAlert(deps, "payments.review.oracle-plan", { collectionId, code: peerCode(detail) }, "payments.review.oracle-plan");
|
|
470
|
-
}
|
|
471
|
-
else {
|
|
472
|
-
await pauseCollection(db, deps, collectionId, reason, undefined, false);
|
|
473
|
-
}
|
|
474
|
-
deps.logger.warn(`reserve.${reason}`, { collectionId, detail });
|
|
475
|
-
deps.metrics.count("review_billing.paused", 1, { outcome: reason });
|
|
476
|
-
}
|
|
477
|
-
/** The normal path: pull → build → (reserve) → decide → the `decisions` row. */
|
|
478
|
-
async decideFresh(deps, source, db, engine, claimId, billing = null, finalAttempt = false) {
|
|
479
|
-
// The one pull per claim: metadata + body + attachment descriptors (the source's store), and
|
|
480
|
-
// the content the engine binds the decision to — the engine's alone, so it is taken off here
|
|
481
|
-
// and no handler ever sees it. Like the body, it never leaves this step.
|
|
482
|
-
const { content, ...pulled } = await source.pull(claimId);
|
|
483
|
-
const { collectionId } = pulled;
|
|
484
|
-
const fdeps = { ...factDeps(deps, pulled.media, engine, db, this.env, source), finalAttempt };
|
|
485
|
-
// `subject.protocolDid` — where the engine reads `#rub`/`#vct` from, and what its manifest is
|
|
486
|
-
// keyed by, so it is resolved before the build (the manifest is the build's first read).
|
|
487
|
-
const protocolDid = await source.protocolDid(collectionId);
|
|
488
|
-
// The engine names the chain lane's subject `claim` (§8.1); the seam names lanes.
|
|
489
|
-
const kind = (await source.subject(claimId)).kind === "api" ? "api" : "claim";
|
|
490
|
-
// S16 — the payout block, beside the facts (never in them).
|
|
491
|
-
const payout = payoutRequestFor(protocolDid);
|
|
492
|
-
// RESOLVE + FREEZE (lib/facts/builder.ts). Nothing here is decided: a build that cannot
|
|
493
|
-
// complete is recorded as what it is, and the engine is never asked.
|
|
494
|
-
const built = await buildFacts(fdeps, { claim: pulled, protocolDid });
|
|
495
|
-
if (!built.ok) {
|
|
496
|
-
return this.buildFailure(deps, source, db, pulled, built);
|
|
497
|
-
}
|
|
498
|
-
// The claim WILL be decided: only now is a review reserved (never decide without one).
|
|
499
|
-
if (billing && billing !== "reserved") {
|
|
500
|
-
await this.reserve(deps, db, billing, { claimId, collectionId, protocolDid });
|
|
501
|
-
}
|
|
502
|
-
const request = (b) => ({
|
|
503
|
-
subject: { kind, claimId, collectionId, protocolDid },
|
|
504
|
-
rubric: { id: b.rubricId },
|
|
505
|
-
facts: b.facts,
|
|
506
|
-
...(payout ? { payout } : {}),
|
|
507
|
-
claimContent: content,
|
|
508
|
-
});
|
|
509
|
-
let decided;
|
|
510
|
-
try {
|
|
511
|
-
decided = await this.decide(deps, engine, fdeps, pulled, protocolDid, built, request);
|
|
512
|
-
}
|
|
513
|
-
catch (err) {
|
|
514
|
-
// The re-anchored rubric is the one that cannot be built for — recorded exactly as a first
|
|
515
|
-
// build's failure would be (see `decide`), never decided.
|
|
516
|
-
if (err instanceof BuildFailure) {
|
|
517
|
-
return this.buildFailure(deps, source, db, pulled, err.result);
|
|
518
|
-
}
|
|
519
|
-
throw await this.engineFailure(deps, source, db, claimId, collectionId, err);
|
|
520
|
-
}
|
|
521
|
-
const { response } = decided;
|
|
522
|
-
requireReceipt(response, "decision");
|
|
523
|
-
// Both sides hash the same canonical JSON (factset.ts); a drift here means the engine is
|
|
524
|
-
// binding its decision to facts other than the ones we froze — worth knowing, never fatal.
|
|
525
|
-
const ours = await factsDigest(decided.built.facts);
|
|
526
|
-
if (ours !== response.factsDigest) {
|
|
527
|
-
deps.logger.warn("evaluate.facts-digest-drift", {
|
|
528
|
-
claimId,
|
|
529
|
-
ours,
|
|
530
|
-
theirs: response.factsDigest,
|
|
531
|
-
});
|
|
532
|
-
}
|
|
533
|
-
// Null on the engine only when the protocol has no `#rub` — impossible past our own resolve,
|
|
534
|
-
// but never store a null pin for a claim we KNOW the rubric of.
|
|
535
|
-
const rubricId = response.rubricId ?? decided.built.rubricId;
|
|
536
|
-
const result = await this.record(deps, source, db, pulled, response, rubricId, decided.built.facts);
|
|
537
|
-
// The answer archive (lib/archive.ts), here because the body is in hand only inside this step.
|
|
538
|
-
// Fail-soft and returns nothing: the body goes to R2, never into the step result.
|
|
539
|
-
if (this.env.ANSWERS_ARCHIVE) {
|
|
540
|
-
await archiveAnswer(this.env, deps, {
|
|
541
|
-
lane: source.lane,
|
|
542
|
-
claim: pulled,
|
|
543
|
-
content,
|
|
544
|
-
submitter: await source.submitterKey(pulled).catch(() => null),
|
|
545
|
-
response,
|
|
546
|
-
facts: { ext: decided.built.facts.ext, ai: decided.built.facts.ai },
|
|
547
|
-
hooks: getOracle().config.archive,
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
// The reviewer note (lib/review/reviewer-note.ts): only for a claim held for a person, and
|
|
551
|
-
// here for the same reason — the answers are in hand only inside this step. Fail-soft and
|
|
552
|
-
// returns nothing; outside the decision, so it can never move a verdict.
|
|
553
|
-
if (response.decision.status === "pending_review" && reviewerNoteEnabled()) {
|
|
554
|
-
const facts = decided.built.facts;
|
|
555
|
-
await writeReviewerNote({
|
|
556
|
-
config: deps.config,
|
|
557
|
-
logger: deps.logger,
|
|
558
|
-
trace: deps.trace,
|
|
559
|
-
env: this.env,
|
|
560
|
-
cache: deps.cache,
|
|
561
|
-
db,
|
|
562
|
-
engine,
|
|
563
|
-
}, {
|
|
564
|
-
claimId,
|
|
565
|
-
protocolDid,
|
|
566
|
-
pendingTasks: response.decision.pendingTasks,
|
|
567
|
-
answers: facts.answers,
|
|
568
|
-
attachments: facts.attachments,
|
|
569
|
-
facts: { ext: facts.ext, ai: facts.ai },
|
|
570
|
-
});
|
|
571
|
-
}
|
|
572
|
-
return result;
|
|
573
|
-
}
|
|
574
|
-
/** Reserve the review (payments/review-billing.ts), or say why not — a pause, a retry, a stop. */
|
|
575
|
-
async reserve(deps, db, payer, claim) {
|
|
576
|
-
const out = await reserveChainClaim({ ...deps, db }, payer, claim);
|
|
577
|
-
if (out.ok)
|
|
578
|
-
return;
|
|
579
|
-
if ("pause" in out) {
|
|
580
|
-
await this.pausePayer(deps, db, claim.collectionId, out.pause, out.detail);
|
|
581
|
-
throw AppError.unavailable(`${PAYER_BILLING} collection paused ${out.pause} (${out.detail})`);
|
|
582
|
-
}
|
|
583
|
-
if ("fail" in out)
|
|
584
|
-
throw new NonRetryableError(`${PAYER_BILLING} ${out.detail}`);
|
|
585
|
-
throw AppError.unavailable(`${PAYER_BILLING} reservation refused (${out.detail})`);
|
|
586
|
-
}
|
|
587
|
-
/**
|
|
588
|
-
* A decision recovered from the engine's store → its `decisions` row.
|
|
589
|
-
*
|
|
590
|
-
* It is FINAL for this claim — the engine will not judge it twice — so it outranks anything this
|
|
591
|
-
* attempt would have concluded, including a build that failed. What it lacks is the facts it was
|
|
592
|
-
* decided on: the attempt that produced them died before writing the row, and the engine keeps a
|
|
593
|
-
* trace, not a FactSet. So they are rebuilt here — and stand in for the originals only if the
|
|
594
|
-
* rebuild is PROVABLY the same one, which is exactly what the engine's `factsDigest` proves.
|
|
595
|
-
* Anything else (a non-deterministic fact, a re-anchored rubric, a failed rebuild) is written as
|
|
596
|
-
* a park with its reason, at write time: better an explicit blocked row naming the drift than a
|
|
597
|
-
* decided row the act lane refuses later without knowing why.
|
|
598
|
-
*
|
|
599
|
-
* The rebuild may not even be ATTEMPTABLE: the owner's delegation can expire while the claim
|
|
600
|
-
* sits between attempts, and then the pull fail-closes 403. A deterministic "no" there is NOT a
|
|
601
|
-
* reason to lose the decision — the ingest would be marked `failed` with no row at all, while
|
|
602
|
-
* the engine holds a final answer it will never issue again. So it is recorded parked, facts
|
|
603
|
-
* null, and the act lane refuses to sign it (src/act.ts reads those facts). Only a transient
|
|
604
|
-
* failure is rethrown, so the step still retries and a later attempt can record the real facts.
|
|
605
|
-
*/
|
|
606
|
-
async recordRecovered(deps, source, db, engine, claimId, collectionId, recovered) {
|
|
607
|
-
let claim;
|
|
608
|
-
let built;
|
|
609
|
-
try {
|
|
610
|
-
// Nothing is sent here, so the content is dropped with the pull.
|
|
611
|
-
const { content: _content, ...rest } = await source.pull(claimId);
|
|
612
|
-
claim = rest;
|
|
613
|
-
const protocolDid = await source.protocolDid(claim.collectionId);
|
|
614
|
-
built = await buildFacts(factDeps(deps, claim.media, engine, db, this.env, source), {
|
|
615
|
-
claim,
|
|
616
|
-
protocolDid,
|
|
617
|
-
});
|
|
618
|
-
}
|
|
619
|
-
catch (err) {
|
|
620
|
-
// `stepError` is the retry/stop line the step itself is judged by (steps.ts) — reuse it so
|
|
621
|
-
// "deterministic" means the same thing here as it does there. Transient: rethrow untouched
|
|
622
|
-
// and let the step wrapper classify it, exactly as before.
|
|
623
|
-
if (!(stepError(err) instanceof NonRetryableError))
|
|
624
|
-
throw err;
|
|
625
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
626
|
-
deps.logger.warn("evaluate.recovered-facts-unbuildable", {
|
|
627
|
-
claimId,
|
|
628
|
-
collectionId,
|
|
629
|
-
status: recovered.decision.status,
|
|
630
|
-
error: message,
|
|
631
|
-
});
|
|
632
|
-
return this.record(deps, source, db, { claimId, collectionId }, recovered, recovered.rubricId, null, `recovered decision; facts could not be rebuilt: ${message}`);
|
|
633
|
-
}
|
|
634
|
-
const rebuilt = built.ok && (await factsDigest(built.facts)) === recovered.factsDigest ? built.facts : null;
|
|
635
|
-
return this.record(deps, source, db, claim, recovered, recovered.rubricId, rebuilt, rebuilt
|
|
636
|
-
? undefined
|
|
637
|
-
: "recovered decision's factsDigest does not match the facts rebuilt in this attempt");
|
|
638
|
-
}
|
|
639
|
-
/**
|
|
640
|
-
* The engine's answer → the `decisions` row, the collection's rubric pin and the `unusable`
|
|
641
|
-
* pause: ONE path whether the answer came from `POST /v1/decisions` just now or was recovered
|
|
642
|
-
* from the engine's store on a retry (see `evaluate`), so the two can never drift apart.
|
|
643
|
-
*
|
|
644
|
-
* `sent` is the FactSet this decision was made on — its `ext`/`ai` are frozen onto the row for
|
|
645
|
-
* the act policy to read at signing time (schema.ts `decisions.facts`), and nothing else of it
|
|
646
|
-
* is: `answers`/`attachments` are the claim's erasable surface and never persisted here. Null
|
|
647
|
-
* only on the recovery path when the rebuild could not be proven to be the same facts, which is
|
|
648
|
-
* what `park` says: the row is written `blocked` with that reason instead of `pending`, so the
|
|
649
|
-
* claim read explains itself and the act lane never has to guess.
|
|
650
|
-
*/
|
|
651
|
-
async record(deps, source, db, claim, response, rubricId, sent, park) {
|
|
652
|
-
const { claimId, collectionId } = claim;
|
|
653
|
-
// What the act lane could ever sign (chain-write.ts step 1): a `decided` 1/2 now, or a
|
|
654
|
-
// `pending_review` once its fold makes it final (the fold writes `pending` itself). Anything
|
|
655
|
-
// else — an engine `unusable`, a `decided` review (5) — is a recorded non-action, so it is
|
|
656
|
-
// `skipped` from the start rather than `pending` for ever.
|
|
657
|
-
const { status, evaluationStatus } = response.decision;
|
|
658
|
-
const actable = !!source.after.act &&
|
|
659
|
-
(status === "pending_review" ||
|
|
660
|
-
(status === "decided" && (evaluationStatus === 1 || evaluationStatus === 2)));
|
|
661
|
-
const row = {
|
|
662
|
-
claimId,
|
|
663
|
-
collectionId,
|
|
664
|
-
rubricId,
|
|
665
|
-
factsDigest: response.factsDigest,
|
|
666
|
-
verdict: response.decision.verdict,
|
|
667
|
-
status: response.decision.status,
|
|
668
|
-
resultDigest: response.resultDigest,
|
|
669
|
-
receiptCid: response.receipt?.cid ?? null,
|
|
670
|
-
receiptJws: response.receipt?.jws ?? null,
|
|
671
|
-
decision: response.decision,
|
|
672
|
-
explanation: response.explanation,
|
|
673
|
-
facts: sent ? { ext: sent.ext, ai: sent.ai } : null,
|
|
674
|
-
erroredFacts: erroredFactsOf(response),
|
|
675
|
-
// The review view's `duplicates` source (`{}` stores nothing, like an older engine's none).
|
|
676
|
-
pendingTaskDetails: Object.keys(response.pendingTaskDetails ?? {}).length > 0
|
|
677
|
-
? (response.pendingTaskDetails ?? null)
|
|
678
|
-
: null,
|
|
679
|
-
// Nothing is ever acted on for a source without an act step (the API lane, §7.1).
|
|
680
|
-
chainStatus: !actable
|
|
681
|
-
? "skipped"
|
|
682
|
-
: park
|
|
683
|
-
? "blocked"
|
|
684
|
-
: "pending",
|
|
685
|
-
chainError: actable ? (park ?? null) : null,
|
|
686
|
-
};
|
|
687
|
-
// The decision and, when it is FINAL, its verdict denormalized onto the claim's duplicate
|
|
688
|
-
// registry rows (stock/unique-store.ts; a no-op for a rubric without `unique`) land in ONE
|
|
689
|
-
// batch: a retry that finds the row is done (the early return above), so the token can never
|
|
690
|
-
// be owed separately. `pending_review` stays NULL (= pending, blockable) until the fold.
|
|
691
|
-
// The claim's connector-result cache rows die in the same batch (fact-cache.ts) — decided,
|
|
692
|
-
// so nothing is left for a retry to reuse. So does the collection's rubric pin, first sight
|
|
693
|
-
// only (NULL only): the chain is the authority and the per-claim pin is the row's own
|
|
694
|
-
// `rubricId`; this one is the dashboard's "which rules govern this collection", never
|
|
695
|
-
// re-written by a later re-anchor. Inside the batch, or a crash after it would lose the pin for
|
|
696
|
-
// good (the retry returns early on the row). Chain lane only — an API scope has no
|
|
697
|
-
// `watched_collections` row.
|
|
698
|
-
const token = registryVerdictToken(row.status, row.verdict);
|
|
699
|
-
await db.batch([
|
|
700
|
-
db.insert(decisions).values(row).onConflictDoNothing(),
|
|
701
|
-
...(token ? [finalVerdictUpdate(db, { collectionId, claimId, token })] : []),
|
|
702
|
-
factCacheDelete(db, claimId),
|
|
703
|
-
...(isChain(source)
|
|
704
|
-
? [
|
|
705
|
-
db
|
|
706
|
-
.update(watchedCollections)
|
|
707
|
-
.set({ rubricId: row.rubricId })
|
|
708
|
-
.where(and(eq(watchedCollections.collectionId, collectionId), isNull(watchedCollections.rubricId))),
|
|
709
|
-
]
|
|
710
|
-
: []),
|
|
711
|
-
]);
|
|
712
|
-
await this.decisionLatency(deps, source, db, claimId, response, sent);
|
|
713
|
-
// A payout scope whose claim names no payee: decided, its receipt without a payment binding
|
|
714
|
-
// (ENGINE-CONTRACT "Payee").
|
|
715
|
-
if (response.decisionToPayRefused) {
|
|
716
|
-
deps.logger.warn("payout.refused", {
|
|
717
|
-
claimId,
|
|
718
|
-
collectionId,
|
|
719
|
-
code: response.decisionToPayRefused,
|
|
720
|
-
});
|
|
721
|
-
}
|
|
722
|
-
// Settle on the receipt (payments spec §4 step 6): fail-soft, the reconcile re-settles a lost
|
|
723
|
-
// one; a claim with no charge (billing mode `none`, a chain claim its evaluation leg pays) is a
|
|
724
|
-
// no-op. An engine `unusable` is a decision with a receipt: settled, as on the API lane. A
|
|
725
|
-
// `pending_review` has no receipt until the fold: hold its reservation through the review
|
|
726
|
-
// instead (extend.ts).
|
|
727
|
-
if (deps.config.X402_FACILITATOR_URL) {
|
|
728
|
-
if (row.receiptJws)
|
|
729
|
-
await settleClaim({ ...deps, db }, claimId, row.receiptJws);
|
|
730
|
-
else if (row.status === "pending_review")
|
|
731
|
-
await extendForReview({ ...deps, db }, claimId);
|
|
732
|
-
}
|
|
733
|
-
// The engine could not bind the rubric to these facts (header): a real, receipted decision
|
|
734
|
-
// for THIS claim — and the same answer for every other claim of the collection.
|
|
735
|
-
if (response.decision.status === "unusable") {
|
|
736
|
-
await pauseUnusable(source, deps.logger, "evaluate", {
|
|
737
|
-
claimId,
|
|
738
|
-
collectionId,
|
|
739
|
-
rubricId: row.rubricId,
|
|
740
|
-
issues: response.decision.reasons,
|
|
741
|
-
receiptCid: row.receiptCid,
|
|
742
|
-
});
|
|
743
|
-
}
|
|
744
|
-
deps.logger.info("evaluate.done", {
|
|
745
|
-
claimId,
|
|
746
|
-
collectionId,
|
|
747
|
-
rubricId: row.rubricId,
|
|
748
|
-
status: response.decision.status,
|
|
749
|
-
verdict: response.decision.verdict,
|
|
750
|
-
pendingTasks: response.decision.pendingTasks,
|
|
751
|
-
resultDigest: response.resultDigest,
|
|
752
|
-
receiptCid: row.receiptCid,
|
|
753
|
-
traceId: response.traceId ?? null,
|
|
754
|
-
});
|
|
755
|
-
return resultOf(row);
|
|
756
|
-
}
|
|
757
|
-
/**
|
|
758
|
-
* `api.decision.latency` — accept → decision row, the SLO metric. Accepted is the API claim's
|
|
759
|
-
* `receivedAt`, or the chain claim's ledger `createdAt` (its first `record-queued`). `status` says
|
|
760
|
-
* whether the protocol's rubric runs AI checks: the frozen FactSet's `ai` keys are exactly the
|
|
761
|
-
* work order's (validate.ts holds them to the manifest), so no extra read; `-` only when a
|
|
762
|
-
* recovered decision's facts could not be rebuilt. Best-effort: a metric never fails the step.
|
|
763
|
-
*/
|
|
764
|
-
async decisionLatency(deps, source, db, claimId, response, sent) {
|
|
765
|
-
try {
|
|
766
|
-
const acceptedAt = isChain(source)
|
|
767
|
-
? (await db.query.claimIngests.findFirst({
|
|
768
|
-
columns: { createdAt: true },
|
|
769
|
-
where: eq(claimIngests.claimId, claimId),
|
|
770
|
-
}))?.createdAt
|
|
771
|
-
: (await db.query.apiClaims.findFirst({
|
|
772
|
-
columns: { receivedAt: true },
|
|
773
|
-
where: eq(apiClaims.claimId, claimId),
|
|
774
|
-
}))?.receivedAt;
|
|
775
|
-
if (!acceptedAt)
|
|
776
|
-
return;
|
|
777
|
-
const { protocolDid } = response.subject;
|
|
778
|
-
deps.metrics.timing("api.decision.latency", Date.now() - acceptedAt.getTime(), {
|
|
779
|
-
op: isChain(source) ? "chain" : "api",
|
|
780
|
-
status: sent ? (Object.keys(sent.ai).length > 0 ? "ai" : "no_ai") : null,
|
|
781
|
-
outcome: response.decision.verdict,
|
|
782
|
-
id: typeof protocolDid === "string" ? protocolDid : null,
|
|
783
|
-
});
|
|
784
|
-
}
|
|
785
|
-
catch (err) {
|
|
786
|
-
deps.logger.warn("metrics.decision-latency-failed", { claimId, detail: String(err) });
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
/**
|
|
790
|
-
* `POST /v1/decisions`, with the one recoverable refusal handled here: `RUBRIC_PIN_MISMATCH` —
|
|
791
|
-
* our pin ≠ the chain's `#rub` proof as the engine read it, i.e. `#rub` was re-anchored between
|
|
792
|
-
* our (KV-cached) manifest read and theirs. Invalidate and re-read the manifest ONCE: a changed
|
|
793
|
-
* pin is a new work order (new facts, so a rebuild — connectors run again; the fact cache is
|
|
794
|
-
* keyed by the pin, fact-cache.ts), the same pin means the engine and the chain disagree, which
|
|
795
|
-
* no retry of ours fixes. The manifest may also have turned `unusable` under the new anchor —
|
|
796
|
-
* then the rebuild records that, and never decides.
|
|
797
|
-
*/
|
|
798
|
-
async decide(deps, engine, fdeps, pulled, protocolDid, built, request) {
|
|
799
|
-
try {
|
|
800
|
-
return { response: await this.ask(deps, engine, request(built)), built };
|
|
801
|
-
}
|
|
802
|
-
catch (err) {
|
|
803
|
-
if (!(err instanceof EngineError) || err.code !== "RUBRIC_PIN_MISMATCH")
|
|
804
|
-
throw err;
|
|
805
|
-
await engine.invalidateManifest(protocolDid);
|
|
806
|
-
const fresh = await engine.getManifest(protocolDid);
|
|
807
|
-
const repinned = isManifestUnusable(fresh) ? null : fresh.rubricId.toLowerCase();
|
|
808
|
-
if (repinned === built.rubricId) {
|
|
809
|
-
throw new NonRetryableError(`Rubric pin mismatch that re-reading the manifest did not fix: ${err.message}`);
|
|
810
|
-
}
|
|
811
|
-
deps.logger.warn("evaluate.rubric-repinned", {
|
|
812
|
-
claimId: pulled.claimId,
|
|
813
|
-
collectionId: pulled.collectionId,
|
|
814
|
-
from: built.rubricId,
|
|
815
|
-
to: repinned,
|
|
816
|
-
});
|
|
817
|
-
const rebuilt = await buildFacts(fdeps, { claim: pulled, protocolDid, manifest: fresh });
|
|
818
|
-
if (!rebuilt.ok) {
|
|
819
|
-
// The re-anchored rubric is the one that cannot be built for — record THAT, no engine call.
|
|
820
|
-
throw new BuildFailure(rebuilt);
|
|
821
|
-
}
|
|
822
|
-
return { response: await this.ask(deps, engine, request(rebuilt)), built: rebuilt };
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
/** One decide call, with the contract's retry-once on `NO_EVALUATE_GRANT` (see `GRANT_RETRY_DELAY`). */
|
|
826
|
-
async ask(deps, engine, req) {
|
|
827
|
-
try {
|
|
828
|
-
return await engine.decide(req);
|
|
829
|
-
}
|
|
830
|
-
catch (err) {
|
|
831
|
-
if (!(err instanceof EngineError) || err.code !== "NO_EVALUATE_GRANT")
|
|
832
|
-
throw err;
|
|
833
|
-
deps.logger.warn("evaluate.no-grant-yet", {
|
|
834
|
-
claimId: req.subject.claimId,
|
|
835
|
-
collectionId: req.subject.collectionId,
|
|
836
|
-
retryInMs: GRANT_RETRY_DELAY.ms,
|
|
837
|
-
});
|
|
838
|
-
await new Promise((resolve) => setTimeout(resolve, GRANT_RETRY_DELAY.ms));
|
|
839
|
-
return engine.decide(req);
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
/**
|
|
843
|
-
* The engine said no (docs/ENGINE-CONTRACT.md "Errors"): do what the table says, then hand back
|
|
844
|
-
* the error to throw — NonRetryable for a deterministic refusal, the original for a retryable
|
|
845
|
-
* one. Non-engine errors pass through for `stepError` to classify.
|
|
846
|
-
*/
|
|
847
|
-
async engineFailure(deps, source, db, claimId, collectionId, err) {
|
|
848
|
-
if (!(err instanceof EngineError))
|
|
849
|
-
return err;
|
|
850
|
-
const scope = { claimId, collectionId, code: err.code, requestId: err.requestId ?? null };
|
|
851
|
-
switch (err.code) {
|
|
852
|
-
case "NO_EVALUATE_GRANT":
|
|
853
|
-
// Twice (see `ask`): not hired (any more) for this collection — no claim of it can be
|
|
854
|
-
// decided until the owner re-grants, so stop enqueuing and say why. The watched row stands.
|
|
855
|
-
// Only the chain lane's subject is grant-checked (§8.1); an API source never gets here.
|
|
856
|
-
if (isChain(source))
|
|
857
|
-
await source.pause(collectionId, "engine-no-grant");
|
|
858
|
-
deps.logger.error("evaluate.not-hired", scope);
|
|
859
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
860
|
-
return new NonRetryableError(`Not hired for collection ${collectionId}: ${err.message}`);
|
|
861
|
-
case "INSUFFICIENT_CREDITS": {
|
|
862
|
-
// The oracle OWNER's credits are exhausted at the engine (an invoice bounced): EVERY
|
|
863
|
-
// collection pauses (the credits are the owner's, shared across every collection we watch
|
|
864
|
-
// for them, never this kit's own wallet), the operator is paged, and this claim keeps
|
|
865
|
-
// retrying. While the pause stands, the cron enqueues nothing new and every in-flight
|
|
866
|
-
// claim's retry stops at the pause check before its pull (`evaluate`), spending nothing;
|
|
867
|
-
// once the billing cron lifts it, the next retry pulls, builds and the engine answers.
|
|
868
|
-
await pauseForEngineCredits(db, deps);
|
|
869
|
-
deps.logger.error("evaluate.credits-exhausted", { ...scope, topUpUrl: err.topUpUrl });
|
|
870
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
871
|
-
return err;
|
|
872
|
-
}
|
|
873
|
-
case "VALIDATION":
|
|
874
|
-
// The engine refused the facts' SHAPE — our connectors' bug, never a claim outcome. The
|
|
875
|
-
// details are Zod paths and messages (identifiers), logged so the fix is one grep away.
|
|
876
|
-
deps.logger.error("evaluate.facts-rejected", { ...scope, details: err.details ?? null });
|
|
877
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
878
|
-
return new NonRetryableError(`${FAILURE.engineRejected}: ${err.message}`);
|
|
879
|
-
case "CONTENT_MISMATCH":
|
|
880
|
-
case "PAYLOAD_TOO_LARGE": {
|
|
881
|
-
// The engine would not bind the decision to the content we sent (company authority): the
|
|
882
|
-
// bytes, the answers, the attachments, the agent — or a body over its cap, which the client
|
|
883
|
-
// also says before sending. No resend changes the claim, so it is recorded like a build
|
|
884
|
-
// failure (`skipped`: nothing to act on), the ingest fails, and the operator is paged —
|
|
885
|
-
// `details.reason` names the check, never an answer value.
|
|
886
|
-
const status = CONTENT_REFUSAL_STATUS[err.code];
|
|
887
|
-
const reason = reasonOf(err.details);
|
|
888
|
-
await db.batch([
|
|
889
|
-
db
|
|
890
|
-
.insert(decisions)
|
|
891
|
-
.values({
|
|
892
|
-
claimId,
|
|
893
|
-
collectionId,
|
|
894
|
-
status,
|
|
895
|
-
explanation: `${err.code}${reason ? ` (${reason})` : ""}: ${err.message}`,
|
|
896
|
-
chainStatus: "skipped",
|
|
897
|
-
})
|
|
898
|
-
.onConflictDoNothing(),
|
|
899
|
-
factCacheDelete(db, claimId),
|
|
900
|
-
]);
|
|
901
|
-
deps.logger.error("evaluate.content-refused", { ...scope, reason });
|
|
902
|
-
await reportAlert(deps, `engine.${status.replaceAll("_", "-")}`, { claimId, collectionId, code: err.code, reason }, `engine.${status}:${claimId}`);
|
|
903
|
-
return new NonRetryableError(`${err.code}: ${reason ?? err.message}`);
|
|
904
|
-
}
|
|
905
|
-
case "CONFLICT":
|
|
906
|
-
// The engine holds this claim's decision under a different `kind` or `collectionId` than
|
|
907
|
-
// we just sent — the same claimId reached it through two lanes or scopes. Our bug, never a
|
|
908
|
-
// claim outcome, and no retry changes the stored row: stop, and page it.
|
|
909
|
-
deps.logger.error("evaluate.subject-conflict", scope);
|
|
910
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
911
|
-
return new NonRetryableError(`${FAILURE.engineConflict}: the engine holds this claim under another subject: ${err.message}`);
|
|
912
|
-
case "RATE_LIMITED":
|
|
913
|
-
// The engine's per-IP limiter said "not now" (a backlog burst): the step's own backoff
|
|
914
|
-
// (1 min doubling) already outlasts its 60 s window, so retry as-is — logged with the
|
|
915
|
-
// engine's `Retry-After` so a throttled backlog is visible, never a permanent failure.
|
|
916
|
-
deps.logger.warn("evaluate.throttled", {
|
|
917
|
-
...scope,
|
|
918
|
-
retryAfterMs: err.retryAfterMs ?? null,
|
|
919
|
-
});
|
|
920
|
-
return err;
|
|
921
|
-
// The three refusals below all come out of the engine's OWN `POST /v1/oracles/register`
|
|
922
|
-
// (decision-engine spec §5/§9), surfacing HERE because `client.ts`'s `registered()` wrapper
|
|
923
|
-
// re-registers whenever its KV "registered" marker (REGISTERED_TTL_SEC, 30d) has expired —
|
|
924
|
-
// i.e. any `decide()` can hit one of these, not just a fresh boot. None pause a collection:
|
|
925
|
-
// unlike `engine-credits`, the engine's `/me` keeps answering `ok` here (billingStatus is a
|
|
926
|
-
// different gate), so `cron/billing.ts`'s auto-resume would never see the fix and un-pause —
|
|
927
|
-
// this is the owner's/operator's fix to make, not a chain-write concern.
|
|
928
|
-
case "NO_SUBSCRIPTION": {
|
|
929
|
-
// The owner has no IXO subscription, or it lapsed — the engine can't find a credit wallet
|
|
930
|
-
// to bill without one. `topUpUrl` is null for a bare "no subscription at all" (nothing to
|
|
931
|
-
// manage a URL for yet) — the kit's own PORTAL_BASE_URL still names somewhere to send them.
|
|
932
|
-
const topUpUrl = err.topUpUrl ?? deps.config.PORTAL_BASE_URL;
|
|
933
|
-
deps.logger.error("evaluate.owner-billing-refused", { ...scope, topUpUrl });
|
|
934
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
935
|
-
return new NonRetryableError(`Owner has no active IXO subscription: ${err.message}. Subscribe in the portal (${topUpUrl}), then register again.`);
|
|
936
|
-
}
|
|
937
|
-
case "DELEGATION_NOT_FOR_ORACLE":
|
|
938
|
-
// `OWNER_DELEGATION_CAR` is a real, unexpired CAR — just bound (`billing/pay with`) to a
|
|
939
|
-
// DIFFERENT oracle's did:web. Re-registering with the same file only fails the same way.
|
|
940
|
-
deps.logger.error("evaluate.owner-billing-refused", scope);
|
|
941
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
942
|
-
return new NonRetryableError(`Owner delegation is bound to a different oracle: ${err.message}. Re-mint the CAR for ` +
|
|
943
|
-
"THIS oracle (decision-oracle delegation --for engine --oracle <this did:web>) and " +
|
|
944
|
-
"set it as OWNER_DELEGATION_CAR.");
|
|
945
|
-
default: {
|
|
946
|
-
// A bad/expired CAR is a plain `400 BAD_REQUEST` (owner-delegation.ts's own checks — not a
|
|
947
|
-
// dedicated code), told apart from any other BAD_REQUEST only by its message naming the
|
|
948
|
-
// field this kit actually sends.
|
|
949
|
-
if (err.code === "BAD_REQUEST" && err.message.includes("ownerDelegation")) {
|
|
950
|
-
deps.logger.error("evaluate.owner-billing-refused", scope);
|
|
951
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
952
|
-
return new NonRetryableError(`Owner delegation rejected: ${err.message}. Re-mint the CAR ` +
|
|
953
|
-
"(decision-oracle delegation --for engine) and register again.");
|
|
954
|
-
}
|
|
955
|
-
return err.retryable ? err : new NonRetryableError(`${err.code}: ${err.message}`);
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
/**
|
|
960
|
-
* Facts that don't match the rubric's work order (lib/facts/validate.ts) — the producer's bug,
|
|
961
|
-
* never a claim outcome: the issue list is codes and paths (identifiers), logged and paged so the
|
|
962
|
-
* fix is one grep away, and the error to throw is NonRetryable (a retry would re-run the same
|
|
963
|
-
* producer). The row is already recorded by the caller.
|
|
964
|
-
*/
|
|
965
|
-
invalidFacts(deps, claimId, collectionId, built) {
|
|
966
|
-
deps.logger.error("evaluate.facts-invalid", {
|
|
967
|
-
claimId,
|
|
968
|
-
collectionId,
|
|
969
|
-
rubricId: built.rubricId ?? null,
|
|
970
|
-
issues: built.issues,
|
|
971
|
-
});
|
|
972
|
-
const err = new NonRetryableError(`Facts do not match the rubric's work order: ${built.issues.map((i) => `${i.code} ${i.path}`).join(", ")}`);
|
|
973
|
-
reportError(err, { lane: "evaluate", step: "evaluate", claimId, collectionId });
|
|
974
|
-
return err;
|
|
975
|
-
}
|
|
976
|
-
/**
|
|
977
|
-
* A claim the facts lane could not build for — recorded WITHOUT a decide call (schema.ts
|
|
978
|
-
* `decisions`): `status` names the reason, `explanation` the issues, and nothing will ever be
|
|
979
|
-
* acted on, so `chainStatus` is `skipped` from the start. Then the reason's consequence:
|
|
980
|
-
* `rubric-unusable` is the engine's manifest saying the rubric cannot be bound — the same
|
|
981
|
-
* handling as its decide-time `unusable` (pause the collection; steps.ts), minus the receipt, and
|
|
982
|
-
* the ingest completes; `invalid-facts` is OUR (or the provider's) bug, never a claim outcome —
|
|
983
|
-
* recorded so the claim read says why, then failed loudly, like the engine's own `VALIDATION`
|
|
984
|
-
* refusal this check exists to catch first.
|
|
985
|
-
*/
|
|
986
|
-
async buildFailure(deps, source, db, claim, built) {
|
|
987
|
-
const { claimId, collectionId } = claim;
|
|
988
|
-
const status = BUILD_FAILURE_STATUS[built.reason];
|
|
989
|
-
const rubricId = built.rubricId ?? null;
|
|
990
|
-
const explanation = built.issues.map((i) => `${i.code}: ${i.message} (${i.path})`).join("; ");
|
|
991
|
-
// A first build never cached anything, but a RUBRIC_PIN_MISMATCH rebuild lands here AFTER the
|
|
992
|
-
// first build's connectors ran — the cache dies with the outcome, same as `record`.
|
|
993
|
-
await db.batch([
|
|
994
|
-
db
|
|
995
|
-
.insert(decisions)
|
|
996
|
-
.values({ claimId, collectionId, rubricId, status, explanation, chainStatus: "skipped" })
|
|
997
|
-
.onConflictDoNothing(),
|
|
998
|
-
factCacheDelete(db, claimId),
|
|
999
|
-
]);
|
|
1000
|
-
// Only a re-pinned rebuild gets here with a review reserved (the first build is checked before
|
|
1001
|
-
// reserving): no receipt will ever come, so it is freed — its cap back too.
|
|
1002
|
-
if (isChain(source)) {
|
|
1003
|
-
await releaseChainClaim({ ...deps, db }, claimId, built.reason === "invalid-facts" ? "facts_invalid" : "facts_unusable");
|
|
1004
|
-
}
|
|
1005
|
-
if (built.reason === "invalid-facts")
|
|
1006
|
-
throw this.invalidFacts(deps, claimId, collectionId, built);
|
|
1007
|
-
await pauseUnusable(source, deps.logger, "evaluate", {
|
|
1008
|
-
claimId,
|
|
1009
|
-
collectionId,
|
|
1010
|
-
rubricId,
|
|
1011
|
-
issues: built.issues,
|
|
1012
|
-
});
|
|
1013
|
-
return { status, verdict: null, resultDigest: null, receiptCid: null };
|
|
1014
|
-
}
|
|
1015
302
|
/**
|
|
1016
303
|
* A terminal failure's class — the API lane's §5 code (lib/api-lane/record.ts `claimErrorCode`,
|
|
1017
304
|
* read off the same message the ledger stores), for either lane. Best-effort like the rest of
|
|
@@ -1033,7 +320,10 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
1033
320
|
}
|
|
1034
321
|
/**
|
|
1035
322
|
* Upsert this claim's ledger row. `error` is null unless `failed` (schema.ts), so a rerun that
|
|
1036
|
-
* succeeds clears the old message.
|
|
323
|
+
* succeeds clears the old message. `queued` never lands on a row already `processing` or
|
|
324
|
+
* `processed`: the inline run (or an earlier run of this instance) got that far, and moving it
|
|
325
|
+
* back would hide an inline run in flight from `evaluate` (two runs, the handlers paid twice) and
|
|
326
|
+
* show a decided claim as `queued` until the run caught up.
|
|
1037
327
|
*/
|
|
1038
328
|
async recordStatus(claimId, event, update) {
|
|
1039
329
|
const fields = {
|
|
@@ -1048,19 +338,13 @@ export class ClaimIngest extends WorkflowEntrypoint {
|
|
|
1048
338
|
.onConflictDoUpdate({
|
|
1049
339
|
target: claimIngests.claimId,
|
|
1050
340
|
set: fields,
|
|
341
|
+
...(update.status === "queued"
|
|
342
|
+
? { setWhere: notInArray(claimIngests.status, ["processing", "processed"]) }
|
|
343
|
+
: {}),
|
|
1051
344
|
// A `failed` never overwrites a `processed`: an orphaned attempt of a claim another run
|
|
1052
345
|
// already finished (seen on mainnet) would otherwise mark a decided claim dead.
|
|
1053
346
|
...(update.status === "failed" ? { setWhere: ne(claimIngests.status, "processed") } : {}),
|
|
1054
347
|
});
|
|
1055
348
|
}
|
|
1056
349
|
}
|
|
1057
|
-
/** The re-anchored rubric could not be built for — carries the result to be recorded (see `decide`, `buildFailure`). */
|
|
1058
|
-
class BuildFailure extends Error {
|
|
1059
|
-
result;
|
|
1060
|
-
constructor(result) {
|
|
1061
|
-
super(`facts lane: ${result.reason}`);
|
|
1062
|
-
this.result = result;
|
|
1063
|
-
this.name = "BuildFailure";
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
350
|
//# sourceMappingURL=claim-ingest.js.map
|