@passioncode-ai/fabric-agent-adapter 0.39.1 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +203 -0
- package/README.md +18 -6
- package/SECURITY.md +27 -6
- package/bin/fabric-agent-adapter.js +71 -6
- package/package.json +2 -2
- package/plugins/fabric-agent-adapter/.claude-plugin/plugin.json +1 -1
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/SKILL.md +42 -66
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/dashboard-links.md +4 -2
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/operator-channel-and-repository.md +55 -0
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/provider-entry.md +9 -3
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/verification.md +15 -0
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/adapt_project.py +181 -26
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/check_dashboard_link.py +66 -5
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_provider.py +81 -23
- package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_secrets.py +1 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/SKILL.md +42 -33
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/assets/dashboard-kit/fabric-button.js +21 -9
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/assets/dashboard-kit/fabric-dashboard.css +1 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/assets/dashboard-kit/fabric-dashboard.js +2 -2
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/assets/dashboard-kit/kit.json +9 -4
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/recipes/support-agent/README.md +104 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/recipes/support-agent/config.example.json +15 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/recipes/support-agent/knowledge.example.json +13 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/recipes/support-agent/support_agent.py +465 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/board.md +21 -5
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/connectors.md +93 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/dashboard-kit.md +2 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/dashboard-links.md +4 -2
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/delegation.md +3 -2
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/estate-record.md +5 -4
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/interop.md +5 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/kit-modules.md +129 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/lifecycle.md +3 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/package.md +5 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/references/usage.md +2 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/check_dashboard_layout.mjs +48 -19
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/check_dashboard_link.py +66 -5
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/check_service.py +11 -184
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/check_service_rules.py +204 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/dashboard_kit.py +53 -19
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-auth.mjs +288 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-backups.mjs +327 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-base.mjs +47 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-board-fixture.mjs +79 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-board-link.mjs +145 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-board-session.mjs +848 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-board-shapes.mjs +340 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-board.mjs +42 -1237
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-ceremonies.mjs +7 -18
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-crypto.mjs +41 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-delegation.mjs +15 -46
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-descriptor.mjs +387 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-estate.mjs +11 -68
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-events.mjs +52 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-http.mjs +231 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-interop.mjs +50 -16
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-jsvalue.mjs +67 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-lock.mjs +235 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-logs.mjs +136 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-openrouter.mjs +335 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-package.mjs +146 -32
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-paths.mjs +69 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-platform.mjs +164 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-profiles.mjs +1 -32
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-protocols.mjs +122 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-service.mjs +177 -2541
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-share.mjs +1 -8
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-supervisors.mjs +404 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-time.mjs +46 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric-usage.mjs +458 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_auth.py +408 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_backups.py +399 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_base.py +91 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_board.py +64 -1398
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_board_fixture.py +116 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_board_link.py +162 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_board_session.py +882 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_board_shapes.py +457 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_ceremonies.py +36 -39
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_crypto.py +336 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_delegation.py +113 -222
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_descriptor.py +586 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_estate.py +44 -405
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_events.py +140 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_http.py +402 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_hub.py +6 -2
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_interop.py +119 -77
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_jsvalue.py +310 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_kit.py +243 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_lock.py +308 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_logs.py +202 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_openrouter.py +431 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_package.py +285 -109
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_paths.py +102 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_platform.py +338 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_profiles.py +32 -39
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_protocols.py +173 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_secrets.py +1 -1
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_service.py +298 -3450
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_share.py +8 -8
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_supervisors.py +437 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_time.py +120 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/fabric_usage.py +697 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/kit.json +1621 -0
- package/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/sample_service.py +25 -7
- package/plugins/fabric-agent-adapter/skills/creating-fabric-agents/SKILL.md +29 -60
- package/plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/dashboard-links.md +4 -2
- package/plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/models-and-providers.md +3 -0
- package/plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/operator-channel-and-repository.md +55 -0
- package/plugins/fabric-agent-adapter/skills/creating-fabric-agents/scripts/check_dashboard_link.py +66 -5
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/SKILL.md +3 -3
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/references/native-host-hooks.md +2 -1
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/references/session-hooks.md +26 -13
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session.py +122 -1392
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_cli.py +278 -0
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_env.py +539 -0
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_hosts.py +324 -0
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_mail.py +173 -0
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_mcp.py +491 -0
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/host_hooks.py +11 -1
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/hosts/context.mjs +25 -2
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/hosts/openclaw/index.mjs +4 -2
- package/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/hosts/opencode/fabric.mjs +2 -1
|
@@ -0,0 +1,848 @@
|
|
|
1
|
+
// The board responder (DEC-0078, FAA-12b): the work context a handler sees and the responder loop. The session, the
|
|
2
|
+
// checkpoint and the request live in fabric-board-link.mjs and are re-exported here.
|
|
3
|
+
// Part of the Node board connector (FAA-65, layer L4); the twin of fabric_board_session.py. Node 20+, no dependency.
|
|
4
|
+
import crypto from 'node:crypto';
|
|
5
|
+
import { Backoff, HubError, normalProject } from './fabric-hub.mjs';
|
|
6
|
+
import { acceptedRevisions, feedDataProblems, negotiateRevision } from './fabric-service.mjs';
|
|
7
|
+
import { BoardError, CAPABILITY, LEASE_SECONDS, REQUEST_STATES, SUMMARY_MAX, Stopped, isInt, isObj, outcomeProblems, receiptProblems } from './fabric-board-shapes.mjs';
|
|
8
|
+
import { BoardRequest, BoardSession, Checkpoint } from './fabric-board-link.mjs';
|
|
9
|
+
|
|
10
|
+
export { BoardRequest, BoardSession, Checkpoint }; // re-exported: callers import them from here since 0.40.0
|
|
11
|
+
|
|
12
|
+
export const RENEW_SECONDS = 20;
|
|
13
|
+
|
|
14
|
+
export const POLL_SECONDS = 15;
|
|
15
|
+
|
|
16
|
+
export const POLL_JITTER_SECONDS = 5;
|
|
17
|
+
|
|
18
|
+
export const DRAIN_SECONDS = 10;
|
|
19
|
+
|
|
20
|
+
export const EVENT_SCHEMA = 'urn:fabric:schema:board-responder-event:1';
|
|
21
|
+
|
|
22
|
+
export const STOP_CODES = Object.freeze(new Set(['fenced', 'lease_expired', 'authority_changed']));
|
|
23
|
+
|
|
24
|
+
const TERMINAL = new Set(['completed', 'failed_known', 'cancelled', 'expired']);
|
|
25
|
+
|
|
26
|
+
/** A lost call is sent again this long before the lease it keeps would lapse (FAA-77, finding 38). */
|
|
27
|
+
export const LEASE_MARGIN_SECONDS = 10;
|
|
28
|
+
|
|
29
|
+
/** How long a held slot waits before it asks again: doubling from the poll interval, never past this. */
|
|
30
|
+
export const HOLD_RETRY_MAX_SECONDS = 900;
|
|
31
|
+
|
|
32
|
+
/** What a handler may do while it works: progress(note), beginEffect(summary) before it spends or acts, and `stopped`. */
|
|
33
|
+
export class WorkContext {
|
|
34
|
+
#responder;
|
|
35
|
+
#entry;
|
|
36
|
+
constructor(responder, entry) {
|
|
37
|
+
this.#responder = responder;
|
|
38
|
+
this.#entry = entry;
|
|
39
|
+
this.stopped = false;
|
|
40
|
+
}
|
|
41
|
+
async progress(note) {
|
|
42
|
+
if (this.stopped) throw new Stopped();
|
|
43
|
+
await this.#responder.progress(this.#entry, note, this);
|
|
44
|
+
}
|
|
45
|
+
async beginEffect(summary) {
|
|
46
|
+
if (this.stopped) throw new Stopped();
|
|
47
|
+
await this.#responder.effectBegin(this.#entry, summary, this);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const clean = (text, max = SUMMARY_MAX) => String(text).split(/\s+/).filter(Boolean).join(' ').slice(0, max);
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* One service's responder for one Project and its capabilities, one request at a time. handler(request, ctx) answers
|
|
55
|
+
* the outcome (may be async); admit(request) is the input gate; observe(entry) reads the world after a restart;
|
|
56
|
+
* emit(kind, level, text, extra) receives events. run({signal}) ticks until aborted; step() is one tick.
|
|
57
|
+
*/
|
|
58
|
+
export class Responder {
|
|
59
|
+
constructor({ hub, projectId, capabilities, credentialFile, checkpointFile, handler, admit, observe, emit, service = 'service',
|
|
60
|
+
now = () => performance.now() / 1000, rand = Math.random, sessionFactory } = {}) {
|
|
61
|
+
if (typeof admit !== 'function' || typeof handler !== 'function') throw new BoardError('invalid_arguments', 'A responder needs the service\'s own input gate (admit) and a handler.');
|
|
62
|
+
const caps = [...(capabilities ?? [])];
|
|
63
|
+
if (!caps.length || caps.some((c) => typeof c !== 'string' || !CAPABILITY.test(c)) || new Set(caps).size !== caps.length) throw new BoardError('invalid_arguments', 'Name each capability once, [a-z][a-z0-9._-]{1,127}.');
|
|
64
|
+
Object.assign(this, { hub, capabilities: caps, credentialFile, handler, admit, observe, emitTo: emit, now, rand });
|
|
65
|
+
this.project = normalProject(projectId);
|
|
66
|
+
this.service = String(service).replace(/[^A-Za-z0-9._-]/g, '-').slice(0, 40) || 'service';
|
|
67
|
+
this.checkpoint = new Checkpoint(checkpointFile);
|
|
68
|
+
this.sessionFactory = sessionFactory ?? ((grant) => new BoardSession(grant));
|
|
69
|
+
this.backoff = new Backoff({ rand });
|
|
70
|
+
this.session = null;
|
|
71
|
+
this.epoch = null;
|
|
72
|
+
this.slots = {};
|
|
73
|
+
this.current = null;
|
|
74
|
+
this.state = 'idle';
|
|
75
|
+
this.waitUntil = 0;
|
|
76
|
+
this.stopping = false;
|
|
77
|
+
this.claimKeys = {};
|
|
78
|
+
this.said = {};
|
|
79
|
+
this.rr = 0;
|
|
80
|
+
this.reconcileAt = 0;
|
|
81
|
+
/** The first pause under an operator's hold (DEC-0079); it doubles to HOLD_RETRY_MAX_SECONDS. */
|
|
82
|
+
this.holdDelay = POLL_SECONDS;
|
|
83
|
+
this.onHold = new Set();
|
|
84
|
+
/** One fenced call at a time, and the entry edits beside them (FAA-77, finding 36). */
|
|
85
|
+
this.fenceQueue = Promise.resolve();
|
|
86
|
+
this.toResume = []; // transferred requests waiting until nothing runs (finding 39)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Runs fn after every fenced call queued before it (finding 36): an await inside a call never lets another in. */
|
|
90
|
+
locked(fn) {
|
|
91
|
+
const run = this.fenceQueue.then(fn, fn);
|
|
92
|
+
this.fenceQueue = run.catch(() => {});
|
|
93
|
+
return run;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// -- events --
|
|
97
|
+
emit(kind, level, text, { notify = false, episode, ...value } = {}) {
|
|
98
|
+
if (episode !== undefined) {
|
|
99
|
+
if (this.said[kind] === episode) return;
|
|
100
|
+
this.said[kind] = episode;
|
|
101
|
+
}
|
|
102
|
+
let data = { schema: EVENT_SCHEMA, value: Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined && v !== null)) };
|
|
103
|
+
if (feedDataProblems(data).length) data = undefined; // FAC-SEM-101: drop the data, keep the sentence
|
|
104
|
+
if (!this.emitTo) return;
|
|
105
|
+
try {
|
|
106
|
+
this.emitTo(kind, level, text, { notify, ...(data ? { data } : {}) });
|
|
107
|
+
} catch {
|
|
108
|
+
// an event that cannot be written never stops the responder
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// -- keys and fences --
|
|
113
|
+
key(op) {
|
|
114
|
+
return `${this.service}.${op}.${crypto.randomBytes(8).toString('hex')}`;
|
|
115
|
+
}
|
|
116
|
+
idem(key) {
|
|
117
|
+
if (this.epoch === null) throw new BoardError('no-epoch', 'Fabric told this board session no idempotency epoch.', false, 'com.status');
|
|
118
|
+
return { epoch: this.epoch, key };
|
|
119
|
+
}
|
|
120
|
+
fence(entry, key) {
|
|
121
|
+
return { request: entry.request, generation: entry.generation, attempt: entry.attempt, digest: entry.digest, expectedRevision: entry.revision, idempotency: this.idem(key) };
|
|
122
|
+
}
|
|
123
|
+
receipt(op, answer) {
|
|
124
|
+
const problems = receiptProblems(answer, op);
|
|
125
|
+
if (problems.length) throw new BoardError('receipt-shape', `Fabric answered ${op} with a receipt the contract does not name (${problems[0]}).`, false, op);
|
|
126
|
+
return answer;
|
|
127
|
+
}
|
|
128
|
+
call(op, args) {
|
|
129
|
+
if (!this.session) return Promise.reject(new BoardError('session-ended', 'No board session.', true, op));
|
|
130
|
+
return this.session.call(op, args);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// -- connecting --
|
|
134
|
+
async status() {
|
|
135
|
+
const status = await this.call('com.status', {});
|
|
136
|
+
const verdict = negotiateRevision(status.protocol, acceptedRevisions('fabric-project-comms'));
|
|
137
|
+
if (verdict.error) throw new BoardError('schema-unsupported', `Fabric's board speaks ${status.protocol}, which this kit does not read.`, false, 'com.status', verdict.error.data);
|
|
138
|
+
if (status.board?.state === 'unavailable') throw new BoardError('not_available', 'Fabric\'s board cannot be read right now.', true, 'com.status');
|
|
139
|
+
const epoch = status.idempotency?.epoch;
|
|
140
|
+
// Fail closed: a guessed epoch would store keys under a namespace the board never named.
|
|
141
|
+
if (!isInt(epoch, 1)) throw new BoardError('no-epoch', 'Fabric told this board session no idempotency epoch (Fabric before its board-session fix); the responder waits rather than guess one.', false, 'com.status');
|
|
142
|
+
this.epoch = epoch;
|
|
143
|
+
return status;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async connect() {
|
|
147
|
+
this.state = 'connecting';
|
|
148
|
+
const grant = await this.hub.joinBoard(this.project, this.credentialFile);
|
|
149
|
+
const missing = this.capabilities.filter((c) => !grant.capabilities.includes(c));
|
|
150
|
+
const session = this.sessionFactory(grant);
|
|
151
|
+
await session.open();
|
|
152
|
+
this.session = session;
|
|
153
|
+
const status = await this.status();
|
|
154
|
+
if (missing.length) {
|
|
155
|
+
this.emit('board.access.needed', 'warn', `The operator has not let this service answer ${missing.join(', ')} on this Project; ask in Fabric (Agents & access).`, { notify: true, episode: missing.join(','), capabilities: missing });
|
|
156
|
+
}
|
|
157
|
+
this.slots = {};
|
|
158
|
+
await this.recover(status, this.capabilities.filter((c) => grant.capabilities.includes(c)));
|
|
159
|
+
this.state = 'ready';
|
|
160
|
+
for (const kind of ['board.down', 'board.blocked', ...(missing.length ? [] : ['board.access.needed'])]) delete this.said[kind];
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async enroll(capability) {
|
|
164
|
+
const args = { idempotency: this.idem(this.key('enroll')), capability };
|
|
165
|
+
let receipt;
|
|
166
|
+
try {
|
|
167
|
+
receipt = this.receipt('com.enroll', await this.call('com.enroll', args));
|
|
168
|
+
} catch (e) {
|
|
169
|
+
if (e.code === 'held') {
|
|
170
|
+
this.pause(capability);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
if (e.code === 'fenced') {
|
|
174
|
+
this.slots[capability] = { state: 'contested', renewedAt: this.now() };
|
|
175
|
+
this.emit('board.slot.contested', 'warn', `Another responder holds the ${capability} slot on this Project; this service waits for it to lapse and never takes it over by itself.`, { notify: true, episode: capability, capability });
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
throw e;
|
|
179
|
+
}
|
|
180
|
+
this.slots[capability] = { state: 'held', generation: receipt.slot.generation, renewedAt: this.now() };
|
|
181
|
+
delete this.said['board.slot.contested'];
|
|
182
|
+
this.lifted(capability);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The operator holds this slot or its Project (DEC-0079): ask again later, the pause doubling each time, and never act
|
|
187
|
+
* around the hold. Accepted work in progress continues; only new work waits.
|
|
188
|
+
*/
|
|
189
|
+
pause(capability) {
|
|
190
|
+
const slot = this.slots[capability] ?? {};
|
|
191
|
+
const pauses = slot.state === 'paused' ? (slot.pauses ?? 0) + 1 : 1;
|
|
192
|
+
const wait = Math.min(HOLD_RETRY_MAX_SECONDS, this.holdDelay * 2 ** (pauses - 1));
|
|
193
|
+
this.slots[capability] = { state: 'paused', pauses, retryAt: this.now() + wait, renewedAt: this.now() };
|
|
194
|
+
if (!this.onHold.has(capability)) { // once per hold, per capability
|
|
195
|
+
this.onHold.add(capability);
|
|
196
|
+
this.emit('board.slot.held', 'info', `The operator holds new work for ${capability} on this Project; the service waits and asks again later. Work already accepted continues.`, { capability });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
lifted(capability) {
|
|
201
|
+
if (!this.onHold.delete(capability)) return;
|
|
202
|
+
this.emit('board.slot.released', 'info', `The operator's hold on ${capability} was lifted; the service takes new work again.`, { capability });
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// -- restart: reconcile and transfer --
|
|
206
|
+
async readState(entry) {
|
|
207
|
+
const message = await this.call('com.get', { message: entry.message });
|
|
208
|
+
const req = isObj(message) ? message.request : null;
|
|
209
|
+
if (!isObj(req) || req.id !== entry.request || !REQUEST_STATES.includes(req.state)) return null;
|
|
210
|
+
entry.revision = req.revision ?? entry.revision;
|
|
211
|
+
return req.state;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async recover(status, capabilities) {
|
|
215
|
+
const transfer = {};
|
|
216
|
+
const reconcile = [];
|
|
217
|
+
for (const entry of Object.values(this.checkpoint.entries)) {
|
|
218
|
+
if (!capabilities.includes(entry.capability)) continue;
|
|
219
|
+
let state;
|
|
220
|
+
try {
|
|
221
|
+
state = await this.readState(entry);
|
|
222
|
+
} catch (e) {
|
|
223
|
+
if (e.code === 'not_authorized') {
|
|
224
|
+
this.checkpoint.drop(entry.request);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
throw e;
|
|
228
|
+
}
|
|
229
|
+
if (state === null || TERMINAL.has(state) || state === 'queued' || state === 'claimed') this.checkpoint.drop(entry.request);
|
|
230
|
+
else if (state === 'accepted') (transfer[entry.capability] ??= []).push(entry);
|
|
231
|
+
else reconcile.push(entry);
|
|
232
|
+
}
|
|
233
|
+
const responders = (status.responders ?? []).filter(isObj);
|
|
234
|
+
const generations = Object.fromEntries(responders.map((r) => [r.capability, r.generation ?? 0]));
|
|
235
|
+
const onHold = new Set(responders.filter((r) => r.held === true).map((r) => r.capability)); // DEC-0079
|
|
236
|
+
for (const capability of capabilities) {
|
|
237
|
+
if (onHold.has(capability)) {
|
|
238
|
+
for (const entry of transfer[capability] ?? []) {
|
|
239
|
+
entry.step = 'lapsed'; // taken back by a transfer once the hold is lifted
|
|
240
|
+
this.checkpoint.put(entry);
|
|
241
|
+
}
|
|
242
|
+
this.pause(capability);
|
|
243
|
+
} else if (transfer[capability]?.length) await this.replace(capability, transfer[capability], generations[capability] ?? 0);
|
|
244
|
+
else await this.enroll(capability);
|
|
245
|
+
}
|
|
246
|
+
for (const entry of reconcile) {
|
|
247
|
+
if (this.slots[entry.capability]?.state === 'held') await this.reconcile(entry);
|
|
248
|
+
else if (entry.step !== 'unknown') {
|
|
249
|
+
entry.step = 'unknown'; // the slot is not this session's yet: a later tick reconciles it (finding 38)
|
|
250
|
+
this.checkpoint.put(entry);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async replace(capability, entries, generation) {
|
|
256
|
+
const args = { idempotency: this.idem(this.key('replace')), capability, expectedGeneration: isInt(generation, 0) ? generation : 0 };
|
|
257
|
+
if (entries.length) args.transfer = { requests: entries.map((e) => e.request).slice(0, 64), checkpoint: { summary: 'Resumed by the same service after a restart; no effect had begun.' } };
|
|
258
|
+
let receipt;
|
|
259
|
+
try {
|
|
260
|
+
receipt = this.receipt('com.responder_replace', await this.call('com.responder_replace', args));
|
|
261
|
+
} catch (e) {
|
|
262
|
+
if (e.code !== 'held') throw e;
|
|
263
|
+
for (const entry of entries) {
|
|
264
|
+
entry.step = 'lapsed'; // nothing moved: taken back by a transfer once the hold is lifted
|
|
265
|
+
this.checkpoint.put(entry);
|
|
266
|
+
}
|
|
267
|
+
this.pause(capability);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
this.slots[capability] = { state: 'held', generation: receipt.slot.generation, renewedAt: this.now() };
|
|
271
|
+
this.lifted(capability);
|
|
272
|
+
const moved = new Map((receipt.transferred ?? []).map((item) => [item.request, item]));
|
|
273
|
+
for (const entry of entries) {
|
|
274
|
+
const item = moved.get(entry.request);
|
|
275
|
+
if (!item) {
|
|
276
|
+
// A board pinned before DEC-0078 hands back no attempt. Never claim it back: it stays held.
|
|
277
|
+
entry.step = 'held';
|
|
278
|
+
this.checkpoint.put(entry);
|
|
279
|
+
this.emit('board.request.held', 'warn', 'An accepted request could not be resumed: this Fabric does not hand back a transferred attempt (DEC-0078). It stays held; nothing was done.', { notify: true, episode: entry.request, request: entry.request, capability });
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
if (entry.step === 'replying' && !entry.replied && entry.keys?.reply && !entry.probe) {
|
|
283
|
+
// Finding 35: whether the old reply was stored is asked under the old fence, but only from the resume queue, after
|
|
284
|
+
// the new attempt is recorded: a second lost answer then leaves the request queued, never stranded.
|
|
285
|
+
entry.probe = { generation: entry.generation, attempt: entry.attempt, digest: entry.digest, revision: entry.revision, key: entry.keys.reply };
|
|
286
|
+
}
|
|
287
|
+
Object.assign(entry, { generation: item.attempt.generation, attempt: item.attempt.id, digest: item.attempt.digest, revision: item.revision,
|
|
288
|
+
step: ['accepting', 'held', 'lapsed'].includes(entry.step) ? 'accepted' : entry.step });
|
|
289
|
+
// Finding 35: a fenced key's digest named the old attempt; under the new one it is idempotency_conflict.
|
|
290
|
+
entry.keys = entry.keys?.reconcile ? { reconcile: entry.keys.reconcile } : {};
|
|
291
|
+
this.checkpoint.put(entry);
|
|
292
|
+
if (!this.toResume.includes(entry.request)) this.toResume.push(entry.request);
|
|
293
|
+
}
|
|
294
|
+
await this.resumeNext();
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* A reply whose answer was lost under the old attempt (finding 35): a stored key answers before the fence is checked
|
|
299
|
+
* (project-comms.md, Idempotency), so the old call again, under the old fence kept in `probe`, says whether it was
|
|
300
|
+
* stored; a refusal says it was not. A lost answer, or a board that cannot be read now, throws and leaves `probe` for
|
|
301
|
+
* the next try: neither says whether the reply was stored.
|
|
302
|
+
*/
|
|
303
|
+
async repliedBefore(entry) {
|
|
304
|
+
const { key, generation, attempt, digest, revision } = entry.probe;
|
|
305
|
+
let replied = true;
|
|
306
|
+
try {
|
|
307
|
+
this.receipt('com.reply', await this.call('com.reply', { ...this.replyExtra(entry), fence: this.fence({ ...entry, generation, attempt, digest, revision }, key) }));
|
|
308
|
+
} catch (e) {
|
|
309
|
+
if (['unreachable', 'session-ended', 'authority_changed', 'not_authorized', 'not_available', 'capacity_exceeded'].includes(e.code) || !(e instanceof BoardError)) throw e;
|
|
310
|
+
replied = false;
|
|
311
|
+
}
|
|
312
|
+
if (replied) entry.replied = true;
|
|
313
|
+
delete entry.probe;
|
|
314
|
+
this.checkpoint.put(entry);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Resume a transferred request while nothing runs; each idle tick takes the next (finding 39). A request leaves the
|
|
319
|
+
* queue only once its probe has answered: a lost answer keeps it first in line for the next tick (finding 35).
|
|
320
|
+
*/
|
|
321
|
+
async resumeNext() {
|
|
322
|
+
while (!this.current && this.toResume.length && !this.stopping) {
|
|
323
|
+
const stored = this.checkpoint.entries[this.toResume[0]];
|
|
324
|
+
if (!stored || this.slots[stored.capability]?.state !== 'held') {
|
|
325
|
+
this.toResume.shift();
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
const entry = JSON.parse(JSON.stringify(stored));
|
|
329
|
+
if (entry.probe) await this.repliedBefore(entry);
|
|
330
|
+
this.toResume.shift();
|
|
331
|
+
await this.resume(entry);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/** An observed outcome: the one already known, else what observe(entry) reads; null when nothing was observed. */
|
|
336
|
+
async observed(entry) {
|
|
337
|
+
let outcome = isObj(entry.outcome) && entry.outcome.observed === true ? entry.outcome : null;
|
|
338
|
+
if (!outcome && this.observe) {
|
|
339
|
+
try {
|
|
340
|
+
outcome = await this.observe({ ...entry });
|
|
341
|
+
} catch {
|
|
342
|
+
outcome = null; // an observation that failed is no observation
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
return isObj(outcome) && outcome.observed === true && !outcomeProblems({ ...outcome, observed: true }, true).length ? outcome : null;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
async reconcile(entry) {
|
|
349
|
+
const state = await this.readState(entry);
|
|
350
|
+
if (state !== 'outcome_unknown') {
|
|
351
|
+
if (state !== null && (TERMINAL.has(state) || state === 'queued' || state === 'claimed')) this.checkpoint.drop(entry.request);
|
|
352
|
+
else if (state === 'in_progress' && entry.step !== 'unknown') {
|
|
353
|
+
entry.step = 'unknown'; // no next holder took the slot yet: a later tick tries again (finding 38)
|
|
354
|
+
this.checkpoint.put(entry);
|
|
355
|
+
}
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
const outcome = await this.observed(entry);
|
|
359
|
+
if (!outcome) {
|
|
360
|
+
entry.step = 'unknown';
|
|
361
|
+
this.checkpoint.put(entry);
|
|
362
|
+
this.emit('board.reconcile.needed', 'warn', 'A request\'s outcome is unknown: the work may have happened. It stays unknown until this service observes what happened.', { notify: true, episode: entry.request, request: entry.request, capability: entry.capability });
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
const sent = Object.fromEntries(['state', 'summary', 'evidence'].filter((k) => k in outcome).map((k) => [k, outcome[k]]));
|
|
366
|
+
sent.observed = true;
|
|
367
|
+
entry.keys ??= {};
|
|
368
|
+
entry.keys.reconcile ??= this.key('reconcile');
|
|
369
|
+
entry.step = 'reconciling';
|
|
370
|
+
this.checkpoint.put(entry);
|
|
371
|
+
this.receipt('com.reconcile', await this.call('com.reconcile', { idempotency: this.idem(entry.keys.reconcile), request: entry.request, digest: entry.digest, outcome: sent }));
|
|
372
|
+
this.checkpoint.drop(entry.request);
|
|
373
|
+
this.emit('board.request.reconciled', 'info', 'A request whose outcome was unknown was settled from what this service observed.', { request: entry.request, capability: entry.capability, state: sent.state });
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// -- one tick --
|
|
377
|
+
interval() {
|
|
378
|
+
return Math.max(1, POLL_SECONDS + POLL_JITTER_SECONDS * (2 * this.rand() - 1));
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** One tick; answers the seconds to wait before the next. */
|
|
382
|
+
async step() {
|
|
383
|
+
if (this.stopping) return DRAIN_SECONDS;
|
|
384
|
+
const now = this.now();
|
|
385
|
+
if (now < this.waitUntil) return this.waitUntil - now;
|
|
386
|
+
try {
|
|
387
|
+
if (!this.session) await this.connect();
|
|
388
|
+
await this.tick();
|
|
389
|
+
this.backoff.reset();
|
|
390
|
+
if (this.state === 'down') { // a lost call was sent again and answered: the session held throughout
|
|
391
|
+
this.state = 'ready';
|
|
392
|
+
delete this.said['board.down'];
|
|
393
|
+
}
|
|
394
|
+
return this.interval();
|
|
395
|
+
} catch (e) {
|
|
396
|
+
if (!(e instanceof BoardError) && !(e instanceof HubError)) throw e;
|
|
397
|
+
return this.fault(e);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Seconds this session's leases still run, by its own clock: the running attempt's, else the oldest held slot's. */
|
|
402
|
+
leaseLeft() {
|
|
403
|
+
const renewed = this.current ? [this.current.renewedAt ?? 0] : Object.values(this.slots).filter((s) => s.state === 'held').map((s) => s.renewedAt ?? 0);
|
|
404
|
+
return renewed.length && this.session ? Math.min(...renewed) + LEASE_SECONDS - this.now() : 0;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
async fault(e) {
|
|
408
|
+
const code = e.code ?? 'unreachable';
|
|
409
|
+
let wait = this.backoff.next();
|
|
410
|
+
const left = this.leaseLeft() - LEASE_MARGIN_SECONDS;
|
|
411
|
+
const keep = code === 'unreachable' && left > 0; // finding 38: the lease holds, so the session and the work stay
|
|
412
|
+
if (keep) wait = Math.max(1, Math.min(wait, left)); // and the call goes again before the lease lapses
|
|
413
|
+
this.waitUntil = this.now() + wait;
|
|
414
|
+
if (['not_authorized', 'authority_changed', 'credential'].includes(code)) {
|
|
415
|
+
this.state = 'not-authorized';
|
|
416
|
+
await this.dropSession();
|
|
417
|
+
this.emit('board.access.needed', 'warn', 'This service may not answer requests for this Project: the operator has to allow it in Fabric (Agents & access). Nothing is claimed until then.', { notify: true, episode: 'access', reason: code });
|
|
418
|
+
} else if (code === 'held') {
|
|
419
|
+
this.state = 'paused';
|
|
420
|
+
this.emit('board.held', 'info', 'The operator holds new work on this Project; the responder waits and asks again with a growing pause.', { episode: 'held', reason: code });
|
|
421
|
+
} else if (['no-epoch', 'schema-unsupported', 'checkpoint', 'receipt-shape', 'idempotency_conflict'].includes(code)) {
|
|
422
|
+
// idempotency_conflict is a key sent with other content, a defect and not an outage (finding 35)
|
|
423
|
+
this.state = 'down';
|
|
424
|
+
await this.dropSession();
|
|
425
|
+
this.emit('board.blocked', 'error', `The responder stopped answering requests: ${e.message}`, { notify: true, episode: code, reason: code });
|
|
426
|
+
} else {
|
|
427
|
+
this.state = 'down';
|
|
428
|
+
if (['session-ended', 'unreachable', 'not-running', 'not-a-hub', 'hub-unavailable', 'protocol'].includes(code) && !keep) await this.dropSession();
|
|
429
|
+
this.emit('board.down', 'warn', 'Fabric\'s board cannot be reached; the responder tries again with a growing pause.', { episode: 'down', reason: code });
|
|
430
|
+
}
|
|
431
|
+
return wait;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async dropSession() {
|
|
435
|
+
const work = this.current;
|
|
436
|
+
this.current = null;
|
|
437
|
+
this.toResume = []; // the next session transfers them again
|
|
438
|
+
if (work) {
|
|
439
|
+
const { entry } = work;
|
|
440
|
+
if (work.done && !work.failure && !outcomeProblems(work.outcome, !!entry.effect).length) {
|
|
441
|
+
entry.outcome = { ...work.outcome, observed: work.outcome.observed ?? !entry.effect };
|
|
442
|
+
try {
|
|
443
|
+
this.checkpoint.put(entry);
|
|
444
|
+
} catch {
|
|
445
|
+
// the next start reruns it
|
|
446
|
+
}
|
|
447
|
+
} else work.ctx.stopped = true;
|
|
448
|
+
}
|
|
449
|
+
const session = this.session;
|
|
450
|
+
this.session = null;
|
|
451
|
+
if (session) await session.close();
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
async tick() {
|
|
455
|
+
const work = this.current;
|
|
456
|
+
if (work) {
|
|
457
|
+
if (work.done) await this.finish(work);
|
|
458
|
+
else await this.renew(work);
|
|
459
|
+
}
|
|
460
|
+
const heldByWork = work && this.current ? work.entry.capability : null;
|
|
461
|
+
for (const [capability, slot] of Object.entries(this.slots)) {
|
|
462
|
+
if (slot.state === 'paused') {
|
|
463
|
+
// still held: a longer pause; lifted: the slot is ours again
|
|
464
|
+
if (capability !== heldByWork && this.now() >= (slot.retryAt ?? 0)) await this.enroll(capability);
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
if (capability !== heldByWork && this.now() - (slot.renewedAt ?? 0) >= RENEW_SECONDS - 0.5) await this.enroll(capability);
|
|
468
|
+
}
|
|
469
|
+
if (!this.current && !this.stopping) {
|
|
470
|
+
await this.continueLapsed();
|
|
471
|
+
await this.resumeNext();
|
|
472
|
+
}
|
|
473
|
+
if (!this.current && !this.stopping) {
|
|
474
|
+
await this.reconcilePending();
|
|
475
|
+
await this.claimNext();
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
async continueLapsed() {
|
|
480
|
+
const lapsed = {};
|
|
481
|
+
for (const entry of Object.values(this.checkpoint.entries)) {
|
|
482
|
+
if (entry.step === 'lapsed' && this.slots[entry.capability]?.state === 'held') (lapsed[entry.capability] ??= []).push(JSON.parse(JSON.stringify(entry)));
|
|
483
|
+
}
|
|
484
|
+
for (const [capability, entries] of Object.entries(lapsed)) {
|
|
485
|
+
const live = [];
|
|
486
|
+
for (const e of entries) {
|
|
487
|
+
if (await this.readState(e) === 'accepted') live.push(e);
|
|
488
|
+
else this.checkpoint.drop(e.request);
|
|
489
|
+
}
|
|
490
|
+
if (live.length) await this.replace(capability, live, this.slots[capability].generation ?? 0);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
async reconcilePending() {
|
|
495
|
+
if (this.now() < this.reconcileAt) return;
|
|
496
|
+
this.reconcileAt = this.now() + 300;
|
|
497
|
+
for (const entry of Object.values(this.checkpoint.entries)) {
|
|
498
|
+
if (entry.step === 'unknown' && this.slots[entry.capability]?.state === 'held') await this.reconcile(entry);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
async claimNext() {
|
|
503
|
+
const held = this.capabilities.filter((c) => this.slots[c]?.state === 'held');
|
|
504
|
+
for (let i = 0; i < held.length; i += 1) {
|
|
505
|
+
const capability = held[(this.rr + i) % held.length];
|
|
506
|
+
this.claimKeys[capability] ??= this.key('claim');
|
|
507
|
+
let answer;
|
|
508
|
+
try {
|
|
509
|
+
answer = this.receipt('com.claim', await this.call('com.claim', { idempotency: this.idem(this.claimKeys[capability]), capability }));
|
|
510
|
+
} catch (e) {
|
|
511
|
+
if (e.code === 'lease_expired') {
|
|
512
|
+
await this.enroll(capability);
|
|
513
|
+
continue;
|
|
514
|
+
}
|
|
515
|
+
if (e.code === 'idempotency_window_expired' || e.code === 'capacity_exceeded') {
|
|
516
|
+
delete this.claimKeys[capability];
|
|
517
|
+
await this.status();
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
if (e.code === 'fenced') {
|
|
521
|
+
this.slots[capability] = { state: 'contested', renewedAt: this.now() };
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
524
|
+
if (e.code === 'held') {
|
|
525
|
+
this.pause(capability);
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
throw e;
|
|
529
|
+
}
|
|
530
|
+
if (!('request' in answer)) continue; // nothing claimable; the same key polls again
|
|
531
|
+
delete this.claimKeys[capability];
|
|
532
|
+
this.rr += 1;
|
|
533
|
+
await this.start(capability, answer);
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// -- a claimed request --
|
|
539
|
+
async start(capability, receipt) {
|
|
540
|
+
if (!receipt.attempt || !receipt.message) throw new BoardError('receipt-shape', 'com.claim answered a request without its attempt or message.', false, 'com.claim');
|
|
541
|
+
const { attempt } = receipt;
|
|
542
|
+
const entry = { request: receipt.request.id, message: receipt.message, capability, generation: attempt.generation, attempt: attempt.id, digest: attempt.digest,
|
|
543
|
+
revision: receipt.request.revision, step: 'claimed', keys: {}, effect: false };
|
|
544
|
+
this.checkpoint.put(entry);
|
|
545
|
+
const message = await this.call('com.get', { message: entry.message });
|
|
546
|
+
const req = isObj(message) ? message.request : null;
|
|
547
|
+
if (message.digest !== entry.digest) {
|
|
548
|
+
this.checkpoint.drop(entry.request);
|
|
549
|
+
this.emit('board.request.skipped', 'warn', 'A claimed request was not accepted: the message read back carries another digest than the claim. Nothing was done; the claim lapses.', { request: entry.request, capability, reason: 'digest-mismatch' });
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
if (!isObj(req) || req.id !== entry.request || req.capability !== capability || req.target !== this.project) {
|
|
553
|
+
this.checkpoint.drop(entry.request);
|
|
554
|
+
this.emit('board.request.skipped', 'warn', 'A claimed request was not accepted: it is not addressed to this Project and capability. Nothing was done; the claim lapses.', { request: entry.request, capability, reason: 'not-addressed' });
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
const request = new BoardRequest(message, capability);
|
|
558
|
+
let reason;
|
|
559
|
+
if (message.redacted === true || typeof request.body !== 'string') reason = 'The request\'s body was redacted, so there is nothing to act on.';
|
|
560
|
+
else {
|
|
561
|
+
try {
|
|
562
|
+
reason = (await this.admit(request)) ?? null;
|
|
563
|
+
} catch {
|
|
564
|
+
reason = 'The service\'s input gate could not read this request.';
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
if (!(await this.accept(entry))) return;
|
|
568
|
+
if (reason !== null && reason !== undefined) {
|
|
569
|
+
entry.outcome = { state: 'failed_known', observed: true, summary: clean(`Refused by this service's input gate: ${reason}`) };
|
|
570
|
+
await this.complete(entry);
|
|
571
|
+
this.emit('board.request.refused', 'info', 'A request was refused by the service\'s input gate and completed as failed.', { request: entry.request, capability });
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
await this.runWork(entry, request);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
async accept(entry) {
|
|
578
|
+
entry.keys.accept ??= this.key('accept');
|
|
579
|
+
entry.step = 'accepting';
|
|
580
|
+
this.checkpoint.put(entry);
|
|
581
|
+
let receipt;
|
|
582
|
+
try {
|
|
583
|
+
receipt = await this.fenced('com.accept', entry, entry.keys.accept);
|
|
584
|
+
} catch (e) {
|
|
585
|
+
if (e.code !== 'held') throw e;
|
|
586
|
+
// A hold came between the claim and the accept (DEC-0079): nothing was done, the claim lapses and the request waits.
|
|
587
|
+
this.checkpoint.drop(entry.request);
|
|
588
|
+
this.emit('board.request.waiting', 'info', 'A claimed request was not accepted: the operator holds it. Nothing was done; it waits on the board.', { request: entry.request, capability: entry.capability });
|
|
589
|
+
this.pause(entry.capability);
|
|
590
|
+
return false;
|
|
591
|
+
}
|
|
592
|
+
if (!receipt) return false; // the claim was lost before it was accepted; nothing was done
|
|
593
|
+
entry.revision = receipt.request.revision;
|
|
594
|
+
entry.step = 'accepted';
|
|
595
|
+
this.checkpoint.put(entry);
|
|
596
|
+
return true;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
async runWork(entry, request) {
|
|
600
|
+
const ctx = new WorkContext(this, entry);
|
|
601
|
+
const work = { entry, request, ctx, done: false, outcome: null, failure: null, renewedAt: this.now() };
|
|
602
|
+
this.current = work;
|
|
603
|
+
work.promise = Promise.resolve().then(() => this.handler(request, ctx)).then((outcome) => { work.outcome = outcome; }, (e) => { work.failure = e; }).finally(() => { work.done = true; });
|
|
604
|
+
// A handler that answers at once is finished in this tick; a longer one is renewed by the next ticks.
|
|
605
|
+
await Promise.race([work.promise, new Promise((resolve) => setImmediate(resolve))]);
|
|
606
|
+
if (work.done && this.current === work) await this.finish(work);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async resume(entry) {
|
|
610
|
+
if (isObj(entry.outcome)) {
|
|
611
|
+
await this.complete(entry);
|
|
612
|
+
return;
|
|
613
|
+
}
|
|
614
|
+
const message = await this.call('com.get', { message: entry.message });
|
|
615
|
+
await this.runWork(entry, new BoardRequest(message, entry.capability));
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* A fenced call, one at a time (finding 36): fence built, sent and receipt applied under the lock. A revision_conflict
|
|
620
|
+
* on com.renew or com.progress (they store nothing) reads the revision and sends once more. On a stop code the attempt
|
|
621
|
+
* is lost: the work stops, and it answers null (authority_changed throws after, so the responder stops claiming).
|
|
622
|
+
*/
|
|
623
|
+
async fenced(op, entry, key, extra = {}, apply = null) {
|
|
624
|
+
let again = op === 'com.renew' || op === 'com.progress';
|
|
625
|
+
for (;;) {
|
|
626
|
+
try {
|
|
627
|
+
return await this.locked(async () => {
|
|
628
|
+
const receipt = this.receipt(op, await this.call(op, { ...extra, fence: this.fence(entry, key) }));
|
|
629
|
+
apply?.(receipt);
|
|
630
|
+
return receipt;
|
|
631
|
+
});
|
|
632
|
+
} catch (e) {
|
|
633
|
+
if (e.code === 'revision_conflict' && again) {
|
|
634
|
+
again = false;
|
|
635
|
+
await this.locked(() => this.readState(entry));
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
if (STOP_CODES.has(e.code) || ['revision_conflict', 'invalid_transition', 'digest_conflict'].includes(e.code)) {
|
|
639
|
+
this.lose(entry, e.code);
|
|
640
|
+
if (e.code === 'authority_changed') throw e;
|
|
641
|
+
return null;
|
|
642
|
+
}
|
|
643
|
+
throw e;
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
lose(entry, code) {
|
|
649
|
+
if (this.current && this.current.entry === entry) this.current.ctx.stopped = true;
|
|
650
|
+
if (entry.effect) {
|
|
651
|
+
entry.step = 'unknown';
|
|
652
|
+
this.checkpoint.put(entry);
|
|
653
|
+
this.emit('board.reconcile.needed', 'warn', 'The attempt was lost after its effect began; the outcome stays unknown until this service observes what happened.', { notify: true, episode: entry.request, request: entry.request, capability: entry.capability, reason: code });
|
|
654
|
+
} else if (code === 'lease_expired' && !['claimed', 'accepting'].includes(entry.step)) {
|
|
655
|
+
entry.step = 'lapsed';
|
|
656
|
+
this.checkpoint.put(entry);
|
|
657
|
+
this.emit('board.attempt.lost', 'warn', 'This service\'s lease on an accepted request lapsed; it stopped working on it and takes it back by a transfer.', { request: entry.request, capability: entry.capability, reason: code });
|
|
658
|
+
} else {
|
|
659
|
+
this.checkpoint.drop(entry.request);
|
|
660
|
+
this.emit('board.attempt.lost', 'warn', `The board took a request back from this service (${code}); it stopped working on it, and nothing had been done outside the service.`, { request: entry.request, capability: entry.capability, reason: code });
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
async renew(work) {
|
|
665
|
+
if (this.now() - (work.renewedAt ?? 0) < RENEW_SECONDS - 0.5) return;
|
|
666
|
+
if (!(await this.fenced('com.renew', work.entry, this.key('renew')))) {
|
|
667
|
+
this.current = null;
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
work.renewedAt = this.now();
|
|
671
|
+
(this.slots[work.entry.capability] ??= {}).renewedAt = this.now();
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
async progress(entry, note) {
|
|
675
|
+
const extra = note !== undefined && note !== null ? { note: { text: clean(note) || 'Working.' } } : {};
|
|
676
|
+
if (!(await this.fenced('com.progress', entry, this.key('progress'), extra))) throw new Stopped();
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
async effectBegin(entry, summary, ctx) {
|
|
680
|
+
entry.keys.effect_begin ??= this.key('effect');
|
|
681
|
+
entry.step = 'effect-beginning';
|
|
682
|
+
entry.effect = true; // from here a lost attempt is an unknown outcome, never "nothing happened"
|
|
683
|
+
this.checkpoint.put(entry);
|
|
684
|
+
const extra = summary ? { summary: clean(summary) } : {};
|
|
685
|
+
const begun = (receipt) => {
|
|
686
|
+
entry.revision = receipt.request.revision;
|
|
687
|
+
entry.step = 'in-progress';
|
|
688
|
+
this.checkpoint.put(entry);
|
|
689
|
+
};
|
|
690
|
+
let receipt = null;
|
|
691
|
+
let lost = 0;
|
|
692
|
+
let delay = this.holdDelay;
|
|
693
|
+
for (;;) { // a lost answer: the same key again tells whether it was stored
|
|
694
|
+
try {
|
|
695
|
+
receipt = await this.fenced('com.effect_begin', entry, entry.keys.effect_begin, extra, begun);
|
|
696
|
+
break;
|
|
697
|
+
} catch (e) {
|
|
698
|
+
if (e.code === 'held') {
|
|
699
|
+
// The operator holds this work (DEC-0079): the effect has not begun. Wait, keep the lease (the tick renews it;
|
|
700
|
+
// com.renew is never held), and ask again with the same key.
|
|
701
|
+
entry.effect = false;
|
|
702
|
+
entry.step = 'effect-held';
|
|
703
|
+
this.checkpoint.put(entry);
|
|
704
|
+
this.emit('board.request.waiting', 'info', 'The operator holds this request before its effect: the service waits and does nothing outside it until the hold is lifted.', { episode: entry.request, request: entry.request, capability: entry.capability });
|
|
705
|
+
const until = Date.now() + delay * 1000;
|
|
706
|
+
while (Date.now() < until) {
|
|
707
|
+
if (ctx?.stopped) throw new Stopped();
|
|
708
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(250, Math.max(1, until - Date.now()))));
|
|
709
|
+
}
|
|
710
|
+
if (ctx?.stopped) throw new Stopped();
|
|
711
|
+
delay = Math.min(HOLD_RETRY_MAX_SECONDS, delay * 2);
|
|
712
|
+
entry.effect = true;
|
|
713
|
+
entry.step = 'effect-beginning';
|
|
714
|
+
this.checkpoint.put(entry);
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
if (e.code !== 'unreachable' || lost === 2) throw new Stopped();
|
|
718
|
+
lost += 1;
|
|
719
|
+
await new Promise((resolve) => setTimeout(resolve, lost * 1000));
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
if (!receipt) throw new Stopped();
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
async finish(work) {
|
|
726
|
+
const { entry } = work;
|
|
727
|
+
if (work.ctx.stopped && !(entry.request in this.checkpoint.entries)) {
|
|
728
|
+
this.current = null;
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
let { outcome } = work;
|
|
732
|
+
if (work.failure) {
|
|
733
|
+
if (entry.effect) {
|
|
734
|
+
await this.settleLive(entry, 'handler-failed');
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
if (work.failure instanceof Stopped) {
|
|
738
|
+
this.current = null;
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
outcome = { state: 'failed_known', observed: true, summary: `The service could not do this (${work.failure?.name ?? 'Error'}).` };
|
|
742
|
+
}
|
|
743
|
+
const problems = outcomeProblems(outcome, !!entry.effect);
|
|
744
|
+
if (problems.length) {
|
|
745
|
+
this.emit('board.outcome.invalid', 'warn', `The handler's outcome was not sent: ${problems[0]}.`, { request: entry.request, capability: entry.capability });
|
|
746
|
+
if (entry.effect) await this.settleLive(entry, problems.some((p) => p.includes('observation_required')) ? 'observation_required' : 'outcome');
|
|
747
|
+
else {
|
|
748
|
+
entry.outcome = { state: 'failed_known', observed: true, summary: 'The service answered in a form it cannot send.' };
|
|
749
|
+
await this.complete(entry);
|
|
750
|
+
}
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
entry.outcome = { ...outcome, observed: outcome.observed ?? !entry.effect };
|
|
754
|
+
await this.complete(entry);
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/**
|
|
758
|
+
* Finding 37: the handler threw after beginEffect, or answered unobserved, while the attempt is live. Only an observed
|
|
759
|
+
* result settles a started effect (project-comms.md, Request lifecycle), so observe(entry) is completed under the live
|
|
760
|
+
* fence. With nothing observed, com.responder_replace on this responder's own slot is the contract's lost-result
|
|
761
|
+
* operation ("an in_progress one becomes outcome_unknown ... when the next holder takes the slot", Slots end with the
|
|
762
|
+
* session); com.reconcile settles it once something is observed.
|
|
763
|
+
*/
|
|
764
|
+
async settleLive(entry, code) {
|
|
765
|
+
const lost = !!this.current?.ctx.stopped; // the attempt is not live: nothing to settle
|
|
766
|
+
const state = lost ? null : await this.readState(entry);
|
|
767
|
+
if (state === 'accepted') { // effect_begin was never stored, so nothing began: a known failure
|
|
768
|
+
entry.effect = false;
|
|
769
|
+
entry.outcome = { state: 'failed_known', observed: true, summary: `The service could not do this (${code}).` };
|
|
770
|
+
await this.complete(entry);
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
const outcome = state === 'in_progress' ? await this.observed(entry) : null;
|
|
774
|
+
if (outcome) {
|
|
775
|
+
entry.outcome = { ...Object.fromEntries(['state', 'summary', 'evidence'].filter((k) => k in outcome).map((k) => [k, outcome[k]])), observed: true };
|
|
776
|
+
await this.complete(entry);
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
this.current = null;
|
|
780
|
+
this.lose(entry, code);
|
|
781
|
+
const slot = this.slots[entry.capability] ?? {};
|
|
782
|
+
if (state === 'in_progress' && slot.state === 'held') await this.replace(entry.capability, [], slot.generation ?? 0);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
replyExtra(entry) {
|
|
786
|
+
const { reply } = entry.outcome;
|
|
787
|
+
return { body: { text: reply.text, ...(reply.format ? { format: reply.format } : {}) }, ...(reply.artifacts?.length ? { artifacts: reply.artifacts } : {}) };
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
async complete(entry) {
|
|
791
|
+
const { outcome } = entry;
|
|
792
|
+
if (outcome.reply && !entry.replied) {
|
|
793
|
+
entry.keys.reply ??= this.key('reply');
|
|
794
|
+
entry.step = 'replying';
|
|
795
|
+
this.checkpoint.put(entry);
|
|
796
|
+
if (!(await this.fenced('com.reply', entry, entry.keys.reply, this.replyExtra(entry)))) {
|
|
797
|
+
this.current = null;
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
entry.replied = true;
|
|
801
|
+
this.checkpoint.put(entry);
|
|
802
|
+
}
|
|
803
|
+
entry.keys.complete ??= this.key('complete');
|
|
804
|
+
entry.step = 'completing';
|
|
805
|
+
this.checkpoint.put(entry);
|
|
806
|
+
const sent = Object.fromEntries(['state', 'observed', 'summary', 'evidence'].filter((k) => k in outcome).map((k) => [k, outcome[k]]));
|
|
807
|
+
const receipt = await this.fenced('com.complete', entry, entry.keys.complete, { outcome: sent });
|
|
808
|
+
this.current = null;
|
|
809
|
+
if (!receipt) return;
|
|
810
|
+
this.checkpoint.drop(entry.request);
|
|
811
|
+
this.emit('board.request.completed', 'info', `A request was answered: ${sent.state === 'completed' ? 'completed' : 'failed, as known'}.`, { request: entry.request, capability: entry.capability, state: sent.state });
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
// -- running --
|
|
815
|
+
/** Tick until `signal` aborts, then shutdown(). */
|
|
816
|
+
async run({ signal, drain = DRAIN_SECONDS } = {}) {
|
|
817
|
+
while (!signal?.aborted) {
|
|
818
|
+
const wait = await this.step();
|
|
819
|
+
await new Promise((resolve) => {
|
|
820
|
+
const timer = setTimeout(resolve, wait * 1000);
|
|
821
|
+
signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
await this.shutdown(drain);
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/** Stop claiming, give running work up to `drain` seconds, then leave the session (its DELETE releases the slots). */
|
|
828
|
+
async shutdown(drain = DRAIN_SECONDS) {
|
|
829
|
+
this.stopping = true;
|
|
830
|
+
const work = this.current;
|
|
831
|
+
if (work && !work.done) {
|
|
832
|
+
await Promise.race([work.promise, new Promise((resolve) => setTimeout(resolve, drain * 1000).unref?.())]);
|
|
833
|
+
if (!work.done) work.ctx.stopped = true;
|
|
834
|
+
}
|
|
835
|
+
if (work && work.done && this.session) {
|
|
836
|
+
try {
|
|
837
|
+
await this.finish(work);
|
|
838
|
+
} catch {
|
|
839
|
+
// the checkpoint keeps it
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
if (this.session) {
|
|
843
|
+
await this.session.close();
|
|
844
|
+
this.session = null;
|
|
845
|
+
}
|
|
846
|
+
this.state = 'stopped';
|
|
847
|
+
}
|
|
848
|
+
}
|