@uuaid/pillar 0.2.0-alpha.8 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOSTING-LIMITS.md +432 -0
- package/INTERACTION-CONVENTION.md +222 -1
- package/README.md +371 -17
- package/bin/carrier.mjs +7 -1
- package/bin/pillar-ask.mjs +1 -0
- package/bin/pillar-blob.mjs +139 -0
- package/bin/pillar-directory.mjs +232 -0
- package/bin/pillar-escrow.mjs +675 -0
- package/bin/pillar-responder.mjs +47 -1
- package/bin/pillar-seed.mjs +113 -13
- package/bin/pillar.mjs +345 -4
- package/package.json +30 -7
- package/src/blob/blob.mjs +510 -0
- package/src/blob/broker.mjs +129 -0
- package/src/blob/http.mjs +334 -0
- package/src/blob/store.mjs +55 -0
- package/src/escrow/box.mjs +43 -0
- package/src/escrow/escrow-service.mjs +538 -0
- package/src/escrow/rails/evm-rail.mjs +427 -0
- package/src/escrow/rails/index.mjs +41 -0
- package/src/escrow/rails/zilligon-rail.mjs +73 -0
- package/src/escrow/terms.mjs +368 -0
- package/src/escrow/token-registry.mjs +231 -0
- package/src/escrow/verify.mjs +119 -0
- package/src/escrow/zilligon.mjs +266 -0
- package/src/identity/keychain.mjs +27 -10
- package/src/identity/tier.mjs +130 -0
- package/src/identity/trust.mjs +63 -0
- package/src/index.mjs +28 -2
- package/src/mailbox/mailbox.mjs +41 -0
- package/src/net/carrier-client.mjs +93 -14
- package/src/net/carrier-server.mjs +212 -15
- package/src/net/delegation.mjs +209 -0
- package/src/net/envelope.mjs +81 -0
- package/src/net/link-offer.mjs +107 -0
- package/src/net/refusals.mjs +126 -0
- package/src/net/seed-registry.mjs +467 -0
- package/src/net/seed-tier.mjs +185 -0
- package/src/node-check.mjs +32 -0
- package/src/observability.mjs +141 -0
- package/src/pillar.mjs +206 -17
- package/src/vda/core.mjs +290 -0
- package/src/vda/directory.mjs +87 -0
- package/src/vda/instruments.mjs +246 -0
- package/src/vda/shims.mjs +77 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observability — metrics/analytics for the Pillar communication system.
|
|
3
|
+
*
|
|
4
|
+
* Zero-dependency, in-memory counters / gauges / histograms with an optional
|
|
5
|
+
* append-only JSONL sink (same pattern as the escrow journal). Every Pillar
|
|
6
|
+
* and EscrowService owns one instance; when none is passed in, a default is
|
|
7
|
+
* created internally so instrumentation sites never null-check.
|
|
8
|
+
*
|
|
9
|
+
* inc("sent.via.carrier") counters — labels compose into dotted
|
|
10
|
+
* keys at the CALL SITE, e.g.
|
|
11
|
+
* `inc(\`received.kind.${kind}\`)`
|
|
12
|
+
* setGauge("escrow.offers.open", 3) gauges — last write wins
|
|
13
|
+
* observe("send.latencyMs", ms) histograms — fixed latency buckets
|
|
14
|
+
*
|
|
15
|
+
* Sink: journal({event, ...}) appends an event line; flush() appends a full
|
|
16
|
+
* state snapshot. An external analytics pipeline can tail the file without
|
|
17
|
+
* a live process. summary() renders the one-glance human view.
|
|
18
|
+
*
|
|
19
|
+
* Keys are sanitized (lowercased, non-alphanumerics folded to "-") so
|
|
20
|
+
* attacker-influenced strings (envelope kinds, error reasons) can never
|
|
21
|
+
* fabricate keys or escape the dotted namespace.
|
|
22
|
+
*/
|
|
23
|
+
import { appendFileSync, mkdirSync } from "node:fs";
|
|
24
|
+
import { dirname } from "node:path";
|
|
25
|
+
|
|
26
|
+
// Fixed latency buckets (ms) — covers fast local sends (<10 ms) up to slow
|
|
27
|
+
// carrier round-trips (>10 s). Values above the top bucket still count into
|
|
28
|
+
// n/sum/max; percentiles interpolate within the top bucket's upper bound.
|
|
29
|
+
export const LATENCY_BUCKETS = [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];
|
|
30
|
+
|
|
31
|
+
const MAX_KEY = 64;
|
|
32
|
+
|
|
33
|
+
export function slugKey(key) {
|
|
34
|
+
return String(key)
|
|
35
|
+
.toLowerCase()
|
|
36
|
+
.replace(/[^a-z0-9.-]+/g, "-")
|
|
37
|
+
.replace(/^-+|-+$/g, "") // a trailing "!!!" run must not leave a dangling dash
|
|
38
|
+
.slice(0, MAX_KEY);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function newHistogram() {
|
|
42
|
+
return { n: 0, sum: 0, min: Infinity, max: -Infinity, buckets: new Array(LATENCY_BUCKETS.length).fill(0) };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function percentile(h, p) {
|
|
46
|
+
if (h.n === 0) return 0;
|
|
47
|
+
const target = h.n * p;
|
|
48
|
+
let acc = 0;
|
|
49
|
+
let lower = 0;
|
|
50
|
+
for (let i = 0; i < LATENCY_BUCKETS.length; i++) {
|
|
51
|
+
acc += h.buckets[i];
|
|
52
|
+
if (acc >= target) {
|
|
53
|
+
const upper = LATENCY_BUCKETS[i];
|
|
54
|
+
const inBucket = h.buckets[i];
|
|
55
|
+
const frac = inBucket === 0 ? 0 : (target - (acc - inBucket)) / inBucket;
|
|
56
|
+
return Math.round(lower + frac * (upper - lower));
|
|
57
|
+
}
|
|
58
|
+
lower = LATENCY_BUCKETS[i];
|
|
59
|
+
}
|
|
60
|
+
return h.max;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export class Observability {
|
|
64
|
+
constructor({ sinkPath = null } = {}) {
|
|
65
|
+
this.sinkPath = sinkPath;
|
|
66
|
+
this.counters = new Map();
|
|
67
|
+
this.gauges = new Map();
|
|
68
|
+
this.histograms = new Map();
|
|
69
|
+
this.startedAt = new Date().toISOString();
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/* ------------------------------------------------ write paths -------- */
|
|
73
|
+
|
|
74
|
+
inc(key, n = 1) {
|
|
75
|
+
const k = slugKey(key);
|
|
76
|
+
this.counters.set(k, (this.counters.get(k) ?? 0) + n);
|
|
77
|
+
return this.counters.get(k);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
setGauge(key, value) {
|
|
81
|
+
this.gauges.set(slugKey(key), value);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
observe(key, ms) {
|
|
85
|
+
const k = slugKey(key);
|
|
86
|
+
let h = this.histograms.get(k);
|
|
87
|
+
if (!h) { h = newHistogram(); this.histograms.set(k, h); }
|
|
88
|
+
const v = Number(ms);
|
|
89
|
+
if (!Number.isFinite(v)) return;
|
|
90
|
+
h.n += 1;
|
|
91
|
+
h.sum += v;
|
|
92
|
+
if (v < h.min) h.min = v;
|
|
93
|
+
if (v > h.max) h.max = v;
|
|
94
|
+
const idx = LATENCY_BUCKETS.findIndex((b) => v <= b);
|
|
95
|
+
h.buckets[idx < 0 ? LATENCY_BUCKETS.length - 1 : idx] += 1;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Append an event line to the sink (no-op without a sinkPath). */
|
|
99
|
+
journal({ event, ...fields }) {
|
|
100
|
+
if (!this.sinkPath) return;
|
|
101
|
+
mkdirSync(dirname(this.sinkPath), { recursive: true });
|
|
102
|
+
appendFileSync(this.sinkPath, JSON.stringify({ at: new Date().toISOString(), event, ...fields }) + "\n");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Append a full state snapshot line to the sink (no-op without a sinkPath). */
|
|
106
|
+
flush() {
|
|
107
|
+
if (!this.sinkPath) return;
|
|
108
|
+
mkdirSync(dirname(this.sinkPath), { recursive: true });
|
|
109
|
+
appendFileSync(this.sinkPath, JSON.stringify({ type: "snapshot", ...this.snapshot() }) + "\n");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/* ------------------------------------------------ read paths --------- */
|
|
113
|
+
|
|
114
|
+
snapshot() {
|
|
115
|
+
const histograms = {};
|
|
116
|
+
for (const [k, h] of this.histograms) {
|
|
117
|
+
histograms[k] = { n: h.n, sum: h.sum, min: h.min, max: h.max, p50: percentile(h, 0.5), p95: percentile(h, 0.95) };
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
at: new Date().toISOString(),
|
|
121
|
+
startedAt: this.startedAt,
|
|
122
|
+
counters: Object.fromEntries(this.counters),
|
|
123
|
+
gauges: Object.fromEntries(this.gauges),
|
|
124
|
+
histograms,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** One-glance human view for CLI / dashboards. */
|
|
129
|
+
summary() {
|
|
130
|
+
const s = this.snapshot();
|
|
131
|
+
const lines = [
|
|
132
|
+
`pillar metrics (since ${s.startedAt}) — ${this.counters.size} counters, ${this.gauges.size} gauges, ${this.histograms.size} histograms`,
|
|
133
|
+
];
|
|
134
|
+
for (const [k, v] of Object.entries(s.counters).sort()) lines.push(` ${k.padEnd(48)} ${v}`);
|
|
135
|
+
for (const [k, v] of Object.entries(s.gauges).sort()) lines.push(` ${k.padEnd(48)} = ${v}`);
|
|
136
|
+
for (const [k, h] of Object.entries(s.histograms).sort()) {
|
|
137
|
+
lines.push(` ${k.padEnd(48)} n=${h.n} min=${h.min}ms p50=${h.p50}ms p95=${h.p95}ms max=${h.max}ms`);
|
|
138
|
+
}
|
|
139
|
+
return lines.join("\n");
|
|
140
|
+
}
|
|
141
|
+
}
|
package/src/pillar.mjs
CHANGED
|
@@ -19,9 +19,14 @@ import { CID } from "multiformats/cid";
|
|
|
19
19
|
import * as raw from "multiformats/codecs/raw";
|
|
20
20
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
21
21
|
import { createTransport, PILLAR_PROTOCOL, uuaidToDhtKey, peerIdFromEd25519Hex } from "./net/transport.mjs";
|
|
22
|
-
import { seal, open, decrypt, envelopeSha, ENVELOPE_VERSION } from "./net/envelope.mjs";
|
|
22
|
+
import { seal, open, decrypt, envelopeSha, mkEnvelopeId, ENVELOPE_VERSION } from "./net/envelope.mjs";
|
|
23
23
|
import { Mailbox } from "./mailbox/mailbox.mjs";
|
|
24
24
|
import { CarrierClient } from "./net/carrier-client.mjs";
|
|
25
|
+
import { Observability } from "./observability.mjs";
|
|
26
|
+
import {
|
|
27
|
+
BLOB_KIND, MAX_INLINE_BODY_BYTES, BlobError, BlobSizeError, envelopeBodyBytes, preflightBodyBytes,
|
|
28
|
+
prepareBlobSend, fetchAndDecryptBlob, prepareBrokerBlobSend, fetchBrokerBlob,
|
|
29
|
+
} from "./blob/blob.mjs";
|
|
25
30
|
|
|
26
31
|
const VALUE_STORE_PREFIX = "/uuaid/1.0";
|
|
27
32
|
|
|
@@ -37,6 +42,13 @@ export class Pillar {
|
|
|
37
42
|
carriers = [],
|
|
38
43
|
enableLibp2p = true,
|
|
39
44
|
cryptoInventory = { ed25519: "active", "ml-dsa-65": "reserved", "slh-dsa-128s": "reserved" },
|
|
45
|
+
trust = null, // { policy, allow, block } from loadTrustPolicy(); null = open
|
|
46
|
+
tierGrant = null, // signed grant object (tier.mjs) sent with carrier delivers
|
|
47
|
+
metrics = null, // Observability instance; a default is created internally
|
|
48
|
+
blobStore = null, // BlobStore (src/blob/store.mjs) for the blob transport
|
|
49
|
+
blobBroker = null, // challenge broker for blob reserve/finalize/redeem
|
|
50
|
+
blobClient = null, // PillarBlobHttpClient for the production S3 broker path
|
|
51
|
+
senderKeyCheck = null, // registry gate for senders not derived from their key: (uuaid, pubkeyHex) => status
|
|
40
52
|
}) {
|
|
41
53
|
if (!keychain?._identity) throw new Error("Pillar: keychain must be loaded before construction");
|
|
42
54
|
this.keychain = keychain;
|
|
@@ -47,11 +59,19 @@ export class Pillar {
|
|
|
47
59
|
this.dhtClientMode = dhtClientMode;
|
|
48
60
|
this.dhtPeerInfoMapper = dhtPeerInfoMapper;
|
|
49
61
|
this.carriers = carriers;
|
|
50
|
-
this.carrierClient = carriers.length ? new CarrierClient({ keychain, carriers }) : null;
|
|
62
|
+
this.carrierClient = carriers.length ? new CarrierClient({ keychain, carriers, tierGrant }) : null;
|
|
51
63
|
this._carrierPoller = null;
|
|
52
64
|
this.enableLibp2p = enableLibp2p;
|
|
53
65
|
this.cryptoInventory = cryptoInventory;
|
|
54
66
|
this.node = null;
|
|
67
|
+
this.trust = trust;
|
|
68
|
+
this.metrics = metrics ?? new Observability();
|
|
69
|
+
this.blobStore = blobStore;
|
|
70
|
+
this.blobBroker = blobBroker;
|
|
71
|
+
this.blobClient = blobClient;
|
|
72
|
+
this.blobTier = tierGrant?.tier ?? "community";
|
|
73
|
+
this.blobMaxBodyBytes = Number.isInteger(tierGrant?.maxBodyBytes) ? tierGrant.maxBodyBytes : null;
|
|
74
|
+
this.senderKeyCheck = senderKeyCheck;
|
|
55
75
|
this._envelopeHandlers = new Set();
|
|
56
76
|
this._openStreams = new Map(); // peerIdString -> pushable
|
|
57
77
|
}
|
|
@@ -234,8 +254,12 @@ export class Pillar {
|
|
|
234
254
|
* fleet deployment we bypass DHT via the address book which is populated
|
|
235
255
|
* once at fleet-up time.
|
|
236
256
|
*/
|
|
237
|
-
async send({ recipient, recipientPublicKey, kind = "message", payload = {}, resolveTimeoutMs = 5000 }) {
|
|
238
|
-
if (!this.node && !this.carrierClient)
|
|
257
|
+
async send({ recipient, recipientPublicKey, kind = "message", payload = {}, envelopeId = null, resolveTimeoutMs = 5000 }) {
|
|
258
|
+
if (!this.node && !this.carrierClient) {
|
|
259
|
+
this.metrics.inc("send.error.not-started");
|
|
260
|
+
throw new Error("pillar not started");
|
|
261
|
+
}
|
|
262
|
+
const t0 = Date.now();
|
|
239
263
|
|
|
240
264
|
const book = this.addressBook?.get(recipient);
|
|
241
265
|
|
|
@@ -251,18 +275,30 @@ export class Pillar {
|
|
|
251
275
|
} catch (_e) { /* fall through to the error below */ }
|
|
252
276
|
}
|
|
253
277
|
if (!pubkey) {
|
|
278
|
+
this.metrics.inc("send.error.no-public-key");
|
|
254
279
|
throw new Error(`send: no public key known for ${recipient} — encryption is mandatory; register the peer (address book / fleet.json) or pass recipientPublicKey`);
|
|
255
280
|
}
|
|
256
281
|
|
|
257
|
-
const envelope = seal(this.keychain, { recipient, recipientPublicKey: pubkey, kind, payload });
|
|
282
|
+
const envelope = seal(this.keychain, { recipient, recipientPublicKey: pubkey, kind, payload, id: envelopeId ?? undefined });
|
|
283
|
+
const hardCap = preflightBodyBytes(envelopeBodyBytes(envelope), { maxBodyBytes: MAX_INLINE_BODY_BYTES });
|
|
284
|
+
if (!hardCap.ok) {
|
|
285
|
+
this.metrics.inc("send.error.direct-cap");
|
|
286
|
+
throw new BlobSizeError(`envelope ${hardCap.bytes}B exceeds the ${MAX_INLINE_BODY_BYTES}B hard cap`, { size: hardCap.bytes, max: MAX_INLINE_BODY_BYTES });
|
|
287
|
+
}
|
|
258
288
|
const persisted = this.mailbox.appendOutbound(envelope);
|
|
259
289
|
|
|
260
290
|
// 1. Address book path: direct libp2p dial (lowest latency, no third party).
|
|
261
291
|
if (this.node && book?.peerId && book?.multiaddr) {
|
|
262
292
|
try {
|
|
263
|
-
|
|
293
|
+
const delivery = await this._deliverToPeer(envelope, book.peerId, book.multiaddr);
|
|
294
|
+
this.metrics.inc("send.via.direct");
|
|
295
|
+
this.metrics.observe("send.latencyMs.via.direct", Date.now() - t0);
|
|
296
|
+
return { envelope, persisted, via: "direct", delivery };
|
|
264
297
|
} catch (e) {
|
|
265
|
-
if (!this.carrierClient)
|
|
298
|
+
if (!this.carrierClient) {
|
|
299
|
+
this.metrics.inc("send.error.direct-delivery-failed");
|
|
300
|
+
throw e;
|
|
301
|
+
} // else fall through to carrier
|
|
266
302
|
}
|
|
267
303
|
}
|
|
268
304
|
|
|
@@ -270,16 +306,27 @@ export class Pillar {
|
|
|
270
306
|
if (this.carrierClient) {
|
|
271
307
|
const receipt = await this.carrierClient.deliver(envelope);
|
|
272
308
|
this.mailbox.markOutboundDelivered(envelope.id, envelopeSha(envelope));
|
|
309
|
+
this.metrics.inc("send.via.carrier");
|
|
310
|
+
this.metrics.observe("send.latencyMs.via.carrier", Date.now() - t0);
|
|
273
311
|
return { envelope, persisted, via: "carrier", delivery: receipt };
|
|
274
312
|
}
|
|
275
313
|
|
|
276
314
|
// 3. DHT path.
|
|
277
|
-
if (!this.node)
|
|
315
|
+
if (!this.node) {
|
|
316
|
+
this.metrics.inc("send.error.no-route");
|
|
317
|
+
throw new Error(`send: cannot resolve ${recipient}: no carrier configured and libp2p not started`);
|
|
318
|
+
}
|
|
278
319
|
if (!dhtAdvert) {
|
|
279
320
|
try { dhtAdvert = await this.resolveUuaid(recipient, { timeoutMs: resolveTimeoutMs }); }
|
|
280
|
-
catch (e) {
|
|
321
|
+
catch (e) {
|
|
322
|
+
this.metrics.inc("send.error.resolve-failed");
|
|
323
|
+
throw new Error(`send: cannot resolve ${recipient}: ${e.message}`);
|
|
324
|
+
}
|
|
281
325
|
}
|
|
282
|
-
|
|
326
|
+
const delivery = await this._deliverToPeer(envelope, dhtAdvert.peerId, dhtAdvert.addrs?.[0]);
|
|
327
|
+
this.metrics.inc("send.via.dht");
|
|
328
|
+
this.metrics.observe("send.latencyMs.via.dht", Date.now() - t0);
|
|
329
|
+
return { envelope, persisted, via: "dht", delivery };
|
|
283
330
|
}
|
|
284
331
|
|
|
285
332
|
/**
|
|
@@ -292,26 +339,90 @@ export class Pillar {
|
|
|
292
339
|
* so every recipient can reply to the whole group without extra lookups.
|
|
293
340
|
*
|
|
294
341
|
* Failures are isolated: one undeliverable member never blocks the rest.
|
|
295
|
-
* Returns { group, results: [{ member, ok, via?, envelopeId?, error? }] }.
|
|
342
|
+
* Returns { group, results: [{ member, ok, via?, envelopeId?, error?, code?, fix? }] }.
|
|
296
343
|
*/
|
|
297
344
|
async sendGroup({ group, members, kind = "message", payload = {}, resolveTimeoutMs = 5000 }) {
|
|
298
345
|
if (!Array.isArray(members) || members.length === 0) {
|
|
299
346
|
throw new Error(`sendGroup: group "${group}" has no members`);
|
|
300
347
|
}
|
|
301
348
|
const others = members.filter(m => m !== this.uuaid);
|
|
302
|
-
|
|
349
|
+
// _group goes last so a payload can never override the group it travels in.
|
|
350
|
+
const wrapped = { ...payload, _group: { name: group, members } };
|
|
303
351
|
const results = [];
|
|
304
352
|
for (const member of others) {
|
|
305
353
|
try {
|
|
306
354
|
const r = await this.send({ recipient: member, kind, payload: wrapped, resolveTimeoutMs });
|
|
307
355
|
results.push({ member, ok: true, via: r.via, envelopeId: r.envelope.id });
|
|
308
356
|
} catch (e) {
|
|
309
|
-
results.push({ member, ok: false, error: e.message });
|
|
357
|
+
results.push({ member, ok: false, error: e.message, ...(typeof e.code === "string" ? { code: e.code } : {}), ...(typeof e.fix === "string" ? { fix: e.fix } : {}) });
|
|
310
358
|
}
|
|
311
359
|
}
|
|
360
|
+
this.metrics.inc("sendGroup");
|
|
361
|
+
const failed = results.filter((r) => !r.ok).length;
|
|
362
|
+
if (failed) this.metrics.inc("sendGroup.failed", failed);
|
|
312
363
|
return { group, results };
|
|
313
364
|
}
|
|
314
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Blob send — the oversized-payload path. Explicit flow (blob.mjs header):
|
|
368
|
+
* broker reserve -> local AES-256-GCM encrypt (fresh DEK + nonce, AAD bound
|
|
369
|
+
* to this envelope) -> store upload -> broker finalize -> seal a kind
|
|
370
|
+
* "blob-ref" envelope carrying the signed ref (DEK inside the sealed
|
|
371
|
+
* payload, never in metadata) -> deliver.
|
|
372
|
+
*
|
|
373
|
+
* Preflight is exact: the placeholder-free body is measured against the
|
|
374
|
+
* tier inline limit and the 2 MiB direct hard cap BEFORE any store or
|
|
375
|
+
* broker call. The envelope id is generated first so the GCM AAD can bind
|
|
376
|
+
* the ciphertext to it.
|
|
377
|
+
*/
|
|
378
|
+
async sendBlob({ recipient, recipientPublicKey, kind = BLOB_KIND, data, ttlMs, resolveTimeoutMs = 5000 }) {
|
|
379
|
+
if (!this.blobClient && !this.blobStore) throw new BlobError("no-store", "sendBlob: configure blobClient or blobStore on the Pillar");
|
|
380
|
+
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
381
|
+
// Reserve an envelope id so the blob ciphertext is AAD-bound to it.
|
|
382
|
+
const envelopeId = mkEnvelopeId();
|
|
383
|
+
const placeholder = { blobRef: { v: 1, envelopeId } };
|
|
384
|
+
// Seal a disposable probe before any broker/S3 side effect. Besides sizing,
|
|
385
|
+
// this enforces recipient UUAID ↔ public-key binding up front.
|
|
386
|
+
const probe = seal(this.keychain, { recipient, recipientPublicKey, kind, payload: placeholder, id: envelopeId });
|
|
387
|
+
const pf = preflightBodyBytes(envelopeBodyBytes(probe), { tier: this.blobTier, maxBodyBytes: this.blobMaxBodyBytes });
|
|
388
|
+
if (!pf.ok) {
|
|
389
|
+
this.metrics.inc(`sendBlob.preflight.${pf.source}`);
|
|
390
|
+
throw new BlobError("blob-too-large", `sendBlob: placeholder body ${pf.bytes}B exceeds ${pf.source} limit ${pf.limit}B`);
|
|
391
|
+
}
|
|
392
|
+
const prepared = this.blobClient
|
|
393
|
+
? await prepareBrokerBlobSend(this.keychain, { data: buf, envelopeId, recipient, ttlMs, client: this.blobClient })
|
|
394
|
+
: await prepareBlobSend(this.keychain, {
|
|
395
|
+
data: buf, envelopeId, recipient, ttlMs,
|
|
396
|
+
store: this.blobStore, broker: this.blobBroker,
|
|
397
|
+
});
|
|
398
|
+
const r = await this.send({
|
|
399
|
+
recipient, recipientPublicKey, kind,
|
|
400
|
+
payload: { blobRef: prepared.ref }, envelopeId, resolveTimeoutMs,
|
|
401
|
+
});
|
|
402
|
+
this.metrics.inc("sendBlob.ok");
|
|
403
|
+
return { ...r, blob: { object: prepared.object, hash: prepared.hash, size: prepared.size, metadata: prepared.metadata } };
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* Redeem a blob-ref from an inbound payload — fetch + hash-check +
|
|
408
|
+
* single-use broker redemption + AES-256-GCM decrypt with the ref's DEK.
|
|
409
|
+
* The sender key gate (derived from its key, else a registry badge) runs
|
|
410
|
+
* BEFORE the broker redemption burns. Returns { data, object, hash, size }.
|
|
411
|
+
*/
|
|
412
|
+
async receiveBlob(payload, { envelopeId = null, now = Date.now() } = {}) {
|
|
413
|
+
const ref = payload?.blobRef;
|
|
414
|
+
if (!ref) throw new BlobError("blob-ref-invalid", "receiveBlob: payload carries no blobRef");
|
|
415
|
+
if (!this.blobClient && !this.blobStore) throw new BlobError("no-store", "receiveBlob: configure blobClient or blobStore on the Pillar");
|
|
416
|
+
const r = this.blobClient
|
|
417
|
+
? await fetchBrokerBlob({ ref, envelopeId, recipient: this.uuaid, now, client: this.blobClient })
|
|
418
|
+
: await fetchAndDecryptBlob({
|
|
419
|
+
ref, store: this.blobStore, envelopeId, now,
|
|
420
|
+
broker: this.blobBroker, senderKeyCheck: this.senderKeyCheck,
|
|
421
|
+
});
|
|
422
|
+
this.metrics.inc("receiveBlob.ok");
|
|
423
|
+
return r;
|
|
424
|
+
}
|
|
425
|
+
|
|
315
426
|
/**
|
|
316
427
|
* Start pulling our inbox from every configured carrier. Received envelopes
|
|
317
428
|
* flow through the same verification + mailbox + handler pipeline as
|
|
@@ -328,8 +439,11 @@ export class Pillar {
|
|
|
328
439
|
onEnvelope: async (envelope, { carrier, seq }) => {
|
|
329
440
|
await this._receive(JSON.stringify(envelope), null);
|
|
330
441
|
this.mailbox.setCarrierCursor?.(carrier, seq);
|
|
442
|
+
this.metrics.inc(`carrier.poll.${carrier}`);
|
|
443
|
+
},
|
|
444
|
+
onError: (_e, meta) => {
|
|
445
|
+
this.metrics.inc(`carrier.poll.error.${meta?.carrier ?? "unknown"}`);
|
|
331
446
|
},
|
|
332
|
-
onError: (_e) => { /* transient poll errors are expected; backoff is in client */ },
|
|
333
447
|
});
|
|
334
448
|
return this._carrierPoller;
|
|
335
449
|
}
|
|
@@ -338,7 +452,38 @@ export class Pillar {
|
|
|
338
452
|
if (this._carrierPoller) { this._carrierPoller.stop(); this._carrierPoller = null; }
|
|
339
453
|
}
|
|
340
454
|
|
|
455
|
+
/**
|
|
456
|
+
* One-shot inbox pull: fetch every pending envelope from every carrier
|
|
457
|
+
* (no long-poll) through the same verification + mailbox + handler
|
|
458
|
+
* pipeline, then return. Complements startCarrierPolling for CLI /
|
|
459
|
+
* cron-style "check my mail now" callers. Returns envelopes fetched.
|
|
460
|
+
*/
|
|
461
|
+
async drainCarrierInbox() {
|
|
462
|
+
if (!this.carrierClient) return 0;
|
|
463
|
+
let total = 0;
|
|
464
|
+
const cursors = this.mailbox.getCarrierCursors?.() ?? {};
|
|
465
|
+
for (const base of this.carriers) {
|
|
466
|
+
let since = cursors[base] ?? 0;
|
|
467
|
+
for (;;) { // carrier pages at 200/fetch — loop until an empty page
|
|
468
|
+
const r = await this.carrierClient.fetchInbox(base, { since, waitS: 0 });
|
|
469
|
+
if (!r.envelopes.length) break;
|
|
470
|
+
for (const { seq, envelope } of r.envelopes) {
|
|
471
|
+
await this._receive(JSON.stringify(envelope), null);
|
|
472
|
+
this.mailbox.setCarrierCursor?.(base, seq);
|
|
473
|
+
since = Math.max(since, seq);
|
|
474
|
+
total++;
|
|
475
|
+
this.metrics.inc(`carrier.drain.${base}`);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return total;
|
|
480
|
+
}
|
|
481
|
+
|
|
341
482
|
async _deliverToPeer(envelope, peerIdStr, multiaddrStr) {
|
|
483
|
+
const bytes = envelopeBodyBytes(envelope);
|
|
484
|
+
if (bytes > MAX_INLINE_BODY_BYTES) {
|
|
485
|
+
throw new BlobSizeError(`direct envelope ${bytes}B exceeds the ${MAX_INLINE_BODY_BYTES}B hard cap`, { size: bytes, max: MAX_INLINE_BODY_BYTES });
|
|
486
|
+
}
|
|
342
487
|
const { peerIdFromString } = await import("@libp2p/peer-id");
|
|
343
488
|
const { multiaddr } = await import("@multiformats/multiaddr");
|
|
344
489
|
const peerId = peerIdFromString(peerIdStr);
|
|
@@ -383,6 +528,10 @@ export class Pillar {
|
|
|
383
528
|
const pubkey = recipientPublicKey ?? this.addressBook?.get(recipient)?.publicKey;
|
|
384
529
|
if (!pubkey) throw new Error(`sendDirect: no public key for ${recipient} — encryption is mandatory`);
|
|
385
530
|
const envelope = seal(this.keychain, { recipient, recipientPublicKey: pubkey, kind, payload });
|
|
531
|
+
const bytes = envelopeBodyBytes(envelope);
|
|
532
|
+
if (bytes > MAX_INLINE_BODY_BYTES) {
|
|
533
|
+
throw new BlobSizeError(`direct envelope ${bytes}B exceeds the ${MAX_INLINE_BODY_BYTES}B hard cap`, { size: bytes, max: MAX_INLINE_BODY_BYTES });
|
|
534
|
+
}
|
|
386
535
|
this.mailbox.appendOutbound(envelope);
|
|
387
536
|
const { peerIdFromString } = await import("@libp2p/peer-id");
|
|
388
537
|
const { multiaddr } = await import("@multiformats/multiaddr");
|
|
@@ -404,8 +553,14 @@ export class Pillar {
|
|
|
404
553
|
async _handleStream(stream, connection) {
|
|
405
554
|
try {
|
|
406
555
|
let buf = "";
|
|
556
|
+
let receivedBytes = 0;
|
|
407
557
|
for await (const chunk of stream) {
|
|
408
558
|
const bytes = chunk?.subarray ? chunk.subarray() : chunk;
|
|
559
|
+
receivedBytes += bytes.byteLength ?? bytes.length;
|
|
560
|
+
if (receivedBytes > MAX_INLINE_BODY_BYTES + 1) {
|
|
561
|
+
this.metrics.inc("received.rejected.direct-cap");
|
|
562
|
+
throw new BlobSizeError(`incoming direct envelope exceeds the ${MAX_INLINE_BODY_BYTES}B hard cap`, { size: receivedBytes, max: MAX_INLINE_BODY_BYTES });
|
|
563
|
+
}
|
|
409
564
|
buf += u8ToString(bytes);
|
|
410
565
|
const newlineIdx = buf.indexOf("\n");
|
|
411
566
|
if (newlineIdx < 0) continue;
|
|
@@ -425,15 +580,20 @@ export class Pillar {
|
|
|
425
580
|
async _receive(rawLine, connection) {
|
|
426
581
|
let envelope;
|
|
427
582
|
try { envelope = JSON.parse(rawLine); }
|
|
428
|
-
catch (_e) {
|
|
583
|
+
catch (_e) {
|
|
584
|
+
this.metrics.inc("received.unparsable");
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
429
587
|
const verdict = open(envelope, { cryptoInventory: this.cryptoInventory });
|
|
430
588
|
if (!verdict.ok) {
|
|
589
|
+
this.metrics.inc(`received.rejected.${verdict.reason}`);
|
|
431
590
|
// eslint-disable-next-line no-console
|
|
432
591
|
console.warn("pillar: rejected envelope from", connection?.remotePeer?.toString(), verdict.reason);
|
|
433
592
|
return;
|
|
434
593
|
}
|
|
435
594
|
if (envelope.recipient !== this.uuaid) {
|
|
436
595
|
// Not for us. Store-and-forward would go here in v0.3.
|
|
596
|
+
this.metrics.inc("received.wrong-recipient");
|
|
437
597
|
return;
|
|
438
598
|
}
|
|
439
599
|
// E2E: decrypt the sealed payload. Ciphertext-at-rest: the mailbox stores
|
|
@@ -442,16 +602,45 @@ export class Pillar {
|
|
|
442
602
|
try {
|
|
443
603
|
payload = decrypt(this.keychain, envelope);
|
|
444
604
|
} catch (e) {
|
|
605
|
+
this.metrics.inc("received.decrypt-failed");
|
|
445
606
|
// eslint-disable-next-line no-console
|
|
446
607
|
console.warn("pillar: envelope decrypt failed (dropping)", envelope.id, e.message);
|
|
447
608
|
return;
|
|
448
609
|
}
|
|
449
610
|
const persisted = this.mailbox.appendInbound(envelope);
|
|
450
|
-
if (persisted.duplicate)
|
|
611
|
+
if (persisted.duplicate) {
|
|
612
|
+
this.metrics.inc("received.duplicate");
|
|
613
|
+
return; // replay: do not re-fire handlers
|
|
614
|
+
}
|
|
615
|
+
this.metrics.inc(`received.kind.${envelope.kind}`);
|
|
616
|
+
// Trust gate (recipient consent): quarantine-not-drop. The envelope is
|
|
617
|
+
// already persisted (chain-hashed evidence); a gated sender just never
|
|
618
|
+
// reaches handlers. An ack/reply passes the gate ONLY if its inReplyTo
|
|
619
|
+
// names an envelope in OUR outbox — a genuine receipt for our own send.
|
|
620
|
+
// (kind alone is spoofable; the outbox check is not.)
|
|
621
|
+
if (this.trust) {
|
|
622
|
+
const isReceipt = (envelope.kind === "ack" || envelope.kind === "reply") &&
|
|
623
|
+
typeof payload?.inReplyTo === "string" && this.mailbox.hasOutbound(payload.inReplyTo);
|
|
624
|
+
if (!isReceipt) {
|
|
625
|
+
const { evaluateTrust } = await import("./identity/trust.mjs");
|
|
626
|
+
const known = new Set(this.addressBook?.keys() ?? []);
|
|
627
|
+
const verdict = evaluateTrust(this.trust, envelope.sender, known);
|
|
628
|
+
if (!verdict.deliver) {
|
|
629
|
+
this.mailbox.markQuarantined(envelope.id, verdict.reason);
|
|
630
|
+
this.metrics.inc(`received.quarantined.${verdict.reason}`);
|
|
631
|
+
// eslint-disable-next-line no-console
|
|
632
|
+
console.warn(`pillar: quarantined ${envelope.id} from ${envelope.sender} (${verdict.reason})`);
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
451
637
|
const view = { ...envelope, payload };
|
|
452
638
|
for (const h of this._envelopeHandlers) {
|
|
453
639
|
try { await h(view, persisted); }
|
|
454
|
-
catch (e) {
|
|
640
|
+
catch (e) {
|
|
641
|
+
this.metrics.inc("received.handler-error");
|
|
642
|
+
console.error("pillar: envelope handler threw", e);
|
|
643
|
+
}
|
|
455
644
|
}
|
|
456
645
|
}
|
|
457
646
|
}
|