@uuaid/pillar 2.0.2 → 2.0.3

package/HOSTING-LIMITS.md

@@ -310,6 +310,21 @@ front of each one.
   clients retry after 2 s). Stored envelopes stay in SQLite. Leave disk for a
   256 MiB store plus its WAL file.
 
+**Secure delete (from 2.0.3).** Expired envelopes are zeroed on disk by
+default. `vacuum()` runs with SQLite `secure_delete` on and follows each
+expiry pass that deleted anything with `wal_checkpoint(TRUNCATE)`, so neither
+the database file nor the WAL keeps a deleted envelope's bytes. Turn it off
+per broker with `--secure-delete off` (`CARRIER_SECURE_DELETE=off` for
+`pillar-carrier`, `SEED_SECURE_DELETE=off` for `pillar-seed`); any other value
+than on/off refuses to start. It is one setting for the whole store: SQLite
+sets it per connection, so it cannot differ by tenant. Two things it does not
+do:
+- Bytes deleted before you upgraded stay in free pages. Run
+  `sqlite3 <db> 'VACUUM;'` once, with the carrier stopped, to clear them.
+- A checkpoint that stays busy (another process holding the database open)
+  is logged as `secure-delete: WAL checkpoint still busy` and retried at the
+  next hourly pass; until then the WAL can still hold those bytes.
+
 **Relay host.** Back up `PUSH_RELAY_SEED_PATH`. Every poll token names the
 relay's key, so losing that file invalidates all of them
 (apps/pillar-push/src/config.ts:19-22). Watch `fcmKey.daysLeft` on
@@ -343,6 +358,7 @@ or an exception message. A parameterised reason such as
 | `keyId-sender-mismatch` | 400 | Sign with `transportSignature.keyId` equal to the sender UUAID, or send under a delegation |
 | `publicKey-sender-mismatch` | 400 | Send from the UUAID your signing key derives, or attach the principal's delegation |
 | `bad-signature` | 400, 401 | Sign again with this identity's own key over the exact bytes; the signature does not verify |
+| `weak-publicKey` (from 2.0.3) | 400 | The sender key is a small-order or non-canonical Ed25519 point, which can "sign" anything; use a real identity key |
 | `not-object`, `bad-version <v>`, `missing-or-bad-alg`, `alg-not-active:<alg>`, `bad-publicKey`, `bad-signature-encoding`, `missing-enc`, `enc-alg-not-supported:<alg>`, `enc-alg-not-active:<alg>`, `bad-epk`, `bad-iv`, `bad-ct`, `bad-canonical`, `verify-threw` | 400 (`verify-threw` also 401) | Re-seal the envelope with `seal()` from `@uuaid/pillar` or `@uuaid/pillar-client`; this one does not verify |
 | `delegation-invalid:<why>` | 403 (send), 401 (inbox) | Mint a fresh delegation from the principal, then resend |
 | `missing-auth-headers` | 401 | Send `x-pillar-pubkey`, `x-pillar-ts` and `x-pillar-sig` |

package/bin/carrier.mjs

@@ -3,6 +3,7 @@
  * carrier — run a standalone carrier server (the 24/7 post office).
  *
  *   carrier --port 8799 --host 127.0.0.1 --db /opt/pillar-carrier/carrier.db
+ *   carrier ... --secure-delete off     # keep deleted envelope bytes (default: on)
  *
  * Defaults to loopback (RedTeam F5) — expose only behind a TLS-terminating
  * proxy (nginx/Caddy).  Pass --host 0.0.0.0 deliberately only if the carrier
@@ -11,7 +12,7 @@
  * about who exists — pure store-and-forward.
  */
 import "../src/node-check.mjs"; // must stay the first import: refuses Node < 22 before any dependency loads
-import { CarrierServer } from "../src/net/carrier-server.mjs";
+import { CarrierServer, parseSecureDelete } from "../src/net/carrier-server.mjs";
 
 function parseArgs(argv) {
   const args = {};
@@ -35,17 +36,29 @@ const ttlDays = Number(args.ttl ?? process.env.CARRIER_TTL_DAYS ?? 14);
 const tierIssuers = String(args["tier-issuers"] ?? process.env.CARRIER_TIER_ISSUERS ?? "")
   .split(",").map(s => s.trim()).filter(s => /^[0-9a-f]{64}$/i.test(s));
 
-const server = new CarrierServer({ dbPath, ttlDays, trustedTierIssuers: tierIssuers });
+// Secure delete of expired envelopes: on unless this broker turns it off.
+let secureDelete;
+try {
+  secureDelete = parseSecureDelete(args["secure-delete"] ?? process.env.CARRIER_SECURE_DELETE);
+} catch (e) {
+  console.error(`carrier: ${e.message}`);
+  process.exit(2);
+}
+
+const server = new CarrierServer({ dbPath, ttlDays, trustedTierIssuers: tierIssuers, secureDelete });
 const addr = await server.listen(port, host);
 console.log(`carrier listening on ${addr.address}:${addr.port}`);
 console.log(`  db:  ${dbPath}`);
 console.log(`  ttl: ${ttlDays} days`);
 console.log(`  tiers: ${tierIssuers.length ? `${tierIssuers.length} trusted issuer(s)` : "disabled (community budgets for all)"}`);
+console.log(`  secure-delete: ${secureDelete ? "on (expired envelopes are zeroed; WAL truncated after each expiry pass)" : "off (expired envelope bytes stay on disk until reused)"}`);
 
 // Vacuum expired envelopes hourly.
 const vacuumTimer = setInterval(() => {
   const n = server.vacuum();
   if (n > 0) console.log(`vacuumed ${n} expired envelopes`);
+  const v = server.lastVacuum;
+  if (v?.scrubbed === false) console.error(`secure-delete: WAL checkpoint still busy after ${v.attempts} tries (busy=${v.checkpoint?.busy}); deleted bytes may remain in the WAL until the next pass`);
 }, 3600_000);
 
 const shutdown = async () => {

package/bin/pillar-seed.mjs

@@ -43,7 +43,7 @@ import "../src/node-check.mjs"; // must stay the first import: refuses Node < 22
 import { mkdirSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { CarrierServer } from "../src/net/carrier-server.mjs";
+import { CarrierServer, parseSecureDelete } from "../src/net/carrier-server.mjs";
 import { Keychain } from "../src/identity/keychain.mjs";
 
 function parseArgs(argv) {
@@ -68,6 +68,14 @@ const publicUrl = args["public-url"] ?? process.env.SEED_PUBLIC_URL ?? null;
 const registryUrl = typeof args.announce === "string" ? args.announce.replace(/\/+$/, "") : null;
 const operator = args.operator ?? process.env.SEED_OPERATOR ?? null;
 const region = args.region ?? process.env.SEED_REGION ?? null;
+// Secure delete of expired envelopes: on unless this seed turns it off.
+let secureDelete;
+try {
+  secureDelete = parseSecureDelete(args["secure-delete"] ?? process.env.SEED_SECURE_DELETE);
+} catch (e) {
+  console.error(`pillar-seed: ${e.message}`);
+  process.exit(2);
+}
 
 /**
  * The seed's identity. Distinct from any agent keychain on the box: it names
@@ -114,12 +122,13 @@ if (args.status) {
   process.exit(0);
 }
 
-const server = new CarrierServer({ dbPath, ttlDays, keychain, publicUrl });
+const server = new CarrierServer({ dbPath, ttlDays, keychain, publicUrl, secureDelete });
 const addr = await server.listen(port, host);
 console.log(`pillar-seed listening on http://${addr.address}:${addr.port}`);
 console.log(`  uuaid: ${keychain._identity.uuaid}`);
 console.log(`  db:  ${dbPath}`);
 console.log(`  ttl: ${ttlDays} days`);
+console.log(`  secure-delete: ${secureDelete ? "on (expired envelopes are zeroed; WAL truncated after each expiry pass)" : "off (expired envelope bytes stay on disk until reused)"}`);
 if (publicUrl) console.log(`  public url: ${publicUrl}`);
 
 // Boot self-check on the address we actually serve — verify before claim.
@@ -171,6 +180,8 @@ if (registryUrl) {
 const vacuumTimer = setInterval(() => {
   const n = server.vacuum();
   if (n > 0) console.log(`vacuumed ${n} expired envelopes`);
+  const v = server.lastVacuum;
+  if (v?.scrubbed === false) console.error(`secure-delete: WAL checkpoint still busy after ${v.attempts} tries (busy=${v.checkpoint?.busy}); deleted bytes may remain in the WAL until the next pass`);
 }, 3600_000);
 
 const shutdown = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uuaid/pillar",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "Pillar (Protocol for Inter-agent Linked Ledger And Relay) — the secure delivery layer of UACP, the Universal Agent Communication Protocol. UACP = UUAID (identity) + Envelope (signed, encrypted container) + Pillar (relay, mailbox, routing, store-and-forward). Reference implementation of the UACP delivery layer; includes VDA-1 (IAASO-3901, in ballot) delivery attestation and a blind escrow for the Zilligon ZGN / EVM rails.",
   "license": "Apache-2.0",
   "type": "module",

package/src/identity/jcs.mjs

@@ -24,11 +24,10 @@ function _ser(v) {
   if (typeof v === "bigint") return v.toString();
   if (Array.isArray(v)) return "[" + v.map(_ser).join(",") + "]";
   if (typeof v === "object") {
-    // UTF-16 code unit sort — the RFC 8785 requirement.
-    const keys = Object.keys(v).sort((a, b) => {
-      const ab = Buffer.from(a, "utf16le"), bb = Buffer.from(b, "utf16le");
-      return Buffer.compare(ab, bb);
-    });
+    // RFC 8785 §3.2.3: keys sort by their UTF-16 code units, which is exactly
+    // JavaScript's default string order. (Comparing UTF-16LE *bytes* is not: it
+    // puts "Ā" (U+0100) before "a", and disagreed with @uuaid/pillar-client.)
+    const keys = Object.keys(v).sort(); // @rule jcs.code-unit-order
     return "{" + keys.map(k => JSON.stringify(k) + ":" + _ser(v[k])).join(",") + "}";
   }
   throw new Error("JCS: unsupported type " + typeof v);

package/src/identity/keychain.mjs

@@ -12,9 +12,43 @@
 import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs";
 import { dirname } from "node:path";
 import { randomBytes, createHash, generateKeyPairSync, sign, verify, createPublicKey, createPrivateKey, hkdfSync, createCipheriv, createDecipheriv } from "node:crypto";
+import { ed25519 } from "@noble/curves/ed25519.js";
 
 const KEYCHAIN_VERSION = 1;
 
+/**
+ * RFC 8032 / FIPS 186-5 rule for an Ed25519 public key: the canonical encoding
+ * of a point that is not of small order and lies in the prime-order subgroup.
+ *
+ * OpenSSL's Ed25519 verify (node `crypto.verify`) does not check this: it uses
+ * the non-cofactored equation, and 5 of the 8 small-order points verify as
+ * keys. With the identity point as the key, R = identity and S = 0 verifies on
+ * every message, so anyone could forge a signature "from" that key's uuaid
+ * (found 2026-10-06). The same rule as `isStrictEd25519PublicKey` in
+ * @uuaid/pillar-client.
+ *
+ * The answer depends on the key alone, so it is cached (bounded).
+ */
+const STRICT_KEY_CACHE_MAX = 4096;
+const strictKeyCache = new Map();
+
+export function isStrictEd25519PublicKey(publicKeyHex) {
+  if (typeof publicKeyHex !== "string" || !/^[0-9a-f]{64}$/i.test(publicKeyHex)) return false;
+  const hex = publicKeyHex.toLowerCase();
+  const cached = strictKeyCache.get(hex);
+  if (cached !== undefined) return cached;
+  let ok = false;
+  try {
+    const point = ed25519.Point.fromHex(hex, false);
+    ok = point.toHex() === hex && !point.isSmallOrder() && point.isTorsionFree();
+  } catch (_e) {
+    ok = false;
+  }
+  if (strictKeyCache.size >= STRICT_KEY_CACHE_MAX) strictKeyCache.delete(strictKeyCache.keys().next().value);
+  strictKeyCache.set(hex, ok);
+  return ok;
+}
+
 /**
  * Derive a stable, publishable local id from a raw Ed25519 public key.  This
  * makes the UUAID self-authenticating: given the UUAID, anyone can
@@ -149,6 +183,7 @@ export class Keychain {
   }
 
   static verifyDetached(publicKeyHex, data, signatureBytes) {
+    if (!isStrictEd25519PublicKey(publicKeyHex)) return false; // @rule keychain.strict-key
     const spkiPrefix = Buffer.from("302a300506032b6570032100", "hex");
     const der = Buffer.concat([spkiPrefix, Buffer.from(publicKeyHex, "hex")]);
     const pub = createPublicKey({ key: der, format: "der", type: "spki" });

package/src/net/carrier-server.mjs

@@ -81,11 +81,39 @@ const RATE_MAP_HARD_CAP = 10_000;          // F2: bound memory under key spray
 const COMMUNITY_DB_BYTES = Math.floor(MAX_DB_BYTES * 0.85);   // last 15% of the store is reserved
 const COMMUNITY_CONCURRENT_WAITS = 48;                        // of MAX_CONCURRENT_WAITS
 const MAX_PRIORITY_GOOD_PER_MIN = 600;                        // priority senders bill to their own per-IP bucket
+const CHECKPOINT_ATTEMPTS = 3;                                // TRUNCATE checkpoint tries per vacuum pass
+const CHECKPOINT_BUSY_MS = 250;                               // per try; a busy pass is retried next hour
+
+/**
+ * Secure delete for the envelope store: one switch per broker (--secure-delete
+ * on|off, CARRIER_SECURE_DELETE / SEED_SECURE_DELETE). ON by default.
+ *
+ * A plain DELETE leaves an expired envelope's bytes readable in the database
+ * file, and, while the database is open, in the WAL (PC-4 delete-residue probe,
+ * 2026-10-08). ON sets PRAGMA secure_delete, which zeroes deleted content, and
+ * vacuum() follows every expiry delete with wal_checkpoint(TRUNCATE), which
+ * empties the WAL of the frames that still hold it. OFF keeps SQLite's default:
+ * deleted bytes stay until the page is reused. secure_delete is per connection,
+ * so this is a per-broker setting, never a per-tenant one.
+ *
+ * Returns true or false; throws on anything that is not a recognised spelling,
+ * so a typo never silently turns the scrub off.
+ */
+export function parseSecureDelete(value) {
+  if (value === undefined || value === null || value === true) return true;
+  const v = String(value).trim().toLowerCase();
+  if (["on", "1", "true", "yes"].includes(v)) return true;
+  if (["off", "0", "false", "no"].includes(v)) return false;
+  throw new Error(`secure-delete must be on or off, not "${value}"`);
+}
 
 export class CarrierServer {
-  constructor({ dbPath = "carrier.db", ttlDays = 14, cryptoInventory, trustedTierIssuers = [], keychain = null, publicUrl = null } = {}) {
+  constructor({ dbPath = "carrier.db", ttlDays = 14, cryptoInventory, trustedTierIssuers = [], keychain = null, publicUrl = null, secureDelete = true } = {}) {
     this.db = new Database(dbPath);
     this.db.pragma("journal_mode = WAL");
+    this.secureDelete = secureDelete !== false;
+    this.db.pragma(`secure_delete = ${this.secureDelete ? "ON" : "OFF"}`);
+    this.lastVacuum = null;
     this.db.exec(`
       create table if not exists envelopes (
         seq         integer primary key autoincrement,
@@ -235,10 +263,35 @@ export class CarrierServer {
     return true;
   }
 
-  /** Remove envelopes older than TTL.  Call periodically. */
+  /**
+   * Remove envelopes older than TTL.  Call periodically.  Returns the number
+   * deleted.  With secure delete on, a pass that deleted anything then runs
+   * wal_checkpoint(TRUNCATE) until it reports busy=0 (at most
+   * CHECKPOINT_ATTEMPTS tries); `lastVacuum` records the outcome, and
+   * `lastVacuum.scrubbed` is false when the checkpoint never completed.
+   */
   vacuum() {
     const cutoff = new Date(Date.now() - this.ttlDays * 86400_000).toISOString();
     const r = this.db.prepare("delete from envelopes where created_at < ?").run(cutoff);
+    const report = { deleted: r.changes, secureDelete: this.secureDelete, checkpoint: null, attempts: 0, scrubbed: null };
+    if (this.secureDelete && r.changes > 0) {
+      // vacuum() runs on the event loop: wait at most CHECKPOINT_BUSY_MS per
+      // try for a reader to finish, never the connection's 5 s default.
+      const busyTimeout = this.db.pragma("busy_timeout", { simple: true });
+      this.db.pragma(`busy_timeout = ${CHECKPOINT_BUSY_MS}`);
+      try {
+        while (report.attempts < CHECKPOINT_ATTEMPTS) {
+          report.attempts++;
+          const [cp] = this.db.pragma("wal_checkpoint(TRUNCATE)");
+          report.checkpoint = cp;
+          if (cp && cp.busy === 0) break;
+        }
+      } finally {
+        this.db.pragma(`busy_timeout = ${busyTimeout}`);
+      }
+      report.scrubbed = report.checkpoint?.busy === 0;
+    }
+    this.lastVacuum = report;
     return r.changes;
   }
 

package/src/net/envelope.mjs

@@ -28,7 +28,7 @@
  */
 import { createHash, randomBytes } from "node:crypto";
 import { jcs } from "../identity/jcs.mjs";
-import { Keychain } from "../identity/keychain.mjs";
+import { Keychain, isStrictEd25519PublicKey } from "../identity/keychain.mjs";
 import { sealPayload, openPayload, E2E_ALG } from "../crypto/e2e.mjs";
 import { verifyDelegationDoc, delegationCoversKind, CAPABILITY_ONLY_KINDS } from "./delegation.mjs";
 
@@ -112,6 +112,7 @@ export function open(envelope, { cryptoInventory } = {}) {
   }
   if (sig.keyId !== envelope.sender) return { ok: false, reason: "keyId-sender-mismatch" };
   if (typeof sig.publicKey !== "string" || !/^[0-9a-f]{64}$/i.test(sig.publicKey)) return { ok: false, reason: "bad-publicKey" };
+  if (!isStrictEd25519PublicKey(sig.publicKey)) return { ok: false, reason: "weak-publicKey" }; // @rule envelope.open-strict-key
   if (typeof sig.signature !== "string" || !/^[0-9a-f]{128}$/i.test(sig.signature)) return { ok: false, reason: "bad-signature-encoding" };
   // Structural checks on the encryption block — an envelope without one is invalid.
   const enc = envelope.enc;
@@ -172,6 +173,7 @@ function openDelegated(envelope, { cryptoInventory } = {}) {
   const reject = (reason) => ({ ok: false, reason: `delegation-invalid:${reason}`, code: 403 });
   const sig = envelope.transportSignature;
   if (typeof sig.publicKey !== "string" || !/^[0-9a-f]{64}$/i.test(sig.publicKey)) return { ok: false, reason: "bad-publicKey" };
+  if (!isStrictEd25519PublicKey(sig.publicKey)) return { ok: false, reason: "weak-publicKey" }; // @rule envelope.open-delegated-strict-key
   if (typeof sig.signature !== "string" || !/^[0-9a-f]{128}$/i.test(sig.signature)) return { ok: false, reason: "bad-signature-encoding" };
   // Structural checks on the encryption block — same rules as the legacy path.
   const enc = envelope.enc;

package/src/net/refusals.mjs

@@ -32,6 +32,7 @@ export const REFUSAL_FIXES = Object.freeze({
   "keyId-sender-mismatch": "sign with transportSignature.keyId equal to the sender UUAID, or send under a delegation",
   "publicKey-sender-mismatch": "send from the UUAID your signing key derives, or attach the principal's delegation",
   "bad-signature": "sign again with this identity's own key over the exact bytes; the signature does not verify",
+  "weak-publicKey": "use a real Ed25519 identity key; this key is a small-order or non-canonical point that can sign anything, so no signature under it counts",
   "delegation-invalid": "mint a fresh delegation from the principal, then resend",
   // Carrier, envelope shape and crypto (envelope.mjs open()): one fix for all
   "not-object": RESEAL,