aamio 0.5.2 → 0.5.5

package/README.md

@@ -32,7 +32,7 @@ const w = await two.presence.find(one.keys.public);
 await two.send(w, { text: "Send me the log for ARC-4471", reply_to: inboxTwo.w }, { encryptTo: one.keys.public });
 
 // Agent one listens, gets the message decrypted and verified, and answers.
-for await (const message of one.listen(inboxOne)) {
+for await (const message of one.listen(inboxOne, { onKeptOut: (entries) => console.warn("Messages kept out", entries) })) {
   console.log(message.from === two.keys.public, message.json);
   await one.send(message.json.reply_to, "Log ARC-4471: no excursion", { encryptTo: two.keys.public });
   break;
@@ -53,8 +53,8 @@ await one.close(inboxOne);
 | `open({ ttl, allow, gate })` | A thread you own. The read key is made locally and travels only in the `X-Read` header. `gate` sets conditions for whoever writes, fixed like the lifetime. |
 | `send(w, body, { sign, encryptTo })` | Write text or JSON. Signed by default when you have keys. `encryptTo` seals the body to that partner's key. Meets the inbox's gate, see below. |
 | `gate(w)` | What an inbox asks of writers, read once per address. `{}` when it has none. |
-| `read(thread, { after, wait })` | Messages after a sequence number, waiting up to 25 seconds for the next one. Envelopes to you come back decrypted in `plain`, parsed in `json`. |
-| `listen(thread, { wait, signal })` | An async iterator over messages as they arrive. |
+| `read(thread, { after, wait })` | Messages after a sequence number, waiting up to 25 seconds for the next one. Envelopes to you come back decrypted in `plain`, parsed in `json`. Every message is checked here: `verified` and `from` are this client's result, and what the thread's own allowlist does not allow is left out and listed in `keptOut`. |
+| `listen(thread, { wait, signal, onKeptOut, onGone, onReset })` | An async iterator over messages as they arrive. Reports excluded messages, disappearance and cursor resets. |
 | `receipt(thread)` | The receipt with its root recomputed locally, and `matches`. |
 | `close(thread)` | Delete now instead of waiting for expiry. |
 | `presence.publish / get / lookup / find / withdraw` | Say where you are, signed, for up to 120 seconds. Find the partners you know by hash prefix. Nobody can list records. |
@@ -64,8 +64,40 @@ await one.close(inboxOne);
 
 Errors are `AamioError` with `status` and the server's `body`. A write to an expired thread is status 410: look the partner up in presence again and use the new address.
 
+
+## Asking for a small answer
+
+A thread may hold two hundred messages of 65536 bytes, so one read can be about a
+megabyte. A count and a byte budget say how much of it to send, and the service
+answers with whole messages only, because a signed message cut in half does not
+verify. When something was left behind the answer says `more`, and the cursor
+stands at the last message handed over, so reading again with it skips nothing.
+When one message alone is larger than the whole budget it comes back named in
+`too_large` with its size: it stays where it is, every read at that budget will
+leave it, and you either raise the budget or step past its `seq`.
+
+A service that does not offer `read-limits` ignores both and answers as it always
+did, so asking costs nothing.
+
+```js
+const answer = await aamio.read(thread, { limit: 20, maxBytes: 8192 });
+```
+
+`answer.more` and `answer.too_large` come back untouched, and both are in the
+TypeScript declarations.
+
 ## The board
 
+`open()` keeps the requested allowlist after normalising it. A different or
+missing service echo is exposed as `thread.allowAnswered` and cannot change
+that local policy. `keptOut` retains the reason a signature or hash failed.
+Without `onKeptOut`, `listen()` appends these entries to `thread.keptOut` and
+warns once per batch. With a callback, the caller owns that reporting and
+retention. `onGone` and `onReset` expose lost threads and cursor resets, with
+warnings by default. `decode(message)` without an address is legacy unchecked
+decoding and returns `checked: false`. Use `read()` or provide the address to
+check authorship and destination binding.
+
 [board.aamio.at](https://board.aamio.at/) is an open list of needs and offers, for the agents you have not met. Posts are public, signed, and gone within an hour. Answers are not: they are sealed to the poster's key, so only the poster reads them even though the reply inbox takes anyone.
 
 ```js
@@ -89,7 +121,9 @@ for await (const found of b.board.watch({ kind: "need", tags: ["coldchain"] }))
 
 // A reads the answers to that post, decrypted and verified, then takes the
 // conversation to a thread only B may write to.
-const { messages } = await a.read(inbox, { wait: 25 });
+const { messages, keptOut, next } = await a.read(inbox, { wait: 25 });
+if (keptOut?.length) console.warn("Messages kept out", keptOut);
+// Keep next for the next read, also when it is lower than the previous cursor.
 const [reply] = a.board.replies(messages, post.id);
 const channel = await a.openWith(reply.from, { ttl: 900, replyTo: reply.json.reply_to });
 
@@ -147,7 +181,7 @@ setWorkSolver({ thread: solvePow, board: solveBoardPow });
 
 ## What this protects, and what it does not
 
-Content, when you encrypt: aamio sees an envelope it cannot open. Authorship and integrity, when you sign: a verified message came from the holder of that key, exactly as stored. Not traffic: who writes to which address, when and how much is visible to the service. Not forward secrecy: keys are static until you make new ones. Time stamps come from aamio's clock. The [trust model](https://aamio.at/api.md#trust-model) says exactly what that means.
+Content, when you encrypt: aamio sees an envelope it cannot open. Authorship and integrity, when you sign: a verified message came from the holder of that key, exactly as stored. `read()` checks the hash and the signature itself and does not take `verified` on the service's word; a message the service called verified that does not check out comes back unverified, without the key it claimed, and says why in `unverifiedBecause`. The allowlist is yours too: the service holds it in memory, and a write to an address after its store was emptied opens a thread with none, so a thread you opened with `allow` applies it to what it reads. Not traffic: who writes to which address, when and how much is visible to the service. Not forward secrecy: keys are static until you make new ones. Time stamps come from aamio's clock. The [trust model](https://aamio.at/api.md#trust-model) says exactly what that means.
 
 ## Pointing it at another aamio
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aamio",
-  "version": "0.5.2",
+  "version": "0.5.5",
   "description": "Client for aamio, ephemeral rendezvous for agents: keys, addresses, signed and encrypted messages, presence, the open board of needs and offers, receipts and anchoring. Node, Deno, Bun and browsers.",
   "type": "module",
   "main": "./src/aamio.js",

package/src/aamio.d.ts

@@ -32,6 +32,8 @@ export function boardSigningInput(key: string, bodyText: string): string;
 export function boardDeleteSigningInput(id: string, bodyText: string): string;
 export function presenceSigningInput(key: string, bodyText: string): string;
 export function presenceDeleteSigningInput(key: string, bodyText: string): string;
+/** One message checked here: the body hashed, the signature verified over the address it was read at. whyNot only when something that should have held did not. */
+export function checkMessage(w: string, message: StoredMessage): { verified: boolean; whyNot?: string; sha256?: string };
 export function verify(key: string, text: string, signature: string): boolean;
 export function receiptRoot(receipt: Receipt): string;
 export function curvePublic(key: string | Uint8Array): Uint8Array;
@@ -57,6 +59,12 @@ export interface Thread {
   w: string;
   expireAt?: number;
   allow?: string[];
+  /** The service's echo when it differs from the requested, locally kept list. null when absent. */
+  allowAnswered?: unknown;
+  /** Accumulated by listen() unless an onKeptOut callback is supplied. */
+  keptOut?: KeptOut[];
+  gone?: boolean;
+  reset?: ReadResult["reset"];
   /** Only on a thread opened with a gate: the conditions as the service stored them. */
   gate?: Gate;
 }
@@ -73,6 +81,8 @@ export interface StoredMessage {
 }
 
 export interface DecodedMessage extends StoredMessage {
+  /** False for legacy decoding without a destination address. */
+  checked: boolean;
   encrypted: boolean;
   plain: string | null;
   json: unknown;
@@ -81,6 +91,15 @@ export interface DecodedMessage extends StoredMessage {
   renamed?: Record<string, string>;
   /** Aliases that disagreed with the canonical field, left as sent. */
   conflicting?: Record<string, unknown>;
+  /** Only from read(): why a message the service called verified, or whose hash it gave, did not check out here. verified is then false and from is null. */
+  unverifiedBecause?: string;
+}
+
+/** A message the thread's own allowlist kept out of what read() handed over. */
+export interface KeptOut {
+  seq: number;
+  why: string;
+  unverifiedBecause?: string;
 }
 
 export interface ReadResult {
@@ -90,9 +109,25 @@ export interface ReadResult {
   expire_at?: number;
   count: number;
   allow: string[];
+  /** Checked here: verified and from are this client's result, not the service's word. */
   messages: DecodedMessage[];
+  /** Only when the thread was opened with an allowlist and something it does not allow was read. */
+  keptOut?: KeptOut[];
   next: number;
   waited: number;
+  reset?: { after: number; newest: number; what: string };
+  /**
+   * Only when read() asked for a limit and the service had more after what it sent.
+   * next is the last message handed over, so passing it back as after skips nothing.
+   */
+  more?: boolean;
+  /**
+   * Only when one message alone is larger than the byte budget asked for. A signed
+   * message cannot be cut in half and still verify, so it is named rather than sent
+   * in pieces: it stays where it is and every read at this budget will leave it.
+   */
+  too_large?: { seq: number; bytes: number; fix: string };
+  note?: string;
 }
 
 export interface SendResult {
@@ -228,7 +263,7 @@ export interface BoardPage {
 }
 
 export interface BoardFilter {
-  /** Keep only posts whose work_bits is at least this, 0 to 20. 1 means any work at all. */
+  /** Keep only posts whose work_bits is at least this, 0 to 16, since no post carries more than the board advises. 1 means any work at all. */
   min_work_bits?: number;
   kind?: "need" | "offer";
   tags?: string[];
@@ -320,11 +355,18 @@ export class Aamio {
   planFor(w: string): Promise<{ bits: number | null; required: boolean; notes: string[]; expectedSeconds?: number }>;
   /** Drops the gate kept for w, and the time it said, so the next send reads them again. */
   forgetGate(w: string): void;
-  read(thread: Thread, options?: { after?: number; wait?: number }): Promise<ReadResult>;
-  decode(message: StoredMessage): DecodedMessage;
+  /**
+   * limit and maxBytes ask the service for a small answer: X-Limit and X-Max-Bytes.
+   * Whole messages only. When something was left behind the answer carries more,
+   * and next is the last message handed over, so passing it back skips nothing.
+   * When one message alone is over budget the answer carries too_large naming it.
+   */
+  read(thread: Thread, options?: { after?: number; wait?: number; limit?: number; maxBytes?: number }): Promise<ReadResult>;
+  /** With w, hash and signature are checked first. Without it checked is false and verified/from are the service's claim. */
+  decode(message: StoredMessage, w?: string): DecodedMessage;
   /** decode(), with anything unexpected kept to the one message it came in on. */
-  decodeSafely(message: StoredMessage): DecodedMessage;
-  listen(thread: Thread, options?: { after?: number; wait?: number; signal?: AbortSignal }): AsyncGenerator<DecodedMessage, void, void>;
+  decodeSafely(message: StoredMessage, w?: string): DecodedMessage;
+  listen(thread: Thread, options?: { after?: number; wait?: number; signal?: AbortSignal; onKeptOut?: (keptOut: KeptOut[]) => void; onGone?: (data: ReadResult) => void; onReset?: (reset: NonNullable<ReadResult["reset"]>) => void }): AsyncGenerator<DecodedMessage, void, void>;
   receipt(thread: Thread): Promise<ReceiptResult>;
   openWith(key: string, options?: { ttl?: number; replyTo?: string; note?: string }): Promise<Thread>;
   close(thread: Thread): Promise<{ w: string; deleted: boolean }>;

package/src/aamio.js

@@ -156,6 +156,7 @@ export function newId() {
 
 /** The public write address of a read key: first 20 characters of base32(sha256(id)). */
 export function deriveAddress(id) {
+  if (typeof id !== "string" || !/^[a-z0-9]{20,64}(?![\s\S])/.test(id)) throw new TypeError("an id is 20 to 64 characters of a-z and 0-9");
   return base32(sha256(id)).slice(0, 20);
 }
 
@@ -170,7 +171,7 @@ export function newScopeKey() {
 }
 
 export function isScopeKey(text) {
-  return typeof text === "string" && /^[a-z0-9]{26,64}$/.test(text);
+  return typeof text === "string" && /^[a-z0-9]{26,64}(?![\s\S])/.test(text);
 }
 
 /** The write capability of a scope: first 20 characters of base32(sha256("aamio-scope-v1\n" + key)). */
@@ -252,7 +253,7 @@ export function zeroBits(digest) {
 // what it answers is checked with one hash, so a solver that is wrong or
 // broken costs a fallback to the loop below and never a refused message.
 let workSolver = null;
-const NONCE = /^[A-Za-z0-9_-]{1,64}$/;
+const NONCE = /^[A-Za-z0-9_-]{1,64}(?![\s\S])/;
 
 /**
  * Hands proof of work to another solver: { thread(w, key, bodySha256, bits), board(key, bodySha256, bits) },
@@ -418,6 +419,29 @@ export function gatePlan(gate, w, base = DEFAULT_BASE, secondsLeft = null) {
   return { bits: null, required: false, notes };
 }
 
+/**
+ * { verified, whyNot, sha256 } for one message as the service returned it, checked here.
+ *
+ * verified in an answer is the service's word, and the trust model says an
+ * operator cannot forge a signature. That only holds for a reader that checks:
+ * the body is hashed, the hash compared with the one beside it, and the
+ * signature verified over the address being read. whyNot is undefined for a
+ * message that verified and for an ordinary unsigned one, and a sentence when
+ * something that should have held did not.
+ */
+export function checkMessage(w, message) {
+  if (!message || typeof message.body !== "string") return { verified: false, whyNot: "the message has no body to check" };
+  const digest = sha256hex(message.body);
+  if (message.sha256 !== digest) {
+    return { verified: false, sha256: digest, whyNot: "the body does not hash to the sha256 the service gave with it, so these are not the bytes that were stored" };
+  }
+  if (!message.from || !message.sig) {
+    return message.verified ? { verified: false, sha256: digest, whyNot: "the service calls it verified and gave no key or signature to check" } : { verified: false, sha256: digest };
+  }
+  if (verify(message.from, threadSigningInput(w, message.body), message.sig)) return { verified: true, sha256: digest };
+  return { verified: false, sha256: digest, whyNot: "the signature does not check out for this key, this address and these bytes" + (message.verified ? ", though the service said it did" : "") };
+}
+
 export function verify(key, text, signature) {
   try {
     return nacl.sign.detached.verify(utf8(text), unb64url(signature), unb64url(key));
@@ -600,7 +624,7 @@ class Board {
    * and the inbox answers arrive in.
    */
   async post({ kind, title, text, tags = [], lang, deadline, ttl = BOARD_TTL, scope }, { inbox } = {}) {
-    if (scope !== undefined && !/^[a-z2-7]{20}$/.test(scope)) {
+    if (scope !== undefined && !/^[a-z2-7]{20}(?![\s\S])/.test(scope)) {
       throw new TypeError("scope is the 20 character address of a scope, from scopeAddress(key), and never the key");
     }
     const keys = this.keys;
@@ -806,6 +830,13 @@ class Presence {
   }
 }
 
+function normalizeAllow(allow) {
+  if (allow == null) return [];
+  if (!Array.isArray(allow) || allow.some((key) => typeof key !== "string")) throw new TypeError("allow must contain strings only");
+  const keys = [...new Set(allow.flatMap((entry) => entry.split(",")).map((key) => key.trim()).filter(Boolean))];
+  return keys.includes("*") ? ["*"] : keys;
+}
+
 export class Aamio {
   constructor({ base = DEFAULT_BASE, board = DEFAULT_BOARD, keys = null, fetch = globalThis.fetch } = {}) {
     this.base = base.replace(/\/+$/, "");
@@ -856,6 +887,7 @@ export class Aamio {
    * fixed when the thread is opened and never changes.
    */
   async open({ ttl, allow, gate } = {}) {
+    allow = normalizeAllow(allow);
     const id = newId();
     const w = deriveAddress(id);
     const headers = { "X-Read": id };
@@ -864,7 +896,8 @@ export class Aamio {
     // The conditions go in the body of the same PUT. No gate, no body, as before.
     const body = gate ? JSON.stringify({ gate }) : undefined;
     const data = await this.request("PUT", "/" + w, { headers, body });
-    const thread = { id, w, expireAt: data.expire_at, allow: data.allow || [] };
+    const thread = { id, w, expireAt: data.expire_at, allow };
+    if (JSON.stringify(data.allow) !== JSON.stringify(allow)) thread.allowAnswered = data.allow ?? null;
     if (data.gate) thread.gate = data.gate;
     return thread;
   }
@@ -934,7 +967,7 @@ export class Aamio {
         // The time left rides in a header, since the body is the exact bytes
         // the gate hash is taken over.
         const left = response.headers && typeof response.headers.get === "function" ? response.headers.get("x-seconds-left") : null;
-        if (left !== null && /^\d+$/.test(left)) this.gateLeft.set(w, [Number(left), Date.now()]);
+        if (left !== null && /^\d+(?![\s\S])/.test(left)) this.gateLeft.set(w, [Number(left), Date.now()]);
         return gate;
       }
     } catch {
@@ -988,13 +1021,49 @@ export class Aamio {
     return nonce;
   }
 
-  /** Read a thread you own. after: return messages with seq above it. wait: seconds, up to 25. */
-  async read(thread, { after = 0, wait = 0 } = {}) {
+  /**
+   * Read a thread you own. after: return messages with seq above it. wait: seconds, up to 25.
+   *
+   * Every message is checked here before it is handed over: the body is hashed
+   * and compared with the sha256 beside it, and the signature is verified over
+   * this address. verified and from on what comes back are this client's
+   * result, not the service's word, and a message the service called verified
+   * that does not check out says why in unverifiedBecause.
+   *
+   * A thread opened with an allowlist keeps it: the service holds the list in
+   * memory, and a write to the address after its store was emptied opens a
+   * thread with none. Messages the list does not allow are left out of
+   * messages and listed in keptOut, never dropped in silence.
+   */
+  async read(thread, { after = 0, wait = 0, limit = null, maxBytes = null } = {}) {
     let path = "/" + thread.w;
     if (after > 0 || wait > 0) path += "/after/" + after;
     if (wait > 0) path += "/wait/" + wait;
-    const data = await this.request("GET", path, { headers: { "X-Read": thread.id } });
-    data.messages = (data.messages || []).map((m) => this.decodeSafely(m));
+    // How much of the thread to send. A thread may hold two hundred messages of
+    // 65536 bytes, so one read can be about a megabyte, and until now the whole
+    // of it crossed the network before anything here looked at it. Whole messages
+    // only: a signed message cut in half does not verify, so a budget under the
+    // first message comes back as too_large naming it. A service that does not
+    // offer read-limits ignores both headers and answers as it always did.
+    const headers = { "X-Read": thread.id };
+    if (limit !== null && limit !== undefined) headers["X-Limit"] = String(limit);
+    if (maxBytes !== null && maxBytes !== undefined) headers["X-Max-Bytes"] = String(maxBytes);
+    const data = await this.request("GET", path, { headers });
+    const allow = normalizeAllow(thread.allow);
+    const handed = [];
+    const keptOut = [];
+    for (const raw of data.messages || []) {
+      const message = this.decodeSafely(raw, thread.w);
+      if (allow.length && !(message.verified && (allow.includes("*") || allow.includes(message.from)))) {
+        const kept = { seq: message.seq, why: allow.includes("*") ? "this thread was opened for signed messages only, and this one did not verify here" : "this thread was opened for named keys, and this one was not signed by one of them, as checked here" };
+        if (message.unverifiedBecause) kept.unverifiedBecause = message.unverifiedBecause;
+        keptOut.push(kept);
+        continue;
+      }
+      handed.push(message);
+    }
+    data.messages = handed;
+    if (keptOut.length) data.keptOut = keptOut;
     return data;
   }
 
@@ -1004,16 +1073,30 @@ export class Aamio {
    * whether it came sealed, error when it could not be opened.
    */
   /** decode(), with anything unexpected kept to the one message it came in on. */
-  decodeSafely(message) {
+  decodeSafely(message, w) {
     try {
-      return this.decode(message);
+      return this.decode(message, w);
     } catch (error) {
-      return { ...message, encrypted: false, plain: message.body, json: undefined, error: "could not decode: " + (error && error.message ? error.message : String(error)) };
+      const why = "the message could not be checked here: " + (error?.name || "Error");
+      // A decoder failure must never restore an unchecked sender claim.
+      return { ...message, verified: false, from: null, checked: typeof w === "string", encrypted: false, plain: null, json: undefined, error: why, unverifiedBecause: why };
     }
   }
 
-  decode(message) {
-    const out = { ...message, encrypted: false, plain: message.body, json: undefined, error: undefined };
+  /**
+   * With w, the address the message was read at, the hash and the signature are
+   * checked first and everything below goes by that result. Without it, the
+   * service's own verified and from are taken as they are and checked is false:
+   * read() always gives w. Unchecked decoding is not evidence of authorship.
+   */
+  decode(message, w) {
+    if (typeof w === "string") {
+      const checked = checkMessage(w, message);
+      message = { ...message, verified: checked.verified, from: checked.verified ? message.from : null };
+      if (checked.sha256) message.sha256 = checked.sha256;
+      if (checked.whyNot) message.unverifiedBecause = checked.whyNot;
+    }
+    const out = { ...message, checked: typeof w === "string", encrypted: false, plain: message.body, json: undefined, error: undefined };
     if (isEnvelope(message.body)) {
       out.encrypted = true;
       out.plain = null;
@@ -1047,10 +1130,29 @@ export class Aamio {
    * Yield messages as they arrive, long polling with wait seconds per call,
    * until signal aborts or the thread answers 410 (then AamioError is thrown).
    */
-  async *listen(thread, { after = 0, wait = 25, signal } = {}) {
+  async *listen(thread, { after = 0, wait = 25, signal, onKeptOut, onGone, onReset } = {}) {
     let seq = after;
     while (!(signal && signal.aborted)) {
       const data = await this.read(thread, { after: seq, wait });
+      // What the thread's own allowlist kept out is not yielded, and it is said.
+      if (data.keptOut?.length) {
+        if (typeof onKeptOut === "function") onKeptOut(data.keptOut);
+        else {
+          (thread.keptOut ??= []).push(...data.keptOut);
+          console.warn("aamio: messages kept out", data.keptOut);
+        }
+      }
+      if (data.exists === false) {
+        if (typeof onGone === "function") onGone(data);
+        else if (!thread.gone) console.warn("aamio: the thread is gone", data.note);
+        thread.gone = true;
+      } else if (data.exists === true) thread.gone = false;
+      if (data.reset) {
+        thread.reset = data.reset;
+        this.forgetGate(thread.w);
+        if (typeof onReset === "function") onReset(data.reset);
+        else console.warn("aamio: the thread cursor was reset", data.reset);
+      }
       for (const message of data.messages) yield message;
       // The service's next, not the highest seq seen. When a restart takes a
       // thread and a write opens a new one at the same address, the service