@trevrpc/trevrpc-js 1.1.0 → 1.3.0

package/src/connection-metadata.js

@@ -0,0 +1,114 @@
+// Connection metadata is request metadata a client sets once for all its calls, such as a bearer
+// token, so that it crosses each connection once instead of in every REQUEST. A call offers the
+// set (RpcRequest.connection_metadata with entries) until an ANSWER confirms that the server
+// stored it for the session, then refers to it by ID. Calls on different streams reach the server
+// in any order, so a reference only follows a confirmation. The server merges the set into each
+// call's metadata, the call's own entries winning, so the client just sends both; see the wiki's
+// Connection Metadata section. This module holds the channel's set and what each session
+// confirmed; call.js puts them on the REQUEST and learns the confirmation from the ANSWER.
+import { normalizeMetadata } from "./metadata.js";
+
+/** The property of a transport that holds its ConnectionMetadataSource. */
+export const ConnectionMetadataKey = Symbol("trevrpc.connectionMetadata");
+
+/** The connection metadata of a transport, or undefined without a source. */
+export function transportConnectionMetadata(transport) {
+  return transport[ConnectionMetadataKey];
+}
+
+/** A request that carries no connection metadata. */
+const NoConnectionMetadata = Object.freeze({ field: undefined, offered: 0 });
+
+/**
+ * A channel's connection metadata, shared by its sessions: the current set, whose ID changes with
+ * it, and the state of each session.
+ */
+export class ConnectionMetadataSource {
+  /** The current set, `{ id, metadata }`, or undefined without connection metadata. */
+  #current;
+  #ids = 0;
+  #sessions = new WeakMap();
+
+  /** Throws InvalidArgument if `metadata` is invalid, like call metadata. */
+  constructor(metadata) {
+    this.set(metadata);
+  }
+
+  /** The current set, or undefined. */
+  get current() {
+    return this.#current;
+  }
+
+  /**
+   * Replaces the connection metadata, with keys normalized like a call's; empty or missing
+   * metadata removes it. The new set has a new ID, so each session offers it again.
+   */
+  set(metadata) {
+    const normalized = metadata == null ? {} : normalizeMetadata(metadata);
+    const keys = Object.keys(normalized);
+    if (keys.length === 0) {
+      this.#current = undefined;
+      return;
+    }
+    // Copied, so a caller changing its bytes later does not change what is sent.
+    for (const key of keys) {
+      normalized[key] = normalized[key].slice();
+    }
+    // IDs are uint32 on the wire and 0 means none: they run from 1 to 2^32 - 1 and wrap.
+    this.#ids = (this.#ids % 0xffffffff) + 1;
+    this.#current = { id: this.#ids, metadata: normalized };
+  }
+
+  /**
+   * The state of one session, keyed like the compression state: what the server confirmed on the
+   * connection that `key` identifies, dropped with it.
+   */
+  session(key) {
+    let state = this.#sessions.get(key);
+    if (state === undefined) {
+      state = new SessionConnectionMetadata(this);
+      this.#sessions.set(key, state);
+    }
+    return state;
+  }
+}
+
+class SessionConnectionMetadata {
+  #source;
+  /** The ID of the set the server confirmed on this session, or zero. */
+  #confirmed = 0;
+
+  constructor(source) {
+    this.#source = source;
+  }
+
+  /**
+   * The connection metadata of one REQUEST as `{ field, offered }`: `field` is the value of
+   * `connectionMetadata`, a reference to the current set once the session confirmed it and an
+   * offer of it otherwise, or undefined without a set; `offered` is the ID the REQUEST offers,
+   * which `learn` checks the ANSWER against, or zero.
+   */
+  prepare() {
+    const set = this.#source.current;
+    if (set === undefined) {
+      return NoConnectionMetadata;
+    }
+    if (this.#confirmed === set.id) {
+      return { field: { id: set.id }, offered: 0 };
+    }
+    return { field: { id: set.id, metadata: set.metadata }, offered: set.id };
+  }
+
+  /** Records that the server stored the `offered` set, once the call's ANSWER confirms it. */
+  learn(offered, answer) {
+    // Only the current set counts: a late confirmation of one since replaced would make calls
+    // offer the current one again.
+    if (
+      offered !== 0 &&
+      answer.connectionMetadata === offered &&
+      this.#source.current?.id === offered
+    ) {
+      this.#confirmed = offered;
+    }
+  }
+}

package/src/datagram.js

@@ -4,6 +4,7 @@
 // DatagramFlow on the endpoint of the session that carries them. QMux over WebSocket has no
 // datagrams, so those calls never get a flow and everything stays on the stream.
 import { byteView } from "./bytes.js";
+import { FrameType } from "./framing.js";
 import { readVarint, varintLength, writeVarint } from "./varint.js";
 
 /**
@@ -19,16 +20,17 @@ export const FallbackMaxDatagramSize = 1100;
 export const FlowQueueMessages = 32;
 export const FlowQueueBytes = 1 << 20;
 
-const MessageFrameType = 0;
-
-/** Encoded size of one message as a MESSAGE frame inside a datagram. */
-export function messageFrameSize(message) {
-  return 1 + varintLength(message.byteLength) + message.byteLength;
+/**
+ * Encoded size of one outgoing message (`{ type, payload }` from encodeMessage) as a MESSAGE or
+ * ZSTD_MESSAGE frame inside a datagram.
+ */
+export function messageFrameSize({ payload }) {
+  return 1 + varintLength(payload.byteLength) + payload.byteLength;
 }
 
 /**
- * Encodes a datagram payload: the flow ID as a QUIC varint, then one MESSAGE frame per message,
- * in order.
+ * Encodes a datagram payload: the flow ID as a QUIC varint, then the frame of each outgoing
+ * message, in order.
  */
 export function encodeDatagram(flow, messages) {
   let size = varintLength(flow);
@@ -37,19 +39,20 @@ export function encodeDatagram(flow, messages) {
   }
   const datagram = new Uint8Array(size);
   let offset = writeVarint(datagram, 0, flow);
-  for (const message of messages) {
-    datagram[offset] = MessageFrameType;
-    offset = writeVarint(datagram, offset + 1, message.byteLength);
-    datagram.set(message, offset);
-    offset += message.byteLength;
+  for (const { type, payload } of messages) {
+    datagram[offset] = type;
+    offset = writeVarint(datagram, offset + 1, payload.byteLength);
+    datagram.set(payload, offset);
+    offset += payload.byteLength;
   }
   return datagram;
 }
 
 /**
  * Decodes a datagram payload as `{ flow, messages }`, or null if it has no complete flow ID. The
- * messages are the payloads of its MESSAGE frames; every other frame is skipped. A datagram whose
- * frames do not exactly fill it decodes to no messages at all.
+ * messages are its MESSAGE and ZSTD_MESSAGE frames as `{ type, payload }`; every other frame is
+ * skipped. A datagram whose frames do not exactly fill it, or that holds a ZSTD_STREAM_MESSAGE,
+ * which needs a stream's ordered delivery, decodes to no messages at all.
  */
 export function decodeDatagram(datagram) {
   const bytes = byteView(datagram);
@@ -69,8 +72,11 @@ export function decodeDatagram(datagram) {
     if (end > bytes.byteLength) {
       return { flow: flow.value, messages: [] };
     }
-    if (type.value === MessageFrameType) {
-      messages.push(bytes.subarray(length.next, end));
+    if (type.value === FrameType.ZstdStreamMessage) {
+      return { flow: flow.value, messages: [] };
+    }
+    if (type.value === FrameType.Message || type.value === FrameType.ZstdMessage) {
+      messages.push({ type: type.value, payload: bytes.subarray(length.next, end) });
     }
     offset = end;
   }
@@ -116,14 +122,15 @@ class DatagramEndpoint {
 
   /**
    * Allocates the next flow of this session. A flow that `receive`s is registered with the
-   * reader, so datagrams for its ID reach it. Returns undefined if the session cannot read
-   * datagrams.
+   * reader, so datagrams for its ID reach it; `inflate` maps each received `{ type, payload }`
+   * message to its payload, decompressed, and throws if it is invalid. Returns undefined if the
+   * session cannot read datagrams.
    */
-  openFlow({ receive }) {
+  openFlow({ receive, inflate }) {
     if (!this.#start()) {
       return undefined;
     }
-    const flow = new DatagramFlow(this, this.#nextFlow);
+    const flow = new DatagramFlow(this, this.#nextFlow, inflate);
     this.#nextFlow += 1;
     if (receive) {
       this.#flows.set(flow.id, flow);
@@ -207,11 +214,7 @@ class DatagramEndpoint {
         const datagram = decodeDatagram(value);
         const flow = datagram === null ? undefined : this.#flows.get(datagram.flow);
         // Unknown and ended flows drop the datagram, and so do malformed ones (no messages).
-        if (flow !== undefined) {
-          for (const message of datagram.messages) {
-            flow.deliver(message);
-          }
-        }
+        flow?.deliverAll(datagram.messages);
       }
     } catch {
       // A failed session ends its calls through their streams.
@@ -253,21 +256,39 @@ class DatagramEndpoint {
  */
 export class DatagramFlow {
   #endpoint;
+  #inflate;
   #handler;
   #buffered = [];
   #bufferedBytes = 0;
   #released = false;
 
-  constructor(endpoint, id) {
+  constructor(endpoint, id, inflate) {
     this.#endpoint = endpoint;
+    this.#inflate = inflate;
     this.id = id;
   }
 
+  /**
+   * Delivers the messages of one received datagram, decompressed. A datagram with an invalid
+   * message is dropped whole.
+   */
+  deliverAll(messages) {
+    let payloads;
+    try {
+      payloads = messages.map(this.#inflate);
+    } catch {
+      return;
+    }
+    for (const payload of payloads) {
+      this.#deliver(payload);
+    }
+  }
+
   /**
    * Delivers one received message. It must not block or throw. Without a handler it is buffered,
    * and dropped when the buffer is full.
    */
-  deliver(message) {
+  #deliver(message) {
     if (this.#handler !== undefined) {
       this.#handler(message);
     } else if (

package/src/framing.js

@@ -1,7 +1,7 @@
 import { byteView } from "./bytes.js";
 import { internal, invalidArgument, resourceExhausted, unavailable } from "./status.js";
 import { readVarint, varintLength, writeVarint } from "./varint.js";
-import { RpcHeaders, RpcRequest, RpcStatus, preflightWireMessage } from "./wire.js";
+import { RpcAnswer, RpcHeaders, RpcRequest, RpcStatus, preflightWireMessage } from "./wire.js";
 
 export const DefaultMaxFrameSize = 4 * 1024 * 1024;
 
@@ -11,6 +11,9 @@ export const FrameType = Object.freeze({
   Request: 1,
   Headers: 2,
   Status: 3,
+  ZstdMessage: 4,
+  ZstdStreamMessage: 5,
+  Answer: 6,
 });
 
 const KnownFrameTypes = new Set(Object.values(FrameType));
@@ -55,8 +58,17 @@ export function encodeMessageFrame(body, maxFrameSize = DefaultMaxFrameSize) {
 }
 
 /**
- * Encodes a REQUEST frame, followed by a MESSAGE frame when message is given, in one buffer so
- * both go out in a single write.
+ * Encodes an outgoing message, `{ type, payload, size }` from encodeMessage, as its MESSAGE or
+ * ZSTD_MESSAGE frame. The message size counts against the limit, not just the payload's.
+ */
+export function encodeOutgoingFrame(message, maxFrameSize = DefaultMaxFrameSize) {
+  checkFrameLength(message.size, maxFrameSize);
+  return encodeFrame(message.type, message.payload, maxFrameSize);
+}
+
+/**
+ * Encodes a REQUEST frame, followed by the frame of message, an outgoing message from
+ * encodeMessage, when it is given, in one buffer so both go out in a single write.
  */
 export function encodeRequestFrame(request, message, maxFrameSize = DefaultMaxFrameSize) {
   const payload = marshalMessage(RpcRequest, request);
@@ -64,10 +76,10 @@ export function encodeRequestFrame(request, message, maxFrameSize = DefaultMaxFr
   if (message === undefined) {
     return encodeFrame(FrameType.Request, payload, maxFrameSize);
   }
-  checkFrameLength(message.byteLength, maxFrameSize);
+  checkFrameLength(message.size, maxFrameSize);
   const requestSize = frameSize(FrameType.Request, payload.byteLength);
-  const frame = new Uint8Array(requestSize + frameSize(FrameType.Message, message.byteLength));
-  writeFrame(frame, requestSize, FrameType.Message, message);
+  const frame = new Uint8Array(requestSize + frameSize(message.type, message.payload.byteLength));
+  writeFrame(frame, requestSize, message.type, message.payload);
   writeFrame(frame, 0, FrameType.Request, payload);
   return frame;
 }
@@ -77,6 +89,11 @@ export function encodeHeadersFrame(headers, maxFrameSize = DefaultMaxFrameSize)
   return encodeFrame(FrameType.Headers, marshalMessage(RpcHeaders, headers), maxFrameSize);
 }
 
+/** Encodes an ANSWER frame from RpcAnswer fields. */
+export function encodeAnswerFrame(answer, maxFrameSize = DefaultMaxFrameSize) {
+  return encodeFrame(FrameType.Answer, marshalMessage(RpcAnswer, answer), maxFrameSize);
+}
+
 /** Encodes a STATUS frame from RpcStatus fields. */
 export function encodeStatusFrame(status, maxFrameSize = DefaultMaxFrameSize) {
   return encodeFrame(FrameType.Status, marshalMessage(RpcStatus, status), maxFrameSize);

package/src/generator.js

@@ -62,6 +62,7 @@ message FieldDescriptorProto {
   optional int32 oneof_index = 9;
   optional string json_name = 10;
   optional bool proto3_optional = 17;
+  optional FieldOptions options = 8;
 }
 
 message OneofDescriptorProto {
@@ -101,6 +102,12 @@ message MethodOptions {
 message TrevrpcMethodOptions {
   optional bool unreliable_requests = 1;
   optional bool unreliable_responses = 2;
+  // trevrpc.Compression, read as its number so that unknown values reach the check.
+  optional int32 compression = 5;
+}
+
+message FieldOptions {
+  optional bool debug_redact = 16;
 }
 
 message FileOptions {
@@ -178,6 +185,9 @@ const FieldLabel = Object.freeze({
   Repeated: 3,
 });
 
+/** The CodeGeneratorResponse features this generator supports. */
+const FEATURE_PROTO3_OPTIONAL = 1;
+
 /** Generic runtime types that generated declarations import only when they use them. */
 const GeneratedTypeImports = [
   "BidirectionalStreamingCall",
@@ -236,6 +246,7 @@ export function generate(request) {
     const protoFiles = request.protoFile ?? [];
     const rootJson = buildProtobufJson(protoFiles);
     const typeNames = declarationTypeNames(protoFiles);
+    const redacted = redactedTypes(protoFiles);
     const files = [];
 
     for (const file of protoFiles) {
@@ -246,17 +257,17 @@ export function generate(request) {
       const fileName = outputFileName(file.name, options.fileSuffix);
       files.push({
         name: fileName,
-        content: generateFile(file, rootJson, options),
+        content: generateFile(file, rootJson, redacted, options),
       });
       files.push({
         name: declarationFileName(fileName),
-        content: generateDeclarationFile(protoFiles, file, typeNames, options),
+        content: generateDeclarationFile(protoFiles, file, typeNames, redacted, options),
       });
     }
 
-    return { file: files };
+    return { file: files, supportedFeatures: FEATURE_PROTO3_OPTIONAL };
   } catch (error) {
-    return { error: error.message };
+    return { error: error.message, supportedFeatures: FEATURE_PROTO3_OPTIONAL };
   }
 }
 
@@ -323,8 +334,10 @@ export function buildProtobufJson(files) {
   return root;
 }
 
-function generateFile(file, rootJson, options) {
-  const services = (file.service ?? []).map((service) => serviceDescriptor(file, service));
+function generateFile(file, rootJson, redacted, options) {
+  const services = (file.service ?? []).map((service) =>
+    serviceDescriptor(file, service, redacted),
+  );
   const serviceExports = services.map((service) => generateService(service)).join("\n");
 
   return `// Code generated by protoc-gen-trevrpc-js. DO NOT EDIT.\n\nimport { createRoot, createServiceClient } from ${JSON.stringify(
@@ -332,13 +345,13 @@ function generateFile(file, rootJson, options) {
   )};\n\nexport const root = createRoot(${JSON.stringify(rootJson, null, 2)});\n\n${serviceExports}`;
 }
 
-function generateDeclarationFile(protoFiles, file, typeNames, options) {
+function generateDeclarationFile(protoFiles, file, typeNames, redacted, options) {
   const messages = collectMessages(protoFiles)
     .filter(({ message }) => !message.options?.mapEntry)
     .map((message) => generateMessageDeclaration(message, typeNames))
     .join("\n\n");
   const services = (file.service ?? [])
-    .map((service) => serviceDescriptor(file, service))
+    .map((service) => serviceDescriptor(file, service, redacted))
     .map((service) => generateServiceDeclaration(service, typeNames))
     .join("\n\n");
   const sections = [messages, services].filter(Boolean).join("\n\n");
@@ -400,7 +413,7 @@ function generateMethodDeclaration(jsName, method, typeNames) {
   }
 }
 
-function serviceDescriptor(file, service) {
+function serviceDescriptor(file, service, redacted) {
   const fullName =
     file.package === "" || file.package == null ? service.name : `${file.package}.${service.name}`;
   const methods = {};
@@ -409,12 +422,17 @@ function serviceDescriptor(file, service) {
   for (const method of service.method ?? []) {
     const jsName = uniqueName(lowerCamelIdentifier(method.name), usedNames);
     const kind = methodKind(method);
+    const inputType = stripLeadingDot(method.inputType);
+    const outputType = stripLeadingDot(method.outputType);
+    const methodName = `${fullName}.${method.name}`;
+    const options = method.options?.trevrpcMethod;
     methods[jsName] = {
       name: method.name,
       kind,
-      inputType: stripLeadingDot(method.inputType),
-      outputType: stripLeadingDot(method.outputType),
-      ...unreliableDirections(`${fullName}.${method.name}`, kind, method.options?.trevrpcMethod),
+      inputType,
+      outputType,
+      ...unreliableDirections(methodName, kind, options),
+      ...compressionDirections(methodName, kind, options, { inputType, outputType }, redacted),
     };
   }
 
@@ -451,6 +469,88 @@ function unreliableDirections(methodName, kind, options) {
   return directions;
 }
 
+/** `trevrpc.Compression` values, from proto/trevrpc/options.proto. */
+const Compression = { unspecified: 0, message: 1, stream: 2 };
+
+/**
+ * The directions a method's `compression` lets compress, as descriptor fields, leaving out message
+ * types with a field marked `debug_redact`. `COMPRESSION_MESSAGE` compresses messages without a
+ * dictionary in both directions. `COMPRESSION_STREAM` does too, for the messages outside the
+ * stream, and stream-compresses the directions that stream; on a unary method it is an error, and
+ * so is a value this generator does not know.
+ */
+function compressionDirections(methodName, kind, options, { inputType, outputType }, redacted) {
+  const compression = options?.compression ?? Compression.unspecified;
+  if (compression === Compression.unspecified) {
+    return {};
+  }
+  if (compression !== Compression.message && compression !== Compression.stream) {
+    throw new Error(`${methodName}: unknown compression ${compression}`);
+  }
+  const directions = {};
+  if (!redacted.has(inputType)) {
+    directions.compressRequests = true;
+  }
+  if (!redacted.has(outputType)) {
+    directions.compressResponses = true;
+  }
+  if (compression === Compression.message) {
+    return directions;
+  }
+  if (kind === "unary") {
+    throw new Error(`${methodName}: COMPRESSION_STREAM requires a streaming method`);
+  }
+  if (
+    (kind === "clientStreaming" || kind === "bidirectionalStreaming") &&
+    !redacted.has(inputType)
+  ) {
+    directions.compressStreamRequests = true;
+  }
+  if (
+    (kind === "serverStreaming" || kind === "bidirectionalStreaming") &&
+    !redacted.has(outputType)
+  ) {
+    directions.compressStreamResponses = true;
+  }
+  return directions;
+}
+
+/**
+ * The full names of the message types that contain a field marked `debug_redact`, directly or in
+ * a message-typed field, however deeply nested.
+ */
+function redactedTypes(files) {
+  const messages = new Map(
+    collectMessages(files).map(({ fullName, message }) => [fullName, message]),
+  );
+  const redacted = new Set();
+  const visiting = new Set();
+  const visit = (name) => {
+    if (redacted.has(name)) {
+      return true;
+    }
+    const message = messages.get(name);
+    if (message === undefined || visiting.has(name)) {
+      return false;
+    }
+    visiting.add(name);
+    const found = (message.field ?? []).some(
+      (field) =>
+        field.options?.debugRedact === true ||
+        (field.typeName != null && visit(stripLeadingDot(field.typeName))),
+    );
+    visiting.delete(name);
+    if (found) {
+      redacted.add(name);
+    }
+    return found;
+  };
+  for (const name of messages.keys()) {
+    visit(name);
+  }
+  return redacted;
+}
+
 function declarationTypeNames(files) {
   const names = new Map();
   const usedNames = new Set();
@@ -508,12 +608,25 @@ function generateMessageDeclaration({ fullName, message }, typeNames) {
 
 function generateFieldDeclaration(field, mapEntries, typeNames, currentFullName) {
   const optional = field.label === FieldLabel.Required ? "" : "?";
+  const nullable = decodesAbsentAsNull(field) ? " | null" : "";
   return `  ${typescriptPropertyName(fieldJsonName(field))}${optional}: ${fieldTypeScriptType(
     field,
     mapEntries,
     typeNames,
     currentFullName,
-  )};`;
+  )}${nullable};`;
+}
+
+/** Reports whether protobuf.js decodes the field as null when it is absent. */
+function decodesAbsentAsNull(field) {
+  if (field.label === FieldLabel.Repeated) {
+    return false;
+  }
+  return (
+    field.proto3Optional === true ||
+    field.type === FieldType.Message ||
+    field.type === FieldType.Group
+  );
 }
 
 function fieldTypeScriptType(field, mapEntries, typeNames, currentFullName) {
@@ -689,6 +802,11 @@ function fieldToJson(field, mapEntries) {
   if (field.defaultValue != null && field.defaultValue !== "") {
     json.options = { default: field.defaultValue };
   }
+  if (field.proto3Optional) {
+    // protobuf.js gives the field presence, decoding it as null when absent,
+    // only when this option marks its synthetic oneof.
+    json.options = { ...json.options, proto3_optional: true };
+  }
 
   return json;
 }
@@ -709,7 +827,9 @@ function oneofsToJson(message) {
   const oneofs = {};
   for (const [index, oneof] of (message.oneofDecl ?? []).entries()) {
     const fields = (message.field ?? [])
-      .filter((field) => field.oneofIndex === index)
+      // A decoded descriptor reads an unset oneof_index as 0 from its
+      // prototype, so only an own property places the field in a oneof.
+      .filter((field) => Object.hasOwn(field, "oneofIndex") && field.oneofIndex === index)
       .map(fieldJsonName);
     if (fields.length > 0) {
       oneofs[oneof.name] = { oneof: fields };