@replit/river 0.21.0 → 0.22.0

package/dist/transport/impls/uds/client.cjs

@@ -109,18 +109,70 @@ function isAck(controlFlag) {
   return (controlFlag & 1 /* AckBit */) === 1 /* AckBit */;
 }
 
+// tracing/index.ts
+var import_api = require("@opentelemetry/api");
+
+// package.json
+var version = "0.22.0";
+
+// tracing/index.ts
+function getPropagationContext(ctx) {
+  const tracing = {
+    traceparent: "",
+    tracestate: ""
+  };
+  import_api.propagation.inject(ctx, tracing);
+  return tracing;
+}
+function createSessionTelemetryInfo(session, propagationCtx) {
+  const ctx = propagationCtx ? import_api.propagation.extract(import_api.context.active(), propagationCtx) : import_api.context.active();
+  const span = tracer.startSpan(
+    `session ${session.id}`,
+    {
+      attributes: {
+        component: "river",
+        "river.session.id": session.id,
+        "river.session.to": session.to,
+        "river.session.from": session.from
+      }
+    },
+    ctx
+  );
+  return { span, ctx };
+}
+function createConnectionTelemetryInfo(connection, sessionSpan) {
+  const ctx = import_api.trace.setSpan(import_api.context.active(), sessionSpan);
+  const span = tracer.startSpan(
+    `connection ${connection.id}`,
+    {
+      attributes: {
+        component: "river",
+        "river.connection.id": connection.id
+      },
+      links: [{ context: sessionSpan.spanContext() }]
+    },
+    ctx
+  );
+  return { span, ctx };
+}
+var tracer = import_api.trace.getTracer("river", version);
+var tracing_default = tracer;
+
 // transport/session.ts
+var import_api2 = require("@opentelemetry/api");
 var nanoid2 = (0, import_nanoid2.customAlphabet)("1234567890abcdefghijklmnopqrstuvxyz", 6);
 var unsafeId = () => nanoid2();
 var Connection = class {
-  debugId;
+  id;
+  telemetry;
   constructor() {
-    this.debugId = `conn-${unsafeId()}`;
+    this.id = `conn-${nanoid2(12)}`;
   }
 };
 var Session = class {
   codec;
   options;
+  telemetry;
   /**
    * The buffer of messages that have been sent but not yet acknowledged.
    */
@@ -167,7 +219,7 @@ var Session = class {
    * The interval for sending heartbeats.
    */
   heartbeat;
-  constructor(conn, from, to, options) {
+  constructor(conn, from, to, options, propagationCtx) {
     this.id = `session-${nanoid2(12)}`;
     this.options = options;
     this.from = from;
@@ -179,13 +231,14 @@ var Session = class {
       () => this.sendHeartbeat(),
       options.heartbeatIntervalMs
     );
+    this.telemetry = createSessionTelemetryInfo(this, propagationCtx);
   }
   get loggingMetadata() {
     return {
       clientId: this.from,
       connectedTo: this.to,
       sessionId: this.id,
-      connId: this.connection?.debugId
+      connId: this.connection?.id
     };
   }
   /**
@@ -230,6 +283,7 @@ var Session = class {
           `closing connection to ${this.to} due to inactivity (missed ${misses} heartbeats which is ${missDuration}ms)`,
           this.loggingMetadata
         );
+        this.telemetry.span.addEvent("closing connection due to inactivity");
         this.closeStaleConnection();
       }
       return;
@@ -251,32 +305,38 @@ var Session = class {
   sendBufferedMessages(conn) {
     log?.info(`resending ${this.sendBuffer.length} buffered messages`, {
       ...this.loggingMetadata,
-      connId: conn.debugId
+      connId: conn.id
     });
     for (const msg of this.sendBuffer) {
       log?.debug(`resending msg`, {
         ...this.loggingMetadata,
         fullTransportMessage: msg,
-        connId: conn.debugId
+        connId: conn.id
       });
       const ok = conn.send(this.codec.toBuffer(msg));
       if (!ok) {
         const errMsg = `failed to send buffered message to ${this.to} (sus, this is a fresh connection)`;
+        conn.telemetry?.span.setStatus({
+          code: import_api2.SpanStatusCode.ERROR,
+          message: errMsg
+        });
         log?.error(errMsg, {
           ...this.loggingMetadata,
           fullTransportMessage: msg,
-          connId: conn.debugId
+          connId: conn.id,
+          tags: ["invariant-violation"]
         });
-        throw new Error(errMsg);
+        conn.close();
+        return;
       }
     }
   }
   updateBookkeeping(ack, seq) {
     if (seq + 1 < this.ack) {
-      log?.error(
-        `received stale seq ${seq} + 1 < ${this.ack}`,
-        this.loggingMetadata
-      );
+      log?.error(`received stale seq ${seq} + 1 < ${this.ack}`, {
+        ...this.loggingMetadata,
+        tags: ["invariant-violation"]
+      });
       return;
     }
     this.sendBuffer = this.sendBuffer.filter((unacked) => unacked.seq >= ack);
@@ -446,7 +506,6 @@ var UdsConnection = class extends Connection {
 
 // transport/transport.ts
 var import_value = require("@sinclair/typebox/value");
-var import_api2 = require("@opentelemetry/api");
 
 // transport/events.ts
 var ProtocolError = {
@@ -482,11 +541,6 @@ var EventDispatcher = class {
   }
 };
 
-// tracing/index.ts
-var import_api = require("@opentelemetry/api");
-var tracer = import_api.trace.getTracer("river");
-var tracing_default = tracer;
-
 // util/stringify.ts
 function coerceErrorString(err) {
   if (err instanceof Error) {
@@ -619,6 +673,7 @@ var NaiveJsonCodec = {
 };
 
 // transport/transport.ts
+var import_api3 = require("@opentelemetry/api");
 var defaultTransportOptions = {
   heartbeatIntervalMs: 1e3,
   heartbeatsUntilDead: 2,
@@ -697,17 +752,22 @@ var Transport = class {
       status: "connect",
       conn
     });
+    conn.telemetry = createConnectionTelemetryInfo(
+      conn,
+      session.telemetry.span
+    );
     if (isReconnect) {
       session.replaceWithNewConnection(conn);
       log?.info(`reconnected to ${connectedTo}`, session.loggingMetadata);
     }
   }
-  createSession(to, conn) {
+  createSession(to, conn, propagationCtx) {
     const session = new Session(
       conn,
       this.clientId,
       to,
-      this.options
+      this.options,
+      propagationCtx
     );
     this.sessions.set(session.to, session);
     this.eventDispatcher.dispatchEvent("sessionStatus", {
@@ -716,11 +776,11 @@ var Transport = class {
     });
     return session;
   }
-  getOrCreateSession(to, conn, sessionId) {
+  getOrCreateSession(to, conn, sessionId, propagationCtx) {
     let session = this.sessions.get(to);
     let isReconnect = session !== void 0;
     if (session?.advertisedSessionId !== void 0 && sessionId !== void 0 && session.advertisedSessionId !== sessionId) {
-      log?.warn(
+      log?.info(
         `session for ${to} already exists but has a different session id (expected: ${session.advertisedSessionId}, got: ${sessionId}), creating a new one`,
         session.loggingMetadata
       );
@@ -729,7 +789,7 @@ var Transport = class {
       session = void 0;
     }
     if (!session) {
-      session = this.createSession(to, conn);
+      session = this.createSession(to, conn, propagationCtx);
       log?.info(
         `no session for ${to}, created a new one`,
         session.loggingMetadata
@@ -742,6 +802,7 @@ var Transport = class {
   }
   deleteSession(session) {
     session.close();
+    session.telemetry.span.end();
     this.sessions.delete(session.to);
     log?.info(
       `session ${session.id} disconnect from ${session.to}`,
@@ -758,12 +819,16 @@ var Transport = class {
    * @param connectedTo The peer we are connected to.
    */
   onDisconnect(conn, session) {
+    conn.telemetry?.span.end();
     this.eventDispatcher.dispatchEvent("connectionStatus", {
       status: "disconnect",
       conn
     });
     session.connection = void 0;
-    session.beginGrace(() => this.deleteSession(session));
+    session.beginGrace(() => {
+      session.telemetry.span.addEvent("session grace period expired");
+      this.deleteSession(session);
+    });
   }
   /**
    * Parses a message from a Uint8Array into a {@link OpaqueTransportMessage}.
@@ -797,9 +862,10 @@ var Transport = class {
       return;
     const session = this.sessions.get(msg.from);
     if (!session) {
-      log?.error(`(invariant violation) no existing session for ${msg.from}`, {
+      log?.error(`no existing session for ${msg.from}`, {
         clientId: this.clientId,
-        fullTransportMessage: msg
+        fullTransportMessage: msg,
+        tags: ["invariant-violation"]
       });
       return;
     }
@@ -818,9 +884,14 @@ var Transport = class {
         const errMsg = `received out-of-order msg (got seq: ${msg.seq}, wanted seq: ${session.nextExpectedSeq})`;
         log?.error(`${errMsg}, marking connection as dead`, {
           clientId: this.clientId,
-          fullTransportMessage: msg
+          fullTransportMessage: msg,
+          tags: ["invariant-violation"]
         });
         this.protocolError(ProtocolError.MessageOrderingViolated, errMsg);
+        session.telemetry.span.setStatus({
+          code: import_api3.SpanStatusCode.ERROR,
+          message: "message order violated"
+        });
         session.close();
       }
       return;
@@ -862,7 +933,8 @@ var Transport = class {
       const err = "transport is destroyed, cant send";
       log?.error(err, {
         clientId: this.clientId,
-        partialTransportMessage: msg
+        partialTransportMessage: msg,
+        tags: ["invariant-violation"]
       });
       this.protocolError(ProtocolError.UseAfterDestroy, err);
       return void 0;
@@ -947,7 +1019,7 @@ var ClientTransport = class extends Transport {
       if (!session) {
         log?.warn(
           `connection to ${to} timed out waiting for handshake, closing`,
-          { clientId: this.clientId, connectedTo: to, connId: conn.debugId }
+          { clientId: this.clientId, connectedTo: to, connId: conn.id }
         );
         conn.close();
       }
@@ -965,6 +1037,10 @@ var ClientTransport = class extends Transport {
       conn.addDataListener((data2) => {
         const parsed = this.parseMsg(data2);
         if (!parsed) {
+          conn.telemetry?.span.setStatus({
+            code: import_api3.SpanStatusCode.ERROR,
+            message: "message parse failure"
+          });
           conn.close();
           return;
         }
@@ -987,6 +1063,10 @@ var ClientTransport = class extends Transport {
       }
     });
     conn.addErrorListener((err) => {
+      conn.telemetry?.span.setStatus({
+        code: import_api3.SpanStatusCode.ERROR,
+        message: "connection error"
+      });
       log?.warn(`error in connection to ${to}: ${coerceErrorString(err)}`, {
         ...session?.loggingMetadata,
         clientId: this.clientId,
@@ -997,6 +1077,10 @@ var ClientTransport = class extends Transport {
   receiveHandshakeResponseMessage(data, conn) {
     const parsed = this.parseMsg(data);
     if (!parsed) {
+      conn.telemetry?.span.setStatus({
+        code: import_api3.SpanStatusCode.ERROR,
+        message: "non-transport message"
+      });
       this.protocolError(
         ProtocolError.HandshakeFailed,
         "received non-transport message"
@@ -1004,6 +1088,10 @@ var ClientTransport = class extends Transport {
       return false;
     }
     if (!import_value.Value.Check(ControlMessageHandshakeResponseSchema, parsed.payload)) {
+      conn.telemetry?.span.setStatus({
+        code: import_api3.SpanStatusCode.ERROR,
+        message: "invalid handshake response"
+      });
       log?.warn(`received invalid handshake resp`, {
         clientId: this.clientId,
         connectedTo: parsed.from,
@@ -1016,7 +1104,11 @@ var ClientTransport = class extends Transport {
       return false;
     }
     if (!parsed.payload.status.ok) {
-      log?.warn(`received invalid handshake resp`, {
+      conn.telemetry?.span.setStatus({
+        code: import_api3.SpanStatusCode.ERROR,
+        message: "handshake rejected"
+      });
+      log?.warn(`received handshake rejection`, {
         clientId: this.clientId,
         connectedTo: parsed.from,
         fullTransportMessage: parsed
@@ -1046,142 +1138,94 @@ var ClientTransport = class extends Transport {
    * @param to The client ID of the node to connect to.
    */
   async connect(to) {
-    return tracing_default.startActiveSpan(
-      "connect",
-      {
-        attributes: {
-          component: "river",
-          "span.kind": "client"
-        },
-        kind: import_api2.SpanKind.CLIENT
-      },
-      async (span) => {
-        try {
-          await this.connectAttempt(to);
-        } catch (e) {
-          if (e instanceof Error) {
-            span.recordException(e);
-          } else {
-            span.recordException(coerceErrorString(e));
-          }
-          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
-        } finally {
-          span.end();
-        }
+    const canProceedWithConnection = () => this.state === "open";
+    if (!canProceedWithConnection()) {
+      log?.info(
+        `transport state is no longer open, cancelling attempt to connect to ${to}`,
+        { clientId: this.clientId, connectedTo: to }
+      );
+      return;
+    }
+    let reconnectPromise = this.inflightConnectionPromises.get(to);
+    if (!reconnectPromise) {
+      const budgetConsumed = this.retryBudget.getBudgetConsumed(to);
+      if (!this.retryBudget.hasBudget(to)) {
+        const errMsg = `tried to connect to ${to} but retry budget exceeded (more than ${budgetConsumed} attempts in the last ${this.retryBudget.totalBudgetRestoreTime}ms)`;
+        log?.warn(errMsg, { clientId: this.clientId, connectedTo: to });
+        this.protocolError(ProtocolError.RetriesExceeded, errMsg);
+        return;
       }
-    );
-  }
-  async connectAttempt(to, attempt = 0) {
-    const retry = await tracing_default.startActiveSpan(
-      "connect",
-      {
-        attributes: {
-          component: "river",
-          "river.attempt": attempt,
-          "span.kind": "client"
-        },
-        kind: import_api2.SpanKind.CLIENT
-      },
-      async (span) => {
+      let sleep = Promise.resolve();
+      const backoffMs = this.retryBudget.getBackoffMs(to);
+      if (backoffMs > 0) {
+        sleep = new Promise((resolve) => setTimeout(resolve, backoffMs));
+      }
+      log?.info(`attempting connection to ${to} (${backoffMs}ms backoff)`, {
+        clientId: this.clientId,
+        connectedTo: to
+      });
+      this.retryBudget.consumeBudget(to);
+      reconnectPromise = tracing_default.startActiveSpan("connect", async (span) => {
         try {
-          const canProceedWithConnection = () => this.state === "open";
+          span.addEvent("backoff", { backoffMs });
+          await sleep;
           if (!canProceedWithConnection()) {
-            log?.info(
-              `transport state is no longer open, cancelling attempt to connect to ${to}`,
-              { clientId: this.clientId, connectedTo: to }
-            );
-            return false;
+            throw new Error("transport state is no longer open");
           }
-          let reconnectPromise = this.inflightConnectionPromises.get(to);
-          if (!reconnectPromise) {
-            const budgetConsumed = this.retryBudget.getBudgetConsumed(to);
-            if (!this.retryBudget.hasBudget(to)) {
-              const errMsg = `tried to connect to ${to} but retry budget exceeded (more than ${budgetConsumed} attempts in the last ${this.retryBudget.totalBudgetRestoreTime}ms)`;
-              log?.warn(errMsg, { clientId: this.clientId, connectedTo: to });
-              this.protocolError(ProtocolError.RetriesExceeded, errMsg);
-              return false;
-            }
-            let sleep = Promise.resolve();
-            const backoffMs = this.retryBudget.getBackoffMs(to);
-            if (backoffMs > 0) {
-              sleep = new Promise((resolve) => setTimeout(resolve, backoffMs));
-            }
-            log?.info(
-              `attempting connection to ${to} (${backoffMs}ms backoff)`,
-              {
-                clientId: this.clientId,
-                connectedTo: to
-              }
-            );
-            this.retryBudget.consumeBudget(to);
-            reconnectPromise = sleep.then(() => {
-              if (!canProceedWithConnection()) {
-                throw new Error("transport state is no longer open");
-              }
-            }).then(() => this.createNewOutgoingConnection(to)).then((conn) => {
-              if (!canProceedWithConnection()) {
-                log?.info(
-                  `transport state is no longer open, closing pre-handshake connection to ${to}`,
-                  {
-                    clientId: this.clientId,
-                    connectedTo: to,
-                    connId: conn.debugId
-                  }
-                );
-                conn.close();
-                throw new Error("transport state is no longer open");
-              }
-              return this.sendHandshake(to, conn).then((ok) => {
-                if (!ok) {
-                  conn.close();
-                  throw new Error("failed to send handshake");
-                }
-                return conn;
-              });
-            });
-            this.inflightConnectionPromises.set(to, reconnectPromise);
-          } else {
+          span.addEvent("connecting");
+          const conn = await this.createNewOutgoingConnection(to);
+          if (!canProceedWithConnection()) {
             log?.info(
-              `attempting connection to ${to} (reusing previous attempt)`,
+              `transport state is no longer open, closing pre-handshake connection to ${to}`,
               {
                 clientId: this.clientId,
-                connectedTo: to
+                connectedTo: to,
+                connId: conn.id
               }
             );
+            conn.close();
+            throw new Error("transport state is no longer open");
           }
-          try {
-            await reconnectPromise;
-          } catch (error) {
-            this.inflightConnectionPromises.delete(to);
-            const errStr = coerceErrorString(error);
-            if (!this.reconnectOnConnectionDrop || !canProceedWithConnection()) {
-              log?.warn(`connection to ${to} failed (${errStr})`, {
-                clientId: this.clientId,
-                connectedTo: to
-              });
-            } else {
-              log?.warn(`connection to ${to} failed (${errStr}), retrying`, {
-                clientId: this.clientId,
-                connectedTo: to
-              });
-              return true;
-            }
-          }
-        } catch (e) {
-          if (e instanceof Error) {
-            span.recordException(e);
-          } else {
-            span.recordException(coerceErrorString(e));
+          span.addEvent("sending handshake");
+          const ok = await this.sendHandshake(to, conn);
+          if (!ok) {
+            conn.close();
+            throw new Error("failed to send handshake");
           }
-          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+          return conn;
+        } catch (err) {
+          const errStr = coerceErrorString(err);
+          span.recordException(errStr);
+          span.setStatus({ code: import_api3.SpanStatusCode.ERROR });
+          throw err;
         } finally {
           span.end();
         }
-        return false;
+      });
+      this.inflightConnectionPromises.set(to, reconnectPromise);
+    } else {
+      log?.info(`attempting connection to ${to} (reusing previous attempt)`, {
+        clientId: this.clientId,
+        connectedTo: to
+      });
+    }
+    try {
+      await reconnectPromise;
+    } catch (error) {
+      this.inflightConnectionPromises.delete(to);
+      const errStr = coerceErrorString(error);
+      if (!this.reconnectOnConnectionDrop || !canProceedWithConnection()) {
+        log?.warn(`connection to ${to} failed (${errStr})`, {
+          clientId: this.clientId,
+          connectedTo: to
+        });
+      } else {
+        log?.warn(`connection to ${to} failed (${errStr}), retrying`, {
+          clientId: this.clientId,
+          connectedTo: to
+        });
+        return this.connect(to);
       }
-    );
-    if (retry) {
-      return this.connectAttempt(to, attempt + 1);
     }
   }
   deleteSession(session) {
@@ -1189,20 +1233,23 @@ var ClientTransport = class extends Transport {
     super.deleteSession(session);
   }
   async sendHandshake(to, conn) {
-    const tracing = { traceparent: "", tracestate: "" };
-    import_api2.propagation.inject(import_api2.context.active(), tracing);
     let metadata;
     if (this.options.handshake) {
       metadata = await this.options.handshake.get();
       if (!import_value.Value.Check(this.options.handshake.schema, metadata)) {
         log?.error(`handshake metadata did not match schema`, {
           clientId: this.clientId,
-          connectedTo: to
+          connectedTo: to,
+          tags: ["invariant-violation"]
         });
         this.protocolError(
           ProtocolError.HandshakeFailed,
           "handshake metadata did not match schema"
         );
+        conn.telemetry?.span.setStatus({
+          code: import_api3.SpanStatusCode.ERROR,
+          message: "handshake meta mismatch"
+        });
         return false;
       }
     }
@@ -1212,7 +1259,7 @@ var ClientTransport = class extends Transport {
       to,
       session.id,
       metadata,
-      tracing
+      getPropagationContext(session.telemetry.ctx)
     );
     log?.debug(`sending handshake request to ${to}`, {
       clientId: this.clientId,