@replit/river 0.19.3 → 0.20.0

package/dist/transport/index.cjs

@@ -33,6 +33,7 @@ module.exports = __toCommonJS(transport_exports);
 
 // transport/transport.ts
 var import_value = require("@sinclair/typebox/value");
+var import_api2 = require("@opentelemetry/api");
 
 // transport/message.ts
 var import_typebox = require("@sinclair/typebox");
@@ -47,6 +48,12 @@ var TransportMessageSchema = (t) => import_typebox.Type.Object({
   procedureName: import_typebox.Type.Optional(import_typebox.Type.String()),
   streamId: import_typebox.Type.String(),
   controlFlags: import_typebox.Type.Integer(),
+  tracing: import_typebox.Type.Optional(
+    import_typebox.Type.Object({
+      traceparent: import_typebox.Type.String(),
+      tracestate: import_typebox.Type.String()
+    })
+  ),
   payload: t
 });
 var ControlMessageAckSchema = import_typebox.Type.Object({
@@ -84,7 +91,7 @@ var ControlMessagePayloadSchema = import_typebox.Type.Union([
 var OpaqueTransportMessageSchema = TransportMessageSchema(
   import_typebox.Type.Unknown()
 );
-function handshakeRequestMessage(from, to, sessionId, metadata) {
+function handshakeRequestMessage(from, to, sessionId, metadata, tracing) {
   return {
     id: (0, import_nanoid.nanoid)(),
     from,
@@ -93,6 +100,7 @@ function handshakeRequestMessage(from, to, sessionId, metadata) {
     ack: 0,
     streamId: (0, import_nanoid.nanoid)(),
     controlFlags: 0,
+    tracing,
     payload: {
       type: "HANDSHAKE_REQ",
       protocolVersion: PROTOCOL_VERSION,
@@ -395,6 +403,11 @@ var Session = class {
   }
 };
 
+// tracing/index.ts
+var import_api = require("@opentelemetry/api");
+var tracer = import_api.trace.getTracer("river");
+var tracing_default = tracer;
+
 // util/stringify.ts
 function coerceErrorString(err) {
   if (err instanceof Error) {
@@ -954,82 +967,142 @@ var ClientTransport = class extends Transport {
    * @param to The client ID of the node to connect to.
    */
   async connect(to) {
-    const canProceedWithConnection = () => this.state === "open";
-    if (!canProceedWithConnection()) {
-      log?.info(
-        `transport state is no longer open, cancelling attempt to connect to ${to}`,
-        { clientId: this.clientId, connectedTo: to }
-      );
-      return;
-    }
-    let reconnectPromise = this.inflightConnectionPromises.get(to);
-    if (!reconnectPromise) {
-      const budgetConsumed = this.retryBudget.getBudgetConsumed(to);
-      if (!this.retryBudget.hasBudget(to)) {
-        const errMsg = `tried to connect to ${to} but retry budget exceeded (more than ${budgetConsumed} attempts in the last ${this.retryBudget.totalBudgetRestoreTime}ms)`;
-        log?.warn(errMsg, { clientId: this.clientId, connectedTo: to });
-        this.protocolError(ProtocolError.RetriesExceeded, errMsg);
-        return;
-      }
-      let sleep = Promise.resolve();
-      const backoffMs = this.retryBudget.getBackoffMs(to);
-      if (backoffMs > 0) {
-        sleep = new Promise((resolve) => setTimeout(resolve, backoffMs));
-      }
-      log?.info(`attempting connection to ${to} (${backoffMs}ms backoff)`, {
-        clientId: this.clientId,
-        connectedTo: to
-      });
-      this.retryBudget.consumeBudget(to);
-      reconnectPromise = sleep.then(() => {
-        if (!canProceedWithConnection()) {
-          throw new Error("transport state is no longer open");
+    return tracing_default.startActiveSpan(
+      "connect",
+      {
+        attributes: {
+          component: "river",
+          "span.kind": "client"
+        },
+        kind: import_api2.SpanKind.CLIENT
+      },
+      async (span) => {
+        try {
+          await this.connectAttempt(to);
+        } catch (e) {
+          if (e instanceof Error) {
+            span.recordException(e);
+          } else {
+            span.recordException(coerceErrorString(e));
+          }
+          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+        } finally {
+          span.end();
         }
-      }).then(() => this.createNewOutgoingConnection(to)).then((conn) => {
-        if (!canProceedWithConnection()) {
-          log?.info(
-            `transport state is no longer open, closing pre-handshake connection to ${to}`,
-            {
-              clientId: this.clientId,
-              connectedTo: to,
-              connId: conn.debugId
+      }
+    );
+  }
+  async connectAttempt(to, attempt = 0) {
+    const retry = await tracing_default.startActiveSpan(
+      "connect",
+      {
+        attributes: {
+          component: "river",
+          "river.attempt": attempt,
+          "span.kind": "client"
+        },
+        kind: import_api2.SpanKind.CLIENT
+      },
+      async (span) => {
+        try {
+          const canProceedWithConnection = () => this.state === "open";
+          if (!canProceedWithConnection()) {
+            log?.info(
+              `transport state is no longer open, cancelling attempt to connect to ${to}`,
+              { clientId: this.clientId, connectedTo: to }
+            );
+            return false;
+          }
+          let reconnectPromise = this.inflightConnectionPromises.get(to);
+          if (!reconnectPromise) {
+            const budgetConsumed = this.retryBudget.getBudgetConsumed(to);
+            if (!this.retryBudget.hasBudget(to)) {
+              const errMsg = `tried to connect to ${to} but retry budget exceeded (more than ${budgetConsumed} attempts in the last ${this.retryBudget.totalBudgetRestoreTime}ms)`;
+              log?.warn(errMsg, { clientId: this.clientId, connectedTo: to });
+              this.protocolError(ProtocolError.RetriesExceeded, errMsg);
+              return false;
             }
-          );
-          conn.close();
-          throw new Error("transport state is no longer open");
-        }
-        return this.sendHandshake(to, conn).then((ok) => {
-          if (!ok) {
-            conn.close();
-            throw new Error("failed to send handshake");
+            let sleep = Promise.resolve();
+            const backoffMs = this.retryBudget.getBackoffMs(to);
+            if (backoffMs > 0) {
+              sleep = new Promise((resolve) => setTimeout(resolve, backoffMs));
+            }
+            log?.info(
+              `attempting connection to ${to} (${backoffMs}ms backoff)`,
+              {
+                clientId: this.clientId,
+                connectedTo: to
+              }
+            );
+            this.retryBudget.consumeBudget(to);
+            reconnectPromise = sleep.then(() => {
+              if (!canProceedWithConnection()) {
+                throw new Error("transport state is no longer open");
+              }
+            }).then(() => this.createNewOutgoingConnection(to)).then((conn) => {
+              if (!canProceedWithConnection()) {
+                log?.info(
+                  `transport state is no longer open, closing pre-handshake connection to ${to}`,
+                  {
+                    clientId: this.clientId,
+                    connectedTo: to,
+                    connId: conn.debugId
+                  }
+                );
+                conn.close();
+                throw new Error("transport state is no longer open");
+              }
+              return this.sendHandshake(to, conn).then((ok) => {
+                if (!ok) {
+                  conn.close();
+                  throw new Error("failed to send handshake");
+                }
+                return conn;
+              });
+            });
+            this.inflightConnectionPromises.set(to, reconnectPromise);
+          } else {
+            log?.info(
+              `attempting connection to ${to} (reusing previous attempt)`,
+              {
+                clientId: this.clientId,
+                connectedTo: to
+              }
+            );
           }
-          return conn;
-        });
-      });
-      this.inflightConnectionPromises.set(to, reconnectPromise);
-    } else {
-      log?.info(`attempting connection to ${to} (reusing previous attempt)`, {
-        clientId: this.clientId,
-        connectedTo: to
-      });
-    }
-    try {
-      await reconnectPromise;
-    } catch (error) {
-      this.inflightConnectionPromises.delete(to);
-      const errStr = coerceErrorString(error);
-      if (!this.reconnectOnConnectionDrop || !canProceedWithConnection()) {
-        log?.warn(`connection to ${to} failed (${errStr})`, {
-          clientId: this.clientId,
-          connectedTo: to
-        });
-      } else {
-        log?.warn(`connection to ${to} failed (${errStr}), retrying`, {
-          clientId: this.clientId,
-          connectedTo: to
-        });
-        return this.connect(to);
+          try {
+            await reconnectPromise;
+          } catch (error) {
+            this.inflightConnectionPromises.delete(to);
+            const errStr = coerceErrorString(error);
+            if (!this.reconnectOnConnectionDrop || !canProceedWithConnection()) {
+              log?.warn(`connection to ${to} failed (${errStr})`, {
+                clientId: this.clientId,
+                connectedTo: to
+              });
+            } else {
+              log?.warn(`connection to ${to} failed (${errStr}), retrying`, {
+                clientId: this.clientId,
+                connectedTo: to
+              });
+              return true;
+            }
+          }
+        } catch (e) {
+          if (e instanceof Error) {
+            span.recordException(e);
+          } else {
+            span.recordException(coerceErrorString(e));
+          }
+          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+        } finally {
+          span.end();
+        }
+        return false;
       }
+    );
+    if (retry) {
+      return this.connectAttempt(to, attempt + 1);
     }
   }
   deleteSession(session) {
@@ -1037,6 +1110,8 @@ var ClientTransport = class extends Transport {
     super.deleteSession(session);
   }
   async sendHandshake(to, conn) {
+    const tracing = { traceparent: "", tracestate: "" };
+    import_api2.propagation.inject(import_api2.context.active(), tracing);
     let metadata;
     if (this.options.handshake) {
       metadata = await this.options.handshake.get();
@@ -1057,7 +1132,8 @@ var ClientTransport = class extends Transport {
       this.clientId,
       to,
       session.id,
-      metadata
+      metadata,
+      tracing
     );
     log?.debug(`sending handshake request to ${to}`, {
       clientId: this.clientId,
@@ -1088,78 +1164,100 @@ var ServerTransport = class extends Transport {
     });
   }
   handleConnection(conn) {
-    if (this.state !== "open")
-      return;
-    log?.info(`new incoming connection`, {
-      clientId: this.clientId,
-      connId: conn.debugId
-    });
-    let session = void 0;
-    const client = () => session?.to ?? "unknown";
-    const handshakeTimeout = setTimeout(() => {
-      if (!session) {
-        log?.warn(
-          `connection to ${client()} timed out waiting for handshake, closing`,
-          {
-            clientId: this.clientId,
-            connectedTo: client(),
-            connId: conn.debugId
-          }
-        );
-        conn.close();
-      }
-    }, this.options.sessionDisconnectGraceMs);
-    const buffer = [];
-    let receivedHandshakeMessage = false;
-    const handshakeHandler = (data) => {
-      if (receivedHandshakeMessage) {
-        buffer.push(data);
-        return;
-      }
-      receivedHandshakeMessage = true;
-      clearTimeout(handshakeTimeout);
-      void this.receiveHandshakeRequestMessage(data, conn).then(
-        (maybeSession) => {
-          if (!maybeSession) {
+    tracing_default.startActiveSpan(
+      "handleConnection",
+      {
+        attributes: {
+          component: "river",
+          "span.kind": "server"
+        },
+        kind: import_api2.SpanKind.SERVER
+      },
+      (span) => {
+        if (this.state !== "open")
+          return;
+        log?.info(`new incoming connection`, {
+          clientId: this.clientId,
+          connId: conn.debugId
+        });
+        let session = void 0;
+        const client = () => session?.to ?? "unknown";
+        const handshakeTimeout = setTimeout(() => {
+          if (!session) {
+            log?.warn(
+              `connection to ${client()} timed out waiting for handshake, closing`,
+              {
+                clientId: this.clientId,
+                connectedTo: client(),
+                connId: conn.debugId
+              }
+            );
+            span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+            span.end();
             conn.close();
+          }
+        }, this.options.sessionDisconnectGraceMs);
+        const buffer = [];
+        let receivedHandshakeMessage = false;
+        const handshakeHandler = (data) => {
+          if (receivedHandshakeMessage) {
+            buffer.push(data);
             return;
           }
-          session = maybeSession;
-          const dataHandler = (data2) => {
-            const parsed = this.parseMsg(data2);
-            if (!parsed) {
-              conn.close();
-              return;
+          receivedHandshakeMessage = true;
+          clearTimeout(handshakeTimeout);
+          void this.receiveHandshakeRequestMessage(data, conn).then(
+            (maybeSession) => {
+              if (!maybeSession) {
+                span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+                span.end();
+                conn.close();
+                return;
+              }
+              session = maybeSession;
+              const dataHandler = (data2) => {
+                const parsed = this.parseMsg(data2);
+                if (!parsed) {
+                  conn.close();
+                  return;
+                }
+                this.handleMsg(parsed);
+              };
+              conn.removeDataListener(handshakeHandler);
+              conn.addDataListener(dataHandler);
+              for (const data2 of buffer) {
+                dataHandler(data2);
+              }
+              buffer.length = 0;
             }
-            this.handleMsg(parsed);
-          };
-          conn.removeDataListener(handshakeHandler);
-          conn.addDataListener(dataHandler);
-          for (const data2 of buffer) {
-            dataHandler(data2);
+          );
+        };
+        conn.addDataListener(handshakeHandler);
+        conn.addCloseListener(() => {
+          if (session) {
+            log?.info(`connection to ${client()} disconnected`, {
+              clientId: this.clientId,
+              connId: conn.debugId
+            });
+            this.onDisconnect(conn, session);
           }
-          buffer.length = 0;
-        }
-      );
-    };
-    conn.addDataListener(handshakeHandler);
-    conn.addCloseListener(() => {
-      if (!session)
-        return;
-      log?.info(`connection to ${client()} disconnected`, {
-        clientId: this.clientId,
-        connId: conn.debugId
-      });
-      this.onDisconnect(conn, session);
-    });
-    conn.addErrorListener((err) => {
-      if (!session)
-        return;
-      log?.warn(
-        `connection to ${client()} got an error: ${coerceErrorString(err)}`,
-        { clientId: this.clientId, connId: conn.debugId }
-      );
-    });
+          span.setStatus({ code: import_api2.SpanStatusCode.OK });
+          span.end();
+        });
+        conn.addErrorListener((err) => {
+          if (session) {
+            log?.warn(
+              `connection to ${client()} got an error: ${coerceErrorString(
+                err
+              )}`,
+              { clientId: this.clientId, connId: conn.debugId }
+            );
+          }
+          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+          span.end();
+        });
+      }
+    );
   }
   async receiveHandshakeRequestMessage(data, conn) {
     const parsed = this.parseMsg(data);
@@ -1170,117 +1268,163 @@ var ServerTransport = class extends Transport {
       );
       return false;
     }
-    if (!import_value.Value.Check(ControlMessageHandshakeRequestSchema, parsed.payload)) {
-      const reason = "received invalid handshake msg";
-      const responseMsg2 = handshakeResponseMessage(this.clientId, parsed.from, {
-        ok: false,
-        reason
-      });
-      conn.send(this.codec.toBuffer(responseMsg2));
-      const logData = typeof parsed.payload === "object" ? { ...parsed, payload: { ...parsed.payload, metadata: "redacted" } } : { ...parsed };
-      log?.warn(`${reason}: ${JSON.stringify(logData)}`, {
-        clientId: this.clientId,
-        connId: conn.debugId
-      });
-      this.protocolError(
-        ProtocolError.HandshakeFailed,
-        "invalid handshake request"
-      );
-      return false;
-    }
-    const gotVersion = parsed.payload.protocolVersion;
-    if (gotVersion !== PROTOCOL_VERSION) {
-      const reason = `incorrect version (got: ${gotVersion} wanted ${PROTOCOL_VERSION})`;
-      const responseMsg2 = handshakeResponseMessage(this.clientId, parsed.from, {
-        ok: false,
-        reason
-      });
-      conn.send(this.codec.toBuffer(responseMsg2));
-      log?.warn(
-        `received handshake msg with incompatible protocol version (got: ${gotVersion}, expected: ${PROTOCOL_VERSION})`,
-        { clientId: this.clientId, connId: conn.debugId }
-      );
-      this.protocolError(ProtocolError.HandshakeFailed, reason);
-      return false;
+    let activeContext = import_api2.context.active();
+    if (parsed.tracing) {
+      activeContext = import_api2.propagation.extract(activeContext, parsed.tracing);
     }
-    const { session, isReconnect } = this.getOrCreateSession(
-      parsed.from,
-      conn,
-      parsed.payload.sessionId
-    );
-    let handshakeMetadata;
-    if (this.options.handshake) {
-      if (!import_value.Value.Check(
-        this.options.handshake.requestSchema,
-        parsed.payload.metadata
-      )) {
-        const reason = "received malformed handshake metadata";
-        const responseMsg2 = handshakeResponseMessage(
-          this.clientId,
+    return tracing_default.startActiveSpan(
+      "receiveHandshakeRequestMessage",
+      {
+        attributes: {
+          component: "river",
+          "span.kind": "server"
+        },
+        kind: import_api2.SpanKind.SERVER
+      },
+      activeContext,
+      async (span) => {
+        if (!import_value.Value.Check(ControlMessageHandshakeRequestSchema, parsed.payload)) {
+          const reason = "received invalid handshake msg";
+          const responseMsg2 = handshakeResponseMessage(
+            this.clientId,
+            parsed.from,
+            {
+              ok: false,
+              reason
+            }
+          );
+          conn.send(this.codec.toBuffer(responseMsg2));
+          const logData = typeof parsed.payload === "object" ? {
+            ...parsed,
+            payload: { ...parsed.payload, metadata: "redacted" }
+          } : { ...parsed };
+          log?.warn(`${reason}: ${JSON.stringify(logData)}`, {
+            clientId: this.clientId,
+            connId: conn.debugId
+          });
+          this.protocolError(
+            ProtocolError.HandshakeFailed,
+            "invalid handshake request"
+          );
+          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+          span.end();
+          return false;
+        }
+        const gotVersion = parsed.payload.protocolVersion;
+        if (gotVersion !== PROTOCOL_VERSION) {
+          const reason = `incorrect version (got: ${gotVersion} wanted ${PROTOCOL_VERSION})`;
+          const responseMsg2 = handshakeResponseMessage(
+            this.clientId,
+            parsed.from,
+            {
+              ok: false,
+              reason
+            }
+          );
+          conn.send(this.codec.toBuffer(responseMsg2));
+          log?.warn(
+            `received handshake msg with incompatible protocol version (got: ${gotVersion}, expected: ${PROTOCOL_VERSION})`,
+            { clientId: this.clientId, connId: conn.debugId }
+          );
+          this.protocolError(ProtocolError.HandshakeFailed, reason);
+          span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+          span.end();
+          return false;
+        }
+        const { session, isReconnect } = this.getOrCreateSession(
           parsed.from,
-          { ok: false, reason }
+          conn,
+          parsed.payload.sessionId
         );
-        conn.send(this.codec.toBuffer(responseMsg2));
-        log?.warn(`received malformed handshake metadata from ${parsed.from}`, {
-          clientId: this.clientId,
-          connId: conn.debugId
-        });
-        this.protocolError(ProtocolError.HandshakeFailed, reason);
-        this.deleteSession(session);
-        return false;
-      }
-      const parsedMetadata = await this.options.handshake.parse(
-        parsed.payload.metadata,
-        session,
-        isReconnect
-      );
-      if (parsedMetadata === false) {
-        const reason = "rejected by server";
-        const responseMsg2 = handshakeResponseMessage(
-          this.clientId,
-          parsed.from,
-          { ok: false, reason }
+        let handshakeMetadata;
+        if (this.options.handshake) {
+          if (!import_value.Value.Check(
+            this.options.handshake.requestSchema,
+            parsed.payload.metadata
+          )) {
+            const reason = "received malformed handshake metadata";
+            const responseMsg2 = handshakeResponseMessage(
+              this.clientId,
+              parsed.from,
+              { ok: false, reason }
+            );
+            conn.send(this.codec.toBuffer(responseMsg2));
+            log?.warn(
+              `received malformed handshake metadata from ${parsed.from}`,
+              {
+                clientId: this.clientId,
+                connId: conn.debugId
+              }
+            );
+            this.protocolError(ProtocolError.HandshakeFailed, reason);
+            this.deleteSession(session);
+            span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+            span.end();
+            return false;
+          }
+          const parsedMetadata = await this.options.handshake.parse(
+            parsed.payload.metadata,
+            session,
+            isReconnect
+          );
+          if (parsedMetadata === false) {
+            const reason = "rejected by server";
+            const responseMsg2 = handshakeResponseMessage(
+              this.clientId,
+              parsed.from,
+              { ok: false, reason }
+            );
+            conn.send(this.codec.toBuffer(responseMsg2));
+            log?.warn(`rejected handshake from ${parsed.from}`, {
+              clientId: this.clientId,
+              connId: conn.debugId
+            });
+            this.protocolError(ProtocolError.HandshakeFailed, reason);
+            this.deleteSession(session);
+            span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+            span.end();
+            return false;
+          }
+          if (!import_value.Value.Check(this.options.handshake.parsedSchema, parsedMetadata)) {
+            const reason = "failed to parse handshake metadata";
+            const responseMsg2 = handshakeResponseMessage(
+              this.clientId,
+              parsed.from,
+              { ok: false, reason }
+            );
+            conn.send(this.codec.toBuffer(responseMsg2));
+            log?.error(`failed to parse handshake metadata`, {
+              clientId: this.clientId,
+              connId: conn.debugId
+            });
+            this.protocolError(ProtocolError.HandshakeFailed, reason);
+            this.deleteSession(session);
+            span.setStatus({ code: import_api2.SpanStatusCode.ERROR });
+            span.end();
+            return false;
+          }
+          handshakeMetadata = parsedMetadata;
+        }
+        handshakeMetadata ??= {};
+        session.metadata = handshakeMetadata;
+        log?.debug(
+          `handshake from ${parsed.from} ok, responding with handshake success`,
+          { clientId: this.clientId, connId: conn.debugId }
         );
-        conn.send(this.codec.toBuffer(responseMsg2));
-        log?.warn(`rejected handshake from ${parsed.from}`, {
-          clientId: this.clientId,
-          connId: conn.debugId
-        });
-        this.protocolError(ProtocolError.HandshakeFailed, reason);
-        this.deleteSession(session);
-        return false;
-      }
-      if (!import_value.Value.Check(this.options.handshake.parsedSchema, parsedMetadata)) {
-        const reason = "failed to parse handshake metadata";
-        const responseMsg2 = handshakeResponseMessage(
+        const responseMsg = handshakeResponseMessage(
           this.clientId,
           parsed.from,
-          { ok: false, reason }
+          {
+            ok: true,
+            sessionId: session.id
+          }
         );
-        conn.send(this.codec.toBuffer(responseMsg2));
-        log?.error(`failed to parse handshake metadata`, {
-          clientId: this.clientId,
-          connId: conn.debugId
-        });
-        this.protocolError(ProtocolError.HandshakeFailed, reason);
-        this.deleteSession(session);
-        return false;
+        conn.send(this.codec.toBuffer(responseMsg));
+        this.onConnect(conn, parsed.from, session, isReconnect);
+        span.end();
+        return session;
       }
-      handshakeMetadata = parsedMetadata;
-    }
-    handshakeMetadata ??= {};
-    session.metadata = handshakeMetadata;
-    log?.debug(
-      `handshake from ${parsed.from} ok, responding with handshake success`,
-      { clientId: this.clientId, connId: conn.debugId }
     );
-    const responseMsg = handshakeResponseMessage(this.clientId, parsed.from, {
-      ok: true,
-      sessionId: session.id
-    });
-    conn.send(this.codec.toBuffer(responseMsg));
-    this.onConnect(conn, parsed.from, session, isReconnect);
-    return session;
   }
 };
 // Annotate the CommonJS export names for ESM import in node: