@ciphyrshq/sdk 2.6.0 → 3.0.1

package/src/tracer.js

@@ -1,4 +1,15 @@
-import { randomUUID } from 'node:crypto';
+import { randomBytes } from 'node:crypto';
+import { availableParallelism } from 'node:os';
+import { spanStorage, traceStorage } from './context.js';
+import { extract, registerInternalOrigin, setDefaultProject, autoInstrument } from './propagation.js';
+import { remoteParent } from './context.js';
+
+// W3C-shaped ids (32 / 16 hex). `traceparent` accepts nothing else, and
+// generating them in that shape is what lets the header carry the REAL ids
+// instead of a hash of them — so an OpenTelemetry-instrumented peer links to
+// this exact trace. (randomUUID's dashes made it unusable as a W3C id.)
+const newTraceId = () => randomBytes(16).toString('hex');
+const newSpanId  = () => randomBytes(8).toString('hex');
 
 // ─────────────────────────────────────────────────────────────────────────────
 // CiphyrsTracer — buffered span/metric collection with auto-flush
@@ -12,6 +23,14 @@ export class CiphyrsTracer {
   #spanBuffer = [];
   #metricBuffer = [];
   #flushInterval;
+  #heartbeatInterval;
+  #knownAgents = new Set();
+  #spansSinceBeat = 0;
+  #errorsSinceBeat = 0;
+  #startedAt = Date.now();
+  #lastCpu = null;
+  #lastCpuAt = null;
+  #lastLag = null;
 
   /**
    * @param {import('./client.js').CiphyrsClient} client
@@ -19,29 +38,160 @@ export class CiphyrsTracer {
    * @param {string}  opts.projectName      — Project name for grouping traces
    * @param {number}  [opts.flushIntervalMs=5000] — Auto-flush interval in ms
    * @param {number}  [opts.batchSize=50]   — Flush when buffer reaches this size
+   * @param {string}  [opts.agentName]      — The agent this process runs. Optional
+   *   (agents are also discovered from your spans), but naming it here makes the
+   *   agent and its heartbeat visible in the fleet before it handles a request.
+   * @param {number}  [opts.heartbeatIntervalMs=60000] — Liveness beat for every
+   *   agent this tracer has seen. 0 disables. The interval is sent with each
+   *   beat so the server sizes THIS agent's down window to it rather than
+   *   applying one global threshold to the whole fleet.
+   * @param {boolean} [opts.heartbeatMetrics=true] — Ship process metrics (RSS,
+   *   event-loop lag, error rate) with each beat, so the fleet can show
+   *   `degraded` before `down`.
+   * @param {boolean} [opts.propagate=true] — Inject W3C traceparent/baggage into
+   *   outbound fetch/http calls made inside a span, so agents in other processes
+   *   join the same trace and the topology edge forms. See propagation.js.
    */
-  constructor(client, { projectName, flushIntervalMs = 5000, batchSize = 50 } = {}) {
+  constructor(client, {
+    projectName, flushIntervalMs = 5000, batchSize = 50,
+    agentName = null, heartbeatIntervalMs = 60_000, heartbeatMetrics = true,
+    propagate = true,
+  } = {}) {
     this.#client = client;
     this.#projectName = projectName;
-    this.#config = { flushIntervalMs, batchSize };
+    this.#config = { flushIntervalMs, batchSize, agentName, heartbeatIntervalMs, heartbeatMetrics };
+    if (agentName) this.#knownAgents.add(agentName);
     this.#flushInterval = setInterval(() => this.flush().catch(() => {}), flushIntervalMs);
     // Prevent the interval from keeping the process alive
     if (this.#flushInterval.unref) this.#flushInterval.unref();
+
+    if (propagate) {
+      setDefaultProject(projectName);
+      if (client?._baseUrl) registerInternalOrigin(client._baseUrl);
+      autoInstrument();
+    }
+    if (heartbeatIntervalMs > 0 && typeof this.#client?.reportHeartbeat === 'function') {
+      this.#startHeartbeat(heartbeatIntervalMs);
+    }
+  }
+
+  /** Agents this tracer has seen (configured, or observed in a span). */
+  get knownAgents() { return [...this.#knownAgents]; }
+
+  #startHeartbeat(intervalMs) {
+    // Heartbeats did not exist in this SDK at all, so a Node agent could never
+    // report liveness: the fleet had to infer health from span traffic, which
+    // makes a quiet agent indistinguishable from a dead one. On by default, and
+    // the first beat is immediate so a process shows up at start-up rather than
+    // one interval later.
+    const beat = () => {
+      for (const agentName of this.#knownAgents) {
+        this.#client.reportHeartbeat({
+          agentName,
+          status: 'up',
+          projectName: this.#projectName,
+          heartbeatIntervalS: Math.round(intervalMs / 1000),
+          metrics: this.#config.heartbeatMetrics ? this.collectMetrics() : undefined,
+        }).catch(() => {});   // a beat must never take down the agent
+      }
+    };
+    setTimeout(beat, 0).unref?.();
+    this.#heartbeatInterval = setInterval(beat, intervalMs);
+    if (this.#heartbeatInterval.unref) this.#heartbeatInterval.unref();
   }
 
   /**
-   * Start a new trace.
+   * Process metrics to ship with a heartbeat. The server turns a
+   * struggling-but-alive process into `degraded`, so the fleet shows trouble
+   * BEFORE the agent stops answering. Only keys the server whitelists are sent.
+   */
+  collectMetrics() {
+    const m = {
+      uptime_s: Math.round((Date.now() - this.#startedAt) / 100) / 10,
+      runtime: `node ${process.versions?.node || '?'}`,
+    };
+    const spans = this.#spansSinceBeat;
+    const errors = this.#errorsSinceBeat;
+    this.#spansSinceBeat = 0;
+    this.#errorsSinceBeat = 0;
+    m.spans_since_last = spans;
+    m.errors_since_last = errors;
+    if (spans > 0) m.error_rate = Math.round((errors / spans) * 1000) / 1000;
+    try {
+      m.rss_mb = Math.round((process.memoryUsage().rss / (1024 * 1024)) * 10) / 10;
+      m.heap_mb = Math.round((process.memoryUsage().heapUsed / (1024 * 1024)) * 10) / 10;
+    } catch { /* not Node — skip */ }
+    try {
+      // Event-loop lag: how late a zero-delay timer actually ran. This is the
+      // signal that separates "busy" from "wedged" in a Node agent.
+      const lag = this.#lastLag;
+      if (typeof lag === 'number') m.event_loop_lag_ms = Math.round(lag * 100) / 100;
+      const t0 = process.hrtime.bigint();
+      setTimeout(() => {
+        this.#lastLag = Number(process.hrtime.bigint() - t0) / 1e6;
+      }, 0).unref?.();
+    } catch { /* skip */ }
+    try {
+      // CPU is a RATE, so it needs a baseline and a window wide enough to
+      // divide by. The first beat fires immediately at start-up, where the
+      // window is ~0ms and cumulative CPU is not: dividing them reported
+      // 1371% and, because a high cpu_pct is one of the server's degraded
+      // signals, marked a perfectly healthy agent degraded the moment it
+      // booted. So: no baseline or no real window means no number.
+      const now = Date.now();
+      const prev = this.#lastCpu;
+      const prevAt = this.#lastCpuAt;
+      this.#lastCpu = process.cpuUsage();
+      this.#lastCpuAt = now;
+      if (prev && prevAt && now - prevAt >= 500) {
+        const usage = process.cpuUsage(prev);
+        // Normalised by core count, so 100 means "this box is saturated"
+        // rather than "one thread is busy" — a 16-core host legitimately
+        // reports 800% per-core for work that is using half the machine.
+        const cores = Math.max(1, availableParallelism());
+        const windowUs = (now - prevAt) * 1000 * cores;
+        const pct = ((usage.user + usage.system) / windowUs) * 100;
+        m.cpu_pct = Math.min(100, Math.max(0, Math.round(pct * 10) / 10));
+      }
+    } catch { /* skip */ }
+    return m;
+  }
+
+  /**
+   * Start a new trace — or CONTINUE the caller's, when this process is serving
+   * a request that carried trace context (expressMiddleware, fastifyPlugin, or
+   * an explicit `withRemoteContext(headers, fn)`). Without continuation every
+   * service began its own trace and the graph showed disconnected agents even
+   * with headers flowing correctly.
+   *
    * @param {string} name — Human-readable trace name
    * @param {object} [opts]
    * @param {string} [opts.traceId] — Provide your own trace ID, or one is generated
+   * @param {object} [opts.headers] — Inbound request headers, to continue their trace
    * @returns {Trace}
    */
   trace(name, opts = {}) {
     return new Trace(this, name, opts);
   }
 
+  /**
+   * Run `fn` inside a trace, with the trace active for everything it awaits.
+   * The scoped form — prefer it when you can:
+   *
+   *   await tracer.withTrace('handle order', async (t) => {
+   *     await t.span('RouterAgent').run(async () => { … })
+   *   })
+   */
+  async withTrace(name, fn, opts = {}) {
+    const t = this.trace(name, opts);
+    return traceStorage.run({ trace_id: t.traceId, name }, () => fn(t));
+  }
+
   /** @internal — called by Span.end() to enqueue completed spans */
   _enqueueSpan(span) {
+    if (span?.agent_name) this.#knownAgents.add(span.agent_name);
+    this.#spansSinceBeat++;
+    if (span?.status === 'error') this.#errorsSinceBeat++;
     this.#spanBuffer.push(span);
     if (this.#spanBuffer.length >= this.#config.batchSize) {
       this.flush().catch(() => {});
@@ -85,7 +235,8 @@ export class CiphyrsTracer {
             { name: this.#projectName },
             { trace_id: traceId, name: traceSpans[0]?.trace_name || traceId },
             traceSpans,
-          ).catch(err => console.error('Ciphyrs flush error:', err.message)),
+          ).catch(err => this.#requeue(this.#spanBuffer, traceSpans,
+            `trace ${traceId}`, err)),
         );
       }
     }
@@ -95,19 +246,53 @@ export class CiphyrsTracer {
         this.#client._request(`${this.#client._baseUrl}/v1/observe/metrics`, {
           method: 'POST',
           body: { metrics },
-        }).catch(err => console.error('Ciphyrs metrics flush error:', err.message)),
+        }).catch(err => this.#requeue(this.#metricBuffer, metrics, 'metrics', err)),
       );
     }
 
     await Promise.allSettled(promises);
   }
 
+  /**
+   * Put a failed batch back rather than dropping it.
+   *
+   * The buffers are spliced empty BEFORE the send, so the previous `.catch`
+   * that only logged meant every failure the client's retries could not
+   * absorb destroyed that batch for good. The customer saw one line of
+   * console noise and a permanent hole in their traces.
+   *
+   * sdk-node's client.js already did it this way; this brings the published
+   * SDK in line. APPEND rather than prepend: spans are posted one trace at a
+   * time, so with traces A and B both failing, prepending would leave the
+   * buffer holding B before A and hand a recovered collector a reversed
+   * timeline.
+   */
+  #requeue(buffer, items, what, err) {
+    buffer.push(...items);
+    // Bounded, or an unreachable collector grows this without limit inside
+    // the customer's process. Oldest go first: during an incident the recent
+    // spans are the ones worth keeping.
+    const cap = Math.max(this.#config.batchSize * 20, 1000);
+    let dropped = 0;
+    if (buffer.length > cap) {
+      dropped = buffer.length - cap;
+      buffer.splice(0, dropped);
+    }
+    console.error(
+      `[ciphyrs] ${what}: flush failed (${err.message}); `
+      + (dropped
+        ? `buffer full, dropped ${dropped} oldest item(s)`
+        : `${items.length} item(s) re-queued for the next flush`),
+    );
+  }
+
   /**
    * Stop the auto-flush timer and flush any remaining data.
    * Always call this before process exit to avoid data loss.
    */
   async shutdown() {
     clearInterval(this.#flushInterval);
+    if (this.#heartbeatInterval) clearInterval(this.#heartbeatInterval);
     await this.flush();
   }
 }
@@ -121,16 +306,41 @@ export class Trace {
   #traceId;
   #name;
   #startTime;
+  #remoteParentSpanId;
+  #remoteAgent;
+  #ownSpanIds = new Set();
 
   constructor(tracer, name, opts = {}) {
     this.#tracer = tracer;
-    this.#traceId = opts.traceId || randomUUID();
+    // Continue the caller's trace when one is active for this async context
+    // (or was handed in as `opts.headers`), unless an explicit traceId wins.
+    const remote = opts.headers ? extract(opts.headers) : remoteParent();
+    this.#traceId = opts.traceId || remote?.trace_id || newTraceId();
+    this.#remoteParentSpanId = (!opts.traceId && remote?.span_id) || null;
+    this.#remoteAgent = remote?.agent_name || null;
     this.#name = name;
     this.#startTime = Date.now();
   }
 
   get traceId() { return this.#traceId; }
   get name()    { return this.#name; }
+  /** True when this trace continues one started by another agent/process. */
+  get isContinuation() { return this.#remoteParentSpanId !== null; }
+  /** The agent that called us, when the inbound request named one. */
+  get callerAgent() { return this.#remoteAgent; }
+  /** @internal — Span.end() reports back so sibling spans can nest correctly. */
+  _ownSpan(id) { this.#ownSpanIds.add(id); }
+  /** @internal — the parent a new span should use when none was given. */
+  _defaultParent() {
+    const ambient = spanStorage.getStore();
+    if (ambient?.span_id && this.#ownSpanIds.has(ambient.span_id)) return ambient.span_id;
+    return this.#remoteParentSpanId;
+  }
+
+  /** Run `fn` with this trace active for everything it awaits. */
+  run(fn) {
+    return traceStorage.run({ trace_id: this.#traceId, name: this.#name }, () => fn(this));
+  }
 
   /**
    * Create a new span within this trace.
@@ -142,10 +352,20 @@ export class Trace {
    * @returns {Span}
    */
   span(name, opts = {}) {
-    return new Span(this.#tracer, this.#traceId, this.#name, name, opts);
+    const parentSpanId = opts.parentSpanId !== undefined ? opts.parentSpanId : this._defaultParent();
+    const s = new Span(this.#tracer, this.#traceId, this.#name, name, { ...opts, parentSpanId });
+    this._ownSpan(s.spanId);
+    // Auto-enter so the SDK's long-standing style nests without a callback:
+    //   const outer = t.span('Router'); const inner = t.span('Billing');
+    // `inner` is a child of `outer`, and an HTTP call made between them
+    // carries `outer`'s (then `inner`'s) trace headers. end() unwinds.
+    // Pass `{ scoped: true }` to opt out and manage context with run().
+    if (opts.scoped !== true) s.enter();
+    return s;
   }
 }
 
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Span — a single timed operation within a trace
 // ─────────────────────────────────────────────────────────────────────────────
@@ -154,12 +374,15 @@ export class Span {
   #tracer;
   #data;
   #startTime;
+  #ended = false;
+  #entered = false;
+  #previousStore = undefined;
 
   constructor(tracer, traceId, traceName, name, opts = {}) {
     this.#tracer = tracer;
     this.#startTime = Date.now();
     this.#data = {
-      span_id: randomUUID(),
+      span_id: newSpanId(),
       trace_id: traceId,
       trace_name: traceName,
       parent_span_id: opts.parentSpanId || null,
@@ -177,6 +400,49 @@ export class Span {
   }
 
   get spanId() { return this.#data.span_id; }
+  get agentName() { return this.#data.agent_name; }
+
+  /**
+   * Run `fn` with this span active, then end it — the scoped form, and the one
+   * to prefer: everything `fn` awaits sees this span as its parent, and an
+   * outbound HTTP call made inside it carries this span's trace headers.
+   * The span is ended (and marked errored) even if `fn` throws.
+   */
+  async run(fn) {
+    // `kind` rides in the ambient store so a caller inside this span can tell
+    // WHAT span it is in, not merely that one is open. protectTool needs that
+    // distinction: the gateway matches a tool span to its gate call by span
+    // id, so sending the id of an enclosing AGENT span can never match while
+    // still telling the gateway this producer is correlated.
+    const store = { trace_id: this.#data.trace_id, span_id: this.#data.span_id, agent_name: this.#data.agent_name, kind: this.#data.kind };
+    try {
+      return await spanStorage.run(store, () => fn(this));
+    } catch (err) {
+      this.setError(err?.message || String(err));
+      throw err;
+    } finally {
+      if (!this.#ended) this.end();
+    }
+  }
+
+  /**
+   * Make this span the ambient one for the rest of the current async context,
+   * without a callback. This is what keeps the SDK's long-standing
+   * `const s = trace.span(); … s.end()` style nesting correctly — `end()`
+   * restores whatever was ambient before.
+   */
+  enter() {
+    if (this.#entered) return this;
+    this.#entered = true;
+    this.#previousStore = spanStorage.getStore();
+    spanStorage.enterWith({
+      trace_id: this.#data.trace_id, span_id: this.#data.span_id, agent_name: this.#data.agent_name,
+      // Same store shape as run() above — the unscoped style must not be the
+      // one that leaves protectTool unable to tell a tool span from an agent span.
+      kind: this.#data.kind,
+    });
+    return this;
+  }
 
   /** Set the input text for this span. */
   setInput(text)  { this.#data.input_text = text; return this; }
@@ -211,9 +477,12 @@ export class Span {
    * @returns {this}
    */
   end() {
+    if (this.#ended) return this;      // double end() must not double-count
+    this.#ended = true;
     this.#data.started_at = new Date(this.#startTime).toISOString();
     this.#data.ended_at = new Date().toISOString();
     this.#data.duration_ms = Date.now() - this.#startTime;
+    if (this.#entered) spanStorage.enterWith(this.#previousStore);
     this.#tracer._enqueueSpan(this.#data);
     return this;
   }