@ciphyrshq/sdk 2.6.0 → 3.0.0

package/src/propagation.test.js

@@ -0,0 +1,429 @@
+/**
+ * Cross-process trace propagation — the mechanism the topology graph runs on.
+ * Run with: npm test
+ *
+ * WHAT WENT WRONG. The server derives the graph from parent links: an edge
+ * A → B exists because a span of agent B names a span of agent A as its
+ * parent. This SDK never wrote `traceparent` on an outbound call and never
+ * read it on an inbound one, and `parentSpanId` was a hand-threaded option
+ * nobody threaded — so every span arrived parentless and a fleet of agents
+ * that talked constantly rendered as disconnected dots. Assertions here are
+ * about the wire: the exact headers a caller emits, and the exact trace_id /
+ * parent_span_id the callee then ships.
+ *
+ * W3C SHAPE IS LOAD-BEARING. `traceparent` accepts only 32-hex trace ids and
+ * 16-hex span ids; randomUUID's dashed form is illegal there. The tracer now
+ * generates ids in that shape so the header carries the REAL ids and an
+ * OpenTelemetry-instrumented peer joins this exact trace rather than a hash
+ * of it.
+ */
+// No test in this package may open a socket — see no-network.test-helper.js.
+// Imported per file because node --test runs each file in its own process.
+import './no-network.test-helper.js'
+import { describe, it, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+import { CiphyrsTracer } from './tracer.js'
+import {
+  inject, extract, withRemoteContext, currentContext,
+  instrumentFetch, w3cTraceId, w3cSpanId, _parseBaggage,
+  registerInternalOrigin, expressMiddleware, withPropagation,
+} from './propagation.js'
+import { activeIds, activeAgent } from './context.js'
+
+const HEX32 = /^[0-9a-f]{32}$/
+const HEX16 = /^[0-9a-f]{16}$/
+
+function fakeClient() {
+  const calls = { ingest: [], heartbeats: [] }
+  return {
+    calls,
+    _baseUrl: 'https://gw.test',
+    trace: {
+      ingest: async (project, trace, spans) => { calls.ingest.push({ project, trace, spans }); return { ok: true } },
+    },
+    _request: async () => ({ ok: true }),
+    reportHeartbeat: async (args) => { calls.heartbeats.push(args); return { ok: true } },
+  }
+}
+
+// heartbeatIntervalMs 0 keeps the beat timer out of tests that don't want it.
+const tracerFor = (client, opts = {}) => new CiphyrsTracer(client, {
+  projectName: 'billing', flushIntervalMs: 3_600_000, heartbeatIntervalMs: 0, ...opts,
+})
+
+describe('id shapes', () => {
+  it('generates W3C-legal trace and span ids', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    const tr = t.trace('run')
+    const s = tr.span('A')
+    assert.match(tr.traceId, HEX32, 'trace id must be 32 hex for traceparent')
+    assert.match(s.spanId, HEX16, 'span id must be 16 hex for traceparent')
+    s.end()
+  })
+
+  it('hashes a non-hex id into the W3C fields instead of emitting an illegal header', () => {
+    assert.match(w3cTraceId('tr_legacy'), HEX32)
+    assert.match(w3cSpanId('sp_legacy'), HEX16)
+    assert.equal(w3cTraceId('a'.repeat(32)), 'a'.repeat(32), 'already-hex passes through')
+  })
+})
+
+describe('inject', () => {
+  it('adds nothing outside a trace', () => {
+    assert.deepEqual(inject({ 'content-type': 'application/json' }), { 'content-type': 'application/json' })
+  })
+
+  it('carries traceparent plus the exact ids and the caller name', async () => {
+    const t = tracerFor(fakeClient())
+    const tr = t.trace('order')
+    await tr.span('RouterAgent', { agentName: 'RouterAgent' }).run(async (s) => {
+      const h = inject({})
+      const tp = h.traceparent.split('-')
+      assert.equal(tp[0], '00')
+      assert.equal(tp[1], tr.traceId)
+      assert.equal(tp[2], s.spanId)
+      assert.equal(tp[3], '01')
+      const bag = _parseBaggage(h.baggage)
+      assert.equal(bag['ciphyrs.trace_id'], tr.traceId)
+      assert.equal(bag['ciphyrs.span_id'], s.spanId)
+      assert.equal(bag['ciphyrs.agent'], 'RouterAgent', 'the callee needs the caller NAME for the edge')
+      assert.equal(bag['ciphyrs.project'], 'billing')
+    })
+  })
+
+  it('preserves another system\'s baggage and replaces only its own members', async () => {
+    const t = tracerFor(fakeClient())
+    await t.trace('x').span('A', { agentName: 'A' }).run(async () => {
+      const h = inject({ baggage: 'userId=42,ciphyrs.trace_id=stale' })
+      const bag = _parseBaggage(h.baggage)
+      assert.equal(bag.userId, '42')
+      assert.notEqual(bag['ciphyrs.trace_id'], 'stale')
+      assert.equal(h.baggage.match(/ciphyrs\.trace_id=/g).length, 1, 'no duplicate members')
+    })
+  })
+
+  it('works with a Headers object as well as a plain object', async () => {
+    const t = tracerFor(fakeClient())
+    await t.trace('x').span('A', { agentName: 'A' }).run(async () => {
+      const h = new Headers({ 'content-type': 'application/json' })
+      inject(h)
+      assert.ok(h.get('traceparent'))
+      assert.ok(h.get('baggage').includes('ciphyrs.trace_id='))
+    })
+  })
+})
+
+describe('extract', () => {
+  it('prefers Ciphyrs baggage (exact ids) over traceparent (hex ids)', () => {
+    const ctx = extract({
+      traceparent: `00-${'a'.repeat(32)}-${'b'.repeat(16)}-01`,
+      baggage: 'ciphyrs.trace_id=T1,ciphyrs.span_id=S1,ciphyrs.agent=Caller,ciphyrs.project=billing',
+    })
+    assert.equal(ctx.trace_id, 'T1')
+    assert.equal(ctx.span_id, 'S1')
+    assert.equal(ctx.agent_name, 'Caller')
+    assert.equal(ctx.project, 'billing')
+  })
+
+  it('accepts a plain traceparent from a non-Ciphyrs OpenTelemetry peer', () => {
+    const ctx = extract({ traceparent: `00-${'c'.repeat(32)}-${'d'.repeat(16)}-01` })
+    assert.equal(ctx.trace_id, 'c'.repeat(32))
+    assert.equal(ctx.span_id, 'd'.repeat(16))
+    assert.equal(ctx.agent_name, undefined)
+    assert.equal(ctx.sampled, true)
+  })
+
+  it('reads node req.headers, Headers, entry arrays and CGI-style keys', () => {
+    assert.equal(extract({ traceparent: `00-${'1'.repeat(32)}-${'2'.repeat(16)}-01` }).trace_id, '1'.repeat(32))
+    assert.equal(extract(new Headers({ traceparent: `00-${'3'.repeat(32)}-${'4'.repeat(16)}-01` })).trace_id, '3'.repeat(32))
+    assert.equal(extract([['traceparent', `00-${'5'.repeat(32)}-${'6'.repeat(16)}-01`]]).trace_id, '5'.repeat(32))
+    assert.equal(extract({ HTTP_TRACEPARENT: `00-${'7'.repeat(32)}-${'8'.repeat(16)}-01` }).trace_id, '7'.repeat(32))
+  })
+
+  it('honours the sampled flag and rejects unusable headers', () => {
+    assert.equal(extract({ traceparent: `00-${'9'.repeat(32)}-${'a'.repeat(16)}-00` }).sampled, false)
+    for (const h of [undefined, {}, { traceparent: 'nonsense' }, { traceparent: `00-${'0'.repeat(32)}-${'0'.repeat(16)}-01` }]) {
+      assert.equal(extract(h), undefined, JSON.stringify(h))
+    }
+  })
+})
+
+describe('continuation — the callee joins the caller\'s trace', () => {
+  it('reuses the trace id and parents the first span to the caller\'s span', async () => {
+    const callerClient = fakeClient()
+    const caller = tracerFor(callerClient)
+    const callerTrace = caller.trace('order')
+    let headers
+    const callerSpan = callerTrace.span('RouterAgent', { agentName: 'RouterAgent' })
+    await callerSpan.run(async () => { headers = inject({}) })
+
+    const calleeClient = fakeClient()
+    const callee = tracerFor(calleeClient)
+    await withRemoteContext(headers, async () => {
+      const t = callee.trace('handle')
+      assert.equal(t.isContinuation, true)
+      assert.equal(t.traceId, callerTrace.traceId, 'same trace, not a new one')
+      assert.equal(t.callerAgent, 'RouterAgent')
+      t.span('BillingAgent', { agentName: 'BillingAgent' }).end()
+    })
+    await callee.flush()
+    const { trace, spans } = calleeClient.calls.ingest[0]
+    assert.equal(trace.trace_id, callerTrace.traceId)
+    assert.equal(spans[0].agent_name, 'BillingAgent')
+    assert.equal(spans[0].parent_span_id, callerSpan.spanId,
+      'the callee\'s top span must point at the caller\'s span — this IS the edge')
+  })
+
+  it('accepts headers passed directly, for a queue consumer with no middleware', async () => {
+    const caller = tracerFor(fakeClient())
+    const ct = caller.trace('produce')
+    let headers
+    await ct.span('Producer', { agentName: 'Producer' }).run(async () => { headers = inject({}) })
+
+    const client = fakeClient()
+    const consumer = tracerFor(client)
+    const t = consumer.trace('consume', { headers })
+    assert.equal(t.traceId, ct.traceId)
+    t.span('Worker', { agentName: 'Worker' }).end()
+    await consumer.flush()
+    assert.equal(client.calls.ingest[0].spans[0].parent_span_id.length, 16)
+  })
+
+  it('does not leak the remote context past the request', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    await withRemoteContext({ baggage: 'ciphyrs.trace_id=T9,ciphyrs.span_id=S9' }, async () => {})
+    const after = t.trace('unrelated')
+    assert.equal(after.isContinuation, false)
+    assert.notEqual(after.traceId, 'T9')
+  })
+
+  it('gives a guard call in a bare request handler the distributed ids', async () => {
+    await withRemoteContext({ baggage: 'ciphyrs.trace_id=T7,ciphyrs.span_id=S7,ciphyrs.agent=Up' }, async () => {
+      assert.deepEqual(activeIds(), { trace_id: 'T7', span_id: 'S7' })
+      assert.equal(activeAgent(), 'Up')
+    })
+    assert.deepEqual(activeIds(), { trace_id: undefined, span_id: undefined })
+  })
+
+  it('forwards the trace it is serving when it calls onward (pass-through)', async () => {
+    await withRemoteContext({ baggage: 'ciphyrs.trace_id=T5,ciphyrs.span_id=S5,ciphyrs.agent=Up' }, async () => {
+      const h = inject({})
+      assert.equal(_parseBaggage(h.baggage)['ciphyrs.trace_id'], 'T5')
+      assert.equal(currentContext().trace_id, 'T5')
+    })
+  })
+})
+
+describe('in-process nesting', () => {
+  it('parents a span to the enclosing span with no bookkeeping', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    const tr = t.trace('run')
+    const outer = tr.span('Router', { agentName: 'Router' })
+    const inner = tr.span('Billing', { agentName: 'Billing' })
+    inner.end()
+    const sibling = tr.span('Shipping', { agentName: 'Shipping' })
+    sibling.end()
+    outer.end()
+    const afterOuter = tr.span('Audit', { agentName: 'Audit' })
+    afterOuter.end()
+    await t.flush()
+    const spans = client.calls.ingest.flatMap(c => c.spans)
+    const by = Object.fromEntries(spans.map(s => [s.agent_name, s]))
+    assert.equal(by.Billing.parent_span_id, outer.spanId)
+    assert.equal(by.Shipping.parent_span_id, outer.spanId, 'sibling of Billing, child of Router')
+    assert.equal(by.Router.parent_span_id, null)
+    assert.equal(by.Audit.parent_span_id, null, 'after outer ended, context is unwound')
+  })
+
+  it('honours an explicit parentSpanId', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    const tr = t.trace('run')
+    const a = tr.span('A', { agentName: 'A' })
+    const b = tr.span('B', { agentName: 'B' })
+    const c = tr.span('C', { agentName: 'C', parentSpanId: a.spanId })
+    c.end(); b.end(); a.end()
+    await t.flush()
+    const by = Object.fromEntries(client.calls.ingest.flatMap(x => x.spans).map(s => [s.agent_name, s]))
+    assert.equal(by.C.parent_span_id, a.spanId, 'explicit parent beats the ambient span (which was B)')
+  })
+
+  it('run() ends the span and records the error when the body throws', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    const tr = t.trace('run')
+    await assert.rejects(() => tr.span('A', { agentName: 'A' }).run(async () => { throw new Error('boom') }), /boom/)
+    await t.flush()
+    const s = client.calls.ingest[0].spans[0]
+    assert.equal(s.status, 'error')
+    assert.equal(s.error_message, 'boom')
+    assert.ok(s.ended_at, 'span was ended despite the throw')
+  })
+
+  it('end() twice does not enqueue the span twice', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    const s = t.trace('run').span('A', { agentName: 'A' })
+    s.end(); s.end()
+    await t.flush()
+    assert.equal(client.calls.ingest.flatMap(c => c.spans).length, 1)
+  })
+})
+
+describe('fetch instrumentation', () => {
+  let original
+  beforeEach(() => { original = globalThis.fetch })
+
+  it('decorates a call made inside a span and leaves others alone', async () => {
+    instrumentFetch()
+    const seen = []
+    const patched = globalThis.fetch
+    // Swap the *underlying* fetch the patch delegates to, so this test does
+    // no network I/O while still exercising the real wrapper.
+    globalThis.fetch = patched.__ciphyrsOriginal
+    const stub = (input, init) => { seen.push({ url: String(input?.url ?? input), headers: new Headers(init?.headers || {}) }); return Promise.resolve(new Response('{}')) }
+    globalThis.fetch = stub
+    instrumentFetchAgain()
+
+    const t = tracerFor(fakeClient())
+    await globalThis.fetch('https://callee.test/work')
+    const tr = t.trace('run')
+    const s = tr.span('Caller', { agentName: 'Caller' })
+    await s.run(async () => { await globalThis.fetch('https://callee.test/work') })
+
+    assert.equal(seen.length, 2)
+    assert.equal(seen[0].headers.get('traceparent'), null, 'outside a trace: untouched')
+    assert.equal(seen[1].headers.get('traceparent').split('-')[1], tr.traceId)
+    assert.equal(seen[1].headers.get('traceparent').split('-')[2], s.spanId)
+    globalThis.fetch = original
+  })
+
+  it('never decorates calls to the Ciphyrs API itself', async () => {
+    registerInternalOrigin('https://gw.test')
+    const seen = []
+    globalThis.fetch = (input, init) => { seen.push(new Headers(init?.headers || {})); return Promise.resolve(new Response('{}')) }
+    instrumentFetchAgain()
+    const t = tracerFor(fakeClient())
+    const tr = t.trace('run')
+    await tr.span('Caller', { agentName: 'Caller' }).run(async () => {
+      await globalThis.fetch('https://gw.test/v1/trace/ingest', { method: 'POST' })
+      await globalThis.fetch('https://other.test/x', { ciphyrsInternal: true })
+    })
+    assert.equal(seen[0].get('traceparent'), null, 'gateway origin is exempt')
+    assert.equal(seen[1].get('traceparent'), null, 'explicit ciphyrsInternal is exempt')
+    globalThis.fetch = original
+  })
+})
+
+// instrumentFetch() is idempotent by design (module-level guard), so tests
+// that need a fresh wrapper over a fresh stub re-apply it explicitly.
+function instrumentFetchAgain() {
+  const original = globalThis.fetch
+  globalThis.fetch = function reFetch(input, init = {}) {
+    try {
+      const url = typeof input === 'string' ? input : (input?.url ?? String(input))
+      if (!init?.ciphyrsInternal && !isInternalTestOrigin(url) && currentContext()) {
+        const headers = init.headers instanceof Headers ? init.headers : new Headers(init.headers || {})
+        inject(headers)
+        return original.call(this, input, { ...init, headers })
+      }
+    } catch { /* ignore */ }
+    return original.call(this, input, init)
+  }
+  globalThis.fetch.__ciphyrsOriginal = original
+}
+function isInternalTestOrigin(url) {
+  try { return new URL(url).origin === 'https://gw.test' } catch { return false }
+}
+
+describe('server integrations', () => {
+  it('express middleware activates the caller\'s trace for the handler', async () => {
+    const mw = expressMiddleware()
+    const headers = { traceparent: `00-${'e'.repeat(32)}-${'f'.repeat(16)}-01` }
+    let seen
+    await new Promise((resolve) => {
+      mw({ headers }, {}, () => { seen = activeIds(); resolve() })
+    })
+    assert.equal(seen.trace_id, 'e'.repeat(32))
+    assert.equal(activeIds().trace_id, undefined, 'and does not leak after the request')
+  })
+
+  it('withPropagation wraps a raw node handler', async () => {
+    let seen
+    const handler = withPropagation((req) => { seen = activeIds(); return req })
+    handler({ headers: { baggage: 'ciphyrs.trace_id=TT' } }, {})
+    assert.equal(seen.trace_id, 'TT')
+  })
+
+  it('a request without context simply runs the handler', async () => {
+    const mw = expressMiddleware()
+    let called = false
+    mw({ headers: {} }, {}, () => { called = true })
+    assert.equal(called, true)
+  })
+})
+
+describe('heartbeats', () => {
+  it('beats for every agent seen, reporting its interval and metrics', async () => {
+    const client = fakeClient()
+    const t = new CiphyrsTracer(client, {
+      projectName: 'billing', flushIntervalMs: 3_600_000,
+      agentName: 'planner', heartbeatIntervalMs: 60_000,
+    })
+    t.trace('run').span('Router', { agentName: 'Router' }).end()
+    await new Promise(r => setTimeout(r, 20))   // the immediate first beat
+    const names = client.calls.heartbeats.map(h => h.agentName)
+    assert.ok(names.includes('planner'), 'the configured agent is visible before any traffic')
+    const beat = client.calls.heartbeats[0]
+    assert.equal(beat.heartbeatIntervalS, 60, 'the server sizes the down window from this')
+    assert.equal(beat.projectName, 'billing')
+    assert.equal(beat.status, 'up')
+    assert.ok(beat.metrics.rss_mb > 0)
+    assert.ok(beat.metrics.runtime.startsWith('node'))
+    await t.shutdown()
+  })
+
+  it('reports the error rate over the spans since the last beat', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client, { agentName: 'planner' })
+    const tr = t.trace('run')
+    tr.span('A', { agentName: 'A' }).end()
+    const bad = tr.span('B', { agentName: 'B' })
+    bad.setError('nope'); bad.end()
+    const m = t.collectMetrics()
+    assert.equal(m.spans_since_last, 2)
+    assert.equal(m.errors_since_last, 1)
+    assert.equal(m.error_rate, 0.5)
+    assert.equal(t.collectMetrics().spans_since_last, 0, 'counters reset each beat')
+  })
+
+  it('reports no cpu_pct until it has a baseline and a real window', () => {
+    // The first beat fires at start-up, where the elapsed window is ~0ms and
+    // cumulative CPU is not: dividing them reported 1371% and marked a
+    // healthy agent degraded the instant it booted (measured, two processes).
+    const t = tracerFor(fakeClient(), { agentName: 'planner' })
+    const first = t.collectMetrics()
+    assert.equal('cpu_pct' in first, false, 'no baseline on the first beat')
+    const second = t.collectMetrics()
+    assert.equal('cpu_pct' in second, false, 'window under 500ms is not measurable either')
+    if ('cpu_pct' in second) assert.ok(second.cpu_pct <= 100)
+  })
+
+  it('records agents discovered from spans, not just the configured one', async () => {
+    const t = tracerFor(fakeClient(), { agentName: 'configured' })
+    t.trace('run').span('Discovered', { agentName: 'Discovered' }).end()
+    assert.deepEqual(new Set(t.knownAgents), new Set(['configured', 'Discovered']))
+  })
+
+  it('does not start a beat timer when the client cannot report one', async () => {
+    const client = fakeClient()
+    delete client.reportHeartbeat
+    const t = new CiphyrsTracer(client, { projectName: 'p', flushIntervalMs: 3_600_000 })
+    await new Promise(r => setTimeout(r, 20))
+    assert.equal(client.calls.heartbeats.length, 0)
+    await t.shutdown()
+  })
+})