cachegate 1.3.0 → 1.4.0

package/streaming.js

@@ -1,77 +1,77 @@
-// model-router/streaming.js
-//
-// OpenAI-compatible SSE chunk framing, shared by both providers so
-// server.js has exactly one wire format to write regardless of which
-// provider actually answered - the provider adapters (chatStream()) do
-// their own event-format translation and hand server.js plain text
-// deltas plus a final usage/cost summary; this file turns that into the
-// bytes that go on the wire.
-//
-// Scope for this increment: PLAIN TEXT CONTENT ONLY. Tool-call
-// streaming (accumulating partial JSON arguments across chunks, one or
-// more calls in flight at once) is a genuinely harder, separate
-// problem - server.js rejects stream:true + tools with a clear error
-// rather than attempt a half-working version of it.
-//
-// The final chunk carries extra fields (cost_usd, provider, cached,
-// cache_type) beyond real OpenAI's wire format - the same deviation the
-// non-streaming JSON response already makes. This proxy is
-// OpenAI-COMPATIBLE in request/response SHAPE, not a byte-for-byte
-// clone of OpenAI's actual API; MemoCode's own callers need the cost
-// data, and no spec-compliant client chokes on unknown extra JSON
-// fields it doesn't look for.
-
-const crypto = require('crypto');
-
-function chunkFrame(payload) {
-  return `data: ${JSON.stringify(payload)}\n\n`;
-}
-
-function doneFrame() {
-  return 'data: [DONE]\n\n';
-}
-
-function genId() {
-  return 'chatcmpl-' + crypto.randomBytes(12).toString('hex');
-}
-
-function baseChunk(id, model, choice) {
-  return {
-    id,
-    object: 'chat.completion.chunk',
-    created: Math.floor(Date.now() / 1000),
-    model,
-    choices: [choice]
-  };
-}
-
-function roleChunk({ id, model }) {
-  return chunkFrame(baseChunk(id, model, { index: 0, delta: { role: 'assistant' }, finish_reason: null }));
-}
-
-function deltaChunk({ id, model, content }) {
-  return chunkFrame(baseChunk(id, model, { index: 0, delta: { content }, finish_reason: null }));
-}
-
-function finalChunk({ id, model, usage, cost_usd, provider, cached, cache_type }) {
-  const frame = baseChunk(id, model, { index: 0, delta: {}, finish_reason: 'stop' });
-  frame.usage = usage;
-  frame.cost_usd = cost_usd;
-  frame.provider = provider;
-  frame.cached = !!cached;
-  if (cache_type) frame.cache_type = cache_type;
-  return chunkFrame(frame);
-}
-
-function errorFrame(message) {
-  return chunkFrame({ error: { message } });
-}
-
-function startSse(res) {
-  res.setHeader('Content-Type', 'text/event-stream');
-  res.setHeader('Cache-Control', 'no-cache');
-  res.setHeader('Connection', 'keep-alive');
-  if (typeof res.flushHeaders === 'function') res.flushHeaders();
-}
-
-module.exports = { chunkFrame, doneFrame, genId, roleChunk, deltaChunk, finalChunk, errorFrame, startSse };
+// model-router/streaming.js
+//
+// OpenAI-compatible SSE chunk framing, shared by both providers so
+// server.js has exactly one wire format to write regardless of which
+// provider actually answered - the provider adapters (chatStream()) do
+// their own event-format translation and hand server.js plain text
+// deltas plus a final usage/cost summary; this file turns that into the
+// bytes that go on the wire.
+//
+// Scope for this increment: PLAIN TEXT CONTENT ONLY. Tool-call
+// streaming (accumulating partial JSON arguments across chunks, one or
+// more calls in flight at once) is a genuinely harder, separate
+// problem - server.js rejects stream:true + tools with a clear error
+// rather than attempt a half-working version of it.
+//
+// The final chunk carries extra fields (cost_usd, provider, cached,
+// cache_type) beyond real OpenAI's wire format - the same deviation the
+// non-streaming JSON response already makes. This proxy is
+// OpenAI-COMPATIBLE in request/response SHAPE, not a byte-for-byte
+// clone of OpenAI's actual API; MemoCode's own callers need the cost
+// data, and no spec-compliant client chokes on unknown extra JSON
+// fields it doesn't look for.
+
+const crypto = require('crypto');
+
+function chunkFrame(payload) {
+  return `data: ${JSON.stringify(payload)}\n\n`;
+}
+
+function doneFrame() {
+  return 'data: [DONE]\n\n';
+}
+
+function genId() {
+  return 'chatcmpl-' + crypto.randomBytes(12).toString('hex');
+}
+
+function baseChunk(id, model, choice) {
+  return {
+    id,
+    object: 'chat.completion.chunk',
+    created: Math.floor(Date.now() / 1000),
+    model,
+    choices: [choice]
+  };
+}
+
+function roleChunk({ id, model }) {
+  return chunkFrame(baseChunk(id, model, { index: 0, delta: { role: 'assistant' }, finish_reason: null }));
+}
+
+function deltaChunk({ id, model, content }) {
+  return chunkFrame(baseChunk(id, model, { index: 0, delta: { content }, finish_reason: null }));
+}
+
+function finalChunk({ id, model, usage, cost_usd, provider, cached, cache_type }) {
+  const frame = baseChunk(id, model, { index: 0, delta: {}, finish_reason: 'stop' });
+  frame.usage = usage;
+  frame.cost_usd = cost_usd;
+  frame.provider = provider;
+  frame.cached = !!cached;
+  if (cache_type) frame.cache_type = cache_type;
+  return chunkFrame(frame);
+}
+
+function errorFrame(message) {
+  return chunkFrame({ error: { message } });
+}
+
+function startSse(res) {
+  res.setHeader('Content-Type', 'text/event-stream');
+  res.setHeader('Cache-Control', 'no-cache');
+  res.setHeader('Connection', 'keep-alive');
+  if (typeof res.flushHeaders === 'function') res.flushHeaders();
+}
+
+module.exports = { chunkFrame, doneFrame, genId, roleChunk, deltaChunk, finalChunk, errorFrame, startSse };

package/tracing.js

@@ -0,0 +1,97 @@
+// model-router/tracing.js
+//
+// OpenTelemetry export (roadmap step 36.2). Builds on the request-scoped
+// `trace_id` that server.js generates (step 36.1): that id is set as a span
+// attribute so a trace in an OTel backend and a metrics.js row for the same
+// request can be cross-referenced by the same value even though they are two
+// different systems.
+//
+// Gated off by default (OTEL_ENABLED). The @opentelemetry/api no-op tracer
+// is always present (it is the API surface and costs nothing when the SDK is
+// not initialized), but the heavy SDK + exporter are lazily required and only
+// started when BOTH OTEL_ENABLED=true AND OTEL_EXPORTER_OTLP_ENDPOINT is set -
+// "never initializes at all" when unused, not "initializes and exports
+// nowhere". Spans here are MANUAL ONLY: auto-instrumentation is explicitly out
+// of scope (spec 220), so no registerInstrumentations() call.
+
+const { trace, context } = require('@opentelemetry/api');
+
+const TRACER_NAME = 'cachegate';
+
+let initialized = false;
+
+/**
+ * Initialize the OTel SDK exactly once. A no-op unless both gates are open.
+ * Any SDK failure is logged and swallowed (fail-open): tracing must never be
+ * the reason a real request fails.
+ */
+function initTracing() {
+  if (initialized) return;
+  initialized = true;
+  if (process.env.OTEL_ENABLED !== 'true') return;
+  const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+  if (!endpoint) return;
+  try {
+    // Lazy-require the SDK + exporter only on the enabled path, so a
+    // deployment that never opts in never loads them.
+    const { NodeSDK } = require('@opentelemetry/sdk-node');
+    const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+    const sdk = new NodeSDK({
+      traceExporter: new OTLPTraceExporter({ url: endpoint })
+      // No instrumentations: manual spans only (spec 220, out of scope).
+    });
+    sdk.start();
+    process.on('SIGTERM', () => {
+      sdk.shutdown().finally(() => process.exit(0));
+    });
+  } catch (err) {
+    console.warn('⚠️ OTel SDK failed to initialize, tracing disabled:', err.message);
+  }
+}
+
+/**
+ * Start a span as a child of whatever span is currently active. With a no-op
+ * tracer (SDK off) this is a no-op object whose methods are all safe no-ops.
+ */
+function startSpan(name, attributes = {}) {
+  return trace.getTracer(TRACER_NAME).startSpan(name, { attributes });
+}
+
+/**
+ * Run `fn` inside a span named `name`, ending it when `fn` settles. The span
+ * is a child of the active span (the request's root span, set via
+ * withRootSpan). Does NOT make the new span itself active - none of the
+ * wrapped operations start their own nested spans, so there is nothing to
+ * nest. Re-throws on failure after recording the exception on the span.
+ */
+async function withSpan(name, attributes, fn) {
+  const span = startSpan(name, attributes);
+  try {
+    return await fn();
+  } catch (err) {
+    span.recordException(err);
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+
+/**
+ * Run `fn` inside a NEW root span (the one span per request), making it the
+ * active context so every `withSpan`/`startSpan` inside nests under it.
+ * `trace_id` is carried as a span attribute so this trace can be joined to
+ * metrics rows for the same request.
+ */
+async function withRootSpan(name, attributes, fn) {
+  const span = startSpan(name, attributes);
+  try {
+    return await context.with(trace.setSpan(context.active(), span), fn);
+  } catch (err) {
+    span.recordException(err);
+    throw err;
+  } finally {
+    span.end();
+  }
+}
+
+module.exports = { initTracing, startSpan, withSpan, withRootSpan };