neuralos 3.2.12 → 3.2.14

package/bin/gybackend.cjs

@@ -367973,6 +367973,8 @@ var AgentService_v2 = class {
   /** sessionId → ledger run id for the in-flight run (set for every run,
    * unlike activeAgentRunIdsBySession which requires caller metadata). */
   ledgerRunIdsBySession = /* @__PURE__ */ new Map();
+  /** v3.2.13: gateway runId per session — used to correlate LLM trace spans. */
+  currentRunIdBySession = /* @__PURE__ */ new Map();
   constructor(terminalService, commandPolicyService, mcpToolService, skillService, memoryService, uiHistoryService, chatHistoryService, imageAttachmentService, fileTransferService) {
     this.terminalService = terminalService;
     this.chatHistoryService = chatHistoryService;
@@ -368619,6 +368621,7 @@ var AgentService_v2 = class {
       let partialText = "";
       let reasoningContent = "";
       let debugRawChunks = [];
+      const llmTraceStart = Date.now();
       const fullResponse = await invokeWithRetryAndSanitizedInput({
         helpers: this.helpers,
         messages: modelInputMessages,
@@ -368816,6 +368819,26 @@ var AgentService_v2 = class {
         debugRawChunks
       );
       let currentTokens = state.token_state.current_tokens;
+      try {
+        const traceModelName = getStreamedResponseModelName(fullResponse, debugRawChunks) || baseModel?.modelName || baseModel?.model || "unknown";
+        const breakdown = usageInfo ? extractUsageTokenBreakdown(usageInfo.usage) : void 0;
+        this.observability?.llmTrace?.record(
+          {
+            runId: state.runId,
+            sessionId,
+            operation: shouldUseThinkingModelOnThisPass ? "thinking" : "chat",
+            model: traceModelName
+          },
+          {
+            durationMs: Date.now() - llmTraceStart,
+            ...breakdown?.promptTokens !== void 0 ? { inputTokens: breakdown.promptTokens } : {},
+            ...breakdown?.completionTokens !== void 0 ? { outputTokens: breakdown.completionTokens } : {},
+            ...usageInfo ? { totalTokens: usageInfo.totalTokens } : {},
+            finishReason: describeStreamedResponseFinish(fullResponse, debugRawChunks) || void 0
+          }
+        );
+      } catch {
+      }
       if (usageInfo) {
         currentTokens = usageInfo.totalTokens;
         const modelName = getStreamedResponseModelName(fullResponse, debugRawChunks) || baseModel?.modelName || baseModel?.model || "unknown";
@@ -370899,60 +370922,99 @@ ${reminder}`;
     const processedMessages = buildDynamicRequestHistory(messages, {
       modelSupportsImage: sessionBinding.readFileSupport.image
     });
+    const llmTraceStart = Date.now();
+    const traceResult = (error40) => {
+      try {
+        this.observability?.llmTrace?.record(
+          {
+            runId: this.currentRunIdBySession.get(sessionId),
+            sessionId,
+            operation: `audit.${decisionName}`,
+            model: model?.modelName || model?.model || "unknown"
+          },
+          {
+            durationMs: Date.now() - llmTraceStart,
+            ...error40 ? { error: error40 } : {}
+          }
+        );
+      } catch {
+      }
+    };
     if (sessionBinding.thinkingModelSupportsStructuredOutput) {
       const structuredModel = model.withStructuredOutput(schema2, {
         method: "jsonSchema"
       });
-      return await invokeWithRetryAndSanitizedInput({
-        helpers: this.helpers,
-        messages: processedMessages,
-        modelSupportsImage: sessionBinding.readFileSupport.image,
-        signal,
-        operation: async (sanitizedMessages) => {
-          return await structuredModel.invoke(sanitizedMessages, {
-            signal
-          });
-        },
-        onRetry: (attempt) => {
-          console.log(
-            `[AgentService_v2] Retrying thinking model decision for ${decisionName} (attempt ${attempt + 1})...`
-          );
-        },
-        maxRetries: MODEL_RETRY_MAX,
-        delaysMs: MODEL_RETRY_DELAYS_MS
-      });
+      try {
+        const result = await invokeWithRetryAndSanitizedInput({
+          helpers: this.helpers,
+          messages: processedMessages,
+          modelSupportsImage: sessionBinding.readFileSupport.image,
+          signal,
+          operation: async (sanitizedMessages) => {
+            return await structuredModel.invoke(sanitizedMessages, {
+              signal
+            });
+          },
+          onRetry: (attempt) => {
+            console.log(
+              `[AgentService_v2] Retrying thinking model decision for ${decisionName} (attempt ${attempt + 1})...`
+            );
+          },
+          maxRetries: MODEL_RETRY_MAX,
+          delaysMs: MODEL_RETRY_DELAYS_MS
+        });
+        traceResult();
+        return result;
+      } catch (err) {
+        traceResult(err instanceof Error ? err.message : String(err));
+        throw err;
+      }
     }
     if (sessionBinding.thinkingModelSupportsObjectToolChoice) {
       const functionCallingModel = model.withStructuredOutput(schema2, {
         method: "functionCalling"
       });
-      return await invokeWithRetryAndSanitizedInput({
-        helpers: this.helpers,
-        messages: processedMessages,
-        modelSupportsImage: sessionBinding.readFileSupport.image,
+      try {
+        const result = await invokeWithRetryAndSanitizedInput({
+          helpers: this.helpers,
+          messages: processedMessages,
+          modelSupportsImage: sessionBinding.readFileSupport.image,
+          signal,
+          operation: async (sanitizedMessages) => {
+            return await functionCallingModel.invoke(sanitizedMessages, {
+              signal
+            });
+          },
+          onRetry: (attempt) => {
+            console.log(
+              `[AgentService_v2] Retrying tool-call thinking decision for ${decisionName} (attempt ${attempt + 1})...`
+            );
+          },
+          maxRetries: MODEL_RETRY_MAX,
+          delaysMs: MODEL_RETRY_DELAYS_MS
+        });
+        traceResult();
+        return result;
+      } catch (err) {
+        traceResult(err instanceof Error ? err.message : String(err));
+        throw err;
+      }
+    }
+    try {
+      const result = await this.invokeModelDecisionByPlainToolCall(
+        sessionId,
+        processedMessages,
+        schema2,
         signal,
-        operation: async (sanitizedMessages) => {
-          return await functionCallingModel.invoke(sanitizedMessages, {
-            signal
-          });
-        },
-        onRetry: (attempt) => {
-          console.log(
-            `[AgentService_v2] Retrying tool-call thinking decision for ${decisionName} (attempt ${attempt + 1})...`
-          );
-        },
-        maxRetries: MODEL_RETRY_MAX,
-        delaysMs: MODEL_RETRY_DELAYS_MS
-      });
+        decisionName,
+        "thinking"
+      );
+      traceResult(result);
+      return result;
+    } catch (err) {
+      traceResult(err instanceof Error ? err.message : String(err));
+      throw err;
     }
-    return await this.invokeModelDecisionByPlainToolCall(
-      sessionId,
-      processedMessages,
-      schema2,
-      signal,
-      decisionName,
-      "thinking"
-    );
   }
   async getCompactionModelDecision(sessionId, messages, schema2, signal, decisionName) {
     const sessionBinding = this.getSessionModelBinding(sessionId);
@@ -371103,6 +371165,7 @@ ${reminder}`;
     }
     const ledgerRunId = agentRunId ?? `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
     this.ledgerRunIdsBySession.set(sessionId, ledgerRunId);
+    if (runId) this.currentRunIdBySession.set(sessionId, runId);
     const ledgerInputPreview = typeof input === "string" ? input : input?.text ?? "";
     this.agentRunLedger?.startRun({
       runId: ledgerRunId,
@@ -371208,6 +371271,7 @@ ${reminder}`;
       if (this.ledgerRunIdsBySession.get(sessionId) === ledgerRunId) {
         this.ledgerRunIdsBySession.delete(sessionId);
       }
+      this.currentRunIdBySession.delete(sessionId);
       this.agentRunLedger?.finishRun(ledgerRunId, ledgerExitStatus, ledgerExitError);
       await this.clearCheckpoint(sessionId);
     }
@@ -387049,6 +387113,88 @@ var DriftDetector = class {
 // ../../packages/backend/src/services/observability.ts
 init_spanLedger();
 
+// ../../packages/backend/src/services/observability/llmTrace.ts
+init_spanLedger();
+var PROVIDER_FROM_MODEL_RE = /^([a-z0-9-]+)\//;
+function inferProvider(model) {
+  const m2 = PROVIDER_FROM_MODEL_RE.exec(model);
+  return m2 ? m2[1] : "unknown";
+}
+function traceIdFromRun(runId) {
+  const stripped = runId.replace(/[^0-9a-fA-F]/g, "");
+  if (stripped.length >= 32) return stripped.slice(0, 32);
+  if (stripped.length === 0) return "0".repeat(32);
+  let out = stripped;
+  while (out.length < 32) out += stripped;
+  return out.slice(0, 32);
+}
+var LlmTraceRecorder = class {
+  spanLedger = null;
+  otlpForward = null;
+  enabled = true;
+  setSpanLedger(ledger) {
+    this.spanLedger = ledger;
+  }
+  /** Optional forwarder to an external OTLP endpoint (injected by observability.ts). */
+  setOtlpTraceExporter(forward) {
+    this.otlpForward = forward;
+  }
+  setEnabled(on) {
+    this.enabled = on;
+  }
+  isEnabled() {
+    return this.enabled && this.spanLedger !== null;
+  }
+  /**
+   * Record one completed LLM call. Fire-and-forget safe: never throws.
+   */
+  record(input, result) {
+    try {
+      if (!this.enabled || !this.spanLedger) return;
+      const now = Date.now();
+      const startMs = now - Math.max(0, result.durationMs);
+      const traceId = input.runId ? traceIdFromRun(input.runId) : traceIdFromRun(input.sessionId);
+      const spanId = newSpanId();
+      const isError3 = Boolean(result.error);
+      this.spanLedger.ingest({
+        traceId,
+        spanId,
+        service: `llm.${input.operation}`,
+        name: `${input.model} ${input.operation}`,
+        startMs,
+        durationMs: Math.max(0, result.durationMs),
+        status: isError3 ? "error" : "ok"
+      });
+      if (this.otlpForward) {
+        const attributes = [
+          { key: "gen_ai.system", value: { stringValue: input.provider || inferProvider(input.model) } },
+          { key: "gen_ai.request.model", value: { stringValue: input.model } },
+          { key: "gen_ai.operation.name", value: { stringValue: input.operation } },
+          { key: "session.id", value: { stringValue: input.sessionId } }
+        ];
+        if (typeof result.inputTokens === "number") attributes.push({ key: "gen_ai.usage.prompt_tokens", value: { intValue: result.inputTokens } });
+        if (typeof result.outputTokens === "number") attributes.push({ key: "gen_ai.usage.completion_tokens", value: { intValue: result.outputTokens } });
+        if (typeof result.totalTokens === "number") attributes.push({ key: "gen_ai.usage.total_tokens", value: { intValue: result.totalTokens } });
+        if (result.finishReason) attributes.push({ key: "gen_ai.response.finish_reasons", value: { stringValue: result.finishReason } });
+        if (result.error) attributes.push({ key: "error.message", value: { stringValue: result.error.slice(0, 500) } });
+        this.otlpForward([
+          {
+            traceId,
+            spanId,
+            parentSpanId: "",
+            name: `${input.operation} ${input.model}`,
+            startTimeUnixNano: String(BigInt(Math.floor(startMs)) * 1000000n),
+            endTimeUnixNano: String(BigInt(Math.floor(now)) * 1000000n),
+            attributes,
+            status: { code: isError3 ? 2 : 1 }
+          }
+        ]);
+      }
+    } catch {
+    }
+  }
+};
+
 // ../../packages/backend/src/services/dem/rumLedger.ts
 var import_crypto16 = require("crypto");
 var DEFAULT_LIMIT5 = 5e4;
@@ -392597,6 +392743,8 @@ function createObservability(deps) {
     getActual: async () => ""
   });
   const spanLedger = new SpanLedger({});
+  const llmTrace = new LlmTraceRecorder();
+  llmTrace.setSpanLedger(spanLedger);
   const rumLedger = new RumLedger({});
   const infraMonitor = new InfraMonitor({});
   const etwService = new EtwService({});
@@ -392737,7 +392885,7 @@ function createObservability(deps) {
     endpoint: otelEndpoint,
     resourceAttributes: { "service.name": "rterm", "service.version": process.env.GYBACKEND_VERSION ?? "dev" }
   }) : null;
-  const renderPrometheus = () => {
+  const buildHostMetricsRegistry = () => {
     const series = [];
     for (const host of metricsLedger.hosts()) {
       const latest = metricsLedger.latest(host);
@@ -392747,9 +392895,9 @@ function createObservability(deps) {
         if (typeof v === "number" && Number.isFinite(v)) series.push({ host, metric: `host_${k}`, value: v });
       }
     }
-    const reg = registryFromHostMetrics(series, { prefix: "rterm", helpPrefix: "RTerm host metric" });
-    return reg.render();
+    return registryFromHostMetrics(series, { prefix: "rterm", helpPrefix: "RTerm host metric" });
   };
+  const renderPrometheus = () => buildHostMetricsRegistry().render();
   const secretsVault = new SecretsVault({
     masterKey: process.env.RTERM_SECRETS_MASTER_KEY,
     onAudit: (action, key) => {
@@ -393094,8 +393242,7 @@ function createObservability(deps) {
     const intervalMs = Number(process.env.OTEL_EXPORTER_OTLP_INTERVAL_MS ?? 3e4) || 3e4;
     const pushOnce = async () => {
       try {
-        renderPrometheus();
-        await otelExporter.push(prometheusRegistry);
+        await otelExporter.push(buildHostMetricsRegistry());
       } catch {
       }
     };
@@ -393105,6 +393252,28 @@ function createObservability(deps) {
     if (typeof otelPushTimer.unref === "function") otelPushTimer.unref();
     void pushOnce();
   }
+  if (otelEndpoint) {
+    llmTrace.setOtlpTraceExporter((spans) => {
+      void fetch(otelEndpoint.replace(/\/v1\/metrics$/, "/v1/traces"), {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          resourceSpans: [
+            {
+              resource: {
+                attributes: [
+                  { key: "service.name", value: { stringValue: "rterm-agent" } },
+                  { key: "service.version", value: { stringValue: process.env.GYBACKEND_VERSION ?? "dev" } }
+                ]
+              },
+              scopeSpans: [{ spans }]
+            }
+          ]
+        })
+      }).catch(() => {
+      });
+    });
+  }
   const oncallTickTimer = setInterval(() => {
     void escalationService.tick().catch(() => {
     });
@@ -393129,6 +393298,7 @@ function createObservability(deps) {
     anomalyDetector,
     earlyWarning,
     behaviorLedger,
+    llmTrace,
     dagu: { parseDaguYaml, parseDaguWorkflow, daguExecutionPlan },
     notify: { slackChannel, teamsChannel, smtpChannel, telegramChannel },
     aperf: { service: aperfService, toMetricPoint: aperfSummaryToMetricPoint },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "neuralos",
-  "version": "3.2.12",
-  "description": "Headless AI-native backend for RTerm / neuralOS — v3.2.12: chat console fixed (task-completion guard bounded, answers no longer cleared + re-answered, DONE deduped, auditor prompt rebalanced). 11 plugins, 55 plugin tools, parallel tool execution, captureStatus. Transports (SSH/serial/local), SQLite, and NATS libs install automatically.",
+  "version": "3.2.14",
+  "description": "Headless AI-native backend for RTerm / neuralOS — v3.2.14: OTLP metrics push fixed (was always empty) + OpenLLMetry-style LLM tracing (every model call becomes an APM span with gen_ai attributes; optional forwarding to Jaeger/Tempo/Datadog). Chat console fixes from 3.2.12 included. 11 plugins, 55 plugin tools, parallel tool execution, captureStatus.",
   "main": "bin/gybackend.cjs",
   "bin": { "gybackend": "bin/gybackend.cjs" },
   "license": "MIT",