@turmind/halo 1.2.1 → 1.3.1

package/dist/index.js

@@ -521,6 +521,13 @@ function generalSection() {
       { key: "sandbox.hidden_files", globalOnly: true, description: "Comma-separated files hidden by bwrap (Linux only)", description_zh: "\u901A\u8FC7 /dev/null bind \u9690\u85CF\u7684\u6587\u4EF6\uFF08\u9017\u53F7\u5206\u9694\uFF0C\u4EC5 Linux\uFF09", default: "~/.npmrc,~/.bash_history,~/.gitconfig,~/.git-credentials,~/.netrc,~/.halo/global/evo.db,~/.halo/global/evo.db-wal,~/.halo/global/evo.db-shm,~/.halo/global/cron.db,~/.halo/global/cron.db-wal,~/.halo/global/cron.db-shm,~/.halo/global/runs.db,~/.halo/global/runs.db-wal,~/.halo/global/runs.db-shm" },
       // logging
       { key: "logging.level", type: "enum", options: ["debug", "info", "warn", "error"], globalOnly: true, description: "Log level", description_zh: "\u65E5\u5FD7\u7EA7\u522B", default: "warn" },
+      // observability (OpenTelemetry). Read once at boot by observability/otel.ts
+      // and mapped onto the standard OTEL_* env vars — hence "takes effect on
+      // restart" everywhere below.
+      { key: "observability.endpoint", type: "string", globalOnly: true, description: "OTLP base URL of an OpenTelemetry collector (e.g. http://localhost:4318). Traces, metrics and logs are all exported there over OTLP http/protobuf \u2014 http:// or https://, on the collector's OTLP/HTTP port (4318 by default); gRPC (4317) is not supported. Empty = observability off. Takes effect on server restart.", description_zh: "OpenTelemetry collector \u7684 OTLP \u57FA\u7840\u5730\u5740\uFF08\u5982 http://localhost:4318\uFF09\uFF0Ctrace / metrics / \u65E5\u5FD7\u4E09\u7C7B\u4FE1\u53F7\u90FD\u4EE5 OTLP http/protobuf \u53D1\u5230\u8FD9\u91CC\u3002http:// \u6216 https:// \u5747\u53EF\uFF0C\u7AEF\u53E3\u987B\u662F collector \u7684 OTLP/HTTP \u7AEF\u53E3\uFF08\u9ED8\u8BA4 4318\uFF09\uFF0C\u4E0D\u652F\u6301 gRPC\uFF084317\uFF09\u3002\u7559\u7A7A = \u5173\u95ED\u3002\u91CD\u542F\u670D\u52A1\u540E\u751F\u6548\u3002", default: "" },
+      { key: "observability.service_name", type: "string", globalOnly: true, description: "OTel resource service.name reported for this server. Takes effect on restart.", description_zh: "\u672C\u670D\u52A1\u5728 OTel \u4E2D\u4E0A\u62A5\u7684 service.name\u3002\u91CD\u542F\u540E\u751F\u6548\u3002", default: "halo" },
+      { key: "observability.headers", type: "string", secret: true, globalOnly: true, description: "Extra headers for every OTLP request, comma-separated k=v (e.g. authorization=Bearer \u2026). Takes effect on restart.", description_zh: "\u9644\u52A0\u5230\u6BCF\u4E2A OTLP \u8BF7\u6C42\u7684 header\uFF0C\u9017\u53F7\u5206\u9694\u7684 k=v\uFF08\u5982 authorization=Bearer \u2026\uFF09\u3002\u91CD\u542F\u540E\u751F\u6548\u3002", default: "" },
+      { key: "observability.capture_content", type: "boolean", globalOnly: true, description: "Put prompt / completion / tool argument and result text on spans (gen_ai.*.messages etc.). Off = only metadata (model, tokens, latency, tool names). Takes effect on restart.", description_zh: "\u662F\u5426\u628A\u63D0\u793A\u8BCD / \u6A21\u578B\u8F93\u51FA / \u5DE5\u5177\u53C2\u6570\u4E0E\u7ED3\u679C\u6B63\u6587\u5199\u5230 span \u4E0A\uFF08gen_ai.*.messages \u7B49\uFF09\u3002\u5173 = \u53EA\u4E0A\u62A5\u5143\u6570\u636E\uFF08\u6A21\u578B\u3001token\u3001\u8017\u65F6\u3001\u5DE5\u5177\u540D\uFF09\u3002\u91CD\u542F\u540E\u751F\u6548\u3002", default: "false" },
       // self-evolution (see plans/self-evolution.md). All evo settings are
       // global-only — the worker / ticker live in the server process and
       // would have to reload mid-flight if a workspace could override them.
@@ -753,7 +760,7 @@ function loadYamlFile(filePath) {
   }
 }
 function loadModelsRegistry() {
-  const providers = [];
+  const providers2 = [];
   const modelsDir = path5.join(HALO_GLOBAL_DIR, "models");
   try {
     const entries = fs5.readdirSync(modelsDir, { withFileTypes: true });
@@ -764,7 +771,7 @@ function loadModelsRegistry() {
         const raw = fs5.readFileSync(path5.join(modelsDir, entry.name), "utf-8");
         const parsed = YAML3.parse(raw);
         if (parsed && typeof parsed === "object" && typeof parsed.id === "string") {
-          providers.push(parsed);
+          providers2.push(parsed);
         } else {
           console.log(`[config] Skipping ${entry.name}: missing provider id`);
         }
@@ -774,7 +781,7 @@ function loadModelsRegistry() {
     }
   } catch {
   }
-  return { providers };
+  return { providers: providers2 };
 }
 function modelsRegistry() {
   if (!_modelsRegistryCache)
@@ -953,9 +960,9 @@ function reloadSandboxConfig() {
 }
 function findModelEntry(modelId) {
   try {
-    const providers = modelsRegistry().providers;
-    if (providers) {
-      for (const provider of providers) {
+    const providers2 = modelsRegistry().providers;
+    if (providers2) {
+      for (const provider of providers2) {
         for (const model of provider.models ?? []) {
           if (model.id === modelId)
             return model;
@@ -1203,6 +1210,14 @@ var init_config = __esm({
         // 10MB
         maxFiles: systemInt("HALO_LOG_MAX_FILES", "logging.max_files", 5)
       },
+      // OpenTelemetry export. Read once at boot (observability/otel.ts maps these
+      // onto OTEL_* env vars) — changes take effect on restart.
+      observability: {
+        endpoint: settingsStr("general.observability.endpoint", ""),
+        serviceName: settingsStr("general.observability.service_name", "halo"),
+        headers: settingsStr("general.observability.headers", ""),
+        captureContent: settingsBool("general.observability.capture_content", false)
+      },
       // Self-evolution (see plans/self-evolution.md). Getter-based so user edits
       // in settings.yaml take effect on the next ticker run without restart.
       evolution: {
@@ -1600,20 +1615,20 @@ Error: ${err instanceof Error ? err.message : String(err)}`;
             const resultTextFull = resultText.length > uiCap ? resultText.slice(0, uiCap) + `
 
 [Content truncated: ${resultText.length} chars total, showing first ${uiCap}. Use file_read for the complete content.]` : resultText;
-            const cap = tu.name === "activate_skill" ? Infinity : config.limits.toolResultMax;
+            const cap2 = tu.name === "activate_skill" ? Infinity : config.limits.toolResultMax;
             const truncationNote = (origLen) => `
 
-[Content truncated: ${origLen} chars total, showing first ${cap}. Re-run with narrower scope (e.g. grep / file_read with offset+limit) to see specific sections.]`;
-            if (typeof resultContent === "string" && resultContent.length > cap) {
+[Content truncated: ${origLen} chars total, showing first ${cap2}. Re-run with narrower scope (e.g. grep / file_read with offset+limit) to see specific sections.]`;
+            if (typeof resultContent === "string" && resultContent.length > cap2) {
               const orig = resultContent.length;
-              resultContent = resultContent.slice(0, cap) + truncationNote(orig);
+              resultContent = resultContent.slice(0, cap2) + truncationNote(orig);
               resultText = resultContent;
             } else if (Array.isArray(resultContent)) {
               let mutated = false;
               resultContent = resultContent.map((b) => {
-                if (b.type === "text" && b.text.length > cap) {
+                if (b.type === "text" && b.text.length > cap2) {
                   mutated = true;
-                  return { type: "text", text: b.text.slice(0, cap) + truncationNote(b.text.length) };
+                  return { type: "text", text: b.text.slice(0, cap2) + truncationNote(b.text.length) };
                 }
                 return b;
               });
@@ -3411,8 +3426,8 @@ ${report}`;
   await deliver(host, gId, worker.id, header, report);
 }
 async function deliver(host, gId, workerId, header, report) {
-  const cap = config.limits.autoReportMax;
-  const body = report.length > cap ? report.slice(0, cap) + `
+  const cap2 = config.limits.autoReportMax;
+  const body = report.length > cap2 ? report.slice(0, cap2) + `
 
 [Report truncated: ${report.length} chars total. Use get_session_output("${workerId}") for the full text.]` : report;
   await host.querySession(gId, workerId, `${header}
@@ -5387,8 +5402,8 @@ async function deliverRelayReport(host, session) {
 
 ${report}`;
   }
-  const cap = config.limits.autoReportMax;
-  const body = report.length > cap ? report.slice(0, cap) + `
+  const cap2 = config.limits.autoReportMax;
+  const body = report.length > cap2 ? report.slice(0, cap2) + `
 
 [Report truncated: ${report.length} chars total. Use relay_read("${host.workspaceRoot}", "${session.id}") for the full text.]` : report;
   const header = `[Relay report \xB7 workspace ${host.workspaceRoot} \xB7 session ${session.id}]`;
@@ -11273,6 +11288,348 @@ var init_session_state_store = __esm({
   }
 });
 
+// ../server/dist/observability/otel-sdk.js
+var otel_sdk_exports = {};
+__export(otel_sdk_exports, {
+  registerSdk: () => registerSdk
+});
+import { metrics } from "@opentelemetry/api";
+import { logs } from "@opentelemetry/api-logs";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { LoggerProvider, BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
+function registerSdk(serviceName, serviceVersion) {
+  const resource = resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: serviceName,
+    [ATTR_SERVICE_VERSION]: serviceVersion
+  });
+  const tracerProvider = new NodeTracerProvider({
+    resource,
+    spanProcessors: [new BatchSpanProcessor(new OTLPTraceExporter())]
+  });
+  tracerProvider.register();
+  const meterProvider = new MeterProvider({
+    resource,
+    readers: [new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter(), exportIntervalMillis: 15e3 })]
+  });
+  metrics.setGlobalMeterProvider(meterProvider);
+  const loggerProvider = new LoggerProvider({
+    resource,
+    processors: [new BatchLogRecordProcessor({ exporter: new OTLPLogExporter() })]
+  });
+  logs.setGlobalLoggerProvider(loggerProvider);
+  return [tracerProvider, meterProvider, loggerProvider];
+}
+var init_otel_sdk = __esm({
+  "../server/dist/observability/otel-sdk.js"() {
+    "use strict";
+  }
+});
+
+// ../server/dist/observability/otel.js
+import { trace, metrics as metrics2, ProxyTracerProvider } from "@opentelemetry/api";
+import { logs as logs2 } from "@opentelemetry/api-logs";
+function externalSdkRegistered() {
+  const provider = trace.getTracerProvider();
+  if (!(provider instanceof ProxyTracerProvider))
+    return true;
+  return provider.getDelegate() !== new ProxyTracerProvider().getDelegate();
+}
+async function initObservability() {
+  const { endpoint, serviceName, headers } = config.observability;
+  if (endpoint)
+    process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??= endpoint;
+  if (headers)
+    process.env.OTEL_EXPORTER_OTLP_HEADERS ??= headers;
+  process.env.OTEL_SERVICE_NAME ??= serviceName;
+  process.env.OTEL_EXPORTER_OTLP_PROTOCOL ??= "http/protobuf";
+  if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT)
+    return false;
+  enabled = true;
+  if (externalSdkRegistered()) {
+    console.log("[Observability] external OpenTelemetry SDK already registered \u2014 using it, skipping built-in exporters");
+    return true;
+  }
+  const { registerSdk: registerSdk2 } = await Promise.resolve().then(() => (init_otel_sdk(), otel_sdk_exports));
+  providers = registerSdk2(process.env.OTEL_SERVICE_NAME, "1.3.1");
+  console.log(`[Observability] OTLP export \u2192 ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT} (service.name=${process.env.OTEL_SERVICE_NAME}, capture_content=${config.observability.captureContent})`);
+  return true;
+}
+async function shutdownObservability() {
+  if (providers.length === 0)
+    return;
+  const all = Promise.all(providers.map(async (p) => {
+    try {
+      await p.forceFlush();
+    } catch {
+    }
+    try {
+      await p.shutdown();
+    } catch {
+    }
+  }));
+  await Promise.race([all, new Promise((resolve2) => setTimeout(resolve2, 3e3))]);
+}
+var SCOPE, enabled, tracer, getMeter, otelLogger, captureContent, providers;
+var init_otel = __esm({
+  "../server/dist/observability/otel.js"() {
+    "use strict";
+    init_config();
+    SCOPE = "opentelemetry.instrumentation.halo";
+    enabled = false;
+    tracer = trace.getTracer(SCOPE, "1");
+    getMeter = () => metrics2.getMeter(SCOPE, "1");
+    otelLogger = logs2.getLogger(SCOPE, "1");
+    captureContent = () => config.observability.captureContent;
+    providers = [];
+  }
+});
+
+// ../server/dist/observability/genai-spans.js
+import { context, trace as trace2, SpanStatusCode } from "@opentelemetry/api";
+function instruments() {
+  const meter = getMeter();
+  return {
+    tokenUsage: meter.createHistogram("gen_ai.client.token.usage", { unit: "{token}", description: "Tokens per model call, split by gen_ai.token.type" }),
+    operationDuration: meter.createHistogram("gen_ai.client.operation.duration", { unit: "s", description: "Model call latency" }),
+    toolDuration: meter.createHistogram("halo.tool.duration", { unit: "s", description: "Tool execution latency" }),
+    turnDuration: meter.createHistogram("halo.turn.duration", { unit: "s", description: "Agent turn latency (invoke_agent)" }),
+    modelRetries: meter.createCounter("halo.model.retries", { description: "Model-call retries by kind" })
+  };
+}
+function cap(text6, max) {
+  return text6.length > max ? `${text6.slice(0, max)}\u2026[truncated ${text6.length - max} chars]` : text6;
+}
+function imagePlaceholder(source) {
+  return `[image mimeType=${source.media_type} size=${Math.floor(source.data.length * 3 / 4)}B]`;
+}
+function inputText(message) {
+  if (typeof message === "string")
+    return message;
+  return message.map((b) => {
+    if (b.type === "text")
+      return b.text;
+    if (b.type === "image")
+      return imagePlaceholder(b.source);
+    return "";
+  }).filter(Boolean).join("\n");
+}
+function toSemconvMessage(m) {
+  if (typeof m.content === "string")
+    return { role: m.role, parts: [{ type: "text", content: cap(m.content, PART_CAP) }] };
+  const parts = [];
+  let onlyToolResults = m.content.length > 0;
+  for (const b of m.content) {
+    if (b.type !== "tool_result")
+      onlyToolResults = false;
+    switch (b.type) {
+      case "text":
+        parts.push({ type: "text", content: cap(b.text, PART_CAP) });
+        break;
+      case "image":
+        parts.push({ type: "text", content: imagePlaceholder(b.source) });
+        break;
+      case "tool_use":
+        parts.push({ type: "tool_call", id: b.id, name: b.name, arguments: b.input });
+        break;
+      case "tool_result": {
+        const text6 = typeof b.content === "string" ? b.content : b.content.map((c) => c.type === "text" ? c.text : imagePlaceholder(c.source)).join("\n");
+        parts.push({ type: "tool_call_response", id: b.tool_use_id, response: cap(text6, PART_CAP) });
+        break;
+      }
+    }
+  }
+  return { role: onlyToolResults ? "tool" : m.role, parts };
+}
+function messagesAttr(messages2) {
+  const parts = messages2.map((m) => JSON.stringify(toSemconvMessage(m)));
+  let total = parts.reduce((n, p) => n + p.length + 1, 1);
+  let dropped = 0;
+  while (total > ATTR_CAP - 100 && dropped < parts.length)
+    total -= parts[dropped++].length + 1;
+  if (dropped === 0)
+    return `[${parts.join(",")}]`;
+  const marker = JSON.stringify({ role: "user", parts: [{ type: "text", content: `[${dropped} earlier messages omitted]` }] });
+  return `[${[marker, ...parts.slice(dropped)].join(",")}]`;
+}
+function beginTurn(session, message) {
+  if (!enabled)
+    return;
+  const startMs = Date.now();
+  const span = tracer.startSpan(`invoke_agent ${session.agentName}`, {
+    startTime: startMs,
+    attributes: {
+      "gen_ai.operation.name": "invoke_agent",
+      "gen_ai.system": SYSTEM,
+      "gen_ai.agent.name": session.agentName,
+      "gen_ai.request.model": session.currentModelId,
+      "session.id": session.id
+    }
+  });
+  if (captureContent())
+    span.setAttribute("gen_ai.system_instructions", cap(session.systemPrompt, SYSTEM_PROMPT_CAP));
+  turns.set(session.id, {
+    span,
+    ctx: trace2.setSpan(context.active(), span),
+    startMs,
+    input: inputText(message),
+    finalText: "",
+    allText: "",
+    cycleToolCalls: [],
+    pendingTools: /* @__PURE__ */ new Map(),
+    // The user message is pushed by agent.run() after beginTurn, so the first
+    // chat span's delta starts exactly at it.
+    messageCursor: session.agent.messages.length
+  });
+}
+function onAgentEvent(session, event) {
+  if (!enabled)
+    return;
+  const turn = turns.get(session.id);
+  if (!turn)
+    return;
+  switch (event.type) {
+    case "text":
+      turn.allText += event.text ?? "";
+      if (event.final)
+        turn.finalText += event.text ?? "";
+      return;
+    case "tool_call":
+      turn.cycleToolCalls.push({ id: event.toolUseId ?? "", name: event.toolName ?? "", args: event.toolInput });
+      return;
+    case "usage":
+      recordModelCall(session, turn, event);
+      return;
+    case "tool_result":
+      recordToolCall(session, turn, event);
+      return;
+    default:
+      return;
+  }
+}
+function recordModelCall(session, turn, event) {
+  const now = Date.now();
+  const durationMs = event.durationMs ?? 0;
+  const model = session.currentModelId;
+  const span = tracer.startSpan(`chat ${model}`, {
+    startTime: now - durationMs,
+    attributes: {
+      "gen_ai.operation.name": "chat",
+      "gen_ai.system": SYSTEM,
+      "gen_ai.request.model": model,
+      "gen_ai.usage.input_tokens": event.usage?.inputTokens ?? 0,
+      "gen_ai.usage.output_tokens": event.usage?.outputTokens ?? 0,
+      "gen_ai.response.finish_reasons": [turn.cycleToolCalls.length > 0 ? "tool_use" : "end_turn"],
+      "session.id": session.id
+    }
+  }, turn.ctx);
+  if (captureContent()) {
+    const messages2 = session.agent.messages;
+    const last = messages2[messages2.length - 1];
+    const hasAssistantTail = last?.role === "assistant";
+    const inputEnd = hasAssistantTail ? messages2.length - 1 : messages2.length;
+    span.setAttribute("gen_ai.input.messages", messagesAttr(messages2.slice(turn.messageCursor, inputEnd)));
+    if (hasAssistantTail)
+      span.setAttribute("gen_ai.output.messages", messagesAttr([last]));
+    turn.messageCursor = messages2.length;
+  }
+  span.end(now);
+  const chatCtx = trace2.setSpan(turn.ctx, span);
+  for (const tc of turn.cycleToolCalls) {
+    turn.pendingTools.set(tc.id, { name: tc.name, args: tc.args, parentCtx: chatCtx, startMs: now });
+  }
+  turn.cycleToolCalls = [];
+  const modelAttr = { "gen_ai.request.model": model };
+  const m = metricsFor();
+  m.tokenUsage.record(event.usage?.inputTokens ?? 0, { ...modelAttr, "gen_ai.token.type": "input" });
+  m.tokenUsage.record(event.usage?.outputTokens ?? 0, { ...modelAttr, "gen_ai.token.type": "output" });
+  m.operationDuration.record(durationMs / 1e3, { ...modelAttr, "gen_ai.operation.name": "chat" });
+}
+function recordToolCall(session, turn, event) {
+  const id = event.toolUseId ?? "";
+  const pending = turn.pendingTools.get(id);
+  if (!pending)
+    return;
+  turn.pendingTools.delete(id);
+  const now = Date.now();
+  const durationMs = event.durationMs ?? 0;
+  const span = tracer.startSpan(`execute_tool ${pending.name}`, {
+    startTime: now - durationMs,
+    attributes: {
+      "gen_ai.operation.name": "execute_tool",
+      "gen_ai.system": SYSTEM,
+      "gen_ai.tool.name": pending.name,
+      "gen_ai.tool.call.id": id,
+      "session.id": session.id
+    }
+  }, pending.parentCtx);
+  if (captureContent()) {
+    span.setAttribute("gen_ai.tool.call.arguments", cap(JSON.stringify(pending.args ?? null), ATTR_CAP));
+    span.setAttribute("gen_ai.tool.call.result", cap(event.toolResult || "[no output]", ATTR_CAP));
+  }
+  span.end(now);
+  metricsFor().toolDuration.record(durationMs / 1e3, { "gen_ai.tool.name": pending.name });
+}
+function endTurn(session, outcome) {
+  if (!enabled)
+    return;
+  const turn = turns.get(session.id);
+  if (!turn)
+    return;
+  turns.delete(session.id);
+  const now = Date.now();
+  for (const [id, pending] of turn.pendingTools) {
+    const orphan = tracer.startSpan(`execute_tool ${pending.name}`, {
+      startTime: pending.startMs,
+      attributes: {
+        "gen_ai.operation.name": "execute_tool",
+        "gen_ai.system": SYSTEM,
+        "gen_ai.tool.name": pending.name,
+        "gen_ai.tool.call.id": id,
+        "session.id": session.id,
+        "halo.tool.orphaned": true
+      }
+    }, pending.parentCtx);
+    orphan.end(now);
+  }
+  if (captureContent()) {
+    turn.span.setAttribute("gen_ai.task.input", cap(turn.input, ATTR_CAP));
+    turn.span.setAttribute("gen_ai.task.output", cap(turn.finalText || turn.allText, ATTR_CAP));
+  }
+  if (outcome.error) {
+    turn.span.setStatus({ code: SpanStatusCode.ERROR, message: outcome.error });
+    const errName = outcome.error.match(/^([A-Za-z]\w*(?:Exception|Error))\b/)?.[1];
+    turn.span.setAttribute("error.type", errName ?? "_OTHER");
+  }
+  turn.span.end(now);
+  metricsFor().turnDuration.record((now - turn.startMs) / 1e3, { "gen_ai.agent.name": session.agentName, outcome: outcome.error ? "error" : "ok" });
+}
+function recordRetry(kind) {
+  if (!enabled)
+    return;
+  metricsFor().modelRetries.add(1, { kind });
+}
+var turns, SYSTEM, PART_CAP, SYSTEM_PROMPT_CAP, ATTR_CAP, cachedInstruments, metricsFor;
+var init_genai_spans = __esm({
+  "../server/dist/observability/genai-spans.js"() {
+    "use strict";
+    init_otel();
+    turns = /* @__PURE__ */ new Map();
+    SYSTEM = "halo";
+    PART_CAP = 2 * 1024;
+    SYSTEM_PROMPT_CAP = 8 * 1024;
+    ATTR_CAP = 32 * 1024;
+    cachedInstruments = null;
+    metricsFor = () => cachedInstruments ??= instruments();
+  }
+});
+
 // ../server/dist/agents/session-manager.js
 import fsSync7 from "node:fs";
 import path30 from "node:path";
@@ -11403,6 +11760,7 @@ var init_session_manager = __esm({
     init_session_agent_builder();
     init_session_skill_commands();
     init_session_state_store();
+    init_genai_spans();
     SessionManager = class _SessionManager {
       // The fields below are marked `internal:` because they're consumed by
       // helpers in this directory (session-tools.ts). Treat them as
@@ -12091,6 +12449,7 @@ var init_session_manager = __esm({
           const i = message.findIndex((b) => b.type === "text");
           message = i === -1 ? [{ type: "text", text: stamp }, ...message] : message.map((b, j) => j === i && b.type === "text" ? { type: "text", text: stamped(b.text) } : b);
         }
+        beginTurn(session, message);
         for (let attempt = 0; attempt < maxRetries; attempt++) {
           session.abortController = new AbortController();
           const signal = session.abortController.signal;
@@ -12110,6 +12469,7 @@ var init_session_manager = __esm({
               if (event.type === "text") {
                 resultText += event.text ?? "";
               }
+              onAgentEvent(session, event);
               this.processSessionEvent(session, event);
               if (session.interruptRequested && event.type === "tool_result") {
                 console.debug(`[SessionManager] Graceful interrupt after tool result in ${session.id}`);
@@ -12142,8 +12502,10 @@ var init_session_manager = __esm({
               if (result.compacted) {
                 session.agent.messages = result.messages;
                 this.emitEvent(session.id, { type: "system", text: `Session ${session.agentId} context compacted (${result.messages.length} messages remaining, local fallback)` });
-                if (attempt + 1 < maxRetries)
+                if (attempt + 1 < maxRetries) {
+                  recordRetry("context_overflow");
                   continue;
+                }
               }
             }
             const isAccountError = httpStatus !== void 0 ? httpStatus === 401 || httpStatus === 402 || httpStatus === 403 : msg.includes("insufficient balance") || msg.includes("suspended") || msg.includes("invalid api key") || msg.includes("Invalid API Key") || msg.includes("Unauthorized") || msg.includes("authentication");
@@ -12160,6 +12522,7 @@ var init_session_manager = __esm({
                 const jitter = Math.random() * 1e3;
                 const delay = Math.min(baseDelay + jitter, 6e4);
                 console.debug(`[SessionManager] Session ${session.id} rate limited, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries})`);
+                recordRetry("throttle");
                 this.emitEvent(session.id, { type: "system", text: `Session ${session.agentId} rate limited, retrying in ${Math.round(delay / 1e3)}s...` });
                 await sleep(delay);
                 continue;
@@ -12171,6 +12534,7 @@ var init_session_manager = __esm({
                 const jitter = Math.random() * 1e3;
                 const delay = Math.min(baseDelay + jitter, 6e4);
                 console.debug(`[SessionManager] Session ${session.id} transient server error ${errName || httpStatus}, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries})`);
+                recordRetry("server_error");
                 this.emitEvent(session.id, { type: "system", text: `Session ${session.agentId} hit a transient server error, retrying in ${Math.round(delay / 1e3)}s...` });
                 await sleep(delay);
                 continue;
@@ -12180,6 +12544,7 @@ var init_session_manager = __esm({
               if (attempt + 1 < maxRetries) {
                 const delay = 1e3 * Math.pow(2, attempt) + Math.random() * 500;
                 console.warn(`[SessionManager] Session ${session.id} transient network error, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries}): ${msg}`);
+                recordRetry("network");
                 this.emitEvent(session.id, { type: "system", text: `Network hiccup, retrying in ${Math.round(delay / 1e3)}s...` });
                 await sleep(delay);
                 continue;
@@ -12189,6 +12554,7 @@ var init_session_manager = __esm({
               if (attempt + 1 < maxRetries) {
                 const delay = 1e3 * Math.pow(2, attempt) + Math.random() * 500;
                 console.debug(`[SessionManager] Session ${session.id} Mantle empty response, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries})`);
+                recordRetry("empty_response");
                 await sleep(delay);
                 continue;
               }
@@ -12197,6 +12563,7 @@ var init_session_manager = __esm({
               console.debug(`[SessionManager] Session ${session.id} corrupted conversation (attempt ${attempt + 1}/${maxRetries})`);
               session.agent.messages = repairConversationMessages(session.agent.messages, `[Session:${session.id}]`);
               if (attempt + 1 < maxRetries) {
+                recordRetry("corrupted");
                 this.emitEvent(session.id, { type: "system", text: "Repairing conversation state..." });
                 continue;
               }
@@ -12211,6 +12578,7 @@ var init_session_manager = __esm({
                 this.saveAgentState(session);
                 console.warn(`[SessionManager] Session ${session.id} multimodal 4xx \u2014 replaced ${replaced} image block(s) with placeholders (attempt ${attempt + 1}/${maxRetries})`);
                 if (attempt + 1 < maxRetries) {
+                  recordRetry("multimodal_4xx");
                   this.emitEvent(session.id, { type: "system", text: `Model rejected image data \u2014 removed ${replaced} image(s) from history, retrying...` });
                   continue;
                 }
@@ -12225,6 +12593,7 @@ var init_session_manager = __esm({
           }
         }
         session.abortController = null;
+        endTurn(session, { error: session.turnError ?? void 0 });
         session.agent.messages = repairConversationMessages(session.agent.messages, `[Session:${session.id}]`);
         this.db.update(agentSessions).set({ updatedAt: Date.now() }).where(eq9(agentSessions.id, session.id)).run();
         await this.maybeAutoCompact(session);
@@ -13320,12 +13689,12 @@ ${result.summary}`, agentName: "user", report: true, taskId });
        *  caller wants and also eats the trailing JSON fields. Self-cap here,
        *  tail-kept, under the generic cap so the envelope always survives. */
       truncateOutputTail(output) {
-        const cap = config.limits.toolResultMax - 500;
-        if (output.length <= cap)
+        const cap2 = config.limits.toolResultMax - 500;
+        if (output.length <= cap2)
           return output;
-        return `[Output truncated: ${output.length} chars total, showing LAST ${cap}. Earlier text omitted.]
+        return `[Output truncated: ${output.length} chars total, showing LAST ${cap2}. Earlier text omitted.]
 
-` + output.slice(-cap);
+` + output.slice(-cap2);
       }
       getSessionOutput(sessionId) {
         const session = this.sessions.get(sessionId);
@@ -13495,6 +13864,7 @@ var init_session_manager_registry = __esm({
 import fs25 from "node:fs";
 import path31 from "node:path";
 import { homedir as homedir13 } from "node:os";
+import { SeverityNumber } from "@opentelemetry/api-logs";
 function ensureDir() {
   fs25.mkdirSync(logDir, { recursive: true });
 }
@@ -13556,6 +13926,18 @@ function shouldLog(level) {
   const threshold = LEVEL_ORDER[config.logging.level] ?? LEVEL_ORDER.warn;
   return LEVEL_ORDER[level] >= threshold;
 }
+function emitOtelLog(level, args) {
+  if (!enabled)
+    return;
+  const body = formatArgs(args);
+  const module = body.match(/^\[([^\]]+)\]/)?.[1];
+  otelLogger.emit({
+    severityNumber: OTEL_SEVERITY[level],
+    severityText: level.toUpperCase(),
+    body,
+    attributes: module ? { "halo.module": module } : void 0
+  });
+}
 function initLogger() {
   const origLog = console.log.bind(console);
   const origError = console.error.bind(console);
@@ -13566,39 +13948,50 @@ function initLogger() {
       return;
     origDebug(...args);
     writeToFile(formatLine("debug", args));
+    emitOtelLog("debug", args);
   };
   console.log = (...args) => {
     if (!shouldLog("info"))
       return;
     origLog(...args);
     writeToFile(formatLine("info", args));
+    emitOtelLog("info", args);
   };
   console.warn = (...args) => {
     if (!shouldLog("warn"))
       return;
     origWarn(...args);
     writeToFile(formatLine("warn", args));
+    emitOtelLog("warn", args);
   };
   console.error = (...args) => {
     if (!shouldLog("error"))
       return;
     origError(...args);
     writeToFile(formatLine("error", args));
+    emitOtelLog("error", args);
   };
   if (shouldLog("info")) {
     writeToFile(formatLine("info", [`[Logger] File logging initialized: ${logFile} (level: ${config.logging.level})`]));
   }
 }
-var LOG_FILENAME, logDir, logFile, currentSize, LEVEL_ORDER;
+var LOG_FILENAME, logDir, logFile, currentSize, LEVEL_ORDER, OTEL_SEVERITY;
 var init_logger = __esm({
   "../server/dist/logger.js"() {
     "use strict";
     init_config();
+    init_otel();
     LOG_FILENAME = "server.log";
     logDir = path31.join(homedir13(), ".halo", "global", "logs");
     logFile = path31.join(logDir, LOG_FILENAME);
     currentSize = -1;
     LEVEL_ORDER = { debug: 0, info: 1, warn: 2, error: 3 };
+    OTEL_SEVERITY = {
+      debug: SeverityNumber.DEBUG,
+      info: SeverityNumber.INFO,
+      warn: SeverityNumber.WARN,
+      error: SeverityNumber.ERROR
+    };
   }
 });
 
@@ -15326,7 +15719,7 @@ function filterVerbs(items, rawValue) {
   const partial = m[2].toLowerCase();
   return cmd.verbs.filter((v) => v.name.startsWith(partial) && v.name !== partial).map((v) => ({ slashName: `${cmd.slashName} ${v.name}`, description: v.desc ?? "" }));
 }
-function InputBox({ enabled, placeholder, history, onSubmit, hint, commands = [], workspace, onPopupOpenChange }) {
+function InputBox({ enabled: enabled2, placeholder, history, onSubmit, hint, commands = [], workspace, onPopupOpenChange }) {
   const [edit, setEdit] = useState2({ value: "", cursor: 0 });
   const { value, cursor } = edit;
   const [historyIdx, setHistoryIdx] = useState2(null);
@@ -15353,7 +15746,7 @@ function InputBox({ enabled, placeholder, history, onSubmit, hint, commands = []
       insertion = normalized;
     }
     applyEdit((e) => insertAt(e, insertion));
-  }, { isActive: enabled });
+  }, { isActive: enabled2 });
   const expandPastes = (v) => v.replace(/\[#(\d+) pasted \d+ lines?\]/g, (m, id) => pendingPastes.current.get(Number(id)) ?? m);
   const cmdMatches = useMemo(() => filterCommands(commands, value), [commands, value]);
   const verbMatches = useMemo(() => filterVerbs(commands, value), [commands, value]);
@@ -15539,17 +15932,17 @@ function InputBox({ enabled, placeholder, history, onSubmit, hint, commands = []
     if (input.length === 0) return;
     const clean = input.replace(/[\r\n]+/g, " ");
     applyEdit((e) => insertAt(e, clean));
-  }, { isActive: enabled });
+  }, { isActive: enabled2 });
   useEffect2(() => {
     if (historyIdx !== null && value === "") setHistoryIdx(null);
   }, [value, historyIdx]);
   return /* @__PURE__ */ jsxs7(Box7, { flexDirection: "column", children: [
     suggestOpen ? /* @__PURE__ */ jsx8(SlashSuggest, { items: filtered, selectedIdx: suggestIdx }) : null,
     !suggestOpen && pathOpen ? /* @__PURE__ */ jsx8(PathSuggest, { items: pathScan.items, selectedIdx: pathIdx, cwdLabel: pathScan.cwdLabel }) : null,
-    /* @__PURE__ */ jsxs7(Box7, { flexDirection: "column", borderStyle: "round", borderColor: enabled ? "cyan" : "gray", paddingX: 1, children: [
+    /* @__PURE__ */ jsxs7(Box7, { flexDirection: "column", borderStyle: "round", borderColor: enabled2 ? "cyan" : "gray", paddingX: 1, children: [
       /* @__PURE__ */ jsxs7(Box7, { children: [
-        /* @__PURE__ */ jsx8(Text8, { color: enabled ? "cyan" : "gray", children: "> " }),
-        enabled ? /* @__PURE__ */ jsx8(TextInputView, { value, cursor, placeholder }) : /* @__PURE__ */ jsx8(Text8, { color: "gray", dimColor: true, children: placeholder })
+        /* @__PURE__ */ jsx8(Text8, { color: enabled2 ? "cyan" : "gray", children: "> " }),
+        enabled2 ? /* @__PURE__ */ jsx8(TextInputView, { value, cursor, placeholder }) : /* @__PURE__ */ jsx8(Text8, { color: "gray", dimColor: true, children: placeholder })
       ] }),
       hint ? /* @__PURE__ */ jsx8(Box7, { justifyContent: "flex-end", children: /* @__PURE__ */ jsx8(Text8, { color: "gray", dimColor: true, children: hint }) }) : null
     ] })
@@ -18374,10 +18767,10 @@ function parseAgentYaml(content) {
 }
 function buildScaffoldModelBlock() {
   const registry2 = getModelsRegistry();
-  const providers = registry2.providers ?? [];
+  const providers2 = registry2.providers ?? [];
   const configured = config.agent.defaultProvider;
-  const explicit = configured ? providers.find((p) => p.id === configured) : void 0;
-  const provider = explicit ?? providers.find((p) => p.id === FALLBACK_SCAFFOLD_PROVIDER) ?? providers[0];
+  const explicit = configured ? providers2.find((p) => p.id === configured) : void 0;
+  const provider = explicit ?? providers2.find((p) => p.id === FALLBACK_SCAFFOLD_PROVIDER) ?? providers2[0];
   if (!provider) {
     return { provider: FALLBACK_SCAFFOLD_PROVIDER };
   }
@@ -29771,6 +30164,7 @@ async function gracefulShutdown(signal) {
     server.close();
   }
   console.log("[Server] Shutdown complete");
+  await shutdownObservability();
   process.exit(0);
 }
 var __dirname3, PORT, HALO_VERSION, PROJECT_ROOT, HALO_HOME4, GIT_SHA, AGENTCORE, app, allowlist, authRoutes, fileRoutes, dataPreviewRoutes, gitRoutes, agentConfigRoutes, skillRoutes, settingsRoutes, evolutionRoutes, cronRoutes, channelDb, registry, sessionRoutes, sessionArchiveRoutes, showRoutes, metricsRoutes, commandRoutes, FRONTEND_DIR, server, wss;
@@ -29810,13 +30204,14 @@ var init_dist2 = __esm({
     init_descriptors();
     init_auth();
     init_logger();
+    init_otel();
     init_config();
     init_sandbox();
     init_init();
     init_git_ssh();
     __dirname3 = path67.dirname(fileURLToPath3(import.meta.url));
     PORT = config.server.port;
-    HALO_VERSION = "1.2.1";
+    HALO_VERSION = "1.3.1";
     PROJECT_ROOT = findProjectRoot();
     HALO_HOME4 = path67.join(homedir24(), ".halo");
     GIT_SHA = readGitSha();
@@ -29855,6 +30250,7 @@ var init_dist2 = __esm({
         throw new Error(`[Server] Dispatch cases without a descriptor: ${orphanDispatch.join(", ")}. Add a registerDescriptor entry in commands/index.ts or remove the dispatch case.`);
       }
     }
+    await initObservability();
     initLogger();
     await initBwrapCheck();
     setSandboxHiddenPaths(config.sandbox.hiddenDirs, config.sandbox.hiddenFiles, config.sandbox.writableDirs);
@@ -30750,7 +31146,7 @@ import path68 from "node:path";
 import fs61 from "node:fs";
 import { homedir as homedir25 } from "node:os";
 import { parseArgs as parseArgs2 } from "node:util";
-var VERSION = "1.2.1";
+var VERSION = "1.3.1";
 var HELP_TOP = `Usage: halo <command> [options]
 
 Commands:
@@ -31130,8 +31526,8 @@ async function cmdSetup(argv = []) {
 `);
   }
   async function stepModelProviders() {
-    const providers = listModelProviders2();
-    if (providers.length === 0) return;
+    const providers2 = listModelProviders2();
+    if (providers2.length === 0) return;
     const enter = await promptSelect2("Configure model provider keys now? (you can also re-run `halo setup` later)", [
       { value: "skip", label: "Skip" },
       { value: "go", label: "Configure..." }
@@ -31142,7 +31538,7 @@ async function cmdSetup(argv = []) {
       return;
     }
     while (true) {
-      const opts = providers.map((p) => ({
+      const opts = providers2.map((p) => ({
         value: p.id,
         label: p.displayName,
         hint: providerStatusHint(p.id, p.fields)
@@ -31151,7 +31547,7 @@ async function cmdSetup(argv = []) {
       const pick = await promptSelect2("Pick a provider to configure (Done when finished):", opts);
       if (pick == null) abort();
       if (pick === "__done__") break;
-      const provider = providers.find((p) => p.id === pick);
+      const provider = providers2.find((p) => p.id === pick);
       if (!provider) continue;
       await configureSecretGroup(provider.id, provider.bucket, provider.fields);
     }