@meridiona/meridian-darwin-arm64 1.55.0 → 1.57.0

package/VERSION

@@ -1 +1 @@
-1.55.0
+1.57.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meridiona/meridian-darwin-arm64",
-  "version": "1.55.0",
+  "version": "1.57.0",
   "description": "Prebuilt Meridian app for macOS arm64 (daemon binary + dashboard + Python services). Installed via @meridiona/meridian.",
   "homepage": "https://github.com/Meridiona/meridian",
   "repository": {

package/services/agents/observability.py

@@ -3,10 +3,11 @@
 A single `setup(agent_name)` call wires up:
 
   * an OTel `TracerProvider` with `service.name=agent_name`
-  * a `BatchSpanProcessor` exporting OTLP/HTTP-protobuf spans to OpenObserve
-  * a `LoggerProvider` + OTLP-logs handler so every `logging.LogRecord` is also
-    shipped to OpenObserve (correlated to the active span), mirroring the Rust
-    daemon's `OpenTelemetryTracingBridge`
+  * a `BatchSpanProcessor` writing OTLP/HTTP-protobuf spans to the durable
+    telemetry spool (the Rust daemon's shipper drains it to OpenObserve)
+  * a `LoggerProvider` + spool log handler so every `logging.LogRecord` is also
+    spooled (correlated to the active span), mirroring the Rust daemon's
+    `OpenTelemetryTracingBridge`
   * W3C `TraceContextTextMapPropagator` as the global propagator so each
     agent can pick up the Rust daemon's `traceparent` and continue the trace
   * `LoggingInstrumentor` so every `logging.LogRecord` carries
@@ -15,11 +16,11 @@ A single `setup(agent_name)` call wires up:
     under `~/.meridian/logs/{agent_name}.jsonl` plus stderr — both ingestable
     by OpenObserve's log pipeline without further parsing.
 
-Export config (endpoint + Basic-auth credentials) is resolved from the SAME
-`~/.meridian/settings.json` the Rust daemon reads — `otlp_enabled`,
-`otlp_endpoint`, `oo_email`, `oo_password` — so the dashboard Settings page is
-the single source of truth for both processes. The legacy `MERIDIAN_OO_AUTH`
-env credential is deprecated and ignored, matching the daemon.
+Export is gated by the SAME `~/.meridian/settings.json` the Rust daemon reads —
+the `otlp_enabled` toggle — so the dashboard Settings page is the single source
+of truth for both processes. Delivery (endpoint + Basic-auth credentials) is
+owned entirely by the Rust shipper; Python only ever writes to the spool. The
+legacy `MERIDIAN_OO_AUTH` env credential is deprecated and ignored.
 
 `extract_parent_context(traceparent)` is the helper agents use to continue
 a span emitted by another process — typically the Rust ETL or another
@@ -28,46 +29,182 @@ agent stage.
 Idempotent: calling `setup` twice is a no-op for the second call (returns
 the existing tracer). This matters because both the daemon and the
 single-shot CLI paths funnel through the same module.
+
+Spool durability: when `otlp_enabled` is true (regardless of whether
+credentials are present), spans and logs are written atomically to
+`~/.meridian/telemetry/pending/<signal>-<unix_micros>-<seq>.otlp` via
+`SpoolSpanExporter` and `SpoolLogExporter`.  The Rust daemon's shipper
+task drains that shared directory to OpenObserve — Python does NOT need
+its own shipper, and credential resolution lives there, not here.
 """
 from __future__ import annotations
 
-import base64
 import json
 import logging
 import logging.handlers
 import os
 import sys
+import threading
+import time
 from pathlib import Path
-from typing import NamedTuple, Optional
+from typing import Optional
 
 from opentelemetry import trace
 from opentelemetry._logs import set_logger_provider
 from opentelemetry.context import Context
-from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
-    OTLPSpanExporter,
-)
 from opentelemetry.instrumentation.logging import LoggingInstrumentor
 from opentelemetry.propagate import set_global_textmap
 from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
 from opentelemetry.trace.propagation.tracecontext import (
     TraceContextTextMapPropagator,
 )
 from pythonjsonlogger import jsonlogger
 
 
+# ──────────────────────── Spool exporters ──────────────────────────────────────
+
+_spool_seq_lock = threading.Lock()
+_spool_seq = 0
+
+
+def _next_spool_seq() -> int:
+    global _spool_seq
+    with _spool_seq_lock:
+        val = _spool_seq
+        _spool_seq += 1
+    return val
+
+
+def _resolve_telemetry_dir() -> Path:
+    """Mirror of the Rust writer's resolve_telemetry_dir().
+
+    Precedence: MERIDIAN_TELEMETRY_DIR env → ~/.meridian/telemetry.
+    """
+    env = os.environ.get("MERIDIAN_TELEMETRY_DIR", "").strip()
+    if env:
+        return Path(env).expanduser()
+    home = Path.home()
+    return home / ".meridian" / "telemetry"
+
+
+def _write_spool(signal: str, payload: bytes) -> None:
+    """Atomically write payload to ~/.meridian/telemetry/pending/.
+
+    Filename: <signal>-<unix_micros>-<seq>.otlp
+    Write via <name>.tmp then rename so the Rust shipper never sees partial files.
+    """
+    base = _resolve_telemetry_dir()
+    pending = base / "pending"
+    pending.mkdir(parents=True, exist_ok=True)
+
+    micros = int(time.time() * 1_000_000)
+    seq = _next_spool_seq()
+    filename = f"{signal}-{micros}-{seq}.otlp"
+    final_path = pending / filename
+    tmp_path = pending / f"{filename}.tmp"
+
+    try:
+        # fsync the tmp file before the rename so a power loss can't leave a
+        # rename (metadata) durable while the data blocks are still in page
+        # cache — that would surface a truncated .otlp the shipper POSTs (→ a
+        # 400). Mirrors the Rust writer's sync_all() + dir fsync.
+        with open(tmp_path, "wb") as fh:
+            fh.write(payload)
+            fh.flush()
+            os.fsync(fh.fileno())
+        tmp_path.rename(final_path)
+        try:
+            dir_fd = os.open(str(pending), os.O_RDONLY)
+            try:
+                os.fsync(dir_fd)
+            finally:
+                os.close(dir_fd)
+        except OSError:
+            # Directory fsync is best-effort (not all FS/platforms support it);
+            # the tmp-file fsync already guarantees the data is on disk.
+            pass
+    except Exception as exc:
+        logging.getLogger(__name__).warning(
+            "telemetry spool write failed — payload dropped",
+            extra={"signal": signal, "error": str(exc)},
+        )
+        # Best-effort cleanup so a failed write never strands a .tmp orphan
+        # (the Rust shipper sweeps these, but don't rely on it).
+        try:
+            tmp_path.unlink(missing_ok=True)
+        except OSError as cleanup_exc:
+            # Cleanup failure is non-fatal: write already failed and we avoid
+            # raising secondary errors from best-effort orphan removal.
+            logging.getLogger(__name__).debug(
+                "telemetry spool tmp cleanup failed",
+                extra={"tmp_path": str(tmp_path), "error": str(cleanup_exc)},
+            )
+
+
+class SpoolSpanExporter:
+    """Span exporter that writes serialised OTLP payloads to the spool dir.
+
+    Wraps the SDK's encode_spans() to produce the same wire bytes the real
+    OTLPSpanExporter would POST.  The Rust shipper drains pending/ to OO.
+    """
+
+    def export(self, spans):  # type: ignore[override]
+        try:
+            from opentelemetry.exporter.otlp.proto.common.trace_encoder import (
+                encode_spans,
+            )
+            payload = encode_spans(spans).SerializeToString()
+            _write_spool("traces", payload)
+        except Exception as exc:
+            logging.getLogger(__name__).warning(
+                "SpoolSpanExporter.export failed", extra={"error": str(exc)}
+            )
+        return SpanExportResult.SUCCESS
+
+    def shutdown(self) -> None:
+        pass
+
+    def force_flush(self, timeout_millis: int = 30_000) -> bool:
+        return True
+
+
+class SpoolLogExporter:
+    """Log exporter that writes serialised OTLP payloads to the spool dir.
+
+    Mirrors SpoolSpanExporter for the log signal.
+    """
+
+    def export(self, log_data):  # type: ignore[override]
+        try:
+            from opentelemetry.exporter.otlp.proto.common._log_encoder import (
+                encode_logs,
+            )
+            payload = encode_logs(log_data).SerializeToString()
+            _write_spool("logs", payload)
+        except Exception as exc:
+            logging.getLogger(__name__).warning(
+                "SpoolLogExporter.export failed", extra={"error": str(exc)}
+            )
+        return LogExportResult.SUCCESS
+
+    def shutdown(self) -> None:
+        pass
+
+    def force_flush(self, timeout_millis: int = 30_000) -> bool:
+        return True
+
+
 # ──────────────────────── Config ───────────────────────────────────────────────
-DEFAULT_TRACES_ENDPOINT = "http://localhost:5080/api/default/v1/traces"
-DEFAULT_LOGS_ENDPOINT   = "http://localhost:5080/api/default/v1/logs"
-DEFAULT_LOG_DIR         = Path.home() / ".meridian" / "logs"
-# Single source of truth for OpenObserve export config — the SAME file the Rust
-# daemon reads (see `src/observability.rs::resolve_otlp_target`). Keeps the two
-# processes credential-aligned: the dashboard Settings page writes here and both
-# the daemon and this MLX server pick it up with no env plumbing.
+DEFAULT_LOG_DIR = Path.home() / ".meridian" / "logs"
+# Single source of truth for the export TOGGLE — the SAME file the Rust daemon
+# reads (see `src/observability.rs::resolve_otlp_target`). Delivery credentials
+# live there too: the dashboard Settings page writes here and the Rust shipper
+# picks them up. This process only reads `otlp_enabled` to decide whether to
+# spool — it never delivers, so it needs no endpoint/credentials of its own.
 _SETTINGS_PATH = Path(
     os.environ.get("MERIDIAN_SETTINGS_PATH")
     or (Path.home() / ".meridian" / "settings.json")
@@ -80,18 +217,6 @@ _INITIALISED: dict[str, trace.Tracer] = {}
 _PROCESS_SERVICE_NAME: str | None = None
 # Held so shutdown() can flush log records the same way it flushes spans.
 _LOGGER_PROVIDER: LoggerProvider | None = None
-# One-time guard so an export misconfiguration (enabled-but-no-creds, or a
-# schemeless endpoint) warns once per process instead of on every resolve.
-_WARNED_EXPORT_MISCONFIG: bool = False
-
-
-# ──────────────────────── OTLP target resolution ───────────────────────────────
-class _OtlpTarget(NamedTuple):
-    """Resolved OTLP export target: signal endpoints + Basic-auth header value."""
-
-    traces_endpoint: str
-    logs_endpoint: str
-    headers: dict[str, str]
 
 
 def _load_settings() -> dict[str, object]:
@@ -104,85 +229,17 @@ def _load_settings() -> dict[str, object]:
         return {}
 
 
-def _resolve_otlp_target() -> Optional[_OtlpTarget]:
-    """Mirror of the Rust daemon's `resolve_otlp_target()`.
+def _is_otlp_enabled() -> bool:
+    """Return True when the otlp_enabled toggle is on and tracing is not disabled.
 
-    Returns `None` (→ export disabled) when the toggle is off or credentials
-    are missing. Endpoint precedence: settings.json `otlp_endpoint` → the
-    `MERIDIAN_OTLP_TRACES_ENDPOINT`/`MERIDIAN_OTLP_ENDPOINT` env override →
-    the localhost default. Auth is `base64(oo_email:oo_password)` — settings.json
-    only; the legacy `MERIDIAN_OO_AUTH` env path is deprecated and ignored, the
-    same decision the daemon made.
+    Deliberately does NOT check credentials — used to gate the spool exporters
+    so telemetry is captured even when OO credentials are absent (the Rust
+    shipper delivers when they are provided later, and warns once if a user
+    leaves the toggle on with no credentials).
     """
-    global _WARNED_EXPORT_MISCONFIG
-
     if os.environ.get("MERIDIAN_TRACING_DISABLED", "").lower() in ("1", "true", "yes"):
-        return None
-
-    settings = _load_settings()
-    if not settings.get("otlp_enabled"):
-        return None
-
-    # Resolve the endpoint up front so we can warn (not silently disable) when
-    # export is enabled but unusable. Precedence: settings → env → localhost.
-    configured = str(settings.get("otlp_endpoint") or "").strip()
-    env_endpoint = (
-        os.environ.get("MERIDIAN_OTLP_TRACES_ENDPOINT", "").strip()
-        or os.environ.get("MERIDIAN_OTLP_ENDPOINT", "").strip()
-    )
-    traces_endpoint = configured or env_endpoint or DEFAULT_TRACES_ENDPOINT
-
-    def _warn_once(msg: str, *args: object) -> None:
-        global _WARNED_EXPORT_MISCONFIG
-        if not _WARNED_EXPORT_MISCONFIG:
-            _WARNED_EXPORT_MISCONFIG = True
-            logging.getLogger(__name__).warning(msg, *args)
-
-    email = str(settings.get("oo_email") or "")
-    password = str(settings.get("oo_password") or "")
-    if not email or not password:
-        # otlp_enabled but no usable credentials → export OFF. Warn once so an
-        # env-only (MERIDIAN_OO_AUTH) install that predates the settings.json
-        # credential move doesn't go dark silently — mirrors the daemon, which
-        # also warns. MERIDIAN_OO_AUTH is no longer read here.
-        _warn_once(
-            "OpenObserve export enabled but oo_email/oo_password missing in %s — "
-            "traces+logs export DISABLED. Set credentials in the dashboard Settings "
-            "(the MERIDIAN_OO_AUTH env var is no longer used).",
-            _SETTINGS_PATH,
-        )
-        return None
-    # Guard against HTTP header injection / malformed user:password splits —
-    # matches the daemon's same-named check.
-    if any(c in email for c in "\r\n:") or any(c in password for c in "\r\n"):
-        return None
-    auth = base64.standard_b64encode(f"{email}:{password}".encode()).decode()
-
-    # Validate scheme — only http/https are valid OTLP transports. The daemon
-    # disables export + warns on a schemeless endpoint; mirror that exactly so the
-    # two processes don't disagree on whether export is on.
-    if not (traces_endpoint.startswith("http://") or traces_endpoint.startswith("https://")):
-        _warn_once(
-            "OTLP endpoint %r has no http/https scheme — export DISABLED.",
-            traces_endpoint,
-        )
-        return None
-
-    # OpenObserve serves logs at the sibling `…/v1/logs` path. Derive it from the
-    # traces endpoint by swapping the trailing signal segment so a custom host or
-    # base (incl. a trailing slash, e.g. `…/v1/traces/`) carries to BOTH signals —
-    # never silently fall back to localhost for logs while traces go remote.
-    t = traces_endpoint.rstrip("/")
-    if t.endswith("/v1/traces"):
-        logs_endpoint = t[: -len("/v1/traces")] + "/v1/logs"
-    elif t.endswith("/traces"):
-        logs_endpoint = t[: -len("/traces")] + "/logs"
-    elif "traces" in t:
-        logs_endpoint = t.rsplit("traces", 1)[0] + "logs"
-    else:
-        logs_endpoint = t + "/v1/logs"
-
-    return _OtlpTarget(traces_endpoint, logs_endpoint, {"Authorization": f"Basic {auth}"})
+        return False
+    return bool(_load_settings().get("otlp_enabled"))
 
 
 # ──────────────────────── Public API ───────────────────────────────────────────
@@ -207,13 +264,8 @@ def setup(agent_name: str) -> trace.Tracer:
 
     if _PROCESS_SERVICE_NAME is None:
         _PROCESS_SERVICE_NAME = agent_name
-        # Resolve the export target ONCE and pass it to both configurers — a
-        # second read could see a settings.json the dashboard rewrote mid-setup
-        # (TOCTOU), leaving traces enabled while logs resolve disabled (or with
-        # different creds/endpoint) in the same process.
-        target = _resolve_otlp_target()
-        _configure_tracing(agent_name, target)
-        _configure_logging(agent_name, target)
+        _configure_tracing(agent_name)
+        _configure_logging(agent_name)
         logging.getLogger(agent_name).info(
             "observability initialised",
             extra={"service.name": agent_name},
@@ -257,15 +309,14 @@ def extract_parent_context(traceparent: Optional[str]) -> Optional[Context]:
 
 
 # ──────────────────────── Tracing setup ────────────────────────────────────────
-def _configure_tracing(agent_name: str, target: Optional[_OtlpTarget]) -> None:
+def _configure_tracing(agent_name: str) -> None:
     resource = Resource.create({"service.name": agent_name})
     provider = TracerProvider(resource=resource)
 
-    if target is not None:
-        exporter = OTLPSpanExporter(
-            endpoint=target.traces_endpoint, headers=target.headers
-        )
-        provider.add_span_processor(BatchSpanProcessor(exporter))
+    # Wire the spool exporter when otlp_enabled is true (even without creds).
+    # The Rust shipper drains the shared spool dir when a target is available.
+    if _is_otlp_enabled():
+        provider.add_span_processor(BatchSpanProcessor(SpoolSpanExporter()))
 
     # Set as the global provider. OTel's `set_tracer_provider` warns if
     # someone already configured a provider in-process; we accept that and
@@ -274,32 +325,32 @@ def _configure_tracing(agent_name: str, target: Optional[_OtlpTarget]) -> None:
     set_global_textmap(TraceContextTextMapPropagator())
 
 
-def _configure_log_export(
-    agent_name: str, target: Optional[_OtlpTarget]
-) -> Optional[logging.Handler]:
-    """Build an OTLP-logs handler so every `log.*` record reaches OpenObserve,
+def _configure_log_export(agent_name: str) -> Optional[logging.Handler]:
+    """Build a spool-logs handler so every `log.*` record reaches OpenObserve,
     correlated to the active span by trace_id/span_id — the Python counterpart
     of the Rust daemon's `OpenTelemetryTracingBridge`.
 
     Returns the handler (caller attaches it to root) or `None` when export is
     disabled, in which case logs still go to the JSONL file + stdout/stderr.
+
+    When `otlp_enabled` is true the spool log exporter is always wired (even
+    without credentials) — the Rust shipper handles delivery.
     """
     global _LOGGER_PROVIDER
 
-    if target is None:
+    if not _is_otlp_enabled():
         return None
 
     resource = Resource.create({"service.name": agent_name})
     provider = LoggerProvider(resource=resource)
-    exporter = OTLPLogExporter(endpoint=target.logs_endpoint, headers=target.headers)
-    provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+    provider.add_log_record_processor(BatchLogRecordProcessor(SpoolLogExporter()))
     set_logger_provider(provider)
     _LOGGER_PROVIDER = provider
     return LoggingHandler(level=logging.NOTSET, logger_provider=provider)
 
 
 # ──────────────────────── Logging setup ────────────────────────────────────────
-def _configure_logging(agent_name: str, target: Optional[_OtlpTarget]) -> None:
+def _configure_logging(agent_name: str) -> None:
     log_dir = Path(os.environ.get("MERIDIAN_LOG_DIR") or DEFAULT_LOG_DIR)
     log_dir.mkdir(parents=True, exist_ok=True)
     log_path = log_dir / f"{agent_name}.jsonl"
@@ -359,17 +410,17 @@ def _configure_logging(agent_name: str, target: Optional[_OtlpTarget]) -> None:
     root.addHandler(file_h)
     root.addHandler(stdout_h)
     root.addHandler(stderr_h)
-    # Ship every record to OpenObserve via OTLP/HTTP logs too, when export is
-    # configured. The OTel LoggingHandler reads the active span context, so each
+    # Spool every record for OpenObserve via OTLP/HTTP logs too, when export is
+    # enabled. The OTel LoggingHandler reads the active span context, so each
     # OO log row carries the trace_id/span_id that ties it to the classifier's
     # span waterfall. No-op (None) when OTLP is disabled.
-    # The OTLP handler already carries service.name via the OTel Resource, so it
+    # The spool handler already carries service.name via the OTel Resource, so it
     # needs no _ServiceFilter (that would duplicate the attribute on each record).
-    otlp_log_h = _configure_log_export(agent_name, target)
+    otlp_log_h = _configure_log_export(agent_name)
     if otlp_log_h is not None:
-        # Do NOT feed the OTLP exporter's OWN transport logs back into OTLP
-        # export: on export failure httpx/urllib3/opentelemetry emit WARNING+
-        # records which this root handler would try to export → more failures (a
+        # Do NOT feed the spool handler's OWN transport/encoder logs back into
+        # the spool: on a hiccup httpx/urllib3/opentelemetry emit WARNING+
+        # records which this root handler would try to spool → more failures (a
         # log→export→log loop). Drop those from THIS handler only — they still
         # reach the file/stderr handlers.
         _otlp_excluded = ("httpx", "httpcore", "urllib3", "grpc", "opentelemetry")

package/services/agents/pm_worklog_update/agents.py

@@ -62,7 +62,8 @@ def build_synth_agent(
         debug_level: 1 or 2 (verbose).
     """
     from agno.agent import Agent
-    from agno.guardrails import PIIDetectionGuardrail
+    # PII guardrail disabled for now — not needed.
+    # from agno.guardrails import PIIDetectionGuardrail
     from agno.skills import LocalSkills, Skills
 
     return Agent(
@@ -98,11 +99,18 @@ def build_synth_agent(
             SessionBundleSizeGuard(max_tokens=80_000),
         ],
         post_hooks=[
-            PIIDetectionGuardrail(),
+            # PIIDetectionGuardrail disabled — it's an input guardrail and was
+            # wrongly wired into post_hooks (post-hooks get run_output, not
+            # run_input → TypeError). Not needed for now.
             time_spent_sanity_check,
         ],
         output_schema=JiraUpdate,
-        use_json_mode=True,
+        # use_json_mode=False → agno sends the full JiraUpdate json_schema as the
+        # request's response_format (not a bare {"type":"json_object"}). The MLX
+        # /v1/chat/completions handler reads that schema and FSM-constrains
+        # decoding with outlines, so the reasoning model physically cannot leak
+        # chain-of-thought prose instead of the JSON object.
+        use_json_mode=False,
         add_datetime_to_context=True,
         add_history_to_context=False,
         markdown=False,