@cybermem/mcp 0.1.0 → 0.5.0

package/templates/monitoring/grafana/provisioning/dashboards/default.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'Default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

package/templates/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

package/templates/monitoring/log_exporter/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-alpine
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY exporter.py .
+RUN chmod +x exporter.py
+
+EXPOSE 8001
+
+CMD ["python", "-u", "exporter.py"]

package/templates/monitoring/log_exporter/exporter.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+Traefik Access Log Exporter for Prometheus
+Parses Traefik JSON access logs and exports per-request metrics
+Writes aggregates to cybermem_stats table for persistence
+"""
+
+import json
+import os
+import sqlite3
+import time
+from pathlib import Path
+
+# Configuration
+LOG_FILE = os.environ.get("LOG_FILE", "/var/log/traefik/access.log")
+EXPORTER_PORT = int(os.environ.get("EXPORTER_PORT", "8001"))
+SCRAPE_INTERVAL = int(os.environ.get("SCRAPE_INTERVAL", "5"))
+DB_PATH = os.environ.get("DB_PATH", "/data/openmemory.sqlite")
+
+# No Prometheus metrics here - we write to DB instead
+# db_exporter will read from cybermem_stats and export as Gauge
+
+
+def init_db():
+    """Initialize cybermem_stats and cybermem_access_log tables in OpenMemory database"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+
+    # Create aggregate stats table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS cybermem_stats (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            client_name TEXT NOT NULL,
+            operation TEXT NOT NULL,
+            count INTEGER DEFAULT 0,
+            errors INTEGER DEFAULT 0,
+            last_updated INTEGER NOT NULL,
+            UNIQUE(client_name, operation)
+        )
+    """)
+
+    cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_cybermem_stats_client_op
+        ON cybermem_stats(client_name, operation)
+    """)
+
+    # Create access log table for detailed request history
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS cybermem_access_log (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            timestamp INTEGER NOT NULL,
+            client_name TEXT NOT NULL,
+            client_version TEXT,
+            method TEXT NOT NULL,
+            endpoint TEXT NOT NULL,
+            operation TEXT NOT NULL,
+            status TEXT NOT NULL,
+            is_error INTEGER DEFAULT 0
+        )
+    """)
+
+    cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_cybermem_access_log_timestamp
+        ON cybermem_access_log(timestamp DESC)
+    """)
+
+    cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_cybermem_access_log_client
+        ON cybermem_access_log(client_name, timestamp DESC)
+    """)
+
+    conn.commit()
+    conn.close()
+    print(f"[DB] Initialized cybermem tables in {DB_PATH}")
+
+
+def increment_stat(client_name: str, operation: str, is_error: bool = False):
+    """Increment counter in cybermem_stats table"""
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+
+        ts = int(time.time() * 1000)
+
+        # Upsert: increment if exists, insert if not
+        if is_error:
+            cursor.execute(
+                """
+                INSERT INTO cybermem_stats (client_name, operation, count, errors, last_updated)
+                VALUES (?, ?, 1, 1, ?)
+                ON CONFLICT(client_name, operation)
+                DO UPDATE SET
+                    count = count + 1,
+                    errors = errors + 1,
+                    last_updated = ?
+            """,
+                [client_name, operation, ts, ts],
+            )
+        else:
+            cursor.execute(
+                """
+                INSERT INTO cybermem_stats (client_name, operation, count, errors, last_updated)
+                VALUES (?, ?, 1, 0, ?)
+                ON CONFLICT(client_name, operation)
+                DO UPDATE SET
+                    count = count + 1,
+                    last_updated = ?
+            """,
+                [client_name, operation, ts, ts],
+            )
+
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"[DB] Error updating stats: {e}")
+
+
+def log_access(
+    client_name: str,
+    client_version: str,
+    method: str,
+    endpoint: str,
+    operation: str,
+    status: str,
+    is_error: bool,
+):
+    """Log individual request to access_log table"""
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+
+        ts = int(time.time() * 1000)
+
+        cursor.execute(
+            """
+            INSERT INTO cybermem_access_log (timestamp, client_name, client_version, method, endpoint, operation, status, is_error)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+            [
+                ts,
+                client_name,
+                client_version,
+                method,
+                endpoint,
+                operation,
+                status,
+                1 if is_error else 0,
+            ],
+        )
+
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"[DB] Error logging access: {e}")
+
+
+def tail_log_file(filepath, poll_interval=1):
+    """Tail a log file, yielding new lines as they appear"""
+    path = Path(filepath)
+
+    # Wait for file to exist
+    while not path.exists():
+        print(f"Waiting for log file: {filepath}")
+        time.sleep(poll_interval)
+
+    with open(filepath, "r") as f:
+        # Start from the end of file
+        f.seek(0, 2)
+
+        while True:
+            line = f.readline()
+            if line:
+                yield line
+            else:
+                time.sleep(poll_interval)
+
+
+def parse_and_export():
+    """Parse Traefik access logs and export metrics"""
+    print(f"Starting Traefik Log Exporter on port {EXPORTER_PORT}")
+    print(f"Watching log file: {LOG_FILE}")
+
+    for line in tail_log_file(LOG_FILE, poll_interval=SCRAPE_INTERVAL):
+        try:
+            data = json.loads(line.strip())
+
+            # Extract relevant fields
+            method = data.get("RequestMethod", "UNKNOWN")
+            path = data.get("RequestPath", "/unknown")
+            status = str(data.get("DownstreamStatus", 0))
+
+            # Extract MCP client info from custom headers
+            client_name = data.get("request_X-Client-Name", "unknown")
+            client_version = data.get("request_X-Client-Version", "unknown")
+
+            # Fallback to User-Agent if client_name is unknown
+            if client_name == "unknown":
+                ua = data.get("request_User-Agent", "")
+                if ua and ua != "-":
+                    # Simple heuristic: take the first part before '/' or space
+                    # e.g. "curl/7.64.1" -> "curl", "Mozilla/5.0" -> "Mozilla"
+                    parts = ua.split("/")
+                    if len(parts) > 0:
+                        potential_name = parts[0].split(" ")[0].strip()
+                        if potential_name:
+                            client_name = potential_name
+
+            # Remove query params first
+            endpoint = path.split("?")[0]
+
+            # Determine operation type from endpoint BEFORE normalization
+            if endpoint == "/memory/add":
+                operation = "create"
+            elif endpoint == "/memory/query":
+                operation = "read"
+            elif endpoint.startswith("/memory/") and method == "PATCH":
+                operation = "update"
+            elif endpoint.startswith("/memory/") and method == "DELETE":
+                operation = "delete"
+            elif endpoint.startswith("/mcp"):
+                operation = "create"  # MCP operations are typically POST
+            else:
+                operation = "other"
+
+            # NOW normalize endpoint to remove IDs (e.g., /memory/123 -> /memory/:id)
+            if (
+                endpoint.startswith("/memory/")
+                and len(endpoint) > 8
+                and operation in ["update", "delete"]
+            ):
+                endpoint = "/memory/:id"
+
+            # Only track requests to OpenMemory API (/memory/* and /mcp endpoints)
+            # Exclude /health checks - they pollute Top/Last Reader metrics
+            if endpoint.startswith("/memory") or endpoint.startswith("/mcp"):
+                # Check if it's an error (4xx or 5xx)
+                is_error = status.startswith("4") or status.startswith("5")
+
+                # Write aggregate stats
+                increment_stat(client_name, operation, is_error)
+
+                # Log individual request to access_log
+                log_access(
+                    client_name,
+                    client_version,
+                    method,
+                    endpoint,
+                    operation,
+                    status,
+                    is_error,
+                )
+
+                print(
+                    f"[{time.strftime('%H:%M:%S')}] {client_name}/{client_version} {method} {endpoint} ({operation}) -> {status}"
+                )
+
+        except json.JSONDecodeError:
+            # Skip invalid JSON lines
+            continue
+        except Exception as e:
+            print(f"Error processing log line: {e}")
+            continue
+
+
+if __name__ == "__main__":
+    # Initialize database table
+    init_db()
+
+    # Note: No Prometheus HTTP server here
+    # Metrics are written to DB and exported by db_exporter
+    print(f"Writing metrics to database: {DB_PATH}")
+
+    # Start parsing logs
+    parse_and_export()

package/templates/monitoring/log_exporter/requirements.txt

@@ -0,0 +1 @@
+prometheus-client==0.19.0

package/templates/monitoring/postgres_exporter/queries.yml

@@ -0,0 +1,22 @@
+pg_openmemory_stored_memories:
+  query: "SELECT user_id as client, COUNT(*) as count FROM openmemory_memories GROUP BY user_id"
+  metrics:
+    - client:
+        usage: "LABEL"
+        description: "Client identifier"
+    - count:
+        usage: "GAUGE"
+        description: "Total memories stored per client"
+
+pg_openmemory_stored_memories_by_sector:
+  query: "SELECT user_id as client, primary_sector as sector, COUNT(*) as count FROM openmemory_memories GROUP BY user_id, primary_sector"
+  metrics:
+    - client:
+        usage: "LABEL"
+        description: "Client identifier"
+    - sector:
+        usage: "LABEL"
+        description: "Memory sector"
+    - count:
+        usage: "GAUGE"
+        description: "Memories stored per client per sector"

package/templates/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,22 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  # Main metrics source: Database Exporter (memory state)
+  - job_name: "db-exporter"
+    static_configs:
+      - targets: ["db-exporter:8000"]
+
+  # Request logs from Traefik access log
+  # Request logs from Traefik access log -> written to DB -> exported by db-exporter
+  # - job_name: 'log-exporter'
+  #   static_configs:
+  #     - targets: ['log-exporter:8001']
+
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres_exporter:9187"]
+
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]

package/templates/monitoring/traefik/traefik.yml

@@ -0,0 +1,32 @@
+log:
+  level: INFO
+
+accessLog:
+  filePath: "/var/log/traefik/access.log"
+  format: json
+  fields:
+    defaultMode: keep
+    headers:
+      defaultMode: keep
+      names:
+        Authorization: keep
+        Content-Type: keep
+        User-Agent: keep
+        X-Client-Name: keep
+        X-Client-Version: keep
+
+api:
+  dashboard: true
+  insecure: true
+
+providers:
+  docker:
+    endpoint: "unix:///var/run/docker.sock"
+    exposedByDefault: false
+  file:
+    directory: /etc/traefik/dynamic
+    watch: true
+
+entryPoints:
+  web:
+    address: ":8626"

package/templates/monitoring/vector/vector.toml/vector.yaml

@@ -0,0 +1,77 @@
+[sources.traefik_logs]
+type = "file"
+include = ["/var/log/traefik/access.log"]
+read_from = "beginning"
+
+[transforms.parse_logs]
+type = "remap"
+inputs = ["traefik_logs"]
+source = '''
+  . = parse_json!(.message)
+'''
+
+[transforms.extract_metrics]
+type = "remap"
+inputs = ["parse_logs"]
+source = '''
+  # Extract client_id from Authorization header (Bearer <client_id>.<secret>)
+  # If no auth header or invalid format, use "anonymous"
+  auth_header = .request_Authorization
+  client_id = "anonymous"
+  
+  if !is_null(auth_header) {
+    # Try to capture everything between "Bearer " and the first dot
+    parts = split(auth_header, ".")
+    if length(parts) > 0 {
+        # Remove "Bearer " prefix if present
+        token_start = replace(parts[0], "Bearer ", "")
+        # Basic validation to ensure it looks like a client ID
+        if length(token_start) > 0 {
+            client_id = token_start
+        }
+    }
+  }
+  
+  # Standardize endpoint (remove query params, maybe aggregate IDs)
+  endpoint = .RequestPath
+  method = .RequestMethod
+  status = .DownstreamStatus
+  duration_seconds = to_float!(.Duration) / 1000000000.0
+  
+  .tags = {
+    "client": client_id,
+    "method": method,
+    "endpoint": endpoint,
+    "status": to_string(status)
+  }
+  .duration = duration_seconds
+'''
+
+[transforms.generate_metrics]
+type = "log_to_metric"
+inputs = ["extract_metrics"]
+
+  [[transforms.generate_metrics.metrics]]
+  type = "counter"
+  field = "status"
+  name = "requests_total"
+  namespace = "openmemory"
+  tags = ["client", "method", "endpoint", "status"]
+
+  [[transforms.generate_metrics.metrics]]
+  type = "histogram"
+  field = "duration"
+  name = "request_duration_seconds"
+  namespace = "openmemory"
+  tags = ["client", "endpoint"]
+  buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
+
+[sinks.prometheus]
+type = "prometheus_exporter"
+inputs = ["generate_metrics"]
+address = "0.0.0.0:9091"
+
+[sinks.console]
+type = "console"
+inputs = ["extract_metrics"]
+encoding.codec = "json"

package/templates/monitoring/vector/vector.yaml

@@ -0,0 +1,106 @@
+sources:
+  traefik_logs:
+    type: file
+    include:
+      - /var/log/traefik/access.log
+    read_from: end
+
+transforms:
+  parse_logs:
+    type: remap
+    inputs:
+      - traefik_logs
+    source: |
+      . = parse_json!(.message)
+
+  extract_metrics:
+    type: remap
+    inputs:
+      - parse_logs
+    source: |
+      auth_header = .request_Authorization
+      client_name = "Anonymous"
+
+      # Extract client from Authorization header and map to human-readable names
+      if !is_null(auth_header) {
+        parts = split!(auth_header, ".")
+        if length(parts) > 0 {
+            token_start = replace!(parts[0], "Bearer ", "")
+
+            # Map client tokens to human-readable names
+            if contains(token_start, "claude") || contains(token_start, "anthropic") {
+                client_name = "Claude Desktop"
+            } else if contains(token_start, "cursor") {
+                client_name = "Cursor"
+            } else if contains(token_start, "vscode") || contains(token_start, "code") {
+                client_name = "Visual Studio Code"
+            } else if contains(token_start, "copilot") || contains(token_start, "github") {
+                client_name = "GitHub Copilot"
+            } else if contains(token_start, "windsurf") {
+                client_name = "Windsurf"
+            } else if contains(token_start, "cline") {
+                client_name = "Cline"
+            } else if contains(token_start, "continue") {
+                client_name = "Continue"
+            } else if contains(token_start, "zed") {
+                client_name = "Zed"
+            } else if length(token_start) > 0 {
+                # For dev/testing, use the token itself
+                client_name = token_start
+            }
+        }
+      }
+
+      endpoint = .RequestPath
+      method = .RequestMethod
+      status = .DownstreamStatus
+      duration_seconds = to_float!(.Duration) / 1000000000.0
+
+      .tags = {
+        "client": client_name,
+        "method": method,
+        "endpoint": endpoint,
+        "status": to_string!(status)
+      }
+      .duration = duration_seconds
+      .count = 1
+
+  generate_metrics:
+    type: log_to_metric
+    inputs:
+      - extract_metrics
+    metrics:
+      - type: counter
+        field: count
+        name: requests_total
+        namespace: openmemory
+        tags:
+          client: "{{ tags.client }}"
+          method: "{{ tags.method }}"
+          endpoint: "{{ tags.endpoint }}"
+          status: "{{ tags.status }}"
+
+      - type: histogram
+        field: duration
+        name: request_duration_seconds
+        namespace: openmemory
+        tags:
+          client: "{{ tags.client }}"
+          endpoint: "{{ tags.endpoint }}"
+
+sinks:
+  prometheus:
+    type: prometheus_exporter
+    inputs:
+      - generate_metrics
+    address: 0.0.0.0:9091
+    default_namespace: openmemory
+    flush_period_secs: 5
+    suppress_timestamp: true
+
+  console:
+    type: console
+    inputs:
+      - extract_metrics
+    encoding:
+      codec: json

package/requirements.txt

@@ -1,2 +0,0 @@
-mcp>=0.9.0
-httpx>=0.27.0