@mmerterden/multi-agent-pipeline 12.1.1 → 12.3.0

package/pipeline/lib/fetch-figma-annotations.sh

@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+#
+# fetch-figma-annotations.sh
+#
+# Tier-2 (Figma REST) fallback for /multi-agent:analysis annotation ingestion.
+# Fetches Dev Mode annotations for one or more nodes and emits them as the
+# authoritative copy for each node (the visible text layer is a placeholder).
+#
+# Tier 1 (Figma MCP get_design_context) is preferred and handled in the skill;
+# this script is the no-MCP fallback. Generic: no project file keys, no
+# language codes, and no corporate transport are baked in - the caller passes
+# the file key, node ids, and the recognized language prefixes.
+#
+# Usage:
+#   fetch-figma-annotations.sh --file-key <key> --node-id <id>[,<id>...] \
+#                              [--lang-prefixes TR,EN] [--batch 4]
+#   fetch-figma-annotations.sh --url <figma-url> [--lang-prefixes TR,EN]
+#
+# Token source: credential-store.sh get <logical-key>, where <logical-key> is
+# read from prefs.global.keychainMapping.figma_pat. Falls back to the FIGMA_PAT
+# environment variable. No literal keychain service name is embedded here.
+#
+# Label parsing (config-driven via --lang-prefixes, ordered):
+#   prefixed  : lines like "TR: ...", "EN: ..." win regardless of order
+#   two-line  : line 1 -> prefixes[0], line 2 -> prefixes[1]
+#   single    : one line -> prefixes[0]
+#   empty     : no label text
+#
+# Output (stdout, single JSON object):
+#   { "ok": true, "source": "figma-rest", "fileKey": "...", "fetchedAt": "...",
+#     "count": N, "withBase": M,
+#     "annotations": [ {nodeId, name, designText, raw, parsed:{<lang>:val}, mode} ] }
+# On missing/rejected token: { "ok": false, "reason": "missing-token" } and exit 2.
+#
+# Exit codes: 0 success (incl. zero annotations), 1 usage, 2 auth,
+#   4 node not found, 5 rate limited, 6 network/other fatal.
+
+set -uo pipefail
+
+PREFS="$HOME/.claude/multi-agent-preferences.json"
+# shellcheck disable=SC1090,SC1091
+. "$HOME/.claude/lib/credential-store-resolver.sh" 2>/dev/null \
+  || . "$HOME/.copilot/lib/credential-store-resolver.sh" 2>/dev/null \
+  || true
+CRED_STORE="${CRED_STORE:-}"
+FIGMA_API="https://api.figma.com/v1"
+HTTP_TIMEOUT=60
+
+FILE_KEY=""
+NODE_IDS=""
+URL=""
+LANG_PREFIXES="TR,EN"
+BATCH=4
+
+log() { printf '%s\n' "$*" >&2; }
+die() { local code="$1"; shift; log "fetch-figma-annotations: $*"; exit "$code"; }
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --file-key) FILE_KEY="${2:-}"; shift 2 ;;
+    --node-id) NODE_IDS="${2:-}"; shift 2 ;;
+    --url) URL="${2:-}"; shift 2 ;;
+    --lang-prefixes) LANG_PREFIXES="${2:-TR,EN}"; shift 2 ;;
+    --batch) BATCH="${2:-4}"; shift 2 ;;
+    -h|--help) grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'; exit 0 ;;
+    *) die 1 "unknown argument: $1" ;;
+  esac
+done
+
+# Parse fileKey + nodeId from a Figma URL when given.
+if [ -n "$URL" ]; then
+  parsed_key=$(printf '%s' "$URL" | sed -nE 's#.*figma\.com/(design|file)/([A-Za-z0-9]+).*#\2#p')
+  parsed_node=$(printf '%s' "$URL" | sed -nE 's#.*[?&]node-id=([^&]+).*#\1#p' | sed 's/-/:/')
+  [ -z "$FILE_KEY" ] && FILE_KEY="$parsed_key"
+  [ -z "$NODE_IDS" ] && NODE_IDS="$parsed_node"
+fi
+
+[ -z "$FILE_KEY" ] && die 1 "missing --file-key (or a --url to parse it from)"
+[ -z "$NODE_IDS" ] && die 1 "missing --node-id (or a --url to parse it from)"
+
+# Resolve the Figma PAT (keychain via prefs mapping, then FIGMA_PAT env).
+resolve_token() {
+  local token_key=""
+  if [ -f "$PREFS" ] && command -v python3 >/dev/null 2>&1; then
+    token_key=$(python3 -c "
+import json
+try:
+    p = json.load(open('$PREFS'))
+    print(p.get('global', {}).get('keychainMapping', {}).get('figma_pat') or '')
+except Exception:
+    print('')
+" 2>/dev/null || true)
+  fi
+  if [ -n "$token_key" ] && [ -n "$CRED_STORE" ] && [ -x "$CRED_STORE" ]; then
+    local tok
+    tok=$("$CRED_STORE" get "$token_key" 2>/dev/null || true)
+    if [ -n "$tok" ]; then printf '%s' "$tok"; return 0; fi
+  fi
+  if [ -n "${FIGMA_PAT:-}" ]; then printf '%s' "$FIGMA_PAT"; return 0; fi
+  return 1
+}
+
+FIGMA_TOKEN=$(resolve_token || true)
+if [ -z "$FIGMA_TOKEN" ]; then
+  printf '{"ok":false,"reason":"missing-token","source":"figma-rest","fileKey":"%s"}\n' "$FILE_KEY"
+  die 2 "no Figma PAT (map prefs.global.keychainMapping.figma_pat or set FIGMA_PAT)"
+fi
+
+TMPDIR_A=$(mktemp -d "${TMPDIR:-/tmp}/figma-annot.XXXXXX")
+trap 'rm -rf "$TMPDIR_A"' EXIT INT TERM
+
+# curl one /nodes batch (token via -K config so it never lands in argv).
+curl_nodes() {
+  local out="$1" ids="$2" code
+  code=$(curl -sS -o "$out" -w '%{http_code}' \
+            --max-time "$HTTP_TIMEOUT" --connect-timeout 5 \
+            -K <(printf 'header = "X-Figma-Token: %s"\n' "$FIGMA_TOKEN") \
+            "$FIGMA_API/files/$FILE_KEY/nodes?ids=$ids" || echo "000")
+  printf '%s' "$code"
+}
+
+# Split comma ids into batches of $BATCH, fetch each into TMPDIR_A/part-N.json.
+OLD_IFS="$IFS"; IFS=','; set -- $NODE_IDS; IFS="$OLD_IFS"
+part=0; batch_ids=""; n=0
+flush_batch() {
+  [ -z "$batch_ids" ] && return 0
+  part=$((part + 1))
+  local code
+  code=$(curl_nodes "$TMPDIR_A/part-$part.json" "$batch_ids")
+  case "$code" in
+    2*) : ;;
+    401|403) die 2 "auth rejected (HTTP $code)" ;;
+    404) die 4 "node(s) not found (HTTP 404)" ;;
+    429) die 5 "rate limited (HTTP 429)" ;;
+    *) die 6 "figma call failed (HTTP $code)" ;;
+  esac
+  batch_ids=""; n=0
+}
+for id in "$@"; do
+  [ -z "$id" ] && continue
+  if [ -z "$batch_ids" ]; then batch_ids="$id"; else batch_ids="$batch_ids,$id"; fi
+  n=$((n + 1))
+  [ "$n" -ge "$BATCH" ] && flush_batch
+done
+flush_batch
+
+# Walk all node docs, collect + parse annotations. Python only parses local
+# response files; the token is never passed to it.
+LANG_PREFIXES="$LANG_PREFIXES" FILE_KEY="$FILE_KEY" python3 - "$TMPDIR_A" <<'PY'
+import json, os, sys, glob, datetime
+
+parts_dir = sys.argv[1]
+prefixes = [p.strip() for p in os.environ.get("LANG_PREFIXES", "TR,EN").split(",") if p.strip()]
+base = (prefixes[0].lower() if prefixes else "tr")
+
+def parse_label(raw):
+    if not raw or not raw.strip():
+        return {}, "empty"
+    lines = [ln.strip() for ln in raw.splitlines() if ln.strip()]
+    parsed = {}
+    # prefixed: "TR: ...", "EN: ..."
+    prefixed = False
+    for ln in lines:
+        for p in prefixes:
+            lp = p.lower()
+            low = ln.lower()
+            if low.startswith(lp + ":") or low.startswith(lp + " :"):
+                parsed[lp] = ln.split(":", 1)[1].strip()
+                prefixed = True
+    if prefixed:
+        return parsed, "prefixed"
+    if len(lines) >= 2 and len(prefixes) >= 2:
+        return {prefixes[0].lower(): lines[0], prefixes[1].lower(): lines[1]}, "two-line"
+    if len(lines) == 1 and prefixes:
+        return {prefixes[0].lower(): lines[0]}, "single"
+    # fallback: join everything under the base language
+    return {base: " ".join(lines)}, "single"
+
+annotations = []
+
+def walk(node):
+    if not isinstance(node, dict):
+        return
+    anns = node.get("annotations")
+    if isinstance(anns, list) and anns:
+        raw_parts = []
+        for a in anns:
+            if not isinstance(a, dict):
+                continue
+            lbl = a.get("labelMarkdown") or a.get("label") or ""
+            if lbl:
+                raw_parts.append(lbl)
+        raw = "\n".join(raw_parts)
+        parsed, mode = parse_label(raw)
+        design_text = node.get("characters")
+        annotations.append({
+            "nodeId": node.get("id"),
+            "name": node.get("name"),
+            "designText": design_text if isinstance(design_text, str) else None,
+            "raw": raw or None,
+            "parsed": parsed,
+            "mode": mode,
+        })
+    for child in node.get("children", []) or []:
+        walk(child)
+
+for pf in sorted(glob.glob(os.path.join(parts_dir, "part-*.json"))):
+    try:
+        doc = json.load(open(pf))
+    except Exception:
+        continue
+    nodes = doc.get("nodes", {})
+    if isinstance(nodes, dict):
+        for _, entry in nodes.items():
+            if isinstance(entry, dict) and isinstance(entry.get("document"), dict):
+                walk(entry["document"])
+
+with_base = sum(1 for a in annotations if a["parsed"].get(base))
+# Flag base-present-but-a-target-missing to stderr (kick back to content team).
+for a in annotations:
+    if a["parsed"].get(base):
+        missing = [p.lower() for p in prefixes if not a["parsed"].get(p.lower())]
+        if missing:
+            sys.stderr.write("WARN: node %s has %s but missing %s\n" % (
+                a.get("nodeId"), base, ",".join(missing)))
+
+out = {
+    "ok": True,
+    "source": "figma-rest",
+    "fileKey": os.environ.get("FILE_KEY"),
+    "fetchedAt": datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
+    "count": len(annotations),
+    "withBase": with_base,
+    "annotations": annotations,
+}
+print(json.dumps(out, ensure_ascii=False, indent=2))
+PY

package/pipeline/lib/fetch-graylog.sh

@@ -0,0 +1,233 @@
+#!/bin/bash
+#
+# fetch-graylog.sh
+# Query a Graylog instance for the log messages tied to a transaction or
+# conversation id and emit a normalized JSON view the pipeline can attach as
+# analysis context. A log fetch must never block a run: on any network failure
+# (VPN down, host unreachable) the fetcher degrades to an empty result and
+# exits 0.
+#
+# Input forms (at least one id is required):
+#   ./fetch-graylog.sh --trx <transaction-id>
+#   ./fetch-graylog.sh --conv <conversation-id>
+#   ./fetch-graylog.sh --trx <id> --conv <id>
+#
+# Required configuration:
+#   prefs.global.hosts.graylog            -  Graylog host (e.g. logs.example.com)
+#   prefs.global.keychainMapping.graylog  -  keychain key holding the API token
+#
+# Optional env:
+#   GRAYLOG_TIMEOUT_SECONDS    default 25
+#   GRAYLOG_HOST_OVERRIDE      forces the host (used by tests and CI dry-runs)
+#   GRAYLOG_RANGE_SECONDS      relative search window, default 86400 (24h)
+#   GRAYLOG_LIMIT              max messages to return, default 200
+#
+# Auth: Graylog personal access tokens authenticate as HTTP Basic with the
+# token as the username and the literal string "token" as the password. The
+# credential travels through a curl config via process substitution so it
+# never appears in argv (argv is visible to ps).
+#
+# Output (stdout, single JSON object):
+#   {
+#     "fetchedAt": "<ISO8601>",
+#     "source": { "host": "<host>", "transactionId": "..."|null,
+#                 "conversationId": "..."|null, "query": "..." },
+#     "totalResults": <n>,
+#     "messages": [
+#       { "timestamp": "...", "source": "...", "level": <n>|null,
+#         "message": "...", "fields": { ... } }
+#     ],
+#     "degraded": <bool>, "degradeReason": "..."|null
+#   }
+#
+# Exit codes:
+#   0  success, or degraded-empty on a network failure (never blocks a run)
+#   2  missing/expired credential (orchestrator handles the Save Flow)
+#   3  auth rejected on a reachable host (4xx)
+#   4  bad usage
+#   6  host not configured (prefs.global.hosts.graylog is empty)
+
+set -eu
+
+TRX_ID=""
+CONV_ID=""
+TIMEOUT="${GRAYLOG_TIMEOUT_SECONDS:-25}"
+RANGE="${GRAYLOG_RANGE_SECONDS:-86400}"
+LIMIT="${GRAYLOG_LIMIT:-200}"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --trx)  TRX_ID="$2";  shift 2 ;;
+    --conv) CONV_ID="$2"; shift 2 ;;
+    -h|--help)
+      echo "usage: $0 --trx <transaction-id> | --conv <conversation-id>" >&2
+      exit 4 ;;
+    *)
+      echo "ERR: unexpected arg $1" >&2; exit 4 ;;
+  esac
+done
+
+# Host comes from prefs (private config); skill templates only show {GRAYLOG_HOST}.
+PREFS="$HOME/.claude/multi-agent-preferences.json"
+HOST="${GRAYLOG_HOST_OVERRIDE:-}"
+if [ -z "$HOST" ] && [ -f "$PREFS" ]; then
+  HOST=$(python3 -c "
+import json
+try:
+    p = json.load(open('$PREFS'))
+    print(p.get('global', {}).get('hosts', {}).get('graylog') or '')
+except Exception:
+    print('')
+")
+fi
+if [ -z "$HOST" ]; then
+  printf '%s\n' '{"status":"blocked","reason":"host-not-configured","service":"graylog","expected_pref":"global.hosts.graylog"}' >&2
+  exit 6
+fi
+HOST=${HOST%/}  # strip trailing slash if any
+
+if [ -z "$TRX_ID" ] && [ -z "$CONV_ID" ]; then
+  echo "ERR: no id (pass --trx <transaction-id> and/or --conv <conversation-id>)" >&2
+  exit 4
+fi
+
+# Build the Graylog query from whichever ids were supplied.
+QUERY=""
+if [ -n "$TRX_ID" ]; then
+  QUERY="transactionId:\"$TRX_ID\""
+fi
+if [ -n "$CONV_ID" ]; then
+  if [ -n "$QUERY" ]; then
+    QUERY="$QUERY OR conversationId:\"$CONV_ID\""
+  else
+    QUERY="conversationId:\"$CONV_ID\""
+  fi
+fi
+
+# Token via keychain mapping.
+TOKEN_KEY=""
+if [ -f "$PREFS" ]; then
+  TOKEN_KEY=$(python3 -c "
+import json
+try:
+    p = json.load(open('$PREFS'))
+    print(p.get('global', {}).get('keychainMapping', {}).get('graylog') or '')
+except Exception:
+    print('')
+")
+fi
+[ -z "$TOKEN_KEY" ] && TOKEN_KEY="${USER}_Graylog_Access_Token"
+
+# Locate the credential helper via the resolver so Copilot-only installs work.
+# shellcheck disable=SC1090,SC1091
+. "$HOME/.claude/lib/credential-store-resolver.sh" 2>/dev/null \
+  || . "$HOME/.copilot/lib/credential-store-resolver.sh" 2>/dev/null \
+  || { printf '%s\n' '{"status":"blocked","reason":"missing-credential-helper","service":"graylog","expected_key":"'"$TOKEN_KEY"'"}' >&2; exit 2; }
+
+TOKEN=$("$CRED_STORE" get "$TOKEN_KEY" 2>/dev/null || true)
+if [ -z "$TOKEN" ]; then
+  printf '%s\n' '{"status":"blocked","reason":"missing-token","service":"graylog","expected_key":"'"$TOKEN_KEY"'"}' >&2
+  exit 2
+fi
+
+API_BASE="https://$HOST/api"
+
+graylog_get() {
+  # Usage: graylog_get <path-and-query-relative-to-API_BASE>
+  # Prints "<body>\n<http_code>"; the caller splits the last line off.
+  local path="$1"
+  local url="$API_BASE/$path"
+  # PAT basic auth: username=<token>, password=literal "token". The credential
+  # goes through a curl config via process substitution so it never lands in
+  # argv (argv is visible to ps).
+  curl -sS --max-time "$TIMEOUT" --connect-timeout 5 -w "\n%{http_code}" \
+    -K <(printf 'user = "%s:token"\n' "$TOKEN") \
+    -H "Accept: application/json" \
+    -H "X-Requested-By: multi-agent-pipeline" \
+    "$url" 2>/dev/null || true
+}
+
+# Graylog relative universal search.
+ENC_QUERY=$(QUERY_IN="$QUERY" python3 -c "
+import os, urllib.parse as up
+print(up.quote(os.environ['QUERY_IN']))
+")
+SEARCH_PATH="search/universal/relative?query=$ENC_QUERY&range=$RANGE&limit=$LIMIT&sort=timestamp:desc"
+
+RESP=$(graylog_get "$SEARCH_PATH")
+HTTP=$(printf '%s' "$RESP" | tail -n1)
+BODY=$(printf '%s' "$RESP" | sed '$d')
+
+# A genuine auth rejection on a reachable host is the only hard failure.
+case "$HTTP" in
+  401|403)
+    echo "ERR: Graylog auth rejected (HTTP $HTTP)" >&2
+    exit 3 ;;
+esac
+
+# Any other non-200 (empty code on connection failure, 5xx, timeout) degrades
+# to an empty result so a log fetch never blocks a run.
+DEGRADED="false"
+DEGRADE_REASON=""
+if [ "$HTTP" != "200" ]; then
+  DEGRADED="true"
+  DEGRADE_REASON="graylog-unreachable"
+  BODY=""
+fi
+
+SRC_HOST="$HOST" SRC_TRX="$TRX_ID" SRC_CONV="$CONV_ID" SRC_QUERY="$QUERY" \
+DEGRADED_IN="$DEGRADED" DEGRADE_REASON_IN="$DEGRADE_REASON" \
+BODY_IN="$BODY" LIMIT_IN="$LIMIT" \
+python3 - <<'PY'
+import json, os, datetime
+
+def safe_load(s):
+    try:
+        return json.loads(s) if s else None
+    except Exception:
+        return None
+
+body = safe_load(os.environ.get("BODY_IN") or "") or {}
+raw = body.get("messages") or []
+
+try:
+    limit = int(os.environ.get("LIMIT_IN") or "200")
+except Exception:
+    limit = 200
+
+messages = []
+for item in raw[:limit]:
+    m = item.get("message") if isinstance(item, dict) else None
+    if not isinstance(m, dict):
+        continue
+    reserved = ("timestamp", "source", "level", "message")
+    fields = {k: v for k, v in m.items() if k not in reserved}
+    messages.append({
+        "timestamp": m.get("timestamp") or "",
+        "source": m.get("source") or "",
+        "level": m.get("level") if isinstance(m.get("level"), (int, float)) else None,
+        "message": (m.get("message") or "")[:2000],
+        "fields": fields,
+    })
+
+trx = os.environ.get("SRC_TRX") or None
+conv = os.environ.get("SRC_CONV") or None
+total = body.get("total_results")
+if not isinstance(total, int):
+    total = len(messages)
+
+result = {
+    "fetchedAt": datetime.datetime.utcnow().isoformat() + "Z",
+    "source": {
+        "host": os.environ["SRC_HOST"],
+        "transactionId": trx,
+        "conversationId": conv,
+        "query": os.environ.get("SRC_QUERY") or "",
+    },
+    "totalResults": total,
+    "messages": messages,
+    "degraded": os.environ.get("DEGRADED_IN") == "true",
+    "degradeReason": (os.environ.get("DEGRADE_REASON_IN") or None),
+}
+print(json.dumps(result, ensure_ascii=False))
+PY