@fx-data/ai-diagnostics 0.1.20261026070913 → 0.1.20261107142011

package/content/.agents/skills/analyze-engine-metric/scripts/engine_metric_analysis/metric_analyzer.py

@@ -1,791 +0,0 @@
-"""Metric analysis utilities for MPP OLAP execution logs.
-
-Goals:
-- Parse once in streaming mode (large files friendly).
-- Map operator plan id (e.g. `<proc> Project@34255 [1]`) to PHYSICAL node id.
-- Preserve raw values, source lines and explicit structure without diagnosis.
-"""
-from __future__ import annotations
-
-from dataclasses import asdict, dataclass, field
-import hashlib
-import math
-from pathlib import Path
-import re
-from typing import Any, Dict, List, Optional, Tuple
-
-
-_KEY_VALUE_PATTERN = re.compile(r"^(?P<key>[A-Za-z0-9_:@\[\]\-]+)\s*=\s*(?P<value>.+)$")
-_OPERATOR_PATTERN = re.compile(
-    r"^<(?P<kind>[^>]+)>\s+(?P<name>[^@]+)@(?P<node>-?\d+)\s+\[(?P<plan>[^\]]+)\]$"
-)
-_JIT_BACKEND_PATTERN = re.compile(r"^jit_backend\[(?P<name>[^\]]+)\]\s*:$")
-_JOB_FIELD_PATTERN = re.compile(r"([A-Za-z0-9_]+)=('(?:[^'\\]|\\.)*'|[^,\]]+)")
-_TASK_STATE_TIME_PATTERN = re.compile(r",\s*TaskStateTime\[(?P<fields>[^\]]*)\]\]\s*$")
-_PHYSICAL_NODE_PATTERN = re.compile(
-    r"PHYSICAL_(?P<type>[A-Z_]+):\s+\[nodeId:\s*(?P<id>-?\d+),\s*(?P<rest>.+)$"
-)
-_SUBPLAN_HEADER_PATTERN = re.compile(
-    r"SubPlan:\s*\[Id:\s*(?P<id>[^\]\s]+)(?:\s+ParentId:\s*(?P<parent>[^\]\s]+))?"
-)
-_ANALYZED_NODE_PATTERN = re.compile(
-    r"^(?P<prefix>\.*\s*)\[(?P<id>-?\d+)\](?P<name>[A-Za-z0-9_]+Node)\s*(?P<detail>.*)$"
-)
-_SIGNOZ_SUMMARY_KEYS = {
-    "elapsed_time",
-    "execution_time",
-    "total_tasks",
-    "load_balance_cost",
-    "lb_queue_cost",
-    "queue_cost",
-    "optimizer1_cost",
-    "optimizer2_cost",
-    "distribution_cost",
-    "send_cost",
-    "peak_memory",
-}
-
-@dataclass
-class PhysicalNode:
-    node_id: int
-    node_type: str
-    subplan_id: Optional[str]
-    summary: str
-    line_no: int = 0
-
-
-@dataclass
-class SubPlanInfo:
-    subplan_id: str
-    parent_id: Optional[str] = None
-    input_distribution: Optional[str] = None
-    output_distribution: Optional[str] = None
-    line_no: int = 0
-
-
-@dataclass
-class FragmentEdge:
-    source_subplan: str
-    target_subplan: str
-    edge_type: str
-    node_id: int
-    line_no: int
-
-
-@dataclass
-class OperatorMetric:
-    kind: str
-    name: str
-    node_id: int
-    plan_id: str
-    line_no: int = 0
-    metric_lines: Dict[str, int] = field(default_factory=dict)
-    metrics: Dict[str, str] = field(default_factory=dict)
-
-
-@dataclass
-class BackendMetric:
-    name: str
-    line_no: int
-    metric_lines: Dict[str, int] = field(default_factory=dict)
-    metrics: Dict[str, str] = field(default_factory=dict)
-
-
-@dataclass
-class PipelineMetric:
-    stage_id: Optional[str]
-    task_id: Optional[str]
-    pipeline_id: int
-    line_no: int = 0
-    metric_lines: Dict[str, int] = field(default_factory=dict)
-    metrics: Dict[str, str] = field(default_factory=dict)
-    operators: List[OperatorMetric] = field(default_factory=list)
-    backends: List[BackendMetric] = field(default_factory=list)
-
-
-@dataclass
-class ParsedMetric:
-    # Keep last seen job fields for backward compatibility.
-    job_fields: Dict[str, str] = field(default_factory=dict)
-    # Keep all job headers because some files contain multiple tasks/runs.
-    jobs: List[Dict[str, str]] = field(default_factory=list)
-    task_fields: Dict[str, Dict[str, str]] = field(default_factory=dict)
-    # Current native Metric files expose query-level facts before the first pipeline.
-    query_fields: Dict[str, str] = field(default_factory=dict)
-    pipelines: List[PipelineMetric] = field(default_factory=list)
-    physical_nodes: Dict[int, PhysicalNode] = field(default_factory=dict)
-    physical_node_observations: List[PhysicalNode] = field(default_factory=list)
-    subplans: Dict[str, SubPlanInfo] = field(default_factory=dict)
-    subplan_observations: List[SubPlanInfo] = field(default_factory=list)
-    fragment_edges: List[FragmentEdge] = field(default_factory=list)
-    summary_metrics: Dict[str, str] = field(default_factory=dict)
-    analyzed_nodes: List[Dict[str, Any]] = field(default_factory=list)
-    complete_marker: bool = False
-    source_lines: List[Dict[str, Any]] = field(default_factory=list)
-    field_lines: Dict[str, Dict[str, int]] = field(default_factory=dict)
-    field_units: Dict[str, Dict[str, str]] = field(default_factory=dict)
-
-
-def parse_metric_file(path: Path) -> ParsedMetric:
-    """Parse a metric/log file into structured data."""
-    parsed = ParsedMetric()
-
-    current_stage: Optional[str] = None
-    current_subplan: Optional[str] = None
-    current_task: Optional[str] = None
-    current_task_indent = -1
-    current_pipeline: Optional[PipelineMetric] = None
-    current_pipeline_indent = -1
-    current_operator: Optional[OperatorMetric] = None
-    current_operator_indent = -1
-    current_backend: Optional[BackendMetric] = None
-    current_backend_indent = -1
-
-    with path.open("r", encoding="utf-8", errors="replace") as fp:
-        for line_no, raw in enumerate(fp, start=1):
-            line = raw.rstrip("\r\n")
-            if not line:
-                continue
-
-            parsed.source_lines.append({"line": line_no, "text": line})
-            if line.strip() == "end_of_metric":
-                parsed.complete_marker = True
-                continue
-
-            root_kv = _parse_key_value(line.strip())
-            if (
-                root_kv is not None
-                and root_kv[0] == "jsy_cpp_task_name"
-                and not parsed.pipelines
-            ):
-                parsed.query_fields[root_kv[0]] = _strip_wrapping_quotes(root_kv[1])
-                parsed.field_lines.setdefault("query", {})[root_kv[0]] = line_no
-                continue
-
-            if "Job:StandardTask[" in line:
-                job = _parse_job_line(line)
-                if job:
-                    parsed.jobs.append(job)
-                    scope = f"job:{len(parsed.jobs)-1}"
-                    parsed.field_lines[scope] = {key: line_no for key in job}
-                    timer = _TASK_STATE_TIME_PATTERN.search(line)
-                    if timer:
-                        parsed.field_units[scope] = {
-                            match.group(1): "ms"
-                            for match in _JOB_FIELD_PATTERN.finditer(timer.group("fields"))
-                            if match.group(1) in _TASK_STATE_TIME_FIELDS
-                            and re.fullmatch(r"null|-?\d+", match.group(2).strip())
-                        }
-                    parsed.job_fields = job
-
-            analyzed_match = _ANALYZED_NODE_PATTERN.match(line)
-            if analyzed_match:
-                parsed.analyzed_nodes.append(
-                    {
-                        "line_no": line_no,
-                        "node_id": int(analyzed_match.group("id")),
-                        "node_type": analyzed_match.group("name"),
-                        "depth": analyzed_match.group("prefix").count("."),
-                        "detail": analyzed_match.group("detail").strip(),
-                    }
-                )
-                continue
-
-            subplan_info = _parse_subplan_header(line)
-            if subplan_info is not None:
-                subplan_id, parent_id, input_dist, output_dist = subplan_info
-                current_subplan = subplan_id
-                parsed.subplan_observations.append(SubPlanInfo(subplan_id, parent_id, input_dist, output_dist, line_no))
-                existing = parsed.subplans.get(subplan_id)
-                if existing is None:
-                    parsed.subplans[subplan_id] = SubPlanInfo(
-                        subplan_id=subplan_id,
-                        line_no=line_no,
-                        parent_id=parent_id,
-                        input_distribution=input_dist,
-                        output_distribution=output_dist,
-                    )
-                else:
-                    if parent_id and not existing.parent_id:
-                        existing.parent_id = parent_id
-                    if input_dist and not existing.input_distribution:
-                        existing.input_distribution = input_dist
-                    if output_dist and not existing.output_distribution:
-                        existing.output_distribution = output_dist
-
-            physical_match = _PHYSICAL_NODE_PATTERN.search(line)
-            if physical_match:
-                node_id = int(physical_match.group("id"))
-                node_type = physical_match.group("type").strip()
-                rest = physical_match.group("rest")
-                summary = rest.strip()
-                parsed.physical_nodes[node_id] = PhysicalNode(
-                    node_id=node_id,
-                    line_no=line_no,
-                    node_type=node_type,
-                    subplan_id=current_subplan,
-                    summary=summary,
-                )
-                parsed.physical_node_observations.append(parsed.physical_nodes[node_id])
-                if current_subplan:
-                    for source_fragment in _extract_fragment_ids(rest, "sourceFragmentId"):
-                        parsed.fragment_edges.append(
-                            FragmentEdge(
-                                source_subplan=source_fragment,
-                                target_subplan=current_subplan,
-                                edge_type="remote_source",
-                                node_id=node_id,
-                                line_no=line_no,
-                            )
-                        )
-                    for sink_fragment in _extract_fragment_ids(rest, "sinkFragmentId"):
-                        parsed.fragment_edges.append(
-                            FragmentEdge(
-                                source_subplan=current_subplan,
-                                target_subplan=sink_fragment,
-                                edge_type="exchange_sink",
-                                node_id=node_id,
-                                line_no=line_no,
-                            )
-                        )
-
-            bullet = _extract_bullet(line)
-            if bullet is None:
-                continue
-            indent, body = bullet
-
-            # Exit nested context by indentation.
-            if current_backend is not None and indent <= current_backend_indent:
-                current_backend = None
-                current_backend_indent = -1
-            if current_operator is not None and indent <= current_operator_indent:
-                current_operator = None
-                current_operator_indent = -1
-            if current_pipeline is not None and indent <= current_pipeline_indent:
-                current_pipeline = None
-                current_pipeline_indent = -1
-                current_operator = None
-                current_operator_indent = -1
-            if current_task is not None and indent <= current_task_indent:
-                current_task = None
-                current_task_indent = -1
-
-            if body.startswith("stage_info@") and "stage_id=" in body:
-                current_stage = body.split("stage_id=", 1)[1].strip()
-                continue
-
-            if body.startswith("task@"):
-                current_task = body.split()[0][5:]
-                parsed.task_fields.setdefault(current_task, {})
-                current_task_indent = indent
-                continue
-
-            if body.startswith("pipeline@"):
-                pipeline_id = _parse_pipeline_id(body)
-                if pipeline_id is None:
-                    continue
-                current_pipeline = PipelineMetric(
-                    stage_id=current_stage,
-                    task_id=current_task,
-                    pipeline_id=pipeline_id,
-                    line_no=line_no,
-                )
-                parsed.pipelines.append(current_pipeline)
-                current_pipeline_indent = indent
-                current_operator = None
-                current_operator_indent = -1
-                continue
-
-            op_match = _OPERATOR_PATTERN.match(body)
-            if op_match and current_pipeline is not None:
-                current_backend = None
-                current_backend_indent = -1
-                current_operator = OperatorMetric(
-                    line_no=line_no,
-                    kind=op_match.group("kind"),
-                    name=op_match.group("name").strip(),
-                    node_id=int(op_match.group("node")),
-                    plan_id=op_match.group("plan").strip(),
-                )
-                current_pipeline.operators.append(current_operator)
-                current_operator_indent = indent
-                continue
-
-            backend_match = _JIT_BACKEND_PATTERN.match(body)
-            if backend_match and current_pipeline is not None:
-                current_backend = BackendMetric(backend_match.group("name"), line_no)
-                current_pipeline.backends.append(current_backend)
-                current_backend_indent = indent
-                current_operator = None
-                current_operator_indent = -1
-                continue
-
-            kv = _parse_key_value(body)
-            if kv is None:
-                continue
-            key, value = kv
-
-            if key in _SIGNOZ_SUMMARY_KEYS and current_operator is None and current_pipeline is None and current_task is None:
-                parsed.summary_metrics[key] = value
-                parsed.field_lines.setdefault("summary", {})[key] = line_no
-                continue
-
-            if current_operator is not None and indent > current_operator_indent:
-                current_operator.metrics[key] = value
-                current_operator.metric_lines[key] = line_no
-                continue
-
-            if current_backend is not None and indent > current_backend_indent:
-                current_backend.metrics[key] = value
-                current_backend.metric_lines[key] = line_no
-                continue
-
-            if current_pipeline is not None and indent > current_pipeline_indent:
-                current_pipeline.metrics[key] = value
-                current_pipeline.metric_lines[key] = line_no
-                continue
-
-            if current_task is not None:
-                parsed.task_fields.setdefault(current_task, {})[key] = value
-                parsed.field_lines.setdefault(f"task:{current_task}", {})[key] = line_no
-                continue
-
-            if indent == 0 and not parsed.pipelines:
-                parsed.query_fields[key] = _strip_wrapping_quotes(value)
-                parsed.field_lines.setdefault("query", {})[key] = line_no
-
-    return parsed
-
-
-def _extract_bullet(line: str) -> Optional[Tuple[int, str]]:
-    stripped = line.lstrip()
-    if stripped.startswith("- "):
-        indent = len(line) - len(stripped)
-        return indent, stripped[2:].strip()
-
-    marker = " - "
-    idx = line.find(marker)
-    if idx < 0:
-        return None
-    body = line[idx + len(marker) :].strip()
-    if not body:
-        return None
-    return idx, body
-
-
-def _parse_job_line(line: str) -> Dict[str, str]:
-    if "Job:StandardTask[" not in line:
-        return {}
-    start = line.find("Job:StandardTask[")
-    payload = line[start + len("Job:StandardTask[") :]
-    fields: Dict[str, str] = {}
-    for match in _JOB_FIELD_PATTERN.finditer(payload):
-        key = match.group(1)
-        value = match.group(2).strip()
-        if value.startswith("'") and value.endswith("'"):
-            value = value[1:-1]
-        fields[key] = value
-    return fields
-
-
-def _parse_subplan_header(line: str) -> Optional[Tuple[str, Optional[str], Optional[str], Optional[str]]]:
-    match = _SUBPLAN_HEADER_PATTERN.search(line)
-    if not match:
-        return None
-    subplan_id = match.group("id").strip()
-    parent_id = match.group("parent")
-    if parent_id:
-        parent_id = parent_id.strip()
-
-    input_dist: Optional[str] = None
-    output_dist: Optional[str] = None
-    io_match = re.search(r"input:\s*(?P<input>.+?),\s*output:\s*(?P<output>.+)$", line)
-    if io_match:
-        input_dist = io_match.group("input").strip()
-        output_dist = io_match.group("output").strip()
-    return subplan_id, parent_id, input_dist, output_dist
-
-
-def _extract_fragment_ids(rest: str, key: str) -> List[str]:
-    pattern = re.compile(rf"{re.escape(key)}s?:\s*(?P<value>\[[^\]]+\]|[^,\]]+)")
-    match = pattern.search(rest)
-    if not match:
-        return []
-    value = match.group("value").strip()
-    if not value:
-        return []
-    if value.startswith("[") and value.endswith("]"):
-        items = [item.strip() for item in value[1:-1].split(",")]
-        return [item for item in items if item]
-    return [value]
-
-
-def _parse_key_value(body: str) -> Optional[Tuple[str, str]]:
-    match = _KEY_VALUE_PATTERN.match(body)
-    if not match:
-        return None
-    key = match.group("key").strip()
-    value = match.group("value").strip()
-    return key, value
-
-
-def _strip_wrapping_quotes(value: str) -> str:
-    text = value.strip()
-    if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
-        return text[1:-1]
-    return text
-
-
-def _parse_pipeline_id(body: str) -> Optional[int]:
-    m = re.match(r"pipeline@(\d+)$", body)
-    if not m:
-        return None
-    return int(m.group(1))
-
-
-SCHEMA_VERSION = 2
-NORMALIZATION_VERSION = 3
-_NUMBER = re.compile(r"(-?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?)\s*([A-Za-zµμ]*)")
-_TIME_FACTORS = {"ns": 1e-6, "us": 1e-3, "µs": 1e-3, "μs": 1e-3, "ms": 1, "s": 1000, "min": 60000}
-_BYTE_FACTORS = {"bytes": 1, "b": 1, **{u: 1024 ** n for n, units in enumerate((("kb", "kib"), ("mb", "mib"), ("gb", "gib"), ("tb", "tib")), 1) for u in units}}
-_JOB_PHASE_FIELDS = {"queue_time", "wait_resource_time", "plan_time", "start_time", "finish_time"}
-_TASK_STATE_TIME_FIELDS = _JOB_PHASE_FIELDS | {"run_time"}
-
-
-def normalize_value(raw: Any, *, key: str = "", line: int | None = None) -> Dict[str, Any]:
-    """Normalize explicit values; never substitute missing evidence with zero."""
-    result = {"field": key, "raw": raw, "value": None, "unit": None, "aggregation": "raw", "availability": "missing", "line": line}
-    if raw is None or str(raw).strip().lower() in {"", "null", "none", "n/a"}:
-        return result
-    text = str(raw).strip()
-    if text.startswith("{") and text.endswith("}"):
-        entries = {}
-        for part in text[1:-1].split(","):
-            if "=" not in part:
-                return {**result, "availability": "invalid"}
-            name, value = part.split("=", 1)
-            name = name.strip()
-            if name not in {"sum", "max", "min", "avg", "count"} or name in entries:
-                return {**result, "availability": "invalid"}
-            entries[name] = normalize_value(value.strip(), key="count" if name == "count" else key, line=line)
-        # Keep each statistic. Counts describe sample count, not row/byte totals.
-        preference = ("sum", "max", "avg", "min") if "rows" in key or "cols" in key else ("max", "sum", "avg", "min")
-        chosen = next((name for name in preference if name in entries), None)
-        selected = entries.get(chosen, result)
-        return {**result, **{k: selected[k] for k in ("value", "unit", "availability")}, "aggregation": chosen, "statistics": entries}
-    match = _NUMBER.fullmatch(text)
-    if not match:
-        return {**result, "availability": "invalid"}
-    numeric, suffix = match.groups()
-    numeric = numeric.replace(",", "")
-    value = float(numeric) if "." in numeric else int(numeric)
-    suffix = suffix.lower()
-    if suffix in _TIME_FACTORS:
-        value, unit = value * _TIME_FACTORS[suffix], "ms"
-    elif suffix in _BYTE_FACTORS:
-        value, unit = value * _BYTE_FACTORS[suffix], "bytes"
-    elif suffix:
-        return {**result, "availability": "invalid"}
-    elif key in _JOB_PHASE_FIELDS:
-        unit = None
-    else:
-        unit = "ms" if key.endswith("_ms") or key == "run_time" else ("bytes" if key.endswith("_bytes") else "count")
-    if not math.isfinite(value):
-        return {**result, "availability": "invalid"}
-    return {**result, "value": value if value >= 0 else None, "unit": unit, "availability": "observed" if value >= 0 else "not_produced"}
-
-
-def _fields(
-    raw: Dict[str, str],
-    lines: Dict[str, int],
-    source_units: Dict[str, str] | None = None,
-) -> Dict[str, Any]:
-    fields = {key: normalize_value(value, key=key, line=lines.get(key)) for key, value in raw.items()}
-    for key, unit in (source_units or {}).items():
-        fields[key]["unit"] = unit
-    return fields
-
-
-def _calculation(left: Dict[str, Any], right: Dict[str, Any]) -> Dict[str, Any]:
-    """Only compare observations with the same unit and aggregation."""
-    sources = {"left": left, "right": right}
-    if left["availability"] != "observed" or right["availability"] != "observed":
-        return {"delta": None, "ratio": None, "reason": "unavailable_operand", "sources": sources}
-    if left["unit"] is None or right["unit"] is None:
-        return {"delta": None, "ratio": None, "reason": "unknown_unit", "sources": sources}
-    if (left["unit"], left["aggregation"]) != (right["unit"], right["aggregation"]):
-        return {"delta": None, "ratio": None, "reason": "incompatible_units_or_aggregations", "sources": sources}
-    a, b = left["value"], right["value"]
-    return {"delta": b - a, "ratio": b / a if a != 0 else None, "reason": "zero_denominator" if a == 0 else None, "sources": sources}
-
-
-def build_report(path: Path, *, top: int = 5) -> Dict[str, Any]:
-    parsed = parse_metric_file(path)
-    pipelines, transitions = [], []
-    physical_by_id: Dict[int, list] = {}
-    for physical in parsed.physical_node_observations:
-        physical_by_id.setdefault(physical.node_id, []).append(physical)
-    for index, pipeline in enumerate(parsed.pipelines):
-        row = {"stage_id": pipeline.stage_id, "task_id": pipeline.task_id, "pipeline_id": pipeline.pipeline_id,
-               "line_no": pipeline.line_no, "metrics": _fields(pipeline.metrics, pipeline.metric_lines), "operators": []}
-        row["backends"] = [
-            {"name": backend.name, "line_no": backend.line_no,
-             "metrics": _fields(backend.metrics, backend.metric_lines)}
-            for backend in pipeline.backends
-        ]
-        for op_index, operator in enumerate(pipeline.operators):
-            candidates = physical_by_id.get(int(operator.plan_id), []) if operator.plan_id.lstrip("-").isdigit() else []
-            physical = candidates[0] if len(candidates) == 1 else None
-            op = {"kind": operator.kind, "name": operator.name, "node_id": operator.node_id, "plan_id": operator.plan_id,
-                  "line_no": operator.line_no, "metrics": _fields(operator.metrics, operator.metric_lines),
-                  "physical_node": asdict(physical) if physical else None,
-                  "physical_node_candidate_lines": [v.line_no for v in candidates]}
-            row["operators"].append(op)
-            counts = {k: op["metrics"].get(k, normalize_value(None, key=k)) for k in ("input_rows_num", "output_rows_num", "scanned_rows_num", "output_cols_num")}
-            calc = _calculation(counts["input_rows_num"], counts["output_rows_num"])
-            transitions.append({"pipeline_index": index, "operator_index": op_index, "stage_id": pipeline.stage_id,
-                                "task_id": pipeline.task_id, "pipeline_id": pipeline.pipeline_id, "name": operator.name,
-                                "kind": operator.kind, "plan_id": operator.plan_id, "physical_node": op["physical_node"],
-                                "counts": counts, "input_output": calc})
-        pipelines.append(row)
-    known = bool(parsed.jobs or parsed.query_fields or parsed.summary_metrics or parsed.pipelines or parsed.analyzed_nodes or parsed.physical_nodes)
-    detailed = any(p["operators"] for p in pipelines)
-    task_names = list(dict.fromkeys([j["taskName"] for j in parsed.jobs if j.get("taskName")] + ([parsed.query_fields["jsy_cpp_task_name"]] if parsed.query_fields.get("jsy_cpp_task_name") else [])))
-    return {"schema_version": SCHEMA_VERSION, "normalization_version": NORMALIZATION_VERSION,
-            "source": str(path.resolve()), "body_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
-            "task_names": task_names, "input_format": "full_metric" if detailed else ("signoz_metric_log" if known else "unknown"),
-            "complete": parsed.complete_marker, "collection_status": ("format_error" if not known else "body_incomplete" if not parsed.complete_marker else "complete_detail" if detailed else "complete_summary"),
-            "jobs": [{"raw": job, "fields": _fields(job, parsed.field_lines.get(f"job:{i}", {}),
-                                                   parsed.field_units.get(f"job:{i}", {}))}
-                     for i, job in enumerate(parsed.jobs)],
-            "query_fields": _fields(parsed.query_fields, parsed.field_lines.get("query", {})),
-            "summary_metrics": _fields(parsed.summary_metrics, parsed.field_lines.get("summary", {})),
-            "tasks": {task: _fields(fields, parsed.field_lines.get(f"task:{task}", {})) for task, fields in parsed.task_fields.items()},
-            "subplans": [asdict(v) for v in parsed.subplan_observations], "physical_nodes": [asdict(v) for v in parsed.physical_node_observations],
-            "fragment_edges": [asdict(edge) for edge in parsed.fragment_edges], "analyzed_nodes": parsed.analyzed_nodes,
-            "pipelines": pipelines, "data_scale_transitions": transitions, "source_lines": parsed.source_lines,
-            "coverage": {"jobs": len(parsed.jobs), "pipelines": len(pipelines), "operators": len(transitions), "source_lines": len(parsed.source_lines),
-                         "tasks": len(parsed.task_fields), "subplans": len(parsed.subplan_observations), "physical_nodes": len(parsed.physical_node_observations), "analyzed_nodes": len(parsed.analyzed_nodes)}}
-
-
-def build_execution_overview(report: Dict[str, Any], *, top: int = 5) -> Dict[str, Any]:
-    """Expose bounded facts from each recorded scope without merging scopes."""
-    scope_fields = ("query_peak_memory", "query_current_memory", "query_memory_limit", "resultRows", "writtenRows")
-    scopes = []
-    for pointer, fields in [("/query_fields", report["query_fields"]), *[
-        ("/tasks/" + task.replace("~", "~0").replace("/", "~1"), fields)
-        for task, fields in report["tasks"].items()
-    ]]:
-        if any(key in fields for key in scope_fields):
-            scopes.append({"report_pointer": pointer,
-                           "fields": {key: fields.get(key, normalize_value(None, key=key)) for key in scope_fields}})
-
-    pipelines, operator_counts = [], {}
-    resource_fields = ("read_io_file_bytes", "read_block_time", "io_task_pending_time",
-                       "fetch_block_meta_time", "store_tuple_arena_physical", "hash_table_arena_physical",
-                       "arena_physical_allocated", "main_arena_physical_allocated",
-                       "secondary_arena_physical_allocated", "partition_allocated_physical",
-                       "partition_arena_allocated_physical")
-    backend_fields = (
-        "compile_wall_time", "compile_cost_wall_time", "compile_cost_cpu_time",
-        "cache_look_up_time", "llvm_predict_compile_time", "morsel_count",
-        "morsel_active_wall_time", "morsel_active_cpu_time",
-    )
-    for index, pipeline in enumerate(report["pipelines"]):
-        fields = pipeline["metrics"]
-        wall_key = "pipeline_wall_time_ms" if "pipeline_wall_time_ms" in fields else "pipeline_wall_time"
-        wall = fields.get(wall_key, normalize_value(None, key=wall_key))
-        resources = [{"report_pointer": f"/pipelines/{index}/operators/{oi}", "name": operator["name"],
-                      "plan_id": operator["plan_id"],
-                      "fields": {key: operator["metrics"][key] for key in
-                                 ("input_rows_num", "output_rows_num", "output_cols_num", "scanned_rows_num", *resource_fields)
-                                 if key in operator["metrics"]}}
-                     for oi, operator in enumerate(pipeline["operators"])
-                     if any(key in operator["metrics"] for key in resource_fields)]
-        pipelines.append({"report_pointer": f"/pipelines/{index}", "pipeline_id": pipeline["pipeline_id"],
-                          "task_id": pipeline["task_id"], "wall_time": wall,
-                          "parallelism": fields.get("pipeline_parallelism", normalize_value(None, key="pipeline_parallelism")),
-                          "activity": {key: fields[key] for key in
-                                       ("morsel_blocked_time", "morsel_pending_time", "morsel_active_wall_time",
-                                        "morsel_active_cpu_time", "pipeline_task_open_time", "pipeline_task_close_time") if key in fields},
-                          "backend_count": len(pipeline["backends"]),
-                          "backends_returned": min(len(pipeline["backends"]), top),
-                          "backends": [
-                              {
-                                  "name": backend["name"],
-                                  "report_pointer": f"/pipelines/{index}/backends/{bi}",
-                                  "fields": {key: backend["metrics"][key] for key in backend_fields
-                                             if key in backend["metrics"]},
-                              }
-                              for bi, backend in enumerate(pipeline["backends"][:top])
-                          ],
-                          "resource_operator_count": len(resources), "resource_operators_returned": min(len(resources), top),
-                          "resource_operators": resources[:top]})
-        for operator in pipeline["operators"]:
-            name = operator["name"]
-            operator_counts[name] = operator_counts.get(name, 0) + 1
-    pipelines.sort(key=lambda item: item["wall_time"]["value"]
-                   if item["wall_time"]["availability"] == "observed" and item["wall_time"]["unit"] == "ms" else -1,
-                   reverse=True)
-    return {"scope_count": len(scopes), "scopes_returned": min(len(scopes), top), "scopes": scopes[:top],
-            "pipeline_count": len(pipelines), "pipelines_returned": min(len(pipelines), top), "pipelines": pipelines[:top],
-            "operator_occurrences_by_name": operator_counts}
-
-
-def build_memory_overview(report: Dict[str, Any], *, top: int = 5) -> Dict[str, Any]:
-    """Rank known state/allocation fields independently of time and other scopes."""
-    memory_fields = {
-        "store_tuple_arena_logical", "store_tuple_arena_physical",
-        "hash_table_arena_logical", "hash_table_arena_physical",
-        "arena_logical_allocated", "arena_physical_allocated",
-        "main_arena_logical_allocated", "main_arena_physical_allocated",
-        "secondary_arena_logical_allocated", "secondary_arena_physical_allocated",
-        "partition_arena_allocated_logical", "partition_arena_allocated_physical",
-        "partition_allocated_logical", "partition_allocated_physical",
-        "temp_table_logical_allocated", "temp_table_physical_allocated",
-        "allocated_memory", "deallocated_memory", "peak_memory", "current_memory",
-    }
-    groups = {}
-    object_limit = min(top, 3)
-    for pi, pipeline in enumerate(report["pipelines"]):
-        for oi, obj in [(None, pipeline), *enumerate(pipeline["operators"])]:
-            for key, value in obj["metrics"].items():
-                if key not in memory_fields or value["unit"] != "bytes" or value["availability"] != "observed":
-                    continue
-                object_type = "pipeline" if oi is None else "operator"
-                group = (object_type, key, value["aggregation"], pipeline["task_id"], pipeline["stage_id"])
-                pointer = f"/pipelines/{pi}" + (f"/operators/{oi}" if oi is not None else "")
-                item = {"report_pointer": pointer, "pipeline_id": pipeline["pipeline_id"], "metric": value}
-                if oi is not None:
-                    item.update({"name": obj["name"], "plan_id": obj["plan_id"]})
-                groups.setdefault(group, []).append(item)
-    ranked = [
-        {"object_type": key[0], "field": key[1], "unit": "bytes", "aggregation": key[2],
-         "task_id": key[3], "stage_id": key[4], "object_count": len(values),
-         "objects_returned": min(len(values), object_limit),
-         "values": sorted(values, key=lambda item: item["metric"]["value"], reverse=True)[:object_limit]}
-        for key, values in sorted(groups.items(), key=lambda item: str(item[0]))
-    ]
-    return {"group_count": len(ranked), "groups_returned": min(len(ranked), 8), "groups": ranked[:8]}
-
-
-def build_summary(report: Dict[str, Any], report_path: Path, *, top: int = 5) -> Dict[str, Any]:
-    groups: Dict[tuple, list] = {}
-    for pi, pipeline in enumerate(report["pipelines"]):
-        for oi, obj in [(None, pipeline), *enumerate(pipeline["operators"])]:
-            for key, value in obj["metrics"].items():
-                if value["availability"] != "observed":
-                    continue
-                group = ("pipeline" if oi is None else "operator", key, value["unit"], value["aggregation"])
-                pointer = f"/pipelines/{pi}" + (f"/operators/{oi}" if oi is not None else "")
-                groups.setdefault(group, []).append({"object": pointer, "value": value["value"], "line": value["line"]})
-    # Bound default output by field groups as well as objects; all facts remain in report.
-    rankings = [{"object_type": k[0], "field": k[1], "unit": k[2], "aggregation": k[3], "count": len(v),
-                 "values": sorted(v, key=lambda item: item["value"], reverse=True)[:top]} for k, v in sorted(groups.items(), key=lambda item: str(item[0]))]
-    query = {k: v for k, v in report["query_fields"].items() if v["availability"] == "observed"}
-    return {"schema_version": SCHEMA_VERSION, "normalization_version": NORMALIZATION_VERSION,
-            "report_path": str(report_path), "body_sha256": report["body_sha256"],
-            "source": report["source"], "task_names": report["task_names"], "collection_status": report["collection_status"],
-            "complete": report["complete"], "input_format": report["input_format"], "coverage": report["coverage"],
-            "execution_overview": build_execution_overview(report, top=top),
-            "memory_overview": build_memory_overview(report, top=top),
-            "job_snapshots": [{"task_name": j["raw"].get("taskName"), "status": j["raw"].get("taskStatMachine", "UNKNOWN"),
-                               "report_pointer": f"/jobs/{i}", "failure_message_preview": str(j["raw"].get("failedMessage", ""))[:300],
-                               "fields": {k: v for k, v in j["fields"].items() if k in {"run_time", "peakUserMemory", "currentUserMemory", "memCapacity"} | _JOB_PHASE_FIELDS}} for i, j in enumerate(report["jobs"][:top])],
-            "query_metrics": dict(list(query.items())[:20]), "summary_metrics": report["summary_metrics"],
-            "rankings": rankings[:20], "ranking_group_count": len(rankings), "ranking_groups_returned": min(len(rankings), 20)}
-
-
-def _compare_jobs(before: Dict[str, Any], after: Dict[str, Any]) -> Dict[str, Any]:
-    """Compare identified job snapshots without choosing among repeated snapshots."""
-    left, right = before["jobs"], after["jobs"]
-    groups = []
-    for jobs in (left, right):
-        group = {}
-        for index, job in enumerate(jobs):
-            name = job["raw"].get("taskName")
-            if name and name.lower() not in {"null", "unknown", "n/a"}:
-                group.setdefault(name, []).append(index)
-        groups.append(group)
-    lg, rg = groups
-    pairs = [(lg[name][0], rg[name][0], "unique_task_name")
-             for name in sorted(lg.keys() & rg.keys()) if len(lg[name]) == len(rg[name]) == 1]
-    # Different runs normally have different task names. Only a sole identified
-    # job on each selected side is safe to pair without an explicit mapping.
-    if not pairs and len(left) == len(right) == len(lg) == len(rg) == 1:
-        if before["task_names"] == list(lg) and after["task_names"] == list(rg):
-            pairs = [(0, 0, "single_job_per_report")]
-
-    def snapshot(job, index):
-        return {"report_pointer": f"/jobs/{index}", "raw": job["raw"]}
-
-    changes = []
-    for li, ri, basis in pairs:
-        a, b = left[li], right[ri]
-        status_a, status_b = a["raw"].get("taskStatMachine"), b["raw"].get("taskStatMachine")
-        known_status = all(value and value.lower() not in {"null", "unknown", "n/a"}
-                           for value in (status_a, status_b))
-        fields = (a["fields"].keys() | b["fields"].keys()) & {
-            "run_time", "peakUserMemory", "currentUserMemory", "memCapacity"}
-        changes.append({"alignment_basis": basis, "before": snapshot(a, li), "after": snapshot(b, ri),
-                        "status_change": {"before": status_a, "after": status_b,
-                                          "changed": status_a != status_b if known_status else None,
-                                          "reason": None if known_status else "unavailable_operand"},
-                        "metric_changes": {key: _calculation(a["fields"].get(key, normalize_value(None, key=key)),
-                                                            b["fields"].get(key, normalize_value(None, key=key)))
-                                           for key in sorted(fields)}})
-    used_left, used_right = {p[0] for p in pairs}, {p[1] for p in pairs}
-    unmatched_left = [snapshot(job, i) for i, job in enumerate(left) if i not in used_left]
-    unmatched_right = [snapshot(job, i) for i, job in enumerate(right) if i not in used_right]
-    return {"job_changes": changes, "unmatched_jobs_before": unmatched_left, "unmatched_jobs_after": unmatched_right,
-            "job_alignment_ambiguities": [{"reason": "task_correspondence_unresolved",
-                                            "before": unmatched_left, "after": unmatched_right}]
-            if unmatched_left and unmatched_right else []}
-
-
-def compare_reports(before: Dict[str, Any], after: Dict[str, Any], *, top: int = 5) -> Dict[str, Any]:
-    """Match explicit identities; never declare causality or business comparability."""
-    left, right = before["data_scale_transitions"], after["data_scale_transitions"]
-    used_left, used_right, changes = set(), set(), []
-    def operator_fields(report, item):
-        return report["pipelines"][item["pipeline_index"]]["operators"][item["operator_index"]]["metrics"]
-    def field_changes(left_fields, right_fields):
-        return {k: _calculation(left_fields.get(k, normalize_value(None, key=k)), right_fields.get(k, normalize_value(None, key=k)))
-                for k in sorted(left_fields.keys() | right_fields.keys())}
-    def identity(item, basis):
-        if basis == "name_and_plan_id":
-            return (item["name"], item["plan_id"]) if item.get("plan_id") else None
-        physical = item.get("physical_node")
-        return (item["name"], physical["node_type"], physical["summary"]) if physical else None
-    for basis in ("name_and_plan_id", "physical_signature"):
-        lg, rg = {}, {}
-        for group, items, used in ((lg, left, used_left), (rg, right, used_right)):
-            for i, item in enumerate(items):
-                key = identity(item, basis)
-                if i not in used and key is not None:
-                    group.setdefault(key, []).append(i)
-        for key in sorted(lg.keys() & rg.keys(), key=str):
-            if len(lg[key]) != 1 or len(rg[key]) != 1:
-                continue
-            li, ri = lg[key][0], rg[key][0]
-            used_left.add(li); used_right.add(ri)
-            changes.append({"alignment_basis": basis, "before": left[li], "after": right[ri],
-                            "metric_changes": field_changes(operator_fields(before, left[li]), operator_fields(after, right[ri])),
-                            "count_changes": {field: _calculation(left[li]["counts"][field], right[ri]["counts"][field]) for field in left[li]["counts"]}})
-    unmatched_left = [v for i, v in enumerate(left) if i not in used_left]
-    unmatched_right = [v for i, v in enumerate(right) if i not in used_right]
-    ambiguities = []
-    for basis in ("name_and_plan_id", "physical_signature"):
-        keys = {identity(v, basis) for v in unmatched_left} & {identity(v, basis) for v in unmatched_right}
-        for key in sorted(keys - {None}, key=str):
-            lc = sum(identity(v, basis) == key for v in unmatched_left)
-            rc = sum(identity(v, basis) == key for v in unmatched_right)
-            if lc > 1 or rc > 1:
-                ambiguities.append({"basis": basis, "key": key, "before_count": lc, "after_count": rc})
-    return {"schema_version": SCHEMA_VERSION, "before": {k: before[k] for k in ("source", "body_sha256", "task_names", "complete", "collection_status", "coverage")},
-            "after": {k: after[k] for k in ("source", "body_sha256", "task_names", "complete", "collection_status", "coverage")},
-            "operator_data_scale_changes": changes, "operator_alignment_ambiguities": ambiguities,
-            "unmatched_before": unmatched_left, "unmatched_after": unmatched_right,
-            **_compare_jobs(before, after),
-            "query_changes": field_changes(before["query_fields"], after["query_fields"]),
-            "summary_changes": field_changes(before["summary_metrics"], after["summary_metrics"])}