@senpi/runtime 3.0.9-dev.20260710204324 → 3.0.9-dev.feat-strategy-compos.20260716233548

package/dist/composer/validator/graph_checks.py

@@ -0,0 +1,443 @@
+"""graph_checks — stage (d): the graph-semantic checks a JSON Schema cannot express.
+
+These are the checks that LAYER on top of schema validation (supervisor decision): a
+schema can say "this param is an int in [1,1000]" but not "this edge connects a
+``score_map`` output to a ``candles`` input", "this input has no writer", or "these three
+nodes form a cycle". ~All plain Python over the registry — no networkx, no framework (D8):
+the DAG never executes as a DAG, so a few dictionaries and a depth-first search suffice.
+
+Each function appends :class:`TeachingError` s to a shared list and never raises — every
+defect is collected so the author sees them all at once. Checks that depend on a node's
+type being known (ports, types, connectivity) simply skip nodes/edges whose type is
+unknown; the unknown-node check (CMP030) has already named those.
+
+A node's INTERFACE (its typed ports + kind) is resolved uniformly whether it is a registry
+primitive (``NODES_BY_NAME`` — the closed vocabulary) or a per-graph ``pure_fn`` node
+(``composer.pure_fn`` — the open extension point, D2 kind 2). Both present the same
+duck surface (``.name``, ``.kind``, ``.inputs``, ``.outputs``), so all the edge/type/
+connectivity/cycle logic below is written once and reads a pure_fn node's declared ports
+exactly as it reads a primitive's registered ports. The purity/signature/inline-test
+checks on a pure_fn's BODY are a separate stage (``composer.pure_fn.checks``).
+
+Two graph-semantic conventions this validator enforces, both read directly off the
+registry's ``Port`` model (D4 — the registry is the single source of truth; see
+``composer/registry/models.py`` for the field semantics):
+
+  * ``port.merge`` — feature_map inputs are MERGE points (every indicator in the corpus
+    emits a per-asset feature and the scorer consumes a single "merged feature record"),
+    so multiple edges into a ``merge=True`` input are legal. Every other input accepts
+    exactly one writer — a second writer is CMP037.
+  * ``port.required`` — an unwired input with ``required=False`` (an ``asset_list``
+    universe input, ``signal_envelope.equity``, or a stateful node's ``state`` input,
+    which is supplied by the runtime tick boundary rather than a graph edge) is fine;
+    every other unwired input is CMP036.
+"""
+
+from __future__ import annotations
+
+import difflib
+
+from composer.pure_fn import is_pure_fn_entry, parse_pure_fn
+from composer.registry import NODES_BY_NAME, NodeKind, PortType
+
+from .errors import Location, TeachingError
+
+# Tokens that mark an unknown node name as a reach for BESPOKE scoring/classification.
+# When one appears, CMP030 steers to the settled stance: there is no scorer primitive for
+# these — the one generic scorer is `hard_gates_all`, everything else is a `kind: pure_fn`
+# node (registry stance 2026-07-15; the 3 speculative scorers were removed). Deliberately
+# narrow so plain typos (e.g. `rsii`) keep the clean nearest-match hint.
+_SCORING_TOKENS = (
+    "score", "scoring", "scorer", "direction", "confirm", "conviction",
+    "classif", "waterfall", "tier", "gate",
+)
+
+
+def _node_line(line_index: dict, node_id) -> int | None:
+    return line_index.get("nodes", {}).get(node_id)
+
+
+def _split_ref(ref: str) -> tuple[str, str] | None:
+    if not isinstance(ref, str) or ref.count(".") != 1:
+        return None
+    node_id, port = ref.split(".", 1)
+    if not node_id or not port:
+        return None
+    return node_id, port
+
+
+def _describe_hint(iface) -> str:
+    """Where the author should look to fix a bad port reference on this node."""
+    if iface.kind == NodeKind.pure_fn:
+        return "check the node's declared `inputs:`/`outputs:` ports"
+    return f"see `composer describe {iface.name}`"
+
+
+def check(doc: dict, line_index: dict) -> list[TeachingError]:
+    errors: list[TeachingError] = []
+    nodes = doc.get("nodes")
+    if not isinstance(nodes, list):
+        return errors  # schema already flagged a missing/!list nodes field
+    edges = doc.get("edges") or []
+    if not isinstance(edges, list):
+        edges = []
+
+    # ------------------------------------------------------------------ node-level checks
+    # iface_by_id: ids whose interface (ports+kind) is known — primitive OR pure_fn.
+    iface_by_id: dict[str, object] = {}
+    all_ids: set[str] = set()          # every declared id (dangling-edge check uses this)
+    seen_ids: set[str] = set()
+    valid_names = sorted(NODES_BY_NAME)
+
+    for entry in nodes:
+        if not isinstance(entry, dict):
+            continue
+        node_id = entry.get("id")
+        node_type = entry.get("node")
+        line = _node_line(line_index, node_id)
+
+        if isinstance(node_id, str):
+            all_ids.add(node_id)
+            if node_id in seen_ids:
+                errors.append(
+                    TeachingError(
+                        code="CMP031",
+                        message=f"duplicate node id {node_id!r}.",
+                        fix_hint=f"Rename one of the two nodes with id {node_id!r} to a unique id.",
+                        location=Location(node=node_id, line=line),
+                    )
+                )
+            seen_ids.add(node_id)
+
+        if is_pure_fn_entry(entry):
+            # A pure_fn node's interface comes from its declared inputs/outputs (the body
+            # is validated separately). Register it so edges can typecheck against it.
+            if isinstance(node_id, str):
+                iface_by_id[node_id] = parse_pure_fn(entry, line)
+        elif isinstance(node_type, str) and node_type in NODES_BY_NAME:
+            if isinstance(node_id, str):
+                iface_by_id[node_id] = NODES_BY_NAME[node_type]
+        elif isinstance(node_type, str):
+            suggestions = difflib.get_close_matches(node_type, valid_names, n=3, cutoff=0.4)
+            if suggestions:
+                near = ", ".join(f"`{s}`" for s in suggestions)
+                hint = f"did you mean {near}? run `composer describe {suggestions[0]}`"
+            else:
+                hint = "run `composer catalog` to list the available node types"
+            if any(tok in node_type.lower() for tok in _SCORING_TOKENS):
+                hint += (
+                    "; for bespoke scoring/classification there is no such primitive — use "
+                    "`hard_gates_all` for a simple gate-and-weigh, otherwise write a "
+                    "`kind: pure_fn` node"
+                )
+            errors.append(
+                TeachingError(
+                    code="CMP030",
+                    message=f"unknown node type {node_type!r}.",
+                    fix_hint=hint,
+                    location=Location(node=node_id, line=line),
+                )
+            )
+        else:
+            # Neither a registry `node:` nor `kind: pure_fn` — the entry has no interface.
+            errors.append(
+                TeachingError(
+                    code="CMP038",
+                    message=(
+                        f"node {node_id!r} declares neither a registry `node:` type nor "
+                        f"`kind: pure_fn`."
+                    ),
+                    fix_hint=(
+                        "Add a `node:` from `composer catalog`, or declare `kind: pure_fn` "
+                        "with `inputs:`/`outputs:`/`fn:`."
+                    ),
+                    location=Location(node=node_id, line=line),
+                )
+            )
+
+    # ------------------------------------------------------------------ edge-level checks
+    # incoming[(node_id, port)] = list of (from_ref, port_type_or_None)
+    incoming: dict[tuple[str, str], list[tuple[str, object]]] = {}
+    dep_edges: list[tuple[str, str]] = []  # (producer_node, consumer_node) for cycle graph
+
+    for i, edge in enumerate(edges):
+        if not isinstance(edge, dict):
+            continue
+        line = line_index.get("edges", {}).get(i)
+        src = _split_ref(edge.get("from"))
+        dst = _split_ref(edge.get("to"))
+        if src is None or dst is None:
+            continue  # schema's `pattern` check already flagged the malformed ref
+        (from_id, from_port), (to_id, to_port) = src, dst
+
+        # dangling references (against every declared id — a node whose *type* is unknown
+        # still exists, so it is not "dangling"; CMP030 already named its bad type).
+        if from_id not in all_ids:
+            errors.append(
+                TeachingError(
+                    code="CMP032",
+                    message=f"edge `from` references unknown node id {from_id!r}.",
+                    fix_hint=f"Point `from` at an existing node id, or add a node with id {from_id!r}.",
+                    location=Location(node=from_id, port=from_port, line=line),
+                )
+            )
+        if to_id not in all_ids:
+            errors.append(
+                TeachingError(
+                    code="CMP032",
+                    message=f"edge `to` references unknown node id {to_id!r}.",
+                    fix_hint=f"Point `to` at an existing node id, or add a node with id {to_id!r}.",
+                    location=Location(node=to_id, port=to_port, line=line),
+                )
+            )
+        # Port/type checks need a known interface on both ends.
+        if from_id not in iface_by_id or to_id not in iface_by_id:
+            continue
+
+        from_def = iface_by_id[from_id]
+        to_def = iface_by_id[to_id]
+        out_ports = {p.name: p for p in from_def.outputs}
+        in_ports = {p.name: p for p in to_def.inputs}
+
+        out_port = out_ports.get(from_port)
+        in_port = in_ports.get(to_port)
+
+        if out_port is None:
+            avail = ", ".join(f"`{p.name}`" for p in from_def.outputs) or "(none)"
+            errors.append(
+                TeachingError(
+                    code="CMP033",
+                    message=f"node {from_id!r} ({from_def.name}) has no output port {from_port!r}.",
+                    fix_hint=f"Use one of its outputs: {avail} ({_describe_hint(from_def)}).",
+                    location=Location(node=from_id, port=from_port, line=line),
+                )
+            )
+        if in_port is None:
+            avail = ", ".join(f"`{p.name}`" for p in to_def.inputs) or "(none)"
+            errors.append(
+                TeachingError(
+                    code="CMP033",
+                    message=f"node {to_id!r} ({to_def.name}) has no input port {to_port!r}.",
+                    fix_hint=f"Use one of its inputs: {avail} ({_describe_hint(to_def)}).",
+                    location=Location(node=to_id, port=to_port, line=line),
+                )
+            )
+
+        if out_port is not None and in_port is not None:
+            # The `raw` escape type may feed ONLY a pure_fn normalizer (D-wave escape hatch):
+            # a raw wire into any curated node would let opaque, untyped data bypass the typed
+            # port contract. This fires regardless of the in_port's declared type (a curated
+            # node has no raw input, so the mismatch would otherwise surface as CMP034; this
+            # gives the actionable "write a normalizer" lesson instead).
+            raw_to_non_pure_fn = (
+                out_port.type == PortType.raw and to_def.kind != NodeKind.pure_fn
+            )
+            if raw_to_non_pure_fn:
+                errors.append(
+                    TeachingError(
+                        code="CMP039",
+                        message=(
+                            f"raw payload {from_id}.{from_port} feeds {to_id} "
+                            f"({to_def.name}), which is not a pure_fn."
+                        ),
+                        fix_hint=(
+                            "raw may only feed a pure_fn — write a normalizer: add a "
+                            "`kind: pure_fn` node with a `raw` input that reads the payload "
+                            "and returns TYPED outputs (with inline tests), then wire "
+                            f"{from_id}.{from_port} into it."
+                        ),
+                        location=Location(node=to_id, port=to_port, line=line),
+                    )
+                )
+            # CMP034 (type mismatch) would also fire on a raw->curated edge, but CMP039 is the
+            # better lesson there, so suppress the duplicate. A raw->pure_fn edge whose pure_fn
+            # input is NOT declared `raw` still (correctly) reports CMP034.
+            if out_port.type != in_port.type and not raw_to_non_pure_fn:
+                errors.append(
+                    TeachingError(
+                        code="CMP034",
+                        message=(
+                            f"type mismatch: {from_id}.{from_port} is {out_port.type.value} "
+                            f"but {to_id}.{to_port} expects {in_port.type.value}."
+                        ),
+                        fix_hint=(
+                            f"Wire a {in_port.type.value} output into {to_id}.{to_port}, "
+                            f"or feed {from_id}.{from_port} into a port that accepts {out_port.type.value}."
+                        ),
+                        location=Location(node=to_id, port=to_port, line=line),
+                    )
+                )
+            incoming.setdefault((to_id, to_port), []).append(
+                (f"{from_id}.{from_port}", in_port.type)
+            )
+            # Cycle graph: a stateful node's OUTPUTS carry prior-tick data (the state
+            # boundary), so they do not create a forward dependency — this is what breaks
+            # the universal ttl-dedup feedback loop without a false cycle. Omit edges that
+            # originate at a stateful node.
+            if from_def.kind != NodeKind.stateful:
+                dep_edges.append((from_id, to_id))
+
+    # ------------------------------------------------- connectivity + multiple-writer checks
+    for entry in nodes:
+        if not isinstance(entry, dict):
+            continue
+        node_id = entry.get("id")
+        if not isinstance(node_id, str) or node_id not in iface_by_id:
+            continue
+        node_def = iface_by_id[node_id]
+        line = _node_line(line_index, node_id)
+        for port in node_def.inputs:
+            writers = incoming.get((node_id, port.name), [])
+            if len(writers) == 0 and port.required:
+                errors.append(
+                    TeachingError(
+                        code="CMP036",
+                        message=(
+                            f"required input {node_id}.{port.name} ({port.type.value}) "
+                            f"has no incoming edge."
+                        ),
+                        fix_hint=(
+                            f"Add an edge whose `to` is `{node_id}.{port.name}` from a "
+                            f"{port.type.value} output ({_describe_hint(node_def)})."
+                        ),
+                        location=Location(node=node_id, port=port.name, line=line),
+                    )
+                )
+            elif len(writers) > 1 and not port.merge:
+                srcs = ", ".join(sorted(f"`{w[0]}`" for w in writers))
+                errors.append(
+                    TeachingError(
+                        code="CMP037",
+                        message=(
+                            f"input {node_id}.{port.name} ({port.type.value}) has "
+                            f"{len(writers)} writers: {srcs}."
+                        ),
+                        fix_hint=(
+                            f"Keep exactly one edge into `{node_id}.{port.name}` "
+                            f"(only merge-capable inputs may accept multiple writers; "
+                            f"{_describe_hint(node_def)})."
+                        ),
+                        location=Location(node=node_id, port=port.name, line=line),
+                    )
+                )
+
+    # ---------------------------------------------------- escape-hatch (mcp_read) tool checks
+    errors.extend(_check_mcp_read_tools(nodes, line_index))
+
+    # ------------------------------------------------------------------- cycle detection
+    errors.extend(_detect_cycles(iface_by_id, dep_edges, line_index))
+    return errors
+
+
+def _check_mcp_read_tools(nodes: list, line_index: dict) -> list[TeachingError]:
+    """CHECK-time validation for the mcp_read escape hatch: the named `tool` must exist in the
+    committed MCP snapshot (CMP060) and must not be a mutation (CMP061). Both are teaching
+    errors, not runtime surprises. The wrapped-tool / hatch-use ANNOTATIONS are non-blocking and
+    are surfaced by the check verb (composer/verbs/check.py), not here."""
+    errors: list[TeachingError] = []
+    read_nodes = [
+        n for n in nodes
+        if isinstance(n, dict) and n.get("node") == "mcp_read" and isinstance(n.get("id"), str)
+    ]
+    if not read_nodes:
+        return errors
+
+    # Import lazily so the validator has no import-time dependency on the snapshot file.
+    from composer.registry.mcp_closure import MUTATION_TOOLS, snapshot_tool_names
+
+    known = snapshot_tool_names()
+    for entry in read_nodes:
+        node_id = entry["id"]
+        line = _node_line(line_index, node_id)
+        with_block = entry.get("with") or {}
+        tool = with_block.get("tool") if isinstance(with_block, dict) else None
+        if not isinstance(tool, str) or not tool.strip():
+            continue  # a missing/blank required `tool` is CMP013 from the schema stage
+        if tool in MUTATION_TOOLS:
+            errors.append(
+                TeachingError(
+                    code="CMP061",
+                    message=(
+                        f"mcp_read tool {tool!r} is a mutation (write) tool; a scanner is a "
+                        f"read-only producer."
+                    ),
+                    fix_hint=(
+                        "Remove it — mutations (open/close/edit positions, strategy/fund/"
+                        "ratchet writes) are never a data source. The runtime blocks them too."
+                    ),
+                    location=Location(node=node_id, line=line),
+                )
+            )
+        elif tool not in known:
+            near = difflib.get_close_matches(tool, sorted(known), n=3, cutoff=0.5)
+            if near:
+                hint = (
+                    f"did you mean {', '.join(repr(m) for m in near)}? "
+                    "If this is a genuinely-new tool, refresh the snapshot "
+                    "(MCP_SNAPSHOT_REFRESH=1 pytest python-tests/composer/test_mcp_closure.py)."
+                )
+            else:
+                hint = (
+                    "No close match in the committed snapshot. If this is a genuinely-new "
+                    "server-side read, refresh the snapshot (MCP_SNAPSHOT_REFRESH=1 pytest "
+                    "python-tests/composer/test_mcp_closure.py); otherwise fix the tool name."
+                )
+            errors.append(
+                TeachingError(
+                    code="CMP060",
+                    message=f"mcp_read tool {tool!r} is not in the committed MCP snapshot.",
+                    fix_hint=hint,
+                    location=Location(node=node_id, line=line),
+                )
+            )
+    return errors
+
+
+def _detect_cycles(
+    iface_by_id: dict, dep_edges: list[tuple[str, str]], line_index: dict
+) -> list[TeachingError]:
+    adj: dict[str, list[str]] = {nid: [] for nid in iface_by_id}
+    for src, dst in dep_edges:
+        adj.setdefault(src, []).append(dst)
+
+    WHITE, GRAY, BLACK = 0, 1, 2
+    color: dict[str, int] = {nid: WHITE for nid in adj}
+    reported: set[frozenset] = set()
+    errors: list[TeachingError] = []
+
+    def dfs(node: str, stack: list[str]) -> None:
+        color[node] = GRAY
+        stack.append(node)
+        for nxt in adj.get(node, []):
+            if color.get(nxt) == GRAY:
+                # back edge -> cycle is stack[idx:] + nxt
+                idx = stack.index(nxt)
+                cycle = stack[idx:] + [nxt]
+                key = frozenset(cycle[:-1])
+                if key not in reported:
+                    reported.add(key)
+                    path = " -> ".join(cycle)
+                    head = cycle[0]
+                    errors.append(
+                        TeachingError(
+                            code="CMP035",
+                            message=f"cycle detected: {path}.",
+                            fix_hint=(
+                                "Break the loop by removing one edge in the cycle; a "
+                                "strategy graph must be acyclic (state feedback belongs "
+                                "on a stateful node's state ports, not a data edge)."
+                            ),
+                            location=Location(
+                                node=head, line=line_index.get("nodes", {}).get(head)
+                            ),
+                        )
+                    )
+            elif color.get(nxt) == WHITE:
+                dfs(nxt, stack)
+        stack.pop()
+        color[node] = BLACK
+
+    for nid in sorted(adj):
+        if color[nid] == WHITE:
+            dfs(nid, [])
+    return errors

package/dist/composer/validator/loader.py

@@ -0,0 +1,110 @@
+"""loader — safe YAML load, anchor/alias/merge-key rejection, and a line index (D6).
+
+Stage (a) of the pipeline. Three responsibilities:
+
+1. Parse the authored YAML with a *safe* loader (never the full loader — the document is
+   untrusted agent output).
+2. REJECT YAML anchors, aliases, and merge keys with a teaching error. These are banned
+   (D6, non-goal list) because they let a document say the same thing two ways, which
+   defeats the "explainable artifact" guarantee and confuses a small model. Detection is
+   structural — it walks the parse *event* stream and flags any node carrying an anchor,
+   any ``AliasEvent``, and any ``<<`` merge key — not string matching on the source text
+   (which would false-positive on a legitimate ``<<`` inside a quoted string value... and
+   miss anchors written with unusual spacing).
+3. Build a best-effort line index (node id -> line, edge index -> line) so later stages
+   can point the author at the exact line where a defect lives.
+"""
+
+from __future__ import annotations
+
+import yaml
+from yaml import MappingNode, ScalarEvent, SequenceNode
+from yaml.composer import ComposerError
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+
+class YamlLoadError(Exception):
+    """A YAML syntax error (maps to CMP001)."""
+
+    def __init__(self, message: str, line: int | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.line = line
+
+
+class AnchorError(Exception):
+    """An anchor/alias/merge key was used (maps to CMP002)."""
+
+    def __init__(self, what: str, line: int | None = None) -> None:
+        super().__init__(what)
+        self.what = what
+        self.line = line
+
+
+def _mark_line(mark) -> int | None:
+    return (mark.line + 1) if mark is not None else None
+
+
+def _reject_anchors(text: str) -> None:
+    """Raise :class:`AnchorError` on the first anchor, alias, or merge key seen.
+
+    Walks the event stream so this is independent of source formatting and cannot be
+    fooled by whitespace. ``AliasEvent`` covers ``*ref``; a non-null ``event.anchor``
+    covers ``&name``; a bare ``<<`` scalar covers the (alias-less) merge key.
+    """
+    try:
+        events = list(yaml.parse(text))
+    except (ParserError, ScannerError, ComposerError):
+        # A genuine syntax error surfaces from the loader below; not our concern here.
+        return
+    for event in events:
+        if isinstance(event, yaml.AliasEvent):
+            raise AnchorError("alias (`*ref`)", _mark_line(event.start_mark))
+        anchor = getattr(event, "anchor", None)
+        if anchor:
+            raise AnchorError(f"anchor (`&{anchor}`)", _mark_line(event.start_mark))
+        if isinstance(event, ScalarEvent) and event.value == "<<":
+            raise AnchorError("merge key (`<<`)", _mark_line(event.start_mark))
+
+
+def _build_line_index(text: str) -> dict:
+    """Map ``nodes[].id`` -> line and ``edges`` index -> line, best effort."""
+    index: dict = {"nodes": {}, "edges": {}}
+    try:
+        root = yaml.compose(text)
+    except (ParserError, ScannerError, ComposerError):
+        return index
+    if not isinstance(root, MappingNode):
+        return index
+    for key_node, value_node in root.value:
+        key = getattr(key_node, "value", None)
+        if key == "nodes" and isinstance(value_node, SequenceNode):
+            for item in value_node.value:
+                if not isinstance(item, MappingNode):
+                    continue
+                node_id = None
+                for k, v in item.value:
+                    if getattr(k, "value", None) == "id":
+                        node_id = getattr(v, "value", None)
+                if node_id is not None:
+                    index["nodes"][node_id] = _mark_line(item.start_mark)
+        elif key == "edges" and isinstance(value_node, SequenceNode):
+            for i, item in enumerate(value_node.value):
+                index["edges"][i] = _mark_line(item.start_mark)
+    return index
+
+
+def load_graph(text: str) -> tuple[object, dict]:
+    """Load an authored graph document.
+
+    Returns ``(data, line_index)``. Raises :class:`AnchorError` (CMP002) or
+    :class:`YamlLoadError` (CMP001). The caller turns those into teaching errors.
+    """
+    _reject_anchors(text)
+    try:
+        data = yaml.safe_load(text)
+    except (ParserError, ScannerError, ComposerError) as exc:
+        mark = getattr(exc, "problem_mark", None)
+        raise YamlLoadError(str(exc).strip().replace("\n", " "), _mark_line(mark))
+    return data, _build_line_index(text)