@senpi/runtime 3.0.9-dev.20260710204324 → 3.0.9-dev.feat-strategy-compos.20260716233548

package/dist/composer/primitives/stateful.py

@@ -0,0 +1,254 @@
+"""stateful — the ctx.state-backed primitives (D2). Three nodes:
+
+  * ``ttl_dedup_state``    — the universal TTL signal-dedup map (all five trend/breakout
+                             packages). Ported from stag/hawk ``scan.scan``.
+  * ``snapshot_diff_state`` — a rolling window of per-key numeric snapshots exposing the
+                             growth (latest−oldest) between them. Ported from whalehunter
+                             ``scoring.update_ledger`` (per-trader-cohort equity/net growth
+                             over ~10 daily buckets).
+  * ``cohort_cache_state``  — cache an expensive upstream fetch across ticks behind a TTL +
+                             config version, pinning a STABLE cohort for hours and surviving
+                             restarts. Ported from whalehunter ``scan._build_cohorts``.
+
+Each node is compiled onto ``ctx.state`` and SPLIT into a READ half (prologue) and a WRITE
+half (epilogue): the read exposes PRIOR-tick data (the state boundary) BEFORE the pipeline
+runs, and the write records THIS tick's data AFTER it. That is exactly the "stateful outputs
+carry prior-tick data" semantic the validator's cycle check honours — an edge leaving a
+stateful output is not a forward dependency, so wiring a stateful output into the body and a
+body value into its write half never trips CMP035.
+
+MULTIPLE stateful nodes per graph (whalehunter runs all three) each append their OWN
+self-contained, single-key fragment to the shared ctx.state series every tick
+(``{"recent": …}`` / ``{"snapshots": …}`` / ``{"cohort": …}``). A read therefore cannot use
+``state.last()`` blindly — the last append may belong to a SIBLING node — so it scans the
+series newest→oldest for the most-recent entry carrying its own key (``_read_key``). Each
+fragment is self-contained (the whole working map/list), so state-history eviction only drops
+stale fragments; the newest per-key fragment always wins and restart resumes from it.
+
+STATE IS BOUNDED (an unbounded fleet-box state series is a real ops hazard). ttl_dedup prunes
+beyond ``prune_factor × ttl``; snapshot_diff hard-caps at ``max_snapshots`` buckets;
+cohort_cache stores one cohort. The StateManager deque bound (``state_history_max_count``) is a
+second backstop.
+"""
+
+from __future__ import annotations
+
+
+def _read_key(state, key):
+    """Most-recent state entry carrying ``key``, scanning the series newest→oldest.
+
+    Prefers a backward scan (the real StateManager is iterable oldest→newest) so multiple
+    stateful nodes can each append a single-key fragment per tick and still find their own
+    most-recent write. Falls back to ``state.last()`` for a minimal state object that supports
+    neither iteration nor indexing (the unit-test fakes). Returns the value, or None."""
+    if state is None or len(state) == 0:
+        return None
+    try:
+        entries = list(state)
+    except TypeError:
+        entries = None
+    if entries is not None:
+        for entry in reversed(entries):
+            if isinstance(entry, dict) and key in entry:
+                return entry[key]
+        return None
+    last = state.last() or {}
+    return last.get(key) if isinstance(last, dict) else None
+
+
+# ============================================================ ttl_dedup_state (dedup map)
+
+def _last_map(state) -> dict:
+    recent = _read_key(state, "recent")
+    return dict(recent) if isinstance(recent, dict) else {}
+
+
+def ttl_dedup_state_read(state, *, ttl_seconds: int, prune_factor: int, now: float) -> tuple[list, dict]:
+    """READ half: prune the persisted dedup map and return (recent, working_map).
+
+    ``recent`` (the ``recent`` output port, asset_list) = assets signaled within
+    ``ttl_seconds``. ``working_map`` = the pruned {asset: ts} map, carried to the WRITE
+    half (entries older than ``prune_factor * ttl_seconds`` are dropped).
+    """
+    cutoff = now - (ttl_seconds * prune_factor)
+    working = {k.upper(): v for k, v in _last_map(state).items() if v >= cutoff}
+    recent = sorted(a for a, ts in working.items() if (now - ts) < ttl_seconds)
+    return recent, working
+
+
+def ttl_dedup_state_write(state, working_map: dict, *, emitted: list, now: float) -> None:
+    """WRITE half (state_out port): record this tick's emitted signals and persist.
+
+    Stamps each emitted asset with ``now`` in the working map and appends the updated
+    map to ctx.state. A None state (history disabled) is a no-op. The scaffold owns the
+    per-tick commit/rollback around scan().
+    """
+    for sig in emitted:
+        asset = str(sig.get("asset", "")).upper()
+        if asset:
+            working_map[asset] = now
+    if state is not None:
+        state.append({"recent": working_map})
+    return None
+
+
+# ===================================================== snapshot_diff_state (rolling window)
+# Port of whalehunter scoring.update_ledger (scoring.py:73-88): maintain a bounded rolling
+# window of per-key numeric snapshots across ticks and expose the growth between the current
+# (latest-stored) snapshot and the oldest one still in the window. whalehunter tracks each smart
+# cohort's per-coin NET positioning across ~10 UTC-daily buckets to derive the "adding daily"
+# signal; the generalization keeps its COUNT bound (max_snapshots ← whalehunter keep=10) and its
+# per-bucket OVERWRITE (bucket_seconds ← whalehunter's per-UTC-day ledger key, made clock-free).
+#
+# HONEST BOUNDARY (documented in the node doc + AUTHORING §7): the growth is PRIOR-tick — it
+# compares the latest STORED snapshot to the oldest, not this tick's fresh input (which the read
+# half cannot see; it is appended by the write half). This is the exact ttl_dedup read/write
+# split; the one-tick lag is negligible against a multi-bucket (multi-day) window and needs a
+# short warmup (growth is empty until ≥2 buckets exist), just as whalehunter's requireGrowing
+# blocks until ~day 2.
+
+
+def _load_snapshots(state) -> list:
+    snaps = _read_key(state, "snapshots")
+    if not isinstance(snaps, list):
+        return []
+    out = []
+    for s in snaps:
+        if isinstance(s, (list, tuple)) and len(s) == 2 and isinstance(s[1], dict):
+            out.append([float(s[0]), {str(k): v for k, v in s[1].items()}])
+    return out
+
+
+def _extract_values(snapshot, value_field: str) -> dict:
+    """A current snapshot (a feature_map {key: value}) -> {key: float}. When ``value_field`` is
+    set the value is a per-key record and we read that numeric sub-field (whalehunter tracks the
+    ``net`` field of aggregate_bias's per-coin record); "" reads the value as a float directly.
+    Non-numeric / missing -> the key is dropped from this snapshot (never a crash)."""
+    out: dict = {}
+    if not isinstance(snapshot, dict):
+        return out
+    for k in sorted(snapshot):
+        v = snapshot[k]
+        if value_field:
+            v = v.get(value_field) if isinstance(v, dict) else None
+        try:
+            out[str(k)] = float(v)
+        except (TypeError, ValueError):
+            continue
+    return out
+
+
+def snapshot_diff_state_read(
+    state, *, value_field: str, bucket_seconds: int, max_snapshots: int, now: float
+) -> tuple[dict, list, list]:
+    """READ half: load + hard-cap the persisted snapshot window and expose the growth.
+
+    Returns (growth, deltas, working):
+      * ``growth`` (feature_map output) = {key: latest_stored − oldest_stored} — the "adding"
+        signal whalehunter's cohort scoring reads via ``growth.get(coin)``. Empty until ≥2
+        buckets exist (day-1 warmup).
+      * ``deltas`` (record_list output) = per-key {key, current, oldest, growth} rows (sorted),
+        the richer detail for a pure_fn that wants the absolute values behind the delta.
+      * ``working`` = the (count-bounded) snapshot list carried to the WRITE half.
+
+    THE STATE BOUND: keep only the most-recent ``max_snapshots`` buckets (prune the oldest
+    beyond the cap) — state can never exceed max_snapshots × keys.
+    """
+    working = _load_snapshots(state)
+    if max_snapshots > 0 and len(working) > max_snapshots:
+        working = working[-max_snapshots:]
+    growth: dict = {}
+    deltas: list = []
+    if working:
+        oldest = working[0][1]
+        latest = working[-1][1]
+        for key in sorted(set(oldest) | set(latest)):
+            cur = float(latest.get(key, 0.0))
+            old = float(oldest.get(key, 0.0))
+            g = round(cur - old, 6)
+            growth[key] = g
+            deltas.append({"key": key, "current": cur, "oldest": old, "growth": g})
+    return growth, deltas, working
+
+
+def snapshot_diff_state_write(
+    state, working: list, *, snapshot, value_field: str, bucket_seconds: int,
+    max_snapshots: int, now: float,
+) -> None:
+    """WRITE half (state_out port): fold THIS tick's snapshot into the window and persist.
+
+    Within ``bucket_seconds`` of the newest bucket the current bucket is OVERWRITTEN (whalehunter
+    overwrites today's UTC-day ledger entry on every intraday tick); once it has elapsed a NEW
+    bucket opens. Then re-apply the ``max_snapshots`` cap. A None state is a no-op."""
+    values = _extract_values(snapshot, value_field)
+    if working and bucket_seconds > 0 and (now - working[-1][0]) < bucket_seconds:
+        working[-1] = [working[-1][0], values]
+    else:
+        working.append([now, values])
+    if max_snapshots > 0 and len(working) > max_snapshots:
+        del working[: len(working) - max_snapshots]
+    if state is not None:
+        state.append({"snapshots": working})
+    return None
+
+
+# ======================================================= cohort_cache_state (pinned cohort)
+# Port of whalehunter scan._build_cohorts (scan.py:36-85): cache the expensive paged top-trader
+# fetch across ticks behind a TTL + a config VERSION, so an unstable churning cohort does not
+# poison the multi-day ledger. whalehunter's real need is COHORT STABILITY (a cohort pinned for
+# ~24h that survives restarts), not fetch-rate-limiting.
+#
+# HONEST BOUNDARY (documented in the node doc + AUTHORING §7): sources fire EVERY tick — the
+# compiler emits the upstream fetch unconditionally and the scaffold has no conditional-fetch
+# machinery, so this node does NOT skip the fetch (that would need compiler conditional-emission,
+# out of scope). It implements REUSE-AND-REFRESH: the read half serves the last-known-good cohort
+# from ctx.state (pinned + smoothing fetch gaps/failures), and the write half refreshes the stored
+# cohort only when the TTL lapsed, the version changed, or the cache was empty — and only when the
+# fresh fetch actually returned rows (a failed fetch keeps the old cohort, exactly like
+# whalehunter). ctx.state (not a module-level cache) is used precisely because it SURVIVES RESTARTS
+# — whalehunter's stated requirement; a per-process adapter cache would not.
+
+
+def _load_cohort(state) -> dict:
+    cohort = _read_key(state, "cohort")
+    return dict(cohort) if isinstance(cohort, dict) else {}
+
+
+def cohort_cache_state_read(
+    state, *, ttl_hours: float, version: int, now: float
+) -> tuple[list, dict]:
+    """READ half: serve the pinned last-known-good cohort (reuse).
+
+    Returns (cohort, cached): ``cohort`` (the record_list output) is the stored cohort rows
+    whenever the stored config ``version`` still matches (a version change invalidates it — empty
+    until the write half rebuilds). ``cached`` is the raw stored dict carried to the WRITE half so
+    it can make the refresh decision. Cold start (no cache) serves []; downstream then reads an
+    empty cohort for one warmup tick while the write half populates it."""
+    cached = _load_cohort(state)
+    records = cached.get("records", [])
+    active = list(records) if (cached.get("version") == version and isinstance(records, list)) else []
+    return active, cached
+
+
+def cohort_cache_state_write(
+    state, cached: dict, *, fresh, ttl_hours: float, version: int, now: float
+) -> None:
+    """WRITE half (state_out port): refresh the stored cohort when stale, else keep it pinned.
+
+    Refresh (adopt ``fresh``) only when we HAVE a non-empty fresh fetch AND (the cache is empty OR
+    the config version changed OR the TTL lapsed). Otherwise keep the stored cohort — this is what
+    smooths a transient fetch failure and pins the cohort for ``ttl_hours``. A None state is a
+    no-op."""
+    have_fresh = isinstance(fresh, list) and len(fresh) > 0
+    stale = (now - float(cached.get("refreshed_at", 0.0))) > (ttl_hours * 3600.0)
+    version_changed = cached.get("version") != version
+    empty = not cached.get("records")
+    if have_fresh and (empty or version_changed or stale):
+        new = {"records": list(fresh), "version": version, "refreshed_at": now}
+    elif cached:
+        new = cached
+    else:
+        new = {"records": [], "version": version, "refreshed_at": 0.0}
+    if state is not None:
+        state.append({"cohort": new})
+    return None

package/dist/composer/pure_fn/__init__.py

@@ -0,0 +1,34 @@
+"""pure_fn — per-graph pure-function nodes (D2 kind 2): the OPEN extension point.
+
+The registry (``composer.registry``) is the CLOSED vocabulary of runtime-owned primitive
+nodes. A ``pure_fn`` node is the complement: an LLM-written Python function declared inline
+in the graph, with declared typed input/output ports (same closed :class:`PortType`
+vocabulary) and a function body. Full code flexibility is preserved while the telemetry
+loop learns which functions to promote to primitives (D2), but it is fenced by a static
+purity gate — no ctx, no MCP, no I/O — so it can never be a custom data source (D3).
+
+Package surface (kept to the leaf ``models`` so importing this package never pulls the
+validator — the purity/test stages live in submodules imported directly where used, to
+avoid an import cycle with ``composer.validator``):
+
+    models              — PureFnNode / PureFnParam / parse_pure_fn / is_pure_fn_entry.
+    purity   (submodule) — check_purity: the static AST allowlist (CMP04x teaching errors).
+    runtests (submodule) — run_inline_tests: run inline fixture tests (CMP05x).
+    checks   (submodule) — check: the validator stage tying purity + tests together.
+"""
+
+from .models import (
+    PureFnNode,
+    PureFnParam,
+    is_pure_fn_entry,
+    parse_pure_fn,
+    parse_single_def,
+)
+
+__all__ = [
+    "PureFnNode",
+    "PureFnParam",
+    "parse_pure_fn",
+    "parse_single_def",
+    "is_pure_fn_entry",
+]

package/dist/composer/pure_fn/checks.py

@@ -0,0 +1,39 @@
+"""checks — the validator stage for pure_fn nodes (purity CMP04x + inline tests CMP05x).
+
+Runs AFTER the graph-semantic checks. For each ``kind: pure_fn`` node it statically
+purity-checks the body (``purity.check_purity``) and, only if the body is pure, runs its
+inline fixture tests (``runtests.run_inline_tests``). Errors from every node collect into
+one list so the author sees all defects at once (the SERR037 collection rule).
+"""
+
+from __future__ import annotations
+
+from composer.validator.errors import TeachingError
+
+from .models import is_pure_fn_entry, parse_pure_fn
+from .purity import check_purity
+from .runtests import run_inline_tests
+
+
+def check(doc: dict, line_index: dict) -> list[TeachingError]:
+    errors: list[TeachingError] = []
+    nodes = doc.get("nodes")
+    if not isinstance(nodes, list):
+        return errors
+
+    used_fn_names: set[str] = set()
+    for entry in nodes:
+        if not is_pure_fn_entry(entry):
+            continue
+        node_id = entry.get("id")
+        line = line_index.get("nodes", {}).get(node_id) if isinstance(node_id, str) else None
+        pfn = parse_pure_fn(entry, line)
+
+        purity_errors = check_purity(pfn, used_fn_names)
+        errors.extend(purity_errors)
+        # Only run the fixtures once the body is known pure — otherwise the failures are
+        # noise on top of the real (purity) defect.
+        if not purity_errors:
+            errors.extend(run_inline_tests(pfn))
+
+    return errors

package/dist/composer/pure_fn/models.py

@@ -0,0 +1,177 @@
+"""models — the per-graph pure-function node (D2 kind 2).
+
+A ``pure_fn`` node is NOT a registry entry. The registry stays the closed vocabulary of
+runtime-owned primitives (D4); ``pure_fn`` is the OPEN extension point — an LLM-written
+Python function with declared, typed input/output ports (drawn from the SAME closed
+:class:`PortType` vocabulary as the registry) and an inline function body. It is declared
+inline in the graph document:
+
+    - id: score_conviction
+      kind: pure_fn
+      inputs:
+        - { name: trend4h, type: feature_map }
+        - { name: momentum, type: feature_map }
+      outputs:
+        - { name: scores, type: score_map }
+      params:
+        - { name: min_score, type: float, default: 5 }
+      fn: |
+        def score_conviction(trend4h, momentum, min_score):
+            ...
+            return scores
+
+The declared ports (``inputs``/``outputs``) are what the graph WIRES against and are
+independent of the ``fn`` body — port typechecking, connectivity, and cycle detection all
+read them exactly as they read a registry NodeDef's ports (see ``resolve`` in
+``composer.pure_fn.interfaces``). The ``fn`` body is STATICALLY purity-checked and its
+signature is required to match ``inputs`` + ``params`` exactly (see
+``composer.pure_fn.purity``); inline ``tests`` are run at validation time
+(``composer.pure_fn.runtests``).
+
+This module only parses the declaration into a typed object and exposes it through the
+same duck-typed surface a registry NodeDef presents (``.name``, ``.kind``, ``.inputs``,
+``.outputs``, ``.summary``), so the validator's graph checks and the compiler can treat a
+pure_fn node like any other node.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from typing import Optional
+
+from composer.registry import NodeKind, Port, PortType
+
+# The pure_fn param-type vocabulary mirrors the registry ParamType literals.
+_PARAM_TYPES = {"int", "float", "str", "bool", "list[str]", "map_str_float"}
+
+
+def is_pure_fn_entry(entry: object) -> bool:
+    """True when a graph node entry declares ``kind: pure_fn`` (the open extension point)."""
+    return isinstance(entry, dict) and entry.get("kind") == "pure_fn"
+
+
+def parse_single_def(src: str) -> Optional[ast.FunctionDef]:
+    """Return the sole top-level ``def`` in ``src`` (ignoring allow-listed imports), else None.
+
+    Purity (``composer.pure_fn.purity``) is what REJECTS a malformed body with a teaching
+    error; this helper is the tolerant reader the compiler uses once a graph is known valid.
+    """
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return None
+    defs = [n for n in tree.body if isinstance(n, ast.FunctionDef)]
+    return defs[0] if len(defs) == 1 else None
+
+
+@dataclass(frozen=True)
+class PureFnParam:
+    """A typed literal parameter passed to the fn by the compiler (config, not an edge)."""
+
+    name: str
+    type: str
+    default: object = None
+
+
+@dataclass(frozen=True)
+class PureFnNode:
+    """A parsed pure_fn node, presenting the registry-NodeDef duck surface.
+
+    ``inputs``/``outputs`` reuse the registry :class:`Port` model so the validator's
+    existing port/type/connectivity/cycle logic operates on a pure_fn node unchanged.
+    Ports are single-writer, required (``merge=False``, ``required=True``) — a pure_fn
+    declares each typed input explicitly (D2), so there is no feature-merge port here.
+    """
+
+    id: str
+    inputs: tuple[Port, ...]
+    outputs: tuple[Port, ...]
+    params: tuple[PureFnParam, ...]
+    fn_src: str
+    tests: tuple[dict, ...] = ()
+    line: Optional[int] = None
+
+    # -- registry-NodeDef duck surface ------------------------------------------------
+    @property
+    def name(self) -> str:
+        return self.id
+
+    @property
+    def kind(self) -> NodeKind:
+        return NodeKind.pure_fn
+
+    @property
+    def summary(self) -> str:
+        return f"pure_fn node (LLM-written {len(self.inputs)}->{len(self.outputs)} function)"
+
+    # -- fn body ----------------------------------------------------------------------
+    def function_def(self) -> Optional[ast.FunctionDef]:
+        return parse_single_def(self.fn_src)
+
+    @property
+    def fn_name(self) -> Optional[str]:
+        fd = self.function_def()
+        return fd.name if fd is not None else None
+
+
+def _port(entry: dict, *, input_side: bool) -> Optional[Port]:
+    """Build a registry Port from a pure_fn port mapping, or None if the type is not in
+    the closed vocabulary (the schema stage already reports that; we skip it here)."""
+    if not isinstance(entry, dict):
+        return None
+    name = entry.get("name")
+    type_str = entry.get("type")
+    if not isinstance(name, str) or not name.strip():
+        return None
+    try:
+        ptype = PortType(type_str)
+    except ValueError:
+        return None
+    doc = entry.get("doc")
+    if not isinstance(doc, str) or not doc.strip():
+        doc = f"pure_fn {'input' if input_side else 'output'} {name!r}."
+    # pure_fn ports are single-writer, required inputs (no merge point on a pure_fn).
+    return Port(name=name, type=ptype, doc=doc, required=True, merge=False)
+
+
+def parse_pure_fn(entry: dict, line: Optional[int] = None) -> PureFnNode:
+    """Parse a graph node entry (kind: pure_fn) into a :class:`PureFnNode`.
+
+    Tolerant by design — malformed ports/params are dropped (the schema + the purity/test
+    stages carry the teaching errors); this only builds the structure downstream stages
+    walk. Ports whose ``type`` is outside the closed vocabulary are skipped.
+    """
+    node_id = entry.get("id")
+    node_id = node_id if isinstance(node_id, str) else "<pure_fn>"
+
+    inputs = tuple(
+        p for p in (_port(e, input_side=True) for e in (entry.get("inputs") or [])) if p is not None
+    )
+    outputs = tuple(
+        p for p in (_port(e, input_side=False) for e in (entry.get("outputs") or [])) if p is not None
+    )
+
+    params: list[PureFnParam] = []
+    for e in entry.get("params") or []:
+        if not isinstance(e, dict):
+            continue
+        name = e.get("name")
+        ptype = e.get("type")
+        if isinstance(name, str) and ptype in _PARAM_TYPES:
+            params.append(PureFnParam(name=name, type=ptype, default=e.get("default")))
+
+    fn_src = entry.get("fn")
+    fn_src = fn_src if isinstance(fn_src, str) else ""
+
+    tests = tuple(t for t in (entry.get("tests") or []) if isinstance(t, dict))
+
+    return PureFnNode(
+        id=node_id,
+        inputs=inputs,
+        outputs=outputs,
+        params=tuple(params),
+        fn_src=fn_src,
+        tests=tests,
+        line=line,
+    )