okstra 0.30.3 → 0.32.0

package/runtime/python/okstra_token_usage/report.py

@@ -1,93 +1,176 @@
-"""Final-report substitution helpers (writes Phase 7 placeholders into final-report.md)."""
+"""Final-report data.json mutation helpers (Phase 7 token cells)."""
 from __future__ import annotations
 
+import json
+import sys
 from pathlib import Path
+from typing import Any
 
+# Make sibling packages importable when this module runs as part of an
+# in-repo invocation. The installed runtime puts everything under
+# ~/.okstra/lib/python which is already on PYTHONPATH via the bin
+# wrapper, so this path tweak is a no-op there.
+_HERE = Path(__file__).resolve().parent.parent
+if str(_HERE) not in sys.path:
+    sys.path.insert(0, str(_HERE))
 
-def _format_int(n) -> str:
-    try:
-        return f"{int(n):,}"
-    except (TypeError, ValueError):
-        return "0"
-
-
-def _format_usd(v) -> str:
-    try:
-        return f"${float(v):.2f}"
-    except (TypeError, ValueError):
-        return "$0.00"
+from okstra_ctl.render_final_report import (  # noqa: E402
+    RenderError,
+    render_to_file,
+)
 
 
 class SubstituteRefusedError(RuntimeError):
-    """Raised when substitution would write a zero-only Token Usage Summary.
+    """Raised when substitution would write zero-only token cells.
 
     Shipping `0` / `$0.00` in the Lead / Worker / Grand rows is the
     observed silent-failure mode where the collector ran but every
-    session jsonl was empty (or the writer fabricated zeros). The
-    validator catches it post-hoc, but raising here at the substitution
-    boundary surfaces the failure at the exact step where it can still
-    be retried with a re-collection.
+    session jsonl was empty (or the writer fabricated zeros). Surfacing
+    the refusal here lets the caller retry after locating the missing
+    session jsonls instead of baking zeros into the report.
     """
 
 
-def substitute_final_report(report_path: Path, state: dict) -> int:
-    """Replace token-usage placeholders in the final report file with concrete
-    values from the freshly computed usageSummary.
+def _role_match(state_role: str, data_role: str) -> bool:
+    """Loose match between team-state worker role and data.json
+    executionStatus row role. team-state uses tags like ``claude-worker``
+    or ``report-writer``; data.json uses human strings like ``Claude
+    worker``. We normalise both to ``alnum-only-lowercase`` so the two
+    line up across rephrasings.
+    """
+    def normalise(s: str) -> str:
+        return "".join(ch.lower() for ch in s if ch.isalnum())
+    return normalise(state_role) == normalise(data_role)
 
-    Returns the number of placeholder occurrences replaced. If the report file
-    does not exist, returns -1 without raising.
 
-    Raises ``SubstituteRefusedError`` when ``usageSummary.grandTotalTokens``
-    is zero — substituting zeros into the report bakes in the most common
-    silent failure mode (collector ran but found nothing). Callers that
-    want to suppress the refusal (e.g. unit-test fixtures) can pass a
-    summary with ``grandTotalTokens`` > 0 or remove the summary entirely
-    so substitution is skipped.
+def _populate_execution_row(row: dict, source: dict) -> None:
+    """Fill the executionStatus row's token / cost / duration cells from a
+    team-state worker (or lead) entry. ``source`` is the dict that owns
+    a ``usage`` sub-dict and (for workers) a ``status`` field.
     """
-    if not report_path.is_file():
-        return -1
+    usage = source.get("usage") or {}
+    if usage.get("source") == "unavailable":
+        # Leave cells null; renderer emits `--`. A note in the row's
+        # summary is the worker's responsibility, not ours.
+        return
+    if "totalTokens" in usage:
+        row["totalTokens"] = usage["totalTokens"]
+    if "billableEquivalentTokens" in usage:
+        row["billableTokens"] = usage["billableEquivalentTokens"]
+    if "estimatedCostUsd" in usage:
+        row["costUsd"] = usage["estimatedCostUsd"]
+    if "durationMs" in usage:
+        row["durationMs"] = usage["durationMs"]
+    if "cliTotalTokens" in usage:
+        row["cliTotalTokens"] = usage["cliTotalTokens"]
+    if "cliEstimatedCostUsd" in usage:
+        row["cliCostUsd"] = usage["cliEstimatedCostUsd"]
+
+
+def populate_data_token_cells(data_path: Path, team_state: dict) -> int:
+    """Mutate ``data_path`` (final-report data.json) in place: fill
+    ``tokenUsage`` rows and ``executionStatus`` row token cells from
+    ``team_state['usageSummary']`` and ``team_state['workers']`` /
+    ``team_state['lead']``. Returns the number of cells changed (rough
+    count for diagnostics).
 
-    summary = state.get("usageSummary") or {}
+    Raises ``SubstituteRefusedError`` when ``usageSummary.grandTotalTokens``
+    is zero — substituting zeros bakes in the "collector ran but found
+    nothing" failure mode. Callers that intentionally want zeros (e.g.
+    unit-test fixtures) must omit ``usageSummary`` from the team-state.
+    """
+    summary = team_state.get("usageSummary") or {}
     grand_total = summary.get("grandTotalTokens", 0)
     if isinstance(grand_total, (int, float)) and grand_total == 0 and summary:
         raise SubstituteRefusedError(
-            "Refusing to substitute zero-only usageSummary into the final "
-            f"report at {report_path}. grandTotalTokens=0 means the "
-            "collector ran but every session jsonl was empty (or absent). "
-            "Re-run `python3 scripts/okstra-token-usage.py <team-state> "
-            "--write --summary --substitute-final-report <report-path>` "
+            "Refusing to substitute zero-only usageSummary into "
+            f"{data_path}. grandTotalTokens=0 means the collector ran "
+            "but every session jsonl was empty (or absent). Re-run "
             "after locating the missing session jsonls. To intentionally "
-            "ship zeros (test fixtures only), omit `usageSummary` from the "
-            "team-state JSON before calling substitute_final_report."
+            "ship zeros (test fixtures only), omit `usageSummary` from "
+            "team-state before calling this."
         )
+
+    if not data_path.is_file():
+        raise SubstituteRefusedError(f"data file not found: {data_path}")
+
+    data = json.loads(data_path.read_text(encoding="utf-8"))
+    changes = 0
+
+    # Token Usage Summary table — four rows.
     cost = summary.get("estimatedCostUsd") or {}
     lead_cost = cost.get("lead") or 0
     worker_cost = cost.get("claudeWorkers") or 0
     cli_cost = cost.get("cliWorkers") or 0
-    grand_cost = lead_cost + worker_cost  # CLI tracked on its own row per template
-
-    mapping = {
-        "{{LEAD_TOTAL_TOKENS}}": _format_int(summary.get("leadTotalTokens", 0)),
-        "{{LEAD_BILLABLE_TOKENS}}": _format_int(summary.get("leadBillableEquivalentTokens", 0)),
-        "{{LEAD_COST_USD}}": _format_usd(lead_cost),
-        "{{WORKER_TOTAL_TOKENS}}": _format_int(summary.get("workerTotalTokens", 0)),
-        "{{WORKER_BILLABLE_TOKENS}}": _format_int(summary.get("workerBillableEquivalentTokens", 0)),
-        "{{WORKER_COST_USD}}": _format_usd(worker_cost),
-        "{{GRAND_TOTAL_TOKENS}}": _format_int(summary.get("grandTotalTokens", 0)),
-        "{{GRAND_BILLABLE_TOKENS}}": _format_int(summary.get("grandBillableEquivalentTokens", 0)),
-        "{{GRAND_COST_USD}}": _format_usd(grand_cost),
-        "{{CLI_COST_USD}}": _format_usd(cli_cost),
-    }
-
-    text = report_path.read_text()
-    replaced = 0
-    for needle, value in mapping.items():
-        count = text.count(needle)
-        if count:
-            text = text.replace(needle, value)
-            replaced += count
-
-    if replaced:
-        report_path.write_text(text)
-    return replaced
 
+    token_usage = data.setdefault("tokenUsage", {})
+    token_usage.setdefault("lead", {}).update({
+        "totalTokens": summary.get("leadTotalTokens"),
+        "billableTokens": summary.get("leadBillableEquivalentTokens"),
+        "costUsd": lead_cost,
+    })
+    token_usage.setdefault("worker", {}).update({
+        "totalTokens": summary.get("workerTotalTokens"),
+        "billableTokens": summary.get("workerBillableEquivalentTokens"),
+        "costUsd": worker_cost,
+    })
+    token_usage.setdefault("grand", {}).update({
+        "totalTokens": summary.get("grandTotalTokens"),
+        "billableTokens": summary.get("grandBillableEquivalentTokens"),
+        # CLI tracked on its own row per the template — grand here means
+        # lead + claudeWorkers, not lead + claudeWorkers + cli.
+        "costUsd": lead_cost + worker_cost,
+    })
+    token_usage.setdefault("cli", {})["costUsd"] = cli_cost
+    changes += 4
+
+    # Execution Status by Agent — per-row token / cost / duration.
+    lead_entry = team_state.get("lead") or {}
+    workers = team_state.get("workers") or []
+    rows = data.get("executionStatus") or []
+    for row in rows:
+        role = row.get("role", "")
+        if "lead" in role.lower():
+            _populate_execution_row(row, lead_entry)
+            changes += 1
+            continue
+        for worker in workers:
+            if _role_match(worker.get("role", ""), role):
+                _populate_execution_row(row, worker)
+                changes += 1
+                break
+
+    data_path.write_text(
+        json.dumps(data, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+    return changes
+
+
+def populate_and_render(
+    data_path: Path,
+    team_state: dict,
+    *,
+    output_path: Path | None = None,
+) -> tuple[int, int]:
+    """Mutate data.json + render the markdown sibling. Returns
+    ``(cells_changed, bytes_written)``.
+
+    The caller is normally Phase 7 step 1 (token-usage CLI with the
+    ``--substitute-data`` flag). The rendered markdown is the canonical
+    user-facing artifact; data.json is the SSOT.
+    """
+    changes = populate_data_token_cells(data_path, team_state)
+
+    if output_path is None:
+        name = data_path.name
+        if name.endswith(".data.json"):
+            output_path = data_path.with_name(name[: -len(".data.json")] + ".md")
+        else:
+            output_path = data_path.with_suffix(".md")
+
+    try:
+        bytes_written = render_to_file(data_path, output_path)
+    except RenderError as exc:
+        raise SubstituteRefusedError(f"render after substitute failed: {exc}") from exc
+    return changes, bytes_written

package/runtime/python/okstra_vendor/__init__.py

@@ -0,0 +1,37 @@
+"""Vendored third-party dependencies used by okstra runtime scripts.
+
+This package isolates third-party Python libraries that okstra needs at
+runtime so that end users do not need to `pip install` anything separate
+from the npm-distributed runtime. The build system (`tools/build.mjs`)
+copies this package into `runtime/python/okstra_vendor/` and the
+installer (`src/install.mjs`) drops it under `~/.okstra/lib/python/`.
+
+Currently vendored:
+  - jinja2 3.1.6 (BSD-3-Clause) — final-report template rendering.
+  - markupsafe 3.0.3 (BSD-3-Clause) — jinja2 dependency. C extension
+    (`_speedups.c` / `_speedups.pyi`) is stripped; the pure-python
+    `_native` fallback is used at import time.
+
+Source provenance and licenses are preserved verbatim under each
+sub-package. Do not edit vendored files in place — re-vendor from PyPI
+sdists when upgrading.
+
+Import alias rationale: jinja2 uses absolute imports (`from markupsafe
+import ...`, and compiled templates `import jinja2`) internally. Without
+re-routing, those would either fail (no system install) or resolve to
+unrelated user-installed copies. We register the vendored copies in
+`sys.modules` under their canonical top-level names so jinja2's internal
+imports find them deterministically.
+"""
+
+import sys as _sys
+
+# Register markupsafe alias BEFORE importing jinja2 — jinja2's __init__
+# evaluates `from .environment import ...`, which does `from markupsafe
+# import Markup` at module-load time. Reversing the order would crash
+# the import chain before the alias was registered.
+from . import markupsafe as _markupsafe
+_sys.modules.setdefault("markupsafe", _markupsafe)
+
+from . import jinja2 as _jinja2  # noqa: E402
+_sys.modules.setdefault("jinja2", _jinja2)

package/runtime/python/okstra_vendor/jinja2/__init__.py

@@ -0,0 +1,38 @@
+"""Jinja is a template engine written in pure Python. It provides a
+non-XML syntax that supports inline expressions and an optional
+sandboxed environment.
+"""
+
+from .bccache import BytecodeCache as BytecodeCache
+from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache
+from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache
+from .environment import Environment as Environment
+from .environment import Template as Template
+from .exceptions import TemplateAssertionError as TemplateAssertionError
+from .exceptions import TemplateError as TemplateError
+from .exceptions import TemplateNotFound as TemplateNotFound
+from .exceptions import TemplateRuntimeError as TemplateRuntimeError
+from .exceptions import TemplatesNotFound as TemplatesNotFound
+from .exceptions import TemplateSyntaxError as TemplateSyntaxError
+from .exceptions import UndefinedError as UndefinedError
+from .loaders import BaseLoader as BaseLoader
+from .loaders import ChoiceLoader as ChoiceLoader
+from .loaders import DictLoader as DictLoader
+from .loaders import FileSystemLoader as FileSystemLoader
+from .loaders import FunctionLoader as FunctionLoader
+from .loaders import ModuleLoader as ModuleLoader
+from .loaders import PackageLoader as PackageLoader
+from .loaders import PrefixLoader as PrefixLoader
+from .runtime import ChainableUndefined as ChainableUndefined
+from .runtime import DebugUndefined as DebugUndefined
+from .runtime import make_logging_undefined as make_logging_undefined
+from .runtime import StrictUndefined as StrictUndefined
+from .runtime import Undefined as Undefined
+from .utils import clear_caches as clear_caches
+from .utils import is_undefined as is_undefined
+from .utils import pass_context as pass_context
+from .utils import pass_environment as pass_environment
+from .utils import pass_eval_context as pass_eval_context
+from .utils import select_autoescape as select_autoescape
+
+__version__ = "3.1.6"

package/runtime/python/okstra_vendor/jinja2/_identifier.py

@@ -0,0 +1,6 @@
+import re
+
+# generated by scripts/generate_identifier_pattern.py
+pattern = re.compile(
+    r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߽߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛࣓-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣ৾ਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣૺ-૿ଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఄా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഀ-ഃ഻഼ാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳷-᳹᷀-᷹᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꣿꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𐴤-𐽆𐴧-𐽐𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑅅𑅆𑅳𑆀-𑆂𑆳-𑇀𑇉-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌻𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑑞𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑠬-𑠺𑨁-𑨊𑨳-𑨹𑨻-𑨾𑩇𑩑-𑩛𑪊-𑪙𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𑴱-𑴶𑴺𑴼𑴽𑴿-𑵅𑵇𑶊-𑶎𑶐𑶑𑶓-𑶗𑻳-𑻶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+"  # noqa: B950
+)

package/runtime/python/okstra_vendor/jinja2/async_utils.py

@@ -0,0 +1,99 @@
+import inspect
+import typing as t
+from functools import WRAPPER_ASSIGNMENTS
+from functools import wraps
+
+from .utils import _PassArg
+from .utils import pass_eval_context
+
+if t.TYPE_CHECKING:
+    import typing_extensions as te
+
+V = t.TypeVar("V")
+
+
+def async_variant(normal_func):  # type: ignore
+    def decorator(async_func):  # type: ignore
+        pass_arg = _PassArg.from_obj(normal_func)
+        need_eval_context = pass_arg is None
+
+        if pass_arg is _PassArg.environment:
+
+            def is_async(args: t.Any) -> bool:
+                return t.cast(bool, args[0].is_async)
+
+        else:
+
+            def is_async(args: t.Any) -> bool:
+                return t.cast(bool, args[0].environment.is_async)
+
+        # Take the doc and annotations from the sync function, but the
+        # name from the async function. Pallets-Sphinx-Themes
+        # build_function_directive expects __wrapped__ to point to the
+        # sync function.
+        async_func_attrs = ("__module__", "__name__", "__qualname__")
+        normal_func_attrs = tuple(set(WRAPPER_ASSIGNMENTS).difference(async_func_attrs))
+
+        @wraps(normal_func, assigned=normal_func_attrs)
+        @wraps(async_func, assigned=async_func_attrs, updated=())
+        def wrapper(*args, **kwargs):  # type: ignore
+            b = is_async(args)
+
+            if need_eval_context:
+                args = args[1:]
+
+            if b:
+                return async_func(*args, **kwargs)
+
+            return normal_func(*args, **kwargs)
+
+        if need_eval_context:
+            wrapper = pass_eval_context(wrapper)
+
+        wrapper.jinja_async_variant = True  # type: ignore[attr-defined]
+        return wrapper
+
+    return decorator
+
+
+_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)}
+
+
+async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V":
+    # Avoid a costly call to isawaitable
+    if type(value) in _common_primitives:
+        return t.cast("V", value)
+
+    if inspect.isawaitable(value):
+        return await t.cast("t.Awaitable[V]", value)
+
+    return value
+
+
+class _IteratorToAsyncIterator(t.Generic[V]):
+    def __init__(self, iterator: "t.Iterator[V]"):
+        self._iterator = iterator
+
+    def __aiter__(self) -> "te.Self":
+        return self
+
+    async def __anext__(self) -> V:
+        try:
+            return next(self._iterator)
+        except StopIteration as e:
+            raise StopAsyncIteration(e.value) from e
+
+
+def auto_aiter(
+    iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]",
+) -> "t.AsyncIterator[V]":
+    if hasattr(iterable, "__aiter__"):
+        return iterable.__aiter__()
+    else:
+        return _IteratorToAsyncIterator(iter(iterable))
+
+
+async def auto_to_list(
+    value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]",
+) -> t.List["V"]:
+    return [x async for x in auto_aiter(value)]