@pineforge/codegen-pyodide 0.8.1 → 0.10.0

package/pineforge_codegen/codegen/ta.py

@@ -28,7 +28,9 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from ..ast_nodes import (
-    Assignment, BinOp, ExprStmt, FuncCall, Ternary, UnaryOp, VarDecl,
+    Assignment, BinOp, BoolLiteral, ColorLiteral, ExprStmt, FuncCall,
+    Identifier, MemberAccess, NaLiteral, NumberLiteral, StringLiteral,
+    Subscript, Ternary, TupleLiteral, UnaryOp, VarDecl,
 )
 from .tables import TA_IMPLICIT_APPEND, TA_IMPLICIT_COMPUTE_FULL
 
@@ -221,6 +223,90 @@ class TaSiteHelper:
 
         return ""
 
+    # ------------------------------------------------------------------
+    # Precalculation safety
+    # ------------------------------------------------------------------
+
+    _PRECALC_BAR_IDENTIFIERS = {
+        "open", "high", "low", "close", "volume",
+        "hl2", "hlc3", "ohlc4", "hlcc4",
+        "time", "time_close", "bar_index",
+    }
+
+    def _is_precalc_replayed_source_var(self, name: str) -> bool:
+        """True for top-level ``x = input.source(...)`` variables replayed in
+        ``precalculate()``.
+
+        The precompute loop explicitly advances native source series and then
+        replays those source-input assignments before computing static TA
+        sites. Other user aliases, even when they are statically derived from
+        bar data (``src = close`` / ``ha_close = close``), are not replayed
+        there and must therefore use the normal per-bar TA path."""
+        ast = getattr(self.ctx, "ast", None)
+        for stmt in getattr(ast, "body", ()):
+            if (
+                isinstance(stmt, VarDecl)
+                and stmt.name == name
+                and isinstance(stmt.value, FuncCall)
+                and self._is_source_input(stmt.value)
+            ):
+                return True
+        return False
+
+    def _expr_safe_for_ta_precalc(self, expr) -> bool:
+        if expr is None:
+            return True
+        if isinstance(expr, (NumberLiteral, StringLiteral, BoolLiteral, NaLiteral, ColorLiteral)):
+            return True
+        if isinstance(expr, Identifier):
+            if expr.name in self._PRECALC_BAR_IDENTIFIERS:
+                return True
+            if self._is_precalc_replayed_source_var(expr.name):
+                return True
+            if expr.name in getattr(self.ctx, "series_vars", set()):
+                return False
+            return expr.name in getattr(self, "_static_vars", set())
+        if isinstance(expr, MemberAccess):
+            if isinstance(expr.object, Identifier) and (
+                expr.object.name.startswith("input") or expr.object.name in getattr(self, "_enum_defs", {})
+            ):
+                return True
+            return self._expr_safe_for_ta_precalc(expr.object)
+        if isinstance(expr, BinOp):
+            return self._expr_safe_for_ta_precalc(expr.left) and self._expr_safe_for_ta_precalc(expr.right)
+        if isinstance(expr, UnaryOp):
+            return self._expr_safe_for_ta_precalc(expr.operand)
+        if isinstance(expr, Ternary):
+            return (
+                self._expr_safe_for_ta_precalc(expr.condition)
+                and self._expr_safe_for_ta_precalc(expr.true_val)
+                and self._expr_safe_for_ta_precalc(expr.false_val)
+            )
+        if isinstance(expr, Subscript):
+            return self._expr_safe_for_ta_precalc(expr.object) and self._expr_safe_for_ta_precalc(expr.index)
+        if isinstance(expr, TupleLiteral):
+            return all(self._expr_safe_for_ta_precalc(elem) for elem in expr.elements)
+        if isinstance(expr, FuncCall):
+            if isinstance(expr.callee, MemberAccess) and isinstance(expr.callee.object, Identifier):
+                if expr.callee.object.name in ("math", "str", "color"):
+                    return all(self._expr_safe_for_ta_precalc(arg) for arg in expr.args)
+            return False
+        return False
+
+    def _ta_site_uses_precalc(self, site: "TACallSite") -> bool:
+        """Whether a static TA site can safely read from ``_precalc_*``.
+
+        Static-ness from the analyzer means the expression can be represented
+        from bar data and constants, but the standalone precompute loop only
+        replays a narrow subset of per-bar assignments. A user alias such as
+        ``ha_close = close`` is static in that analyzer sense, yet its Series is
+        empty during precompute, so ``ta.stdev(ha_close, 20)`` precalculates as
+        all-``na``. Opting that site out preserves correctness; it simply uses
+        the ordinary stateful TA object during ``on_bar``."""
+        if not getattr(site, "is_static", False):
+            return False
+        return all(self._expr_safe_for_ta_precalc(arg) for arg in site.compute_args)
+
     def _security_ta_compute_args_for_site(
         self,
         sec_id: int,

package/pineforge_codegen/codegen/tables.py

@@ -55,7 +55,7 @@ def tz_time_field_lambda(field_expr: str, ts_arg: str, tz_arg: str) -> str:
     """
     return (
         "[&]() -> int { "
-        f"std::string _tz = ({tz_arg}); "
+        f"std::string _tz = pineforge::normalize_timezone_for_posix(({tz_arg})); "
         f"time_t _secs = (time_t)(({ts_arg}) / 1000); "
         "struct tm tm_buf; "
         "if (_tz.empty() || _tz == \"UTC\" || _tz == \"Etc/UTC\") { "
@@ -76,11 +76,11 @@ def tz_time_field_lambda(field_expr: str, ts_arg: str, tz_arg: str) -> str:
 
 
 BAR_BUILTINS = {
-    "bar_index": "bar_index_",
+    "bar_index": "pine_bar_index()",
     "time": "current_bar_.timestamp",
     "time_close": "time_close()",
     "timenow": "current_bar_.timestamp",
-    "last_bar_index": "last_bar_index_",
+    "last_bar_index": "pine_last_bar_index()",
     "last_bar_time": "last_bar_time_",
     # time_tradingday: Unix-ms of the session-open of the trading day that
     # contains the current bar. Backed by pine_time_tradingday() in the engine.
@@ -96,10 +96,12 @@ BAR_BUILTINS = {
     # ``_bar_hour()`` helpers. For UTC-exchange data (the crypto corpus,
     # SymInfo's constructor default) the lambda takes the gmtime_r fast
     # path and is value-identical to the old emission.
+    # Route through the engine's cached pine_hour/pine_minute/... (session_time.hpp)
+    # instead of an inline per-call setenv+tzset lambda — the lambda's TZ churn
+    # caused a macOS tzset()->notifyd IPC storm (KI-35). The engine helpers are
+    # value-identical (same tm fields + Pine offsets) but tzset-churn-free.
     **{
-        name: tz_time_field_lambda(
-            TIME_FIELD_EXPRS[name], "current_bar_.timestamp", "syminfo_.timezone"
-        )
+        name: f"pine_{name}(current_bar_.timestamp, syminfo_.timezone)"
         for name in ("hour", "minute", "second", "dayofmonth", "dayofweek",
                      "month", "year", "weekofyear")
     },
@@ -113,7 +115,27 @@ BAR_SERIES_PUSH = {
     "ohlc4": "((current_bar_.open + current_bar_.high + current_bar_.low + current_bar_.close) / 4.0)",
 }
 
-# OHLCV identifiers that refer to the *security* (HTF) bar inside ``request.security()``.
+# Bar identifiers that refer to the *security* (HTF) bar inside
+# ``request.security()``. ``time`` is the HTF bar-open timestamp.
+SECURITY_BAR_FIELD_EXPRS = {
+    "open": "bar.open",
+    "high": "bar.high",
+    "low": "bar.low",
+    "close": "bar.close",
+    "volume": "bar.volume",
+    "time": "bar.timestamp",
+}
+SECURITY_BAR_FIELD_TYPES = {
+    "open": "double",
+    "high": "double",
+    "low": "double",
+    "close": "double",
+    "volume": "double",
+    "time": "int64_t",
+}
+SECURITY_BAR_FIELDS = frozenset(SECURITY_BAR_FIELD_EXPRS)
+
+# Backwards-compatible name for consumers that only need the OHLCV subset.
 SECURITY_OHLC_BAR_FIELDS = frozenset({"open", "high", "low", "close", "volume"})
 
 # Generated C++ runtime function names referenced by the codegen as string
@@ -634,6 +656,33 @@ MATRIX_METHODS = {
 # Math / String dispatch
 # ---------------------------------------------------------------------------
 
+def _math_minmax_na_expr(func_name: str, args: list[str]) -> str:
+    """Emit Pine-compatible math.min/math.max with na propagation.
+
+    ``std::min``/``std::max`` do not propagate NaN consistently because their
+    comparison is specified in terms of ``operator<``. Pine math.min/max return
+    ``na`` when any operand is ``na``, so generated clamp-style expressions
+    like ``math.max(-1, math.min(1, na))`` must stay ``na``.
+    """
+    if not args:
+        return "na<double>()"
+    if len(args) == 1:
+        return f"(double)({args[0]})"
+    op = "std::max" if func_name == "max" else "std::min"
+    decls = " ".join(
+        f"double _v{i} = (double)({arg});" for i, arg in enumerate(args)
+    )
+    guard = " || ".join(f"is_na(_v{i})" for i in range(len(args)))
+    updates = " ".join(
+        f"_out = {op}(_out, _v{i});" for i in range(1, len(args))
+    )
+    return (
+        f"([&]() -> double {{ {decls} "
+        f"if ({guard}) return na<double>(); "
+        f"double _out = _v0; {updates} return _out; }}())"
+    )
+
+
 MATH_FUNC_MAP = {
     "abs": "std::abs", "max": "std::max", "min": "std::min",
     "ceil": "std::ceil", "floor": "std::floor", "round": "std::round",

package/pineforge_codegen/codegen/types.py

@@ -210,8 +210,14 @@ class TypeInferer:
         if isinstance(node, StringLiteral):
             return TypeSpec.primitive("string")
         if isinstance(node, Identifier):
+            loop_specs = getattr(self, "_current_loop_var_specs", None)
+            if loop_specs and node.name in loop_specs:
+                return loop_specs[node.name]
             if node.name in self._collection_types:
                 return self._collection_types[node.name]
+            param_specs = getattr(self, "_current_func_param_specs", {})
+            if node.name in param_specs:
+                return param_specs[node.name]
             if node.name in self._udt_var_types:
                 return TypeSpec.udt(self._udt_var_types[node.name])
             # Drawing-typed method/function parameter (L.6d / U.5): a ``line ln``
@@ -323,6 +329,44 @@ class TypeInferer:
     # Method lowering for collection types (used by visit_call paths)
     # ------------------------------------------------------------------
 
+    def _array_receiver_once_expr(
+        self, array_expr: str, args: list[str], lower_receiver,
+    ) -> str:
+        """Lower an array method without duplicating its receiver evaluation.
+
+        ``ARRAY_METHODS`` is intentionally a compact table of expression
+        templates.  Many templates need the receiver more than once (for
+        example ``begin()`` + ``end()``).  Substituting a temporary-producing
+        receiver directly into those slots creates distinct objects, so the
+        resulting iterator range is invalid.  Render first with a fresh token;
+        when the template uses it repeatedly and the real receiver is not a
+        plain identifier lvalue, bind it to one lambda-local forwarding
+        reference and render every use through that binding.  The lambda's
+        deduced ``auto`` return copies scalar results before a temporary
+        receiver dies; mutations still reach lvalue arrays through the
+        forwarding reference.  Single-use and plain-identifier lowerings
+        remain byte-for-byte unchanged.
+
+        This is receiver-only by design: Pine argument evaluation and the
+        separate empty-array semantics are outside this fix.
+        """
+        counter = getattr(self, "_array_receiver_counter", 0)
+        occupied = "\n".join((array_expr, *args))
+        while True:
+            receiver = f"__pf_array_receiver_{counter}"
+            counter += 1
+            if receiver not in occupied:
+                break
+        self._array_receiver_counter = counter
+
+        lowered = lower_receiver(receiver)
+        if lowered.count(receiver) <= 1 or array_expr.isidentifier():
+            return lower_receiver(array_expr)
+        return (
+            f"[&]() {{ auto&& {receiver} = ({array_expr}); "
+            f"return {lowered}; }}()"
+        )
+
     def _array_method_expr(
         self, array_expr: str, method: str, args: list[str], spec: TypeSpec | None = None,
     ) -> str:
@@ -331,34 +375,36 @@ class TypeInferer:
         arr_cpp_type = self._type_spec_to_cpp(spec)
         elem_cpp = self._type_spec_to_cpp(spec.element) if spec.element is not None else "double"
         if method == "copy":
-            return f"{arr_cpp_type}({array_expr})"
-        if method == "slice":
-            return f"{arr_cpp_type}({array_expr}.begin()+(int)({args[0]}),{array_expr}.begin()+(int)({args[1]}))"
-        if method == "join":
+            lower_receiver = lambda recv: f"{arr_cpp_type}({recv})"
+        elif method == "slice":
+            lower_receiver = lambda recv: f"{arr_cpp_type}({recv}.begin()+(int)({args[0]}),{recv}.begin()+(int)({args[1]}))"
+        elif method == "join" and elem_cpp == "std::string":
             sep = args[0] if args else 'std::string(",")'
-            if elem_cpp == "std::string":
-                return f"[&](){{ std::string r; for(size_t i=0;i<{array_expr}.size();i++){{ if(i>0)r+={sep}; r+={array_expr}[i]; }} return r; }}()"
-        numeric_only = {
-            "sum", "avg", "min", "max", "range", "stdev", "variance", "median",
-            "mode", "percentile_linear_interpolation", "percentile_nearest_rank",
-            "percentrank", "abs", "standardize", "covariance", "binary_search",
-            "binary_search_leftmost", "binary_search_rightmost", "sort_indices",
-        }
-        if method in numeric_only and elem_cpp not in ("double", "int"):
-            self._codegen_error(
-                None,
-                f"array.{method} requires a numeric array",
-                hint="Use numeric arrays for aggregate/statistical array functions.",
-            )
-        if method in ARRAY_METHODS:
-            return ARRAY_METHODS[method](array_expr, args)
-        # Defensive: support_checker rejects any array.* method not in
-        # SUPPORTED_ARRAY (derived from ARRAY_METHODS). Reaching here means the
-        # checker was bypassed or the tables drifted.
-        raise ValueError(
-            f"codegen: unhandled array method '{method}' — analyzer should have "
-            f"rejected. Add it to ARRAY_METHODS."
-        )
+            lower_receiver = lambda recv: f"[&](){{ std::string r; for(size_t i=0;i<{recv}.size();i++){{ if(i>0)r+={sep}; r+={recv}[i]; }} return r; }}()"
+        else:
+            numeric_only = {
+                "sum", "avg", "min", "max", "range", "stdev", "variance", "median",
+                "mode", "percentile_linear_interpolation", "percentile_nearest_rank",
+                "percentrank", "abs", "standardize", "covariance", "binary_search",
+                "binary_search_leftmost", "binary_search_rightmost", "sort_indices",
+            }
+            if method in numeric_only and elem_cpp not in ("double", "int"):
+                self._codegen_error(
+                    None,
+                    f"array.{method} requires a numeric array",
+                    hint="Use numeric arrays for aggregate/statistical array functions.",
+                )
+            if method not in ARRAY_METHODS:
+                # Defensive: support_checker rejects any array.* method not in
+                # SUPPORTED_ARRAY (derived from ARRAY_METHODS). Reaching here means the
+                # checker was bypassed or the tables drifted.
+                raise ValueError(
+                    f"codegen: unhandled array method '{method}' — analyzer should have "
+                    f"rejected. Add it to ARRAY_METHODS."
+                )
+            lower_receiver = lambda recv: ARRAY_METHODS[method](recv, args)
+
+        return self._array_receiver_once_expr(array_expr, args, lower_receiver)
 
     def _map_method_expr(
         self, map_expr: str, method: str, args: list[str], spec: TypeSpec | None = None,
@@ -506,18 +552,88 @@ class TypeInferer:
             return True
         return name in self._int64_reassign_targets()
 
+    def _na_reassign_cpp_type(self, name: str) -> str | None:
+        """Declared scalar C++ type of a ``:=`` reassignment target ``name``, so a
+        bare-``na`` RHS (``x := na``) can be spelled ``na<T>()`` matching the
+        member/local type instead of the default ``na<double>()``.
+
+        Assigning a double quiet-NaN into an ``int``/``int64_t``/``bool`` member is
+        undefined behaviour (NaN->int is unspecified; on ARM64 it saturates to 0,
+        which is not the ``na<T>()`` sentinel) and defeats ``is_na<T>()``. Mirrors
+        the member-declaration type logic (``base._emit_class_members`` /
+        ``_typed_na_init``): ``PINE_TYPE_TO_CPP`` plus the int->int64_t epoch
+        promotion. Returns ``None`` for collections / UDT / drawing handles and
+        for ``double`` (already the default lowering), so those paths are
+        unchanged.
+        """
+        # Collections / UDT / drawing handles never take a scalar ``na<T>()``:
+        # leave them to the drawing-na / default lowering in _visit_rhs_value.
+        if (name in self._array_vars
+                or name in self._map_vars
+                or name in getattr(self, "_matrix_specs", {})
+                or name in self._udt_var_types):
+            return None
+        cpp_type: str | None = None
+        # 1. ``var`` member (class-scope OR function-local: both are recorded in
+        #    ctx.var_members). This is the authoritative declaration source.
+        for vname, ptype, _init in self.ctx.var_members:
+            if vname == name:
+                cpp_type = PINE_TYPE_TO_CPP.get(ptype, "double")
+                break
+        # 2. Function-local plain (non-``var``) scalar: its declared type was
+        #    remembered at the VarDecl (``_type_for_decl``).
+        if cpp_type is None:
+            cpp_type = getattr(self, "_current_func_local_types", {}).get(name)
+        # 3. Function parameter.
+        if cpp_type is None:
+            cpp_type = getattr(self, "_current_func_param_types", {}).get(name)
+        # 4. Global-scope non-``var`` class member.
+        if cpp_type is None:
+            for gname, gptype in self.ctx.global_var_decls:
+                if gname == name:
+                    cpp_type = PINE_TYPE_TO_CPP.get(gptype, "double")
+                    break
+        if cpp_type is None:
+            return None
+        # int -> int64_t promotion for epoch-ms builtins, mirroring the member
+        # declaration so the na sentinel width matches the storage width.
+        if cpp_type == "int" and self._is_int64_builtin_init(name):
+            cpp_type = "int64_t"
+        # Only the retypeable scalar types are meaningful; ``double`` already
+        # lowers to ``na<double>()`` and everything else is left untouched.
+        if cpp_type in ("int", "int64_t", "bool", "std::string"):
+            return cpp_type
+        return None
+
     # ------------------------------------------------------------------
     # BUG C: user-defined-UDT lvalue aliasing
     # ------------------------------------------------------------------
 
     def _is_udt_lvalue(self, expr) -> str | None:
         """If ``expr`` is a *user-defined* UDT lvalue (a bare ``Identifier`` that
-        names a class-scope ``var``/global UDT member, e.g. ``wyckoffSwingLow``),
-        return its UDT type name; else ``None``.
+        names a class-scope ``var``/global UDT member, e.g. ``wyckoffSwingLow``,
+        or an element selected from ``array<UDT>``), return its UDT type name;
+        else ``None``.
 
         Pine UDTs are reference types, so a local initialised from such an lvalue
         and then mutated through must write back to the global. Drawing UDTs are
         handled by the separate ``_uses_drawing`` path and are excluded here."""
+        if isinstance(expr, FuncCall):
+            callee = expr.callee
+            func_name, namespace = self._resolve_callee(callee)
+            receiver = None
+            if namespace == "array" and func_name in ("get", "first", "last") and expr.args:
+                receiver = expr.args[0]
+            elif (isinstance(callee, MemberAccess)
+                  and func_name in ("get", "first", "last")):
+                receiver = callee.object
+            if receiver is not None:
+                spec = self._type_spec_from_expr(receiver)
+                elem = spec.element if spec is not None and spec.kind == "array" else None
+                if (elem is not None and elem.kind == "udt" and elem.name in self._udt_defs
+                        and elem.name not in DRAWING_TYPE_TO_CPP):
+                    return elem.name
+            return None
         if not isinstance(expr, Identifier):
             return None
         udt_t = self._udt_var_types.get(expr.name)
@@ -791,6 +907,8 @@ class TypeInferer:
                     return "double"
             if node.name in self._current_func_param_types:
                 return self._current_func_param_types[node.name]
+            if node.name in getattr(self, "_current_func_local_types", {}):
+                return self._current_func_local_types[node.name]
             sym = self.ctx.symbols.resolve(node.name)
             if sym is not None and getattr(sym, "type_spec", None) is not None:
                 return self._type_spec_to_cpp(sym.type_spec)
@@ -829,9 +947,11 @@ class TypeInferer:
             if namespace == "str":
                 if func_name == "split":
                     return "std::vector<std::string>"
+                if func_name in ("contains", "startswith", "endswith"):
+                    return "bool"
                 if func_name == "tonumber":
                     return "double"
-                if func_name == "length":
+                if func_name in ("length", "pos"):
                     return "int"
                 return "std::string"
             if namespace == "ta" and func_name == "pivot_point_levels":
@@ -881,6 +1001,12 @@ class TypeInferer:
             # pine_str_tostring); bare reads must declare std::string.
             if ename == "format":
                 return "std::string"
+            if ename == "timeframe":
+                if node.member in ("period", "main_period"):
+                    return "std::string"
+                if node.member == "multiplier":
+                    return "int"
+                return "bool"
             # syminfo.* type inference: look up in SYMINFO_MEMBER_MAP
             # and derive C++ type from the expression (na<T>() or function call).
             if ename == "syminfo":
@@ -888,6 +1014,9 @@ class TypeInferer:
                 sym_key = f"syminfo.{node.member}"
                 if sym_key in _pf_sigs.SYMINFO_VARIABLES:
                     return PINE_TYPE_TO_CPP.get(_pf_sigs.SYMINFO_VARIABLES[sym_key], "double")
+            spec = self._type_spec_from_expr(node)
+            if spec is not None:
+                return self._type_spec_to_cpp(spec)
         if isinstance(node, Ternary):
             tt = self._infer_type(node.true_val)
             ft = self._infer_type(node.false_val)
@@ -895,6 +1024,12 @@ class TypeInferer:
                 return tt if tt.startswith("std::vector") else ft
             if tt == "std::string" or ft == "std::string":
                 return "std::string"
+            if tt == "double" or ft == "double":
+                return "double"
+            if tt == "int64_t" or ft == "int64_t":
+                return "int64_t"
+            if tt == "bool" and ft == "bool":
+                return "bool"
             return tt
         # Block-as-expression cases: read the type of the last statement of
         # the first branch / case; matches Pine semantics for ``x = if...``.