@pineforge/codegen-pyodide 1.0.0-rc.1 → 1.0.1

package/pineforge_codegen/codegen/collection_history.py

@@ -0,0 +1,200 @@
+"""The C++ of an array's or a matrix's history (``a[k]``, ``m[k]``).
+
+The analyzer annotates every such read with how its value is used and lists
+the variables whose history the script reads (``ctx.collection_history``;
+pineforge_codegen/collection_history.py). Such a variable keeps its own
+``std::vector`` or matrix member, so every other operation on it keeps its
+C++; beside it, ``_pf_collection_hist_<name>`` keeps the copies its
+executions left. Each execution of the declaration -- each bar, for a top
+level ``var`` -- opens a slot (``open``) and the end of the bar closes it with
+a copy (``close``), so ``a[1]`` is the array as the previous bar left it,
+TradingView's read-only history (fixtures/array_history_tv).
+"""
+
+from __future__ import annotations
+
+from ..collection_history import (
+    COLLECTION_HISTORY_CLASS_CPP,
+    COLLECTION_HISTORY_CPP,
+    COLLECTION_HISTORY_GENERIC_MATRIX_CPP,
+    COLLECTION_HISTORY_MATRIX_CPP,
+    history_annotation,
+    literal_offset,
+)
+from ..ast_nodes import FuncCall, Identifier, MemberAccess
+from ..symbols import TypeSpec
+
+
+class CollectionHistoryEmitter:
+    """CodeGen mixin: the history members of array and matrix variables."""
+
+    def _collection_history_variables(self) -> list:
+        """The declarations whose history the script reads, in key order."""
+        history = getattr(self.ctx, "collection_history", None) or {}
+        return [history[key] for key in sorted(history)]
+
+    def _collection_history_member(self, variable) -> str:
+        """``_pf_collection_hist_<name>``; a later declaration of the same
+        name (a sibling block's) gets ``_pf_collection_hist_<ordinal>_<name>``,
+        which no Pine name (none starts with a digit) can spell: ``x``'s
+        second declaration beside a variable ``x_1``."""
+        prefix = f"{variable.ordinal}_" if variable.ordinal else ""
+        return f"_pf_collection_hist_{prefix}{self._safe_name(variable.name)}"
+
+    def _collection_history_of(self, annotation):
+        history = getattr(self.ctx, "collection_history", None) or {}
+        return history.get(annotation.get("member"))
+
+    def _collection_history_cpp_type(self, variable) -> str:
+        """The history's element type: the variable member's own C++ type,
+        whatever its declaration spells (a wide int array included)."""
+        return f"decltype({self._safe_name(variable.name)})"
+
+    def _collection_history_capacity(self, variable) -> str:
+        """The ring's size: the largest literal offset read plus one, or the
+        Series default (a ``max_bars_back`` cap) for a dynamic offset."""
+        if variable.capacity is None:
+            return self._series_decl_suffix()
+        return f"{{{max(2, variable.capacity)}}}"
+
+    def _emit_collection_history_helper(self, lines: list[str]) -> None:
+        variables = self._collection_history_variables()
+        if not variables:
+            return
+        float_spec = TypeSpec.primitive("float")
+        matrices = [v.spec for v in variables if v.kind == "matrix"]
+        lines.append(COLLECTION_HISTORY_CPP.strip("\n"))
+        if any(spec.element == float_spec for spec in matrices):
+            lines.append(COLLECTION_HISTORY_MATRIX_CPP.strip("\n"))
+        if any(spec.element != float_spec for spec in matrices):
+            lines.append(COLLECTION_HISTORY_GENERIC_MATRIX_CPP.strip("\n"))
+        lines.append(COLLECTION_HISTORY_CLASS_CPP.strip("\n"))
+        lines.append("")
+
+    def _collection_history_open(self, variable) -> str:
+        """The statement opening a declaration's slot."""
+        return (f"{self._collection_history_member(variable)}.open("
+                f"{self._safe_name(variable.name)}, history_advances_new_bar());")
+
+    def _collection_history_close(self, variable) -> str:
+        return (f"{self._collection_history_member(variable)}.close("
+                f"{self._safe_name(variable.name)});")
+
+    def _collection_history_decl_statements(self, node) -> list[str]:
+        """What an execution of a declaration does first: close the history
+        of every other declaration writing the same member (a sibling
+        block's ``x``, which keeps the array its block left), then open its
+        own slot when it is not a ``var`` (a block's on the bars it runs)."""
+        statements = []
+        own = None
+        for variable in self._collection_history_variables():
+            if id(node) in variable.closed_by:
+                statements.append(self._collection_history_close(variable))
+            elif variable.decl_node_id == id(node):
+                own = variable
+        if own is not None and not (getattr(node, "is_var", False)
+                                    or getattr(node, "is_varip", False)):
+            statements.append(self._collection_history_open(own))
+        return statements
+
+    def _emit_collection_history_var_opens(self, lines: list[str], pad: str) -> None:
+        """A top-level ``var``'s slot opens on every bar, after its first
+        bar's initialization."""
+        for variable in self._collection_history_variables():
+            if variable.is_var:
+                lines.append(f"{pad}{self._collection_history_open(variable)}")
+
+    def _emit_collection_history_closes(self, lines: list[str], pad: str) -> None:
+        """The bar's end: every open slot keeps a copy of its variable."""
+        for variable in self._collection_history_variables():
+            lines.append(f"{pad}{self._collection_history_close(variable)}")
+
+    @staticmethod
+    def _roots_at_collection_history(value) -> bool:
+        """Whether ``value`` is an array's or a matrix's history or a
+        built-in's call on one (``m[1]``, ``(m[1]).copy()``,
+        ``matrix.transpose(m[1])``)."""
+        if history_annotation(value) is not None:
+            return True
+        if not isinstance(value, FuncCall):
+            return False
+        callee = value.callee
+        if isinstance(callee, MemberAccess) and history_annotation(callee.object) is not None:
+            return True
+        first = value.args[0] if value.args else None
+        return (isinstance(callee, MemberAccess)
+                and isinstance(callee.object, Identifier)
+                and callee.object.name in ("array", "matrix")
+                and first is not None and history_annotation(first) is not None)
+
+    def _collection_history_offset_cpp(self, node) -> str:
+        offset = literal_offset(node)
+        if offset is not None:
+            return str(offset)
+        return self._history_offset_cpp(self._visit_expr(node.index), node.index)
+
+    def _lower_collection_history(self, node) -> str | None:
+        """The C++ of an annotated array or matrix history read, by its use:
+        a ``const`` copy a built-in reads (``at``), a copy for a variable or a
+        parameter (``value``), the receiver of a built-in that changes it
+        (``changed``: the run stops past a zero offset), the variable itself
+        for a for...in loop (TradingView iterates the current array) and at a
+        zero offset. None for any other read."""
+        annotation = history_annotation(node)
+        if annotation is None:
+            return None
+        current = self._visit_expr(node.object)
+        use = annotation["use"]
+        offset = literal_offset(node)
+        if use == "loop" or offset == 0:
+            return current
+        member = self._collection_history_member(self._collection_history_of(annotation))
+        index = self._collection_history_offset_cpp(node)
+        dynamic = offset is None
+        if use == "change":
+            return f"{member}.changed({index}, {current})"
+        if use == "copy":
+            return (f"{member}.value({index}, {current})" if dynamic
+                    else f"{member}.value({index})")
+        return (f"{member}.at({index}, {current})" if dynamic
+                else f"{member}.at({index})")
+
+    def _collection_history_na(self, node) -> str | None:
+        """``na(a[k])``: whether the history holds no copy there (an array
+        variable itself is never na in PineForge; a matrix may be)."""
+        annotation = history_annotation(node)
+        if annotation is None or annotation["use"] != "na":
+            return None
+        current = self._visit_expr(node.object)
+        offset = literal_offset(node)
+        if offset == 0:
+            return f"is_na({current})" if annotation["kind"] == "matrix" else "false"
+        member = self._collection_history_member(self._collection_history_of(annotation))
+        index = self._collection_history_offset_cpp(node)
+        if offset is None:
+            return f"{member}.is_na({index}, {current})"
+        return f"{member}.is_na({index})"
+
+    def _matrix_history_method_expr(self, receiver, method: str, spec, node) -> str:
+        """A matrix method called on a matrix's history, lowered as the
+        variable's own method call is (visit_call: a matrix variable's
+        method)."""
+        from .tables import MATRIX_METHOD_KWARGS, MATRIX_METHODS, _merge_kwargs
+
+        recv = self._visit_expr(receiver)
+        self._check_matrix_method_allowed(method, spec, node)
+        param_names = MATRIX_METHOD_KWARGS.get(method)
+        raw_args = (
+            _merge_kwargs(node.args, node.kwargs, param_names, lambda a: a)
+            if param_names and node.kwargs else list(node.args)
+        )
+        args = self._matrix_bool_value_args(
+            method, [self._visit_expr(a) for a in raw_args], raw_args, spec)
+        try:
+            return MATRIX_METHODS[method](recv, args)
+        except IndexError:
+            self._codegen_error(
+                node,
+                f"matrix.{method}: wrong number of arguments",
+                hint="Check Pine v6 matrix method signature (positional vs keyword).",
+            )

package/pineforge_codegen/codegen/emit_top.py

@@ -1721,7 +1721,12 @@ class TopLevelEmitter:
                                 name,
                                 target_cpp_type=udt_name,
                             )
-                            lines.append(f"            {safe} = {cpp_val};")
+                            if self._binding_is_series(name, safe):
+                                # A history-read var object: its first
+                                # bar's slot holds the reference.
+                                lines.append(f"            {safe}.push({cpp_val});")
+                            else:
+                                lines.append(f"            {safe} = {cpp_val};")
                             break
                     continue
                 if self._binding_is_series(name, safe):
@@ -1797,6 +1802,10 @@ class TopLevelEmitter:
         # c. Push non-var series (they start fresh each bar with a push)
         # (actual push happens in visit_VarDecl when the decl is visited)
 
+        # c1. A top-level var array's or matrix's history slot opens on every
+        # bar (pineforge_codegen/collection_history.py).
+        self._emit_collection_history_var_opens(lines, "        ")
+
         # c2. First-bar TA resize: rebuild any TA object whose ctor args come
         # from input-backed variables so strategy_set_input() actually changes
         # the circular-buffer sizes. Emits nothing when no TA site depends on
@@ -1815,6 +1824,10 @@ class TopLevelEmitter:
             finally:
                 self._clear_lazy_edge_ta_hoists()
 
+        # d2. The bar's end: each array or matrix variable whose history the
+        #     script reads keeps a copy of the value it ends the bar with.
+        self._emit_collection_history_closes(lines, "        ")
+
         # e. ``// @pf-trace`` pragma block — emitted last so trace values
         #    reflect every assignment / strategy call made earlier in the
         #    bar. The block is wrapped in ``if (trace_enabled_)`` so cost
@@ -2103,11 +2116,16 @@ class TopLevelEmitter:
 
             if (
                 receiver_spec is not None
-                and receiver_spec.kind == "primitive"
+                and (receiver_spec.kind == "primitive"
+                     or (receiver_spec.kind == "udt"
+                         and (receiver_spec.name in DRAWING_TYPE_TO_CPP
+                              or receiver_spec.name in self._udt_defs)))
                 and p in func_sv
             ):
                 # A primitive receiver used with history is a Series boundary,
-                # just like an ordinary history-bearing UDF parameter.
+                # just like an ordinary history-bearing UDF parameter; so is
+                # an object or drawing receiver (``this[1]`` is the receiver of
+                # the previous call: fixtures/udt_history_tv udth_method).
                 elem_cpp_t = self._series_param_element_cpp_type(
                     fi, i, call_site_idx
                 )
@@ -2118,6 +2136,9 @@ class TopLevelEmitter:
                 self._current_func_series_param_types[
                     self._safe_name(p)
                 ] = elem_cpp_t
+                if receiver_spec.kind == "udt":
+                    self._udt_param_udt[self._safe_name(p)] = receiver_spec.name
+                    self._udt_param_udt[p] = receiver_spec.name
             elif is_method and i == 0 and fi.udt_type_name:
                 # Receiver pass modes follow Pine's value/ID families. A user
                 # UDT is itself a numeric object-ID handle, so pass it by value:
@@ -2352,10 +2373,16 @@ class TopLevelEmitter:
                 ),
             )
             if (drawing_name in DRAWING_TYPE_TO_CPP
-                and self._current_func_param_types.get(
+                and (self._current_func_param_types.get(
                     param, ""
                 ).removesuffix("&").removesuffix("*")
-                == DRAWING_TYPE_TO_CPP[drawing_name])
+                == DRAWING_TYPE_TO_CPP[drawing_name]
+                # A history-read drawing parameter or receiver is a
+                # ``const Series<Box>&``: ``(x[1]).get_top()`` dispatches on
+                # the handle it holds (udth_drawparam).
+                or (param in self._current_func_series_params
+                    and self._current_func_series_param_types.get(param)
+                    == DRAWING_TYPE_TO_CPP[drawing_name])))
         }
         self._lexical_udt_types = {
             param: (

package/pineforge_codegen/codegen/helpers.py

@@ -345,7 +345,10 @@ CPP_RESERVED = set(
 # ``_nz_v``) or initialized itself from itself (``_pna_l > close`` did not
 # compile). ``_safe_name`` escapes an authored spelling of any of them. A
 # temporary whose name the emitter allocates against the authored spellings
-# itself (``__pf_map_iter_N``, ``_pf_udt_Item__pf2``) is not listed.
+# itself (``__pf_map_iter_N``, ``_pf_udt_Item__pf2``) is not listed. An
+# array's or a matrix's history member (``_pf_collection_hist_<name>``,
+# ``_pf_collection_hist_<n>_<name>``) is reserved the same way: a script
+# variable of that spelling was a second member of one name.
 CPP_TEMPORARY_NAMES = frozenset("""
     _nz_v _nz_y _fixnan_v _pna_l _pna_r _pfc_l _pfc_r _pfc_eq _hv _hidx _sv
     _out _func_ret _pf_v _pf_idx_v _pf_idx_t _pf_bool_v _pf_bool_t
@@ -364,6 +367,7 @@ _CPP_TEMPORARY_PATTERN = re.compile(
     r"_v\d+|_secval_\d+(?:_v\d+)?|_tuple_result_\d+|_tuple_unused_\d+|__switch_val_\d+"
     r"|_for_(?:start|end|end_eval)_\d+|_pf_(?:str|array|round)_a\d+"
     r"|_pf_every_bar_ta_\d+|_pf_shared_\d+_\d+|__pf_array\w*|__pf_raw_\w+"
+    r"|_pf_collection_hist_\w+"
 )
 
 

package/pineforge_codegen/codegen/security.py

@@ -88,7 +88,7 @@ from .helpers import na_preserving_int_cast, unary_sign_cpp
 from ..security_contexts import GLOBAL_ANNOTATION, UNREACHED_ANNOTATION
 from ..symbols import PineType, method_receiver_type_name
 from .tables import (
-    BAR_BUILTINS, MATH_FUNC_MAP, PINE_TYPE_TO_CPP, SECURITY_BAR_FIELDS,
+    BAR_BUILTINS, DRAWING_TYPE_TO_CPP, MATH_FUNC_MAP, PINE_TYPE_TO_CPP, SECURITY_BAR_FIELDS,
     SECURITY_BAR_FIELD_EXPRS, SECURITY_BAR_FIELD_TYPES, TA_TUPLE_FIELDS,
     _math_minmax_na_expr, _math_round_digits_expr, _merge_kwargs,
 )
@@ -6127,11 +6127,66 @@ class SecurityEmitter:
         lines.append("    }")
         lines.append("")
 
+    def _refuse_security_reference_history(self, item: dict) -> None:
+        """A request.security expression that reads the history of an
+        object or drawing reference, itself or in a function or method it
+        calls. TradingView evaluates it on the requested bars, where the
+        variable holds the objects of that timeframe's bars; PineForge keeps
+        such a history on the chart's bars only, and the payload read the
+        chart's (it never compiled before: the history was a
+        ``Series<double>``). Refused."""
+        if item["expr_node"] is None:
+            # ``request.security(sym, tf)`` with no expression: lowered to na
+            # as it always was (TradingView refuses it, CE10165).
+            return
+        pending = [item["expr_node"]]
+        seen: set[str] = set()
+        while pending:
+            for node, _depth in iter_ast_nodes(pending.pop()):
+                if isinstance(node, Subscript):
+                    receiver = node.object
+                    kind = (self._series_handle_type_name(receiver.name)
+                            if isinstance(receiver, Identifier) else None)
+                    if kind is None:
+                        spec = self._type_spec_from_expr(receiver)
+                        if (spec is not None and spec.kind == "udt"
+                                and (spec.name in DRAWING_TYPE_TO_CPP
+                                     or spec.name in self._udt_defs)):
+                            kind = spec.name
+                    if kind is not None:
+                        self._codegen_error(
+                            node,
+                            f"History of a {kind} reference inside a "
+                            "request.security expression is not supported in "
+                            "PineForge: TradingView reads the references the "
+                            "variable held on the requested timeframe's bars, "
+                            "which PineForge does not keep.",
+                            hint="Request the field values the expression "
+                                 "needs (request.security(..., obj.field)) "
+                                 "and read their history on the chart.",
+                        )
+                if not isinstance(node, FuncCall):
+                    continue
+                func_name, namespace = self._resolve_callee(node.callee)
+                keys = []
+                if namespace is None and func_name in self._func_info_map:
+                    keys.append(func_name)
+                if isinstance(node.callee, MemberAccess):
+                    keys.extend(key for key in self._func_info_map
+                                if key.endswith(f".{node.callee.member}"))
+                for key in keys:
+                    body = getattr(self._func_info_map[key], "node", None)
+                    if key not in seen and body is not None:
+                        seen.add(key)
+                        pending.append(body)
+
     def _emit_security_evaluators(self, lines: list[str]) -> None:
         """Emit _eval_security_N() methods and evaluate_security() dispatch."""
         if not self._security_calls:
             return
 
+        for item in self._security_calls:
+            self._refuse_security_reference_history(item)
         for item in self._security_calls:
             self._emit_security_evaluator_requested(item, lines)
 

package/pineforge_codegen/codegen/ta.py

@@ -1131,6 +1131,65 @@ class TaSiteHelper:
             )
         return cache[id(node)]
 
+    def _lazy_ctor_history_call(self, call) -> bool:
+        """A positional call of the one user function of its name whose body
+        is a single ``T.new(...)`` of a user-defined type over
+        ``_lazy_call_history_pure_expr`` arguments of its parameters: its value
+        is a new object of fixed fields, and TradingView runs it on every
+        execution of its scope below a lazy edge when its history is read
+        (fixtures/udt_history_tv udth_expr: ``(mk(bar_index * 7)[1]).v`` in a
+        ternary's arm reads the previous bar's call)."""
+        if not isinstance(call, FuncCall) or not isinstance(call.callee, Identifier):
+            return False
+        overloads = [fi for fi in self.ctx.func_infos if fi.name == call.callee.name]
+        if call.kwargs or len(overloads) != 1 or overloads[0].node is None:
+            return False
+        node = overloads[0].node
+        body = getattr(node, "body", None) or []
+        if len(call.args) != len(node.params) or len(body) != 1:
+            return False
+        ctor = body[0].expr if isinstance(body[0], ExprStmt) else None
+        if not (isinstance(ctor, FuncCall)
+                and isinstance(ctor.callee, MemberAccess)
+                and isinstance(ctor.callee.object, Identifier)
+                and ctor.callee.object.name in self._udt_defs
+                and ctor.callee.member == "new"):
+            return False
+        params = set(node.params)
+        return all(self._lazy_call_history_pure_expr(arg, params)
+                   for arg in [*ctor.args, *ctor.kwargs.values()])
+
+    def _lazy_reference_history_object(self, obj) -> bool:
+        """Whether the history of the object or drawing reference ``obj``
+        below a lazy edge is kept on every execution of its scope, as
+        TradingView keeps it (udth_expr): a pure call or a
+        ``_lazy_ctor_history_call`` over pure arguments, a ternary over
+        names, literals and operators, or a field of a named object (read
+        na while the object is na, ``_lazy_history_read_cpp``)."""
+        if self._reference_cpp_type(obj) is None:
+            return False
+        if isinstance(obj, FuncCall):
+            return ((self._lazy_call_history_pure_call(obj)
+                     or self._lazy_ctor_history_call(obj))
+                    and all(self._lazy_call_history_pure_expr(arg, None)
+                            for arg in obj.args))
+        if isinstance(obj, Ternary):
+            return self._lazy_call_history_pure_expr(obj, None)
+        return (isinstance(obj, MemberAccess)
+                and isinstance(obj.object, Identifier)
+                and self._reference_cpp_type(obj.object) is not None)
+
+    def _lazy_history_read_cpp(self, obj) -> str:
+        """The value a hoisted history read pushes: the expression, a field of
+        a na object reading na (TradingView stops on the read itself)."""
+        if (isinstance(obj, MemberAccess)
+                and self._reference_cpp_type(obj.object) is not None):
+            handle = self._reference_cpp_type(obj)
+            receiver = self._visit_expr(obj.object)
+            return (f"(is_na({receiver}) ? {handle}{{}} : "
+                    f"{handle}({self._visit_expr(obj)}))")
+        return self._visit_expr(obj)
+
     def _lazy_call_history_units(self, stmt) -> list:
         """The ``pure_call(...)[k]`` reads below a lazy edge of ``stmt``'s
         roots (``_lazy_edge_statement_roots``), in evaluation order, whose
@@ -1159,6 +1218,10 @@ class TaSiteHelper:
                                 for arg in call.args)):
                     units.append(expr)
                     return
+                if (under_lazy and self._registered_history_handle(expr) is not None
+                        and self._lazy_reference_history_object(call)):
+                    units.append(expr)
+                    return
                 scan(expr.object, under_lazy)
                 scan(expr.index, under_lazy)
             elif isinstance(expr, FuncCall):
@@ -1219,7 +1282,8 @@ class TaSiteHelper:
         hoisted = []
         for read in units:
             member = self._inline_history_member("hist_call", read)
-            self._emit_history_series_write(lines, pad, member, self._visit_expr(read.object))
+            self._emit_history_series_write(
+                lines, pad, member, self._lazy_history_read_cpp(read.object))
             self._hoisted_hist_reads[id(read)] = member
             hoisted.append(id(read))
         return hoisted