@pineforge/codegen-pyodide 1.0.0 → 1.1.0

package/glue.py

@@ -40,4 +40,5 @@ def transpile_json(source: str) -> str:
     # A script that transpiled carries its warnings in the same entry format.
     return json.dumps({"ok": True, "cpp": full["cpp"], "inputs": full["inputs"],
                        "strategyParams": full["strategyParams"],
-                       "diagnostics": _diagnostic_entries(full["diagnostics"])})
+                       "diagnostics": _diagnostic_entries(full["diagnostics"]),
+                       "requests": full["requests"]})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pineforge/codegen-pyodide",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Gate-validated Pyodide payload for the PineScript v6 -> C++ transpiler: archive (run in Pyodide), unpacked source, introspected tables, and release metadata.",
   "type": "module",
   "main": "index.mjs",

package/pineforge_codegen/__init__.py

@@ -13,6 +13,7 @@ from .finite_ta_length import expand_finite_choice_extrema_lengths
 from .library_inline import inline_libraries
 from .limits import TimeBudget, check_ast_depth, check_source_size, ensure_recursion_headroom
 from .pragmas import extract_pf_trace_pragmas
+from .request_discovery import discover_requests, request_sites
 from .security_contexts import specialize_security_contexts
 from .block_locals import rename_block_locals
 from .support_checker import check_support as _support_diagnostics
@@ -52,7 +53,9 @@ def _generate(pine_source: str, check_support: bool, filename: str,
     the member of its name cannot hold gets a name of its own, and the
     pipeline runs again (``block_locals``).
 
-    Returns ``(codegen, ctx, cpp, support_diagnostics)``."""
+    Returns ``(codegen, ctx, cpp, support_diagnostics, sites)``: ``sites``
+    are the requests of another symbol's data the support checker lowered
+    (``request_discovery.request_sites``)."""
     budget = None
     clones: frozenset[str] = frozenset()
     renamed: frozenset = frozenset()
@@ -67,6 +70,7 @@ def _generate(pine_source: str, check_support: bool, filename: str,
             if any(d.level == Level.ERROR for d in support_diagnostics):
                 raise CompileError(support_diagnostics)
         budget.check(phase=Phase.ANALYZER)
+        sites = request_sites(ast)
         ast = lower_no_data_requests(ast)
         ast = specialize_security_contexts(ast, filename=filename)
         ast = bind_builtin_keywords(expand_finite_choice_extrema_lengths(ast))
@@ -86,18 +90,18 @@ def _generate(pine_source: str, check_support: bool, filename: str,
         apart = gen.block_locals_needing_names - renamed
         if apart:
             renamed |= apart
-            del ast, ctx, gen, cpp
+            del ast, ctx, gen, cpp, sites
             continue
         needed = gen.session_functions_needing_clones
         if not needed:
-            return gen, ctx, cpp, support_diagnostics
+            return gen, ctx, cpp, support_diagnostics, sites
         # Only uncloned functions ask, so the set grows each time and the
         # loop ends by the time every function that reads a flag at an
         # offset is cloned.
         if needed <= clones:
             raise AssertionError("a cloned function asked for session clones again")
         clones |= needed
-        del ast, ctx, gen, cpp
+        del ast, ctx, gen, cpp, sites
 
 
 def transpile(pine_source: str, *, check_support: bool = True, filename: str = "<input>",
@@ -140,7 +144,8 @@ def transpile(pine_source: str, *, check_support: bool = True, filename: str = "
     Returns:
         Generated C++ source string.
     """
-    _gen, _ctx, cpp, _support = _generate(pine_source, check_support, filename, libraries)
+    _gen, _ctx, cpp, _support, _sites = _generate(pine_source, check_support, filename,
+                                                  libraries)
     return cpp
 
 
@@ -161,26 +166,36 @@ def transpile_full(pine_source: str, *, check_support: bool = True,
       source order). Each has ``title`` / ``type`` /
       ``default`` and optionally ``min`` / ``max`` / ``step`` / ``options``
       (omitted when the corresponding signature argument is absent or
-      references a non-const value). See
-      :meth:`CodeGen.extract_input_manifest`.
+      references a non-const value); an ``input.symbol`` entry also has
+      ``kind: "symbol"``. See :meth:`CodeGen.extract_input_manifest`.
     - ``strategyParams``: the literal ``strategy(...)`` kwargs the analyzer
       surfaced (e.g. ``initial_capital``, ``pyramiding``).
     - ``diagnostics``: the warnings (:class:`~pineforge_codegen.errors.Diagnostic`,
       ``Level.WARNING``) the support checker and the analyzer raised for a
       script that transpiled -- e.g. an approximated ``ta.vwap`` anchor. An
       error still raises ``CompileError``, which carries the warnings too.
+    - ``requests``: every request site that reads another symbol's feed,
+      by line: its symbol and timeframe as registration computes them
+      before the first bar (``literal`` / ``input`` / ``computed`` /
+      ``unresolvable``; ``literal`` / ``chart`` / ``input`` / ``computed``),
+      ``lookahead``, ``gaps`` and ``ignore_invalid_symbol``. A site lowered
+      to ``na`` (its value reaches display sinks only) or in a helper
+      nothing reaches is not listed. See
+      :mod:`pineforge_codegen.request_discovery`.
 
     Args mirror :func:`transpile`.
 
     Returns:
         ``{"cpp": str, "inputs": list[dict], "strategyParams": dict,
-        "diagnostics": list[Diagnostic]}``.
+        "diagnostics": list[Diagnostic], "requests": list[dict]}``.
     """
-    gen, ctx, cpp, support_diagnostics = _generate(pine_source, check_support, filename, libraries)
+    gen, ctx, cpp, support_diagnostics, sites = _generate(
+        pine_source, check_support, filename, libraries)
     return {
         "cpp": cpp,
         "inputs": gen.extract_input_manifest(),
         "strategyParams": dict(ctx.strategy_params),
         "diagnostics": [d for d in (*support_diagnostics, *ctx.diagnostics)
                         if d.level == Level.WARNING],
+        "requests": discover_requests(gen, ctx, sites),
     }

package/pineforge_codegen/analyzer/base.py

@@ -32,6 +32,10 @@ from ..errors import SourceLocation, Diagnostic, CompileError, Level, Phase
 from ..limits import TimeBudget, iter_ast_nodes
 from ..security_contexts import PASS_WARNINGS_ANNOTATION, context_key, reads_bar_series
 from ..session_reads import emitted_session_reads
+from ..collection_history import (
+    COLLECTION_HISTORY_ANNOTATION, LEGACY, REFUSED, SUPPORTED,
+    CollectionHistoryChecker, HistoryRead,
+)
 from ..method_binding import (
     BoundMethodArgs,
     MethodBindError,
@@ -431,6 +435,13 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
         # pass can tell borrowed clones apart from a dead function's own
         # sites (see contracts.TACallSite.owner_func).
         self._enclosing_func_names: list[str] = []
+        # Array and matrix history reads (``a[1]``), each decided where the
+        # visit meets it (pineforge_codegen/collection_history.py), and the
+        # declarations writing each class member, by name (a sibling block's
+        # declaration closes another one's history).
+        self._collection_history_reads: list[HistoryRead] = []
+        self._collection_history_checker: CollectionHistoryChecker | None = None
+        self._member_writer_decls: dict[str, set[int]] = {}
         # Exact TA targets borrowed through nested callable edges while visiting
         # the current callable body. Constructor templates are tracked
         # separately in ``_func_ta_ctor_args`` because state ownership also
@@ -552,6 +563,7 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
         self._ensure_pine_v6()
         self._check_direct_terminal_array_element_callee_shadows()
         self._visit(self._ast)
+        collection_history = self._check_collection_history()
         self._check_direct_terminal_array_temporary_cycles()
         self._register_resolved_direct_terminal_array_forward_calls()
         self._refresh_direct_terminal_array_temporary_returns()
@@ -652,6 +664,7 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
                 for owner, names in self._func_var_storage_names.items()
             },
             func_series_vars=self._func_series_vars,
+            collection_history=collection_history,
             func_global_history_reads={
                 owner: list(names)
                 for owner, names in self._func_global_history_reads.items()
@@ -3784,6 +3797,7 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
                     symbol=sec.symbol,
                     foreign=sec.foreign,
                     ignore_invalid=sec.ignore_invalid,
+                    loc=sec.loc,
                 )
                 new_calls.append(clone)
                 next_sec_id += 1
@@ -4345,6 +4359,7 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
         self._symbols.define(sym)
         setattr(sym, "_pf_decl_node_id", id(node))
         setattr(sym, "_pf_decl_binding_name", node.name)
+        self._note_member_writer(sym, node.name, node)
         if (type_spec is None
                 and self._collection_scope_stack
                 and self._block_node_stack
@@ -4647,6 +4662,7 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
             self._symbols.define(sym)
             setattr(sym, "_pf_decl_node_id", id(node))
             setattr(sym, "_pf_decl_binding_name", name)
+            self._note_member_writer(sym, name, node)
 
             if (self._collection_scope_stack
                     and self._block_node_stack
@@ -6174,6 +6190,19 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
             if self._type_spec_contains_map(object_spec):
                 assert object_spec is not None
                 self._reject_unsupported_map_history(object_spec, node)
+            if (object_spec is not None
+                    and object_spec.kind in ("array", "matrix")):
+                # A parameter given an array or a matrix: its history is a
+                # parameter's, which PineForge does not keep; the earlier
+                # lowering stays where it compiled (collection_history.py).
+                # A selection of one is an expression's history.
+                checker = self._history_checker()
+                if isinstance(node.object, Identifier):
+                    decision = checker.decide_parameter_history(node, object_spec)
+                    if decision.outcome == REFUSED:
+                        self._collection_history_raise(decision, node)
+                else:
+                    self._refuse_collection_expression_history(node, object_spec)
 
         # Propagate concrete caller specs through wrapper calls.  Each edge is
         # identity-keyed from the definition pass, so a local that shadows a
@@ -6428,8 +6457,10 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
     def _visit_Subscript(self, node: Subscript) -> PineType:
         obj_type = self._visit(node.object)
         self._visit(node.index)
+        self._refuse_collection_field_history(node)
 
         object_spec = self._history_receiver_type_spec(node.object)
+        self._refuse_collection_expression_history(node, object_spec)
         if self._type_spec_contains_map(object_spec):
             assert object_spec is not None
             self._reject_unsupported_map_history(object_spec, node)
@@ -6461,7 +6492,17 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
                             "Series buffer for that binding.",
                             node.loc,
                         )
-                    if getattr(sym, "type_spec", None) is None or sym.type_spec.kind not in ("array", "map"):
+                    # An array's or a matrix's history: copies kept beside
+                    # the variable, not a Series of it (collection_history.py);
+                    # a legacy matrix parameter's read keeps the registration
+                    # below, as it had.
+                    keeps_series = not (
+                        getattr(sym, "type_spec", None) is not None
+                        and sym.type_spec.kind in ("array", "matrix")
+                        and not self._note_collection_history_read(node, name, sym)
+                    )
+                    if keeps_series and (getattr(sym, "type_spec", None) is None
+                                         or sym.type_spec.kind not in ("array", "map")):
                         exact_member = getattr(sym, "_pf_var_member_name", None)
                         if exact_member is not None:
                             self._series_var_members.add(exact_member)
@@ -6510,6 +6551,150 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
 
         return obj_type
 
+    def _refuse_collection_field_history(self, node: Subscript) -> None:
+        """The history of an object's array, matrix or map field: TradingView
+        refuses ``h.xs[1]`` and ``(h.xs)[1]`` alike ("Cannot use the
+        history-referencing operator on fields of user-defined types",
+        CE10290: fixtures/array_history_tv ahist_field_noparen,
+        ahist_field_paren, ahist_mfield) and takes ``(h[1]).xs``. The C++
+        indexed the field's current collection."""
+        field = node.object
+        if not isinstance(field, MemberAccess):
+            return
+        owner = self._history_receiver_type_spec(field.object)
+        if (owner is None or owner.kind != "udt"
+                or owner.name not in self._udt_fields):
+            return
+        spec = (self._udt_field_type_specs.get(owner.name) or {}).get(field.member)
+        if spec is None or spec.kind not in ("array", "matrix", "map"):
+            return
+        receiver = (field.object.name if isinstance(field.object, Identifier)
+                    else "object")
+        raise CompileError([Diagnostic(
+            level=Level.ERROR,
+            phase=Phase.ANALYZER,
+            location=node.loc or SourceLocation(self._filename, 1, 1, 1),
+            message=(f"{receiver}.{field.member}[...]: TradingView refuses the "
+                     "history-referencing operator on fields of user-defined "
+                     "types (CE10290)."),
+            hint=(f"Read the field of the object's history: "
+                  f"({receiver}[1]).{field.member}."),
+        )])
+
+    def _history_checker(self) -> CollectionHistoryChecker:
+        """The script's array and matrix history checker, built when the
+        visit first meets such a read."""
+        if self._collection_history_checker is None:
+            self._collection_history_checker = CollectionHistoryChecker(
+                self._ast, self._collection_history_name_kind)
+        return self._collection_history_checker
+
+    def _collection_history_name_kind(self, name: str) -> str | None:
+        """``array`` / ``matrix`` / ``map`` / ``scalar`` for a script
+        variable, else None (collection_history.CollectionHistoryChecker)."""
+        sym = self._symbols.global_scope.symbols.get(name)
+        if sym is None:
+            return None
+        spec = getattr(sym, "type_spec", None)
+        if spec is not None and spec.kind in ("array", "matrix", "map"):
+            return spec.kind
+        if sym.pine_type in (PineType.INT, PineType.FLOAT, PineType.BOOL,
+                             PineType.STRING, PineType.COLOR):
+            return "scalar"
+        return None
+
+    def _note_member_writer(self, sym, name: str, node) -> None:
+        """A declaration writing the class member of its name (the script's
+        top level and its blocks): a sibling block's declaration of the name
+        closes another one's array history before it writes
+        (codegen/collection_history.py)."""
+        if sym.scope == "global":
+            self._member_writer_decls.setdefault(name, set()).add(id(node))
+
+    def _collection_history_raise(self, decision, node) -> None:
+        raise CompileError([Diagnostic(
+            level=Level.ERROR,
+            phase=Phase.ANALYZER,
+            location=(getattr(decision.node, "loc", None) or node.loc
+                      or SourceLocation(self._filename, 1, 1, 1)),
+            message=decision.message,
+            hint=decision.hint,
+        )])
+
+    def _refuse_collection_expression_history(self, node: Subscript, spec) -> None:
+        """The history of an expression whose value is an array or a matrix
+        (a call's result, a selection): TradingView reads the copy the
+        expression produced k executions back (fixtures/array_history_tv
+        ahist_fn ``c1``), which PineForge does not keep. A use whose earlier
+        lowering compiled keeps it; one only a collection fits never compiled
+        and is refused (collection_history.py)."""
+        if (isinstance(node.object, Identifier) or spec is None
+                or spec.kind not in ("array", "matrix")):
+            return
+        checker = self._history_checker()
+        decided = checker.decision_for(node) is not None
+        decision = checker.decide_expression(node, spec)
+        if decision.outcome == REFUSED:
+            self._collection_history_raise(decision, node)
+        if decision.outcome == LEGACY and decision.use == "element" and not decided:
+            self._warn(
+                "(...)[...] reads an element of the selected current array in "
+                "PineForge: no history of a selection is kept, where TradingView "
+                "reads the array it selected that many bars back.",
+                node.loc,
+            )
+
+    def _note_collection_history_read(self, node: Subscript, name: str, sym) -> bool:
+        """Decide ``a[k]`` of an array or a matrix variable
+        (collection_history.py): annotate a supported read, refuse a refused
+        one, and keep a legacy one's earlier lowering. True where that
+        lowering registered a series (a matrix parameter's), which the caller
+        then does as it did."""
+        decl_node_id = getattr(sym, "_pf_decl_node_id", None)
+        read = HistoryRead(
+            node=node,
+            name=name,
+            kind=sym.type_spec.kind,
+            spec=sym.type_spec,
+            in_callable=bool(self._enclosing_func_names),
+            is_parameter=sym.scope != "global" and decl_node_id is None,
+            is_global=sym.scope == "global",
+            is_var=bool(sym.is_var),
+            decl_node_id=decl_node_id,
+            scope_name=sym.scope or "",
+        )
+        decision = self._history_checker().decide(read)
+        if node.annotations and COLLECTION_HISTORY_ANNOTATION in node.annotations:
+            node.annotations = dict(node.annotations)
+            del node.annotations[COLLECTION_HISTORY_ANNOTATION]
+        if decision.outcome == REFUSED:
+            self._collection_history_raise(decision, node)
+        if decision.outcome == SUPPORTED:
+            node.annotations = dict(node.annotations or {})
+            node.annotations[COLLECTION_HISTORY_ANNOTATION] = {
+                "var": name, "kind": read.kind, "use": decision.use,
+                "spec": read.spec, "member": None,
+            }
+            if not any(r.node is node for r in self._collection_history_reads[-1:]):
+                self._collection_history_reads.append(read)
+            return False
+        return decision.outcome == LEGACY and decision.use == "series"
+
+    def _collection_history_is_supported(self, node: Subscript) -> bool:
+        """Whether the checker supports the history read ``node`` (decided
+        where the visit met it; a read not met yet keeps its old typing)."""
+        checker = self._collection_history_checker
+        decision = checker.decision_for(node) if checker is not None else None
+        return decision is not None and decision.outcome == SUPPORTED
+
+    def _check_collection_history(self) -> dict:
+        """The declarations whose history the supported reads need, keyed
+        by their members (pineforge_codegen/collection_history.py)."""
+        if not self._collection_history_reads:
+            return {}
+        return self._history_checker().history_variables(
+            self._collection_history_reads, self._member_writer_decls)
+
     def _note_function_global_history_read(
             self, node: Subscript, name: str, sym) -> None:
         """Record ``x[k]`` in a plain UDF body on a script variable.
@@ -6807,6 +6992,10 @@ class Analyzer(CallHandlers, DiagnosticsHelper, TypeHelper):
             if owner_spec is not None and owner_spec.kind == "udt" and owner_spec.name:
                 field_spec = (self._udt_field_type_specs.get(owner_spec.name) or {}).get(node.member)
                 if field_spec is not None:
+                    # The receiver is read too: ``(o[1]).inner.v`` reads the
+                    # history of ``o`` (fixtures/udt_history_tv udth_ref),
+                    # which only its visit registers.
+                    self._visit(node.object)
                     return self._type_hint_to_pine(str(field_spec))
             self._visit(node.object)
             # strategy.closedtrades.* and strategy.opentrades.* return types

package/pineforge_codegen/analyzer/call_handlers.py

@@ -876,6 +876,7 @@ class CallHandlers:
                 dead=bool(notes.get(DEAD_ANNOTATION)),
                 foreign=notes.get(LOWERING_ANNOTATION) == FEED_LOWERING,
                 ignore_invalid=node.kwargs.get("ignore_invalid_symbol"),
+                loc=node.loc,
             ))
 
             return PineType.STRING if string_result else PineType.FLOAT
@@ -998,6 +999,7 @@ class CallHandlers:
             is_lower_tf_array=True,
             context_resolved=context is not None,
             dead=bool(notes.get(DEAD_ANNOTATION)),
+            loc=node.loc,
         ))
 
         # ``request.security_lower_tf`` returns an array; the value-level

package/pineforge_codegen/analyzer/contracts.py

@@ -196,6 +196,9 @@ class SecurityCallInfo:
     # ``symbol`` string and ``ignore_invalid_symbol`` (the node, or None).
     foreign: bool = False
     ignore_invalid: Any = None
+    # The request call's source location (``request_discovery`` names the
+    # site by its line).
+    loc: Any = None
 
 
 @dataclass
@@ -337,6 +340,9 @@ class AnalyzerContext:
     # per read. TradingView keeps that history per call site, so codegen gives
     # each emitted body its own chart-clocked buffer of the variable.
     func_global_history_reads: dict = field(default_factory=dict)
+    # Array and matrix variables whose history the script reads, by name
+    # (pineforge_codegen/collection_history.py HistoryVariable).
+    collection_history: dict = field(default_factory=dict)
     func_global_history_nodes: dict = field(default_factory=dict)
     # FuncDef owner -> declaration-bound non-persistent history-local names.
     # Parameters are deliberately excluded so a same-named qualified

package/pineforge_codegen/analyzer/types.py

@@ -347,14 +347,25 @@ class TypeHelper:
         if isinstance(value, Subscript):
             # Pine's history operator preserves the value type: a
             # ``Series<line>`` read such as ``h[1]`` is a scalar ``line``
-            # handle, not the legacy numeric fallback.  Keep this refinement
-            # drawing-only; collection subscripts have separate array/map
-            # semantics and primitive history inference already flows through
-            # PineType in ``_visit_Subscript``.
+            # handle and ``c[1]`` of a user-defined object the reference ``c``
+            # held (fixtures/udt_history_tv), not the legacy numeric fallback.
+            # Collection subscripts have separate array/map semantics and
+            # primitive history inference already flows through PineType in
+            # ``_visit_Subscript``.
             receiver_spec = self._type_spec_from_expr(value.object)
             if (receiver_spec is not None
                     and receiver_spec.kind == "udt"
-                    and receiver_spec.name in _DRAWING_TYPE_NAMES):
+                    and (receiver_spec.name in _DRAWING_TYPE_NAMES
+                         or receiver_spec.name in self._udt_fields)):
+                return receiver_spec
+            if (receiver_spec is not None
+                    and receiver_spec.kind in ("array", "matrix")
+                    and isinstance(value.object, Identifier)
+                    and self._collection_history_is_supported(value)):
+                # An array's or a matrix's history is a collection of its
+                # type (fixtures/array_history_tv; pineforge_codegen/
+                # collection_history.py); a legacy read keeps the element
+                # typing it had.
                 return receiver_spec
             return None
         if isinstance(value, IfStmt):

package/pineforge_codegen/ast_nodes.py

@@ -6,6 +6,13 @@ from typing import Any
 from pineforge_codegen.errors import SourceLocation
 
 
+# A MemberAccess (a field, or a method call's callee) written straight after a
+# history reference, ``c[1].v`` / ``b[1].get_top()``: TradingView refuses it
+# (CE10011 / CE10010) and takes ``(c[1]).v``, which parses to the same nodes.
+# Set by the parser, refused by the support checker.
+HISTORY_MEMBER_ANNOTATION = "pf_history_member"
+
+
 # ---------------------------------------------------------------------------
 # Base class
 # ---------------------------------------------------------------------------