@pineforge/codegen-pyodide 0.10.3 → 1.0.0-rc.1

package/pineforge_codegen/pine_libraries.py

@@ -0,0 +1,266 @@
+"""Where the sources of a script's imported Pine libraries come from.
+
+``transpile(source, libraries=...)`` takes them from the caller: a mapping of
+import path (``user/name/version``) to library source text. With
+``libraries=None`` they come from the environment the campaign's case runner
+sets (pineforge-workflow ``docs/xsym-requests.md``, "The environment
+contract"):
+
+- ``PINEFORGE_PINE_LIBRARIES=<dir>``: ``<dir>/libraries.json``
+  (``pineforge-pine-libraries/v1``) and ``<dir>/<user>/<name>/<version>.pine``,
+  the bytes pine-facade served. The directory is case-wide: it holds every
+  library any probe of the case pins.
+- ``PINEFORGE_REQUESTS_ROOT=<dir>``: one ``<slug>/requests.json``
+  (``pineforge-probe-requests/v1``) per probe of the case that pins data, with
+  every file it references under ``<slug>/files/<sha256>``.
+
+A script's imports resolve only through its own manifest, never through the
+case-wide directory alone: otherwise a probe that pins no library would
+transpile on a shard neighbour's pin, and measure differently by case
+composition. ``transpile()`` is handed the source text, not a slug (the frozen
+verifier, pineforge-lab 3bac0b7b ``scripts/verify-engine-local.py:1698``, calls
+``transpile((d / "strategy.pine").read_text())``), so the manifest is found by
+content: the one ``requests.json`` whose ``probe.strategySha256`` is the sha256
+of the script. That sha is of the strategy file's bytes, and ``read_text()``
+folds ``\\r\\n`` and ``\\r`` line ends to ``\\n``: the text matches a manifest
+when its UTF-8 bytes do, or the same bytes with every ``\\n`` spelled
+``\\r\\n``, or ``\\r``. A file mixing line ends matches none. No manifest, or
+more than one, resolves no library: every import keeps its refusal. An import
+the manifest does not pin is refused by name whatever ``libraries.json`` lists,
+and each source is verified against the manifest entry's ``sha256``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+LIBRARIES_ENV = "PINEFORGE_PINE_LIBRARIES"
+REQUESTS_ENV = "PINEFORGE_REQUESTS_ROOT"
+LIBRARIES_SCHEMA = "pineforge-pine-libraries/v1"
+REQUESTS_SCHEMA = "pineforge-probe-requests/v1"
+
+_SHA256_RE = re.compile(r"[0-9a-f]{64}")
+
+
+class LibraryResolveError(Exception):
+    """An import's library source cannot be used; the message names it."""
+
+
+@dataclass(frozen=True)
+class LibrarySource:
+    path: str
+    text: str
+    sha256: str | None
+
+
+def source_digests(text: str) -> set[str]:
+    """sha256 of every byte form ``text`` can have been read from with
+    ``Path.read_text()``: its own UTF-8 bytes and, for a text holding no
+    ``\\r``, the same bytes with CRLF or CR line ends."""
+    forms = [text]
+    if "\r" not in text and "\n" in text:
+        forms += [text.replace("\n", "\r\n"), text.replace("\n", "\r")]
+    digests = set()
+    for form in forms:
+        try:
+            data = form.encode("utf-8", "surrogateescape")
+        except UnicodeEncodeError:
+            continue
+        digests.add(hashlib.sha256(data).hexdigest())
+    return digests
+
+
+class LibraryResolver:
+    """The library sources one transpile may read.
+
+    ``configured`` is False when neither ``libraries=`` nor a requests
+    manifest of the script applies; every import then keeps today's
+    refusal, with ``reason`` (if any) saying why none applied.
+    """
+
+    configured = True
+    reason: str | None = None
+
+    def pinned(self, path: str) -> bool:
+        raise NotImplementedError
+
+    def load(self, path: str) -> LibrarySource:
+        raise NotImplementedError
+
+
+class _Unconfigured(LibraryResolver):
+    configured = False
+
+    def __init__(self, reason: str | None = None) -> None:
+        self.reason = reason
+
+    def pinned(self, path: str) -> bool:
+        return False
+
+    def load(self, path: str) -> LibrarySource:
+        raise LibraryResolveError(self.reason or "no library sources are configured")
+
+
+class _Provided(LibraryResolver):
+    """``transpile(..., libraries={path: source})``: the caller's sources."""
+
+    def __init__(self, sources: Mapping[str, str]) -> None:
+        self._sources: dict[str, str] = {}
+        for path, text in sources.items():
+            if isinstance(text, (bytes, bytearray)):
+                text = bytes(text).decode("utf-8")
+            if not isinstance(text, str):
+                raise TypeError(
+                    f"libraries[{path!r}] must be the library's source text")
+            self._sources[str(path)] = text
+
+    def pinned(self, path: str) -> bool:
+        return path in self._sources
+
+    def load(self, path: str) -> LibrarySource:
+        if path not in self._sources:
+            raise LibraryResolveError(
+                f"library '{path}' is not among the libraries passed to transpile()")
+        text = self._sources[path]
+        return LibrarySource(path, text, None)
+
+
+class _Pinned(LibraryResolver):
+    """The libraries one probe's requests manifest pins."""
+
+    def __init__(self, library_dir: Path, requests_root: Path, slug_dir: Path,
+                 manifest: dict) -> None:
+        self._library_dir = library_dir
+        self._requests_root = requests_root
+        self._slug_dir = slug_dir
+        self._where = f"{slug_dir.name}/requests.json"
+        self._pins: dict[str, dict] = {}
+        for entry in manifest.get("libraries") or []:
+            if isinstance(entry, dict) and isinstance(entry.get("import"), str):
+                self._pins.setdefault(entry["import"], entry)
+        self._index: dict | None = None
+
+    def pinned(self, path: str) -> bool:
+        return path in self._pins
+
+    def _library_index(self) -> dict:
+        if self._index is None:
+            self._index = {}
+            try:
+                doc = json.loads((self._library_dir / "libraries.json")
+                                 .read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                doc = None
+            if (isinstance(doc, dict)
+                    and doc.get("schemaVersion", LIBRARIES_SCHEMA) == LIBRARIES_SCHEMA
+                    and isinstance(doc.get("libraries"), dict)):
+                self._index = {k: v for k, v in doc["libraries"].items()
+                               if isinstance(v, dict)}
+        return self._index
+
+    def load(self, path: str) -> LibrarySource:
+        entry = self._pins.get(path)
+        if entry is None:
+            raise LibraryResolveError(
+                f"library '{path}' is not pinned by this script's requests "
+                f"manifest ({self._where})")
+        access = entry.get("access")
+        if not (isinstance(access, str) and access.startswith("open")):
+            raise LibraryResolveError(
+                f"library '{path}' is not open-source (access {access!r}); "
+                "PineForge inlines open libraries only")
+        sha = entry.get("sha256")
+        if not (isinstance(sha, str) and _SHA256_RE.fullmatch(sha)):
+            raise LibraryResolveError(
+                f"library '{path}' has no valid sha256 in {self._where}")
+        indexed = self._library_index().get(path) or {}
+        indexed_access = indexed.get("access")
+        if indexed_access is not None and not (
+                isinstance(indexed_access, str) and indexed_access.startswith("open")):
+            raise LibraryResolveError(
+                f"library '{path}' is not open-source (access {indexed_access!r} "
+                "in libraries.json); PineForge inlines open libraries only")
+        candidates = [self._slug_dir / "files" / sha]
+        rel = indexed.get("file")
+        if isinstance(rel, str) and rel and not Path(rel).is_absolute() \
+                and ".." not in Path(rel).parts:
+            candidates.append(self._library_dir / rel)
+        candidates.append(self._library_dir / f"{path}.pine")
+        seen: list[str] = []
+        for candidate in dict.fromkeys(candidates):
+            try:
+                data = candidate.read_bytes()
+            except OSError:
+                continue
+            got = hashlib.sha256(data).hexdigest()
+            if got != sha:
+                seen.append(got)
+                continue
+            try:
+                text = data.decode("utf-8")
+            except UnicodeDecodeError as exc:
+                raise LibraryResolveError(
+                    f"library '{path}' source is not UTF-8 text") from exc
+            return LibrarySource(path, text, sha)
+        if seen:
+            raise LibraryResolveError(
+                f"library '{path}' source does not match its pinned sha256 "
+                f"{sha[:12]}... (found {seen[0][:12]}...)")
+        raise LibraryResolveError(
+            f"library '{path}' source is missing (sha256 {sha[:12]}... under "
+            f"${REQUESTS_ENV}/{self._slug_dir.name}/files and ${LIBRARIES_ENV})")
+
+
+def _manifest_for(text: str, requests_root: Path) -> tuple[Path | None, dict | None, str]:
+    """The one requests manifest under ``requests_root`` pinning ``text``."""
+    digests = source_digests(text)
+    matches: list[tuple[Path, dict]] = []
+    try:
+        candidates = sorted(requests_root.glob("*/requests.json"))
+    except OSError:
+        candidates = []
+    for path in candidates:
+        try:
+            doc = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            continue
+        if not isinstance(doc, dict) or doc.get("schemaVersion") != REQUESTS_SCHEMA:
+            continue
+        probe = doc.get("probe")
+        sha = probe.get("strategySha256") if isinstance(probe, dict) else None
+        if isinstance(sha, str) and sha.lower() in digests:
+            matches.append((path.parent, doc))
+    if len(matches) == 1:
+        return matches[0][0], matches[0][1], ""
+    if not matches:
+        return None, None, (
+            f"no requests manifest under ${REQUESTS_ENV} pins this script's "
+            "source")
+    slugs = ", ".join(sorted(p.name for p, _ in matches))
+    return None, None, (
+        f"{len(matches)} requests manifests under ${REQUESTS_ENV} pin this "
+        f"script's source ({slugs})")
+
+
+def library_resolver(source: str, libraries: Mapping[str, str] | None) -> LibraryResolver:
+    """The library sources ``transpile(source, libraries=libraries)`` reads."""
+    if libraries is not None:
+        return _Provided(libraries)
+    library_dir = os.environ.get(LIBRARIES_ENV)
+    if not library_dir:
+        return _Unconfigured()
+    requests_root = os.environ.get(REQUESTS_ENV)
+    if not requests_root:
+        return _Unconfigured(
+            f"${LIBRARIES_ENV} is set but ${REQUESTS_ENV} is not, so no "
+            "requests manifest pins this script's libraries")
+    slug_dir, manifest, reason = _manifest_for(source, Path(requests_root))
+    if manifest is None:
+        return _Unconfigured(reason)
+    return _Pinned(Path(library_dir), Path(requests_root), slug_dir, manifest)

package/pineforge_codegen/pine_spelling.py

@@ -0,0 +1,216 @@
+"""Pine source spellings that the analyzer hands the codegen as strings.
+
+TA constructor arguments (``TACallSite.ctor_args``), user-function call-site
+arguments and class-scope derived lengths travel as Pine source text that the
+codegen scans for identifiers, substitutes into, folds and re-parses. An
+inline ``input.*()`` call is a legitimate leaf of such an expression
+(``ta.ema(close, input.int(9, "fast"))``), and its own argument text -- the
+title string, keyword names, an ``options`` list -- is input metadata, not
+part of the expression. These helpers keep string literals intact and let a
+caller treat each inline input call as one leaf.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import Callable, Iterator
+
+from .ast_nodes import (
+    ASTNode, BinOp, BoolLiteral, FuncCall, FuncDef, Identifier, MemberAccess,
+    MethodDef, NaLiteral, NumberLiteral, StringLiteral, Subscript, Ternary,
+    TupleLiteral, UnaryOp, VarDecl,
+)
+
+_STRING_OR_IDENT = re.compile(
+    r'"(?:[^"\\]|\\.)*"|\'(?:[^\'\\]|\\.)*\'|[A-Za-z_][A-Za-z_0-9]*'
+)
+
+
+def is_input_call(node) -> bool:
+    """``input(...)`` or ``input.<type>(...)``, judged on the spelling alone."""
+    if not isinstance(node, FuncCall):
+        return False
+    callee = node.callee
+    if isinstance(callee, Identifier):
+        return callee.name == "input"
+    return (isinstance(callee, MemberAccess)
+            and isinstance(callee.object, Identifier)
+            and callee.object.name == "input")
+
+
+def pine_string_literal(value: str) -> str:
+    """The double-quoted literal the lexer reads back as ``value``."""
+    return '"' + (value.replace("\\", "\\\\").replace('"', '\\"')
+                  .replace("\n", "\\n").replace("\t", "\\t")) + '"'
+
+
+def spell_input_call(node: FuncCall, title: str | None = None) -> str | None:
+    """An input call spelled with every argument, keywords included, so the
+    text re-parses to the same call. None when an argument is not one of the
+    constant shapes an input takes (literal, name, ``display.none``-style
+    member, signed number, ``options`` list).
+
+    ``title`` spells an untitled call's key as its ``title=`` argument: the
+    re-parsed call has lost the declaration that named it."""
+    parts = [_spell_input_arg(a) for a in node.args]
+    parts += [None if (v := _spell_input_arg(value)) is None else f"{key}={v}"
+              for key, value in node.kwargs.items()]
+    if title is not None:
+        parts.append(f"title={pine_string_literal(title)}")
+    callee = _spell_input_arg(node.callee)
+    if callee is None or None in parts:
+        return None
+    return f"{callee}({', '.join(parts)})"
+
+
+def _spell_input_arg(node) -> str | None:
+    if isinstance(node, NumberLiteral):
+        return str(node.value)
+    if isinstance(node, StringLiteral):
+        return pine_string_literal(node.value)
+    if isinstance(node, BoolLiteral):
+        return "true" if node.value else "false"
+    if isinstance(node, NaLiteral):
+        return "na"
+    if isinstance(node, Identifier):
+        return node.name
+    if isinstance(node, MemberAccess):
+        obj = _spell_input_arg(node.object)
+        return None if obj is None else f"{obj}.{node.member}"
+    if isinstance(node, UnaryOp) and node.op in ("-", "+"):
+        operand = _spell_input_arg(node.operand)
+        return None if operand is None else f"{node.op}{operand}"
+    if isinstance(node, TupleLiteral):
+        elems = [_spell_input_arg(e) for e in node.elements]
+        return None if None in elems else "[" + ", ".join(elems) + "]"
+    return None
+
+
+def expr_start(node: ASTNode) -> ASTNode:
+    """The leftmost sub-node of an expression, whose location is the
+    expression's first token (a call's own location is its ``(``)."""
+    while True:
+        if isinstance(node, FuncCall):
+            child = node.callee
+        elif isinstance(node, (MemberAccess, Subscript)):
+            child = node.object
+        elif isinstance(node, BinOp):
+            child = node.left
+        elif isinstance(node, Ternary):
+            child = node.condition
+        else:
+            return node
+        if getattr(child, "loc", None) is None:
+            return node
+        node = child
+
+
+# Statement fields holding a local block. Pine declares script inputs at
+# global scope only.
+_LOCAL_SCOPE_FIELDS = frozenset({"body", "else_body", "default_body"})
+
+
+def global_input_calls(node) -> Iterator[FuncCall]:
+    """The input calls ``node`` makes at global scope, in source order -- not
+    inside if/for/while/switch blocks or callable bodies."""
+    if isinstance(node, list):
+        for item in node:
+            yield from global_input_calls(item)
+        return
+    if not isinstance(node, ASTNode) or isinstance(node, (FuncDef, MethodDef)):
+        return
+    if is_input_call(node):
+        yield node
+        return
+    for f in dataclasses.fields(node):
+        if f.name in ("loc", "annotations") or f.name in _LOCAL_SCOPE_FIELDS:
+            continue
+        value = getattr(node, f.name)
+        if f.name == "cases":
+            value = [case_expr for case_expr, _stmts in value]
+        elif isinstance(value, dict):
+            value = list(value.values())
+        yield from global_input_calls(value)
+
+
+def input_binding_names(body) -> dict[int, str]:
+    """``id(call) -> name`` for every global-scope input call a declaration
+    holds: bound straight to ``name = input.*()`` or nested anywhere in the
+    declaration's value (``n = input.int(9) * 2``, ``x = ta.ema(close,
+    input.int(9))``, a ``var`` initializer). TradingView keys an input with no
+    title by that variable name ("If not specified, the variable name is used
+    as the input's title"), and so does every PineForge getter and the
+    manifest."""
+    names: dict[int, str] = {}
+    for stmt in body or []:
+        if not isinstance(stmt, VarDecl):
+            continue
+        for node in global_input_calls(stmt):
+            names[id(node)] = stmt.name
+    return names
+
+
+def blank_string_literals(text: str) -> str:
+    """``text`` with every string literal emptied, so an identifier scan
+    never reads a literal's contents (``mode == "Fast"``) as a name."""
+    def _one(match: re.Match) -> str:
+        token = match.group(0)
+        return token[0] * 2 if token[0] in "\"'" else token
+    return _STRING_OR_IDENT.sub(_one, text)
+
+
+def sub_identifiers(text: str, repl: Callable[[re.Match], str]) -> str:
+    """``re.sub`` over identifier tokens only; string literals pass through."""
+    def _one(match: re.Match) -> str:
+        if match.group(0)[0] in "\"'":
+            return match.group(0)
+        return repl(match)
+    return _STRING_OR_IDENT.sub(_one, text)
+
+
+def input_call_spans(text: str) -> list[tuple[int, int]]:
+    """``(start, end)`` of each ``input(...)`` / ``input.<type>(...)`` call in
+    ``text``. Parentheses inside string literals do not count, and a call
+    nested in another input call's arguments is part of the outer span."""
+    spans: list[tuple[int, int]] = []
+    if "input" not in text:
+        return spans
+    for match in _STRING_OR_IDENT.finditer(text):
+        start = match.start()
+        if match.group(0) != "input" or (spans and start < spans[-1][1]):
+            continue
+        if start > 0 and text[start - 1] == ".":
+            continue  # a member named ``input``, not the namespace
+        pos = match.end()
+        member = re.match(r"\.[A-Za-z_][A-Za-z_0-9]*", text[pos:])
+        if member is not None:
+            pos += member.end()
+        if pos >= len(text) or text[pos] != "(":
+            continue
+        end = _matching_paren(text, pos)
+        if end is not None:
+            spans.append((start, end))
+    return spans
+
+
+def _matching_paren(text: str, open_pos: int) -> int | None:
+    """Index just past the ``)`` closing the ``(`` at ``open_pos``."""
+    depth = 0
+    pos = open_pos
+    while pos < len(text):
+        ch = text[pos]
+        if ch in "\"'":
+            literal = _STRING_OR_IDENT.match(text, pos)
+            if literal is None:
+                return None  # unterminated string literal
+            pos = literal.end()
+            continue
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth -= 1
+            if depth == 0:
+                return pos + 1
+        pos += 1
+    return None

package/pineforge_codegen/pragmas.py

@@ -14,7 +14,10 @@ Why a pre-pass?
     :class:`Parser` machinery used for normal Pine expressions. This
     keeps a single source of truth for Pine syntax and ensures pragma
     expressions support the full grammar (logical operators, member
-    access, function calls, ternaries, ...).
+    access, function calls, ternaries, ...). A ``ta.*`` call inside a
+    pragma is not computed, though: no TA state is allocated for it, so
+    the codegen renders it as ``na`` with an ``/* unsupported */`` marker.
+    Trace a script variable that holds the ``ta.*`` value instead.
 
 Pragma syntax (kept deliberately strict so unrelated comments are
 untouched)::
@@ -40,12 +43,14 @@ import re
 from dataclasses import dataclass
 from typing import Any
 
+from .errors import CompileError, SourceLocation
 from .lexer import Lexer
-from .parser import Parser
+from .limits import TimeBudget, check_ast_depth
+from .parser import ParseError, Parser
 
 
-# Anchored to start/end of line so a stray ``// @pf-trace`` substring
-# inside a string literal or block comment cannot match. ``\s+`` after
+# Anchored to start/end of line; lexical string spans are excluded below.
+# ``\s+`` after
 # ``//`` requires at least one space before ``@pf-trace`` (the spec is
 # ``// @pf-trace ``, distinct from Pine's ``//@version=N``).
 _PRAGMA_RE = re.compile(
@@ -53,6 +58,27 @@ _PRAGMA_RE = re.compile(
 )
 
 
+class _StringSpanLexer(Lexer):
+    """Use the Pine lexer itself to locate lines inside string literals."""
+
+    def __init__(self, source: str, filename: str = "<input>",
+                 budget: TimeBudget | None = None) -> None:
+        super().__init__(source, filename=filename, budget=budget)
+        self.string_lines: set[int] = set()
+
+    def _record_string_lines(self, start_line: int) -> None:
+        if self.line > start_line:
+            self.string_lines.update(range(start_line + 1, self.line + 1))
+
+    def _read_multiline(self, quote: str, start_line: int, start_col: int) -> None:
+        super()._read_multiline(quote, start_line, start_col)
+        self._record_string_lines(start_line)
+
+    def _read_quoted(self, quote: str, start_line: int, start_col: int) -> None:
+        super()._read_quoted(quote, start_line, start_col)
+        self._record_string_lines(start_line)
+
+
 @dataclass
 class PfTracePragma:
     """One ``// @pf-trace name=expr`` annotation extracted from Pine source.
@@ -75,7 +101,8 @@ class PfTracePragma:
     line: int
 
 
-def extract_pf_trace_pragmas(source: str) -> list[PfTracePragma]:
+def extract_pf_trace_pragmas(source: str, *, filename: str = "<input>",
+                            budget: TimeBudget | None = None) -> list[PfTracePragma]:
     """Scan ``source`` for ``// @pf-trace`` line comments.
 
     Returns the pragmas in source order. The expression on the
@@ -93,10 +120,22 @@ def extract_pf_trace_pragmas(source: str) -> list[PfTracePragma]:
         scripts) — callers should treat this as the zero-overhead
         path.
     """
+    candidates = [(lineno, match)
+                  for lineno, raw in enumerate(source.splitlines(), start=1)
+                  if (match := _PRAGMA_RE.match(raw)) is not None]
+    if not candidates:
+        return []
+    lexer = _StringSpanLexer(source, filename=filename, budget=budget)
+    try:
+        lexer.tokenize()
+    except CompileError:
+        # This lexical pass only finds string spans. The main Lexer run owns
+        # syntax diagnostics; extraction itself has historically accepted
+        # arbitrary source text, including malformed block comments.
+        pass
     pragmas: list[PfTracePragma] = []
-    for lineno, raw in enumerate(source.splitlines(), start=1):
-        m = _PRAGMA_RE.match(raw)
-        if not m:
+    for lineno, m in candidates:
+        if lineno in lexer.string_lines:
             continue
         name = m.group(1)
         expr_source = m.group(2)
@@ -104,8 +143,23 @@ def extract_pf_trace_pragmas(source: str) -> list[PfTracePragma]:
         # the expression body in isolation. ``Parser._parse_expression``
         # is the same entry the statement parser uses for RHS values,
         # so anything legal in ``x = <expr>`` is legal here.
-        tokens = Lexer(expr_source).tokenize()
-        expr_node = Parser(tokens, source=expr_source)._parse_expression()
+        try:
+            tokens = Lexer(expr_source, filename=filename, budget=budget).tokenize()
+            parser = Parser(tokens, source=expr_source, filename=filename,
+                            budget=budget)
+            try:
+                expr_node = parser._parse_expression()
+                parser._expect_statement_end()
+            except ParseError as error:
+                parser._raise_syntax_error(error)
+            check_ast_depth(expr_node, filename)
+        except CompileError as exc:
+            for diagnostic in exc.diagnostics:
+                loc = diagnostic.location
+                diagnostic.location = SourceLocation(
+                    filename, loc.line + lineno - 1, loc.col, loc.end_col,
+                )
+            raise CompileError(exc.diagnostics) from exc
         pragmas.append(
             PfTracePragma(
                 name=name,