@pineforge/codegen-pyodide 0.10.4 → 1.0.0-rc.1

package/README.md

@@ -1,25 +1,25 @@
 # @pineforge/codegen-pyodide
 
 Gate-validated Pyodide payload for the PineScript v6 → C++ transpiler. Built and
-published from `pineforge-codegen-oss` by `.github/workflows/publish-pyodide.yml`.
+published from `pineforge-codegen-oss` by `.github/workflows/publish-pyodide.yml`,
+with the same version as the matching `pineforge-codegen` release on PyPI.
+Stable releases are on the `latest` dist-tag. A prerelease (for example
+`1.0.0-rc.1`) is published on `next` only.
 
 ## Contents
 - `pineforge_codegen-<version>.tar.gz` — the gate-validated archive (unpack into Pyodide).
 - `pineforge_codegen/` — unpacked Python source (put on `PYTHONPATH` for Node oracle/grammar tooling).
-- `tables.json` — introspected codegen tables (the app renders `tables.generated.ts` from this).
+- `tables.json` — introspected codegen tables (PineForge's web app renders its tables from this).
 - `release.json` — `{ codegen, pyodide, python, emscripten, sha256 }`.
-- `index.mjs` — `release`, `tables`, `archivePath`, `codegenSourceDir`.
+- `glue.py` — the `transpile_json(source)` glue from the repository's `gate/glue.py`; its JSON
+  envelopes are described in the repository's `docs/PUBLIC_CONTRACT.md`.
+- `transpile.worker.mjs` — an ES module worker that runs the glue in Pyodide.
+- `index.mjs` — exports `release`, `tables`, `archivePath`, `sourceRoot`, `codegenSourceDir`,
+  `workerPath` and `glue`.
 
-## One-time bootstrap (maintainer, manual)
-This is a NEW package; npm OIDC Trusted Publishing can only be configured after
-the package exists:
-1. Build locally: `npm run gate:full && node scripts/build-npm-package.mjs`.
-2. From `npm/`, do the first publish with a granular npm token:
-   `npm publish --access=public` (one time).
-3. On npmjs.com, configure the package's Trusted Publisher: GitHub Actions,
-   repo `pineforge-4pass/pineforge-codegen-oss`, workflow `publish-pyodide.yml`.
-4. Thereafter releases publish via OIDC (no token): run `publish-pyodide.yml`
-   via **workflow_dispatch with `dry_run=false`**. NOTE: tag pushes currently run
-   a DRY-RUN only (a safe validation) — the sole real-publish path is dispatch +
-   `dry_run=false`. To switch to tag-driven releases later, change the publish
-   condition in the workflow.
+## Publishing (maintainers)
+A `v*` tag push, which the release workflow makes, publishes through npm OIDC Trusted
+Publishing after the dependency audit and the full parity gate pass; published versions
+carry npm provenance. A manual `workflow_dispatch` defaults to a dry run; `dry_run=false`
+publishes for real. The one-time bootstrap is done: 0.7.0 was the manual first publish,
+and every version since 0.7.1 carries provenance from this workflow.

package/glue.py

@@ -13,23 +13,31 @@ from pineforge_codegen import transpile_full
 from pineforge_codegen.errors import CompileError
 
 
+def _diagnostic_entries(diagnostics) -> list:
+    diags = []
+    for d in diagnostics:
+        loc = d.location
+        message = d.message + " — " + d.hint if getattr(d, "hint", None) else d.message
+        entry = {
+            "line": loc.line if loc else 1,
+            "col": loc.col if loc else 1,
+            "message": message,
+            "severity": getattr(d.level, "value", "error"),
+        }
+        end_col = getattr(loc, "end_col", None) if loc else None
+        if end_col is not None:
+            entry["endCol"] = end_col
+        diags.append(entry)
+    return diags
+
+
 def transpile_json(source: str) -> str:
     try:
         full = transpile_full(source)
     except CompileError as e:
-        diags = []
-        for d in e.diagnostics:
-            loc = d.location
-            message = d.message + " — " + d.hint if getattr(d, "hint", None) else d.message
-            entry = {
-                "line": loc.line if loc else 1,
-                "col": loc.col if loc else 1,
-                "message": message,
-                "severity": getattr(d.level, "value", "error"),
-            }
-            end_col = getattr(loc, "end_col", None) if loc else None
-            if end_col is not None:
-                entry["endCol"] = end_col
-            diags.append(entry)
-        return json.dumps({"ok": False, "error": str(e), "diagnostics": diags})
-    return json.dumps({"ok": True, "cpp": full["cpp"], "inputs": full["inputs"], "strategyParams": full["strategyParams"]})
+        return json.dumps({"ok": False, "error": str(e),
+                           "diagnostics": _diagnostic_entries(e.diagnostics)})
+    # A script that transpiled carries its warnings in the same entry format.
+    return json.dumps({"ok": True, "cpp": full["cpp"], "inputs": full["inputs"],
+                       "strategyParams": full["strategyParams"],
+                       "diagnostics": _diagnostic_entries(full["diagnostics"])})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pineforge/codegen-pyodide",
-  "version": "0.10.4",
+  "version": "1.0.0-rc.1",
   "description": "Gate-validated Pyodide payload for the PineScript v6 -> C++ transpiler: archive (run in Pyodide), unpacked source, introspected tables, and release metadata.",
   "type": "module",
   "main": "index.mjs",

package/pineforge_codegen/__init__.py

@@ -1,15 +1,107 @@
 """PineScript v6 to C++ transpiler."""
 
+from collections.abc import Mapping
+
 from .lexer import Lexer
 from .parser import Parser
 from .analyzer import Analyzer
 from .codegen import CodeGen
+from .errors import CompileError, Level, Phase
+from .external_requests import lower_no_data_requests
+from .builtin_keywords import bind_builtin_keywords
 from .finite_ta_length import expand_finite_choice_extrema_lengths
+from .library_inline import inline_libraries
+from .limits import TimeBudget, check_ast_depth, check_source_size, ensure_recursion_headroom
 from .pragmas import extract_pf_trace_pragmas
+from .security_contexts import specialize_security_contexts
+from .block_locals import rename_block_locals
+from .support_checker import check_support as _support_diagnostics
 from .support_checker import check_support_or_raise
 
 
-def transpile(pine_source: str, *, check_support: bool = True, filename: str = "<input>") -> str:
+def _parse_bounded(pine_source: str, filename: str, budget: TimeBudget | None = None):
+    ensure_recursion_headroom()
+    check_source_size(pine_source, filename)
+    budget = budget or TimeBudget(filename)
+    pragmas = extract_pf_trace_pragmas(
+        pine_source, filename=filename, budget=budget,
+    )
+    budget.check(phase=Phase.LEXER)
+    tokens = Lexer(pine_source, filename=filename, budget=budget).tokenize()
+    budget.check(phase=Phase.LEXER)
+    ast = Parser(tokens, source=pine_source, filename=filename,
+                 budget=budget).parse()
+    check_ast_depth(ast, filename)
+    budget.check(phase=Phase.PARSER)
+    return ast, pragmas, budget
+
+
+def _generate(pine_source: str, check_support: bool, filename: str,
+              libraries: Mapping[str, str] | None = None):
+    """The pipeline, first without the per-call clones of functions that read
+    a ``session.<flag>[k]``, as scripts compiled before such reads were
+    supported; then again with the functions whose reads that C++ holds
+    cloned as well, until the C++ asks for no more (a clone can make a
+    caller's read reach the C++: its callers are cloned per call site, which
+    can retype their parameters). A read in an argument the codegen leaves
+    out never costs a clone (the clones of a function called along a deep
+    call tree grow with every path to it).
+
+    Each pass inlines the libraries the script imports before the support
+    check (``library_inline``). A declaration in a top-level block whose type
+    the member of its name cannot hold gets a name of its own, and the
+    pipeline runs again (``block_locals``).
+
+    Returns ``(codegen, ctx, cpp, support_diagnostics)``."""
+    budget = None
+    clones: frozenset[str] = frozenset()
+    renamed: frozenset = frozenset()
+    while True:
+        ast, pragmas, budget = _parse_bounded(pine_source, filename, budget)
+        ast = inline_libraries(ast, pine_source, libraries=libraries,
+                               filename=filename, budget=budget)
+        rename_block_locals(ast, renamed)
+        support_diagnostics = []
+        if check_support:
+            support_diagnostics = _support_diagnostics(ast, filename=filename)
+            if any(d.level == Level.ERROR for d in support_diagnostics):
+                raise CompileError(support_diagnostics)
+        budget.check(phase=Phase.ANALYZER)
+        ast = lower_no_data_requests(ast)
+        ast = specialize_security_contexts(ast, filename=filename)
+        ast = bind_builtin_keywords(expand_finite_choice_extrema_lengths(ast))
+        check_ast_depth(ast, filename)
+        budget.check(phase=Phase.ANALYZER)
+        ctx = Analyzer(ast, filename=filename, budget=budget,
+                       session_clones=clones).analyze()
+        budget.check(phase=Phase.ANALYZER)
+        # Attach after analysis: pragma expressions are not part of the
+        # program body, so the analyzer never inspects them; the codegen
+        # consumes them directly from the context to emit the on_bar tail
+        # ``if (trace_enabled_) { trace(...); ... }`` block.
+        ctx.pf_trace_pragmas = pragmas
+        gen = CodeGen(ctx, budget=budget)
+        cpp = gen.generate()
+        budget.check(phase=Phase.CODEGEN)
+        apart = gen.block_locals_needing_names - renamed
+        if apart:
+            renamed |= apart
+            del ast, ctx, gen, cpp
+            continue
+        needed = gen.session_functions_needing_clones
+        if not needed:
+            return gen, ctx, cpp, support_diagnostics
+        # Only uncloned functions ask, so the set grows each time and the
+        # loop ends by the time every function that reads a flag at an
+        # offset is cloned.
+        if needed <= clones:
+            raise AssertionError("a cloned function asked for session clones again")
+        clones |= needed
+        del ast, ctx, gen, cpp
+
+
+def transpile(pine_source: str, *, check_support: bool = True, filename: str = "<input>",
+              libraries: Mapping[str, str] | None = None) -> str:
     """Transpile PineScript v6 source code to C++ code.
 
     Semantic rules enforced in :class:`Analyzer` (before codegen), including:
@@ -17,10 +109,10 @@ def transpile(pine_source: str, *, check_support: bool = True, filename: str = "
 
     Before analysis, :func:`support_checker.check_support_or_raise` rejects
     scripts that use language constructs PineForge cannot faithfully execute
-    (e.g. ``indicator()`` declarations, prohibited variables such as
-    ``bar_index``, disallowed ``request.security`` parameters). Pass
-    ``check_support=False`` to bypass this gate (intended for tests of legacy
-    fixtures only).
+    (e.g. ``indicator()`` declarations, ``request.seed``, disallowed
+    ``request.security`` parameters) and warns about approximated ones (e.g.
+    ``bar_index``). Pass ``check_support=False`` to bypass this gate
+    (experimental; intended for tests of legacy fixtures only).
 
     ``// @pf-trace name=expr`` pragmas are extracted from ``pine_source``
     via a pre-pass (the lexer strips comments before the parser sees them)
@@ -36,60 +128,59 @@ def transpile(pine_source: str, *, check_support: bool = True, filename: str = "
             ``file:line:col`` shown in ``CompileError`` (both ``str()`` and
             :meth:`~pineforge_codegen.errors.CompileError.format`) points back
             at the caller's file. Defaults to ``"<input>"``.
+        libraries: The sources of the Pine libraries the script imports,
+            by import path (``{"user/name/version": source_text}``); each
+            import the script uses is inlined from its source (see
+            :mod:`pineforge_codegen.library_inline`). ``None`` (default)
+            reads them from ``$PINEFORGE_PINE_LIBRARIES`` through the
+            script's own requests manifest under ``$PINEFORGE_REQUESTS_ROOT``
+            (see :mod:`pineforge_codegen.pine_libraries`); with neither, an
+            import is refused as before.
 
     Returns:
         Generated C++ source string.
     """
-    pragmas = extract_pf_trace_pragmas(pine_source)
-    tokens = Lexer(pine_source, filename=filename).tokenize()
-    ast = Parser(tokens, source=pine_source, filename=filename).parse()
-    if check_support:
-        check_support_or_raise(ast, filename=filename)
-    ast = expand_finite_choice_extrema_lengths(ast)
-    ctx = Analyzer(ast, filename=filename).analyze()
-    # Attach after analysis: pragma expressions are not part of the
-    # program body, so the analyzer never inspects them; the codegen
-    # consumes them directly from the context to emit the on_bar tail
-    # ``if (trace_enabled_) { trace(...); ... }`` block.
-    ctx.pf_trace_pragmas = pragmas
-    return CodeGen(ctx).generate()
+    _gen, _ctx, cpp, _support = _generate(pine_source, check_support, filename, libraries)
+    return cpp
 
 
 def transpile_full(pine_source: str, *, check_support: bool = True,
-                   filename: str = "<input>") -> dict:
+                   filename: str = "<input>",
+                   libraries: Mapping[str, str] | None = None) -> dict:
     """Transpile like :func:`transpile`, plus the host-UI input manifest.
 
-    Runs ONE pipeline pass (Lexer -> Parser -> support check -> Analyzer ->
-    CodeGen.generate) and returns the generated C++ alongside the data the
+    Runs the pipeline (Lexer -> Parser -> support check -> Analyzer ->
+    CodeGen.generate) once, again while the C++ holds a
+    ``session.<flag>[k]`` read of a function not yet cloned for it (see
+    ``_generate``), and returns the generated C++ alongside the data the
     cloud Studio needs to auto-build a backtest "override params" form:
 
     - ``cpp``: the generated C++ source (identical to :func:`transpile`).
-    - ``inputs``: a list of ``InputDef`` dicts (one per top-level
-      ``var = input.*(...)`` declaration). Each has ``title`` / ``type`` /
+    - ``inputs``: a list of ``InputDef`` dicts (one per global-scope
+      ``input(...)`` / ``input.*(...)`` call, inline calls included, in
+      source order). Each has ``title`` / ``type`` /
       ``default`` and optionally ``min`` / ``max`` / ``step`` / ``options``
       (omitted when the corresponding signature argument is absent or
       references a non-const value). See
       :meth:`CodeGen.extract_input_manifest`.
     - ``strategyParams``: the literal ``strategy(...)`` kwargs the analyzer
       surfaced (e.g. ``initial_capital``, ``pyramiding``).
+    - ``diagnostics``: the warnings (:class:`~pineforge_codegen.errors.Diagnostic`,
+      ``Level.WARNING``) the support checker and the analyzer raised for a
+      script that transpiled -- e.g. an approximated ``ta.vwap`` anchor. An
+      error still raises ``CompileError``, which carries the warnings too.
 
     Args mirror :func:`transpile`.
 
     Returns:
-        ``{"cpp": str, "inputs": list[dict], "strategyParams": dict}``.
+        ``{"cpp": str, "inputs": list[dict], "strategyParams": dict,
+        "diagnostics": list[Diagnostic]}``.
     """
-    pragmas = extract_pf_trace_pragmas(pine_source)
-    tokens = Lexer(pine_source, filename=filename).tokenize()
-    ast = Parser(tokens, source=pine_source, filename=filename).parse()
-    if check_support:
-        check_support_or_raise(ast, filename=filename)
-    ast = expand_finite_choice_extrema_lengths(ast)
-    ctx = Analyzer(ast, filename=filename).analyze()
-    ctx.pf_trace_pragmas = pragmas
-    gen = CodeGen(ctx)
-    cpp = gen.generate()
+    gen, ctx, cpp, support_diagnostics = _generate(pine_source, check_support, filename, libraries)
     return {
         "cpp": cpp,
         "inputs": gen.extract_input_manifest(),
         "strategyParams": dict(ctx.strategy_params),
+        "diagnostics": [d for d in (*support_diagnostics, *ctx.diagnostics)
+                        if d.level == Level.WARNING],
     }

package/pineforge_codegen/analyzer/__init__.py

@@ -36,6 +36,7 @@ from .tables import (
     TA_TUPLE_RETURNS,
     TA_MULTI_CTOR,
     TA_NO_CTOR,
+    TA_LENGTH_ONLY_DEFAULT_SOURCE,
     BUILTIN_VARS,
     BAR_FIELDS,
     SKIP_FUNCS,
@@ -54,6 +55,7 @@ __all__ = [
     "TA_TUPLE_RETURNS",
     "TA_MULTI_CTOR",
     "TA_NO_CTOR",
+    "TA_LENGTH_ONLY_DEFAULT_SOURCE",
     "BUILTIN_VARS",
     "BAR_FIELDS",
     "SKIP_FUNCS",