@pineforge/codegen-pyodide 0.10.4 → 1.0.0-rc.1

package/pineforge_codegen/lexer.py

@@ -21,6 +21,9 @@ from pineforge_codegen.errors import (
     Phase,
     SourceLocation,
 )
+from pineforge_codegen.limits import (
+    MAX_NESTING_DEPTH, TimeBudget, check_source_size, limit_error,
+)
 
 
 class TokenType(Enum):
@@ -173,7 +176,8 @@ class Lexer:
         TokenType.PERCENT_EQUALS,
     }
 
-    def __init__(self, source: str, filename: str = "<input>") -> None:
+    def __init__(self, source: str, filename: str = "<input>",
+                 budget: TimeBudget | None = None) -> None:
         self.source = source
         self.filename = filename
         self.pos = 0
@@ -184,6 +188,7 @@ class Lexer:
         self.paren_depth = 0  # Track () and [] nesting to suppress NEWLINE/INDENT/DEDENT
         self._in_continuation = False  # True when current line is a continuation
         self._diagnostics: list[Diagnostic] = []
+        self._budget = budget
 
     def _peek(self, offset: int = 0) -> str:
         idx = self.pos + offset
@@ -197,6 +202,11 @@ class Lexer:
             self.col = 1
         else:
             self.col += 1
+        if self._budget is not None and self.pos % 1024 == 0:
+            self._budget.check(
+                SourceLocation(self.filename, self.line, self.col, self.col + 1),
+                Phase.LEXER,
+            )
         return ch
 
     def _at_end(self) -> bool:
@@ -223,6 +233,7 @@ class Lexer:
             self._advance()
 
     def tokenize(self) -> list[Token]:
+        check_source_size(self.source, self.filename)
         while not self._at_end():
             self._tokenize_line()
 
@@ -318,6 +329,11 @@ class Lexer:
             current_indent = self.indent_stack[-1]
             if indent_level > current_indent:
                 self.indent_stack.append(indent_level)
+                if len(self.indent_stack) - 1 > MAX_NESTING_DEPTH:
+                    raise limit_error(
+                        f"Block nesting depth exceeds {MAX_NESTING_DEPTH} levels.",
+                        SourceLocation(self.filename, self.line, 1, 2), Phase.LEXER,
+                    )
                 self._emit(TokenType.INDENT, "", self.line, 1)
             elif indent_level < current_indent:
                 while len(self.indent_stack) > 1 and self.indent_stack[-1] > indent_level:
@@ -417,12 +433,12 @@ class Lexer:
             self._read_leading_dot_number(start_line, start_col)
             return
 
-        # Strings
-        if ch == '"':
-            self._read_string(start_line, start_col)
-            return
-        if ch == "'":
-            self._read_string_single(start_line, start_col)
+        # Strings: three quotes of one kind open a multiline string.
+        if ch in "\"'":
+            if self.source.startswith(ch * 3, self.pos):
+                self._read_multiline(ch, start_line, start_col)
+            else:
+                self._read_quoted(ch, start_line, start_col)
             return
 
         # Color literals (#rrggbb or #rrggbbaa)
@@ -492,6 +508,12 @@ class Lexer:
             tt = singles[ch]
             if tt in (TokenType.LPAREN, TokenType.LBRACKET):
                 self.paren_depth += 1
+                if self.paren_depth > MAX_NESTING_DEPTH:
+                    raise limit_error(
+                        f"Delimiter nesting depth exceeds {MAX_NESTING_DEPTH} levels.",
+                        SourceLocation(self.filename, start_line, start_col,
+                                       start_col + 1), Phase.LEXER,
+                    )
             elif tt in (TokenType.RPAREN, TokenType.RBRACKET):
                 self.paren_depth = max(0, self.paren_depth - 1)
             self._emit(tt, ch, start_line, start_col, start_col + 1)
@@ -563,29 +585,89 @@ class Lexer:
         value = "0." + frac
         self._emit(TokenType.NUMBER, value, start_line, start_col, start_col + len(value))
 
-    def _read_string(self, start_line: int, start_col: int) -> None:
-        self._advance()  # consume opening "
+    # Pine v6 User Manual, Strings, "Escape sequences": a backslash makes a
+    # quotation mark or a backslash literal, "The \\n sequence represents the
+    # newline character (U+000A)", "The \\t sequence represents the horizontal
+    # tab character (U+0009)", and "If a backslash applied to a character does
+    # not form a supported escape sequence, the character's meaning does not
+    # change" -- the backslash is dropped and the character kept.
+    _STRING_ESCAPES = {"n": "\n", "t": "\t"}
+
+    def _read_escape(self, buf: list[str]) -> None:
+        self._advance()  # skip backslash
+        ch = self._advance()
+        buf.append(self._STRING_ESCAPES.get(ch, ch))
+
+    # Pine v6 User Manual, Strings: "In Pine v6, programmers can use line
+    # wrapping to define single-line literal strings across multiple code
+    # lines, where each wrapped line has an indentation of one or more spaces.
+    # However, the resulting character sequence adds only one space to the
+    # start of the wrapped lines, and it does not automatically add line
+    # terminators." A line break inside a single-line string therefore reads
+    # as one space when the next line is indented; otherwise the string is
+    # unterminated.
+    def _read_quoted(self, quote: str, start_line: int, start_col: int) -> None:
+        self._advance()  # consume the opening quote
         buf: list[str] = []
-        while not self._at_end() and self.source[self.pos] != '"':
-            if self.source[self.pos] == "\\" and self.pos + 1 < len(self.source):
-                self._advance()  # skip backslash
+        while not self._at_end() and self.source[self.pos] != quote:
+            ch = self.source[self.pos]
+            if ch == "\\" and self.pos + 1 < len(self.source):
+                self._read_escape(buf)
+                continue
+            if ch == "\n":
+                nxt = self.pos + 1
+                indent = nxt
+                while indent < len(self.source) and self.source[indent] in " \t":
+                    indent += 1
+                if indent == nxt or indent >= len(self.source) or self.source[indent] == "\n":
+                    break  # not a wrapped line: the literal is unterminated
+                self._advance_to(indent)
+                buf.append(" ")
+                continue
             buf.append(self._advance())
-        if not self._at_end():
-            self._advance()  # consume closing "
+        if self._at_end() or self.source[self.pos] != quote:
+            self._emit_diagnostic(
+                "Unterminated string literal: the line ends before its closing quote "
+                "and the next line is not indented as a wrapped line.",
+                start_line, start_col, start_col + 1,
+                hint=('Close the string on its line, indent the line it wraps onto, '
+                      'or use a multiline string (\"\"\"...\"\"\").'),
+            )
+        else:
+            self._advance()  # consume the closing quote
         value = "".join(buf)
         self._emit(TokenType.STRING, value, start_line, start_col, self.col)
 
-    def _read_string_single(self, start_line: int, start_col: int) -> None:
-        self._advance()  # consume opening '
+    # Pine v6 User Manual, Strings, "Multiline strings": "A multiline string
+    # is a literal character sequence enclosed by three pairs of ASCII
+    # quotation marks (e.g. \"\"\"...\"\"\") or apostrophes (e.g.,
+    # '''...''')." Everything between the delimiters is literal text -- line
+    # breaks become U+000A, "including any space characters used for
+    # indentation" -- and a quote needs no backslash unless three in a row
+    # would end the string; escapes read as in any Pine string. The string
+    # ends at the first run of three unescaped delimiter quotes.
+    def _read_multiline(self, quote: str, start_line: int, start_col: int) -> None:
+        delimiter = quote * 3
+        self._advance_to(self.pos + 3)  # consume the opening delimiter
         buf: list[str] = []
-        while not self._at_end() and self.source[self.pos] != "'":
+        while not self._at_end():
+            if self.source.startswith(delimiter, self.pos):
+                self._advance_to(self.pos + 3)
+                self._emit(TokenType.STRING, "".join(buf), start_line, start_col, self.col)
+                return
             if self.source[self.pos] == "\\" and self.pos + 1 < len(self.source):
-                self._advance()
+                self._read_escape(buf)
+                continue
             buf.append(self._advance())
-        if not self._at_end():
-            self._advance()  # consume closing '
-        value = "".join(buf)
-        self._emit(TokenType.STRING, value, start_line, start_col, self.col)
+        self._emit_diagnostic(
+            f"Unterminated multiline string literal: no closing {delimiter} before "
+            "the end of the script.",
+            start_line, start_col, start_col + 3,
+            hint=(f"A multiline string ends at the next {chr(34) * 3} (or {chr(39) * 3} "
+                  f"for one opened with {chr(39) * 3}); escape a quote that would end "
+                  "it early."),
+        )
+        self._emit(TokenType.STRING, "".join(buf), start_line, start_col, self.col)
 
     def _read_ident(self, start_line: int, start_col: int) -> None:
         buf: list[str] = []