kopscript 0.9.0 → 0.10.0

package/README.md

@@ -831,6 +831,13 @@ functions/classes.
 language tour above) — the ceiling that's left is what's *inside* those: no async lambdas,
 and no way to build a `task` value by hand outside an `async` function body.
 
+No formatter (`ks fmt`) yet — it was blocked on the lexer discarding every comment entirely,
+which an AST-based pretty-printer would otherwise silently delete on reformat. That's now
+fixed: every `Token` carries its own `leadingComments` (see `src/tokens.ts`'s
+`CommentTrivia`), purely additive trivia the parser/checker/codegen never look at. This is
+groundwork, not the formatter itself — `ks fmt` (walking the AST and re-emitting source with
+comments placed back correctly) is a separate, not-yet-started piece of work.
+
 ### A frontend framework: Kopular
 
 The language shape — modules, closures, DOM interop, `state<T>`, and `extern`/`virtual`

package/dist/lexer.js

@@ -52,6 +52,10 @@ export class Lexer {
         this.pos = 0;
         this.line = 1;
         this.col = 1;
+        // A comment before the very first real token has no "previous token" to
+        // share a line with — without this, skipTrivia()'s default would
+        // incorrectly mark a file-opening comment as sameLineAsPreviousToken.
+        this.hasEmittedToken = false;
     }
     tokenize() {
         const tokens = [];
@@ -64,25 +68,46 @@ export class Lexer {
         return tokens;
     }
     next() {
-        this.skipTrivia();
+        const leadingComments = this.skipTrivia();
         const line = this.line;
         const col = this.col;
-        if (this.isAtEnd())
-            return this.make(TokenKind.EOF, "", line, col);
-        const c = this.peek();
-        if (c === '"')
-            return this.readString(line, col);
-        if (c === "$" && this.peek(1) === '"')
-            return this.readInterpolatedString(line, col);
-        if (c === "r" && this.peek(1) === '"')
-            return this.readRegexLiteral(line, col);
-        if (this.isDigit(c))
-            return this.readNumber(line, col);
-        if (this.isIdentStart(c))
-            return this.readIdentifier(line, col);
-        return this.readOperator(line, col);
+        let token;
+        if (this.isAtEnd()) {
+            token = this.make(TokenKind.EOF, "", line, col);
+        }
+        else {
+            const c = this.peek();
+            if (c === '"')
+                token = this.readString(line, col);
+            else if (c === "$" && this.peek(1) === '"')
+                token = this.readInterpolatedString(line, col);
+            else if (c === "r" && this.peek(1) === '"')
+                token = this.readRegexLiteral(line, col);
+            else if (this.isDigit(c))
+                token = this.readNumber(line, col);
+            else if (this.isIdentStart(c))
+                token = this.readIdentifier(line, col);
+            else
+                token = this.readOperator(line, col);
+        }
+        // Concatenated, not overwritten: `readOperator`'s own error-recovery
+        // path recursively calls `next()` on an illegal character, and that
+        // inner call's `leadingComments` (comments found *after* the bad
+        // character) would otherwise be discarded here in favor of this
+        // (outer) call's own — comments found *before* it.
+        token.leadingComments = [...leadingComments, ...token.leadingComments];
+        this.hasEmittedToken = true;
+        return token;
     }
+    // Comments are otherwise-inert trivia (see tokens.ts's `CommentTrivia`) —
+    // this only decides which token each one attaches to as a "leading"
+    // comment, and whether it looks trailing-style (`sameLineAsPreviousToken`)
+    // for a future formatter to decide how to place it. Only `//` line
+    // comments exist in the language; there's no block-comment syntax to
+    // handle here.
     skipTrivia() {
+        const comments = [];
+        let sameLineAsPreviousToken = this.hasEmittedToken;
         for (;;) {
             const c = this.peek();
             if (c === " " || c === "\t" || c === "\r") {
@@ -90,15 +115,24 @@ export class Lexer {
             }
             else if (c === "\n") {
                 this.advance();
+                sameLineAsPreviousToken = false;
             }
             else if (c === "/" && this.peek(1) === "/") {
+                const line = this.line;
+                const col = this.col;
+                this.advance();
+                this.advance();
+                let text = "";
                 while (!this.isAtEnd() && this.peek() !== "\n")
-                    this.advance();
+                    text += this.advance();
+                comments.push({ text, line, col, sameLineAsPreviousToken });
+                sameLineAsPreviousToken = false; // a `//` comment always eats to end of line
             }
             else {
                 break;
             }
         }
+        return comments;
     }
     readString(line, col) {
         this.advance(); // opening quote
@@ -288,7 +322,7 @@ export class Lexer {
         return this.next();
     }
     make(kind, lexeme, line, col) {
-        return { kind, lexeme, line, col };
+        return { kind, lexeme, line, col, leadingComments: [] };
     }
     isAtEnd() {
         return this.pos >= this.source.length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kopscript",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "KopScript: a small OOP, strongly-typed language that transpiles to JavaScript, with generics and nullable types",
   "type": "module",
   "license": "MIT",