@vesk/compiler 0.2.59 → 0.2.60

package/dist/client-codegen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client-codegen.d.ts","sourceRoot":"","sources":["../src/client-codegen.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAuB,MAAM,uBAAuB,CAAC;AACzE,OAAO,EAGL,cAAc,EAQd,gBAAgB,EASjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAkGxD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;CAC1B;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI1D;AAiDD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,gBAAgB,GAAG,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,CAgJlI;AAID,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAU5E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAuB9F;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,CAY5F;AAoBD,qDAAqD;AACrD,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtC;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,SAAS,CAQtE;AAmiED,iBAAS,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAe3C;AAmJD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAuED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,CAAC,EAAE,SAAS,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,MAAM,CAOpQ;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,UAAU,CAAC,EAAE,MAAM,EACnB,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,GACjD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAC;IAAC,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAyC5J;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAG/E;AAED,OAAO,EAAE,aAAa,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"client-codegen.d.ts","sourceRoot":"","sources":["../src/client-codegen.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAuB,MAAM,uBAAuB,CAAC;AACzE,OAAO,EAGL,cAAc,EAQd,gBAAgB,EASjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAkGxD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;CAC1B;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI1D;AAiDD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,gBAAgB,GAAG,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,CAgJlI;AAID,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAU5E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAuB9F;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,CAY5F;AAoBD,qDAAqD;AACrD,MAAM,MAAM,SAAS,GAAG;IAAE,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtC;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,SAAS,CAQtE;AA0iED,iBAAS,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAe3C;AAmJD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAuED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,CAAC,EAAE,SAAS,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,MAAM,CAOpQ;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,UAAU,CAAC,EAAE,MAAM,EACnB,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,GACjD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAC;IAAC,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAyC5J;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAG/E;AAED,OAAO,EAAE,aAAa,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC"}

package/dist/client-codegen.js

@@ -528,8 +528,16 @@ function emitNode(ctx, node, tracked, effectsVar, parentVar, compPrefix = '__com
             // The children ARE this layout's output, so they take the component's
             // return value rather than being appended to a parent that does not exist.
             if (ctx.hydrate) {
+                const promptExpr = ctx.markerless && (regionBudget || skipK > 0)
+                    ? markerlessSkipExpr(regionBudget, skipK)
+                    : null;
                 ctx.push(`if (props.children !== undefined && props.children !== null) {`);
                 ctx.push(`  if (typeof props.children === 'function') {`);
+                if (promptExpr) {
+                    ctx.push(`    ${ctx.walker}.injectSkipK(${promptExpr});`);
+                    if (regionBudget && ctx.markerless)
+                        ctx.push(`    ${regionBudget} = 0;`);
+                }
                 ctx.push(`    __pendingChild = props.children(__hydrate);`);
                 ctx.push(`  } else if (__hydrate && __hydrate.root) {`);
                 ctx.push(`    __hydrate.root.appendChild(toDomNode(props.children));`);

package/dist/parser.d.ts

@@ -28,8 +28,18 @@ export interface VeskAnnotation {
  * Scans the source and blanks `; key <expr>` / `; index <ident>` clauses
  * found in `for (...)` headers, returning the rewritten code (same length
  * as the input, so every source offset is preserved) plus annotations.
+ *
+ * `delimiters` — the real string/template delimiters from
+ * `collectStringDelimiters` — is optional. Without it the scan assumes every
+ * `"`, `'` and `` ` `` opens a literal, which is wrong for JSX: an apostrophe
+ * in text (`<th>Joe Christopher's Standing</th>`) or in an attribute value
+ * opens a "string" that runs to the next quote anywhere in the file, so every
+ * `for` loop in between is invisible and its `; key …` clause reaches acorn
+ * un-blanked as `Unexpected token` at the header. Callers that can afford the
+ * tokenization pass it in; when it is unavailable the scan falls back to the
+ * plain lexical skip so mid-edit source never crashes it.
  */
-export declare function preprocessForClauses(source: string): {
+export declare function preprocessForClauses(source: string, delimiters?: Set<number> | null): {
     code: string;
     annotations: VeskAnnotation[];
 };

package/dist/parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAOtC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,YAAY,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0BD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,cAAc,EAAE,CAAA;CAAE,CA4LpG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAwFlE;AA2DD,wBAAgB,gBAAgB,IAAI,OAAO,KAAK,CAAC,MAAM,CAEtD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAM/D;AAED,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAuEzE"}
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAOtC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,YAAY,CAAC;IACnB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0BD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAC9B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,cAAc,EAAE,CAAA;CAAE,CAmMjD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAwFlE;AA2DD,wBAAgB,gBAAgB,IAAI,OAAO,KAAK,CAAC,MAAM,CAEtD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAM/D;AAED,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CA6EzE"}

package/dist/parser.js

@@ -3,7 +3,7 @@ import { tsPlugin } from './acorn-ts-plugin/index.js';
 import { VeskParserPlugin } from '@vesk/compiler/src/vesk-plugin';
 import { blankComments, containsForOfIn } from '@vesk/compiler/src/scan';
 import { VeskError, codeFrame } from '@vesk/compiler/src/errors';
-import { findSpecifierExports, hasTopLevelValueDeclaration } from '@vesk/compiler/src/tokens';
+import { findSpecifierExports, hasTopLevelValueDeclaration, collectStringDelimiters, tokenizeCode } from '@vesk/compiler/src/tokens';
 function isIdentChar(code) {
     return ((code >= 97 && code <= 122) || (code >= 65 && code <= 90) || (code >= 48 && code <= 57) || code === 95 || code === 36);
 }
@@ -28,8 +28,18 @@ function offsetToLineCol(source, offset) {
  * Scans the source and blanks `; key <expr>` / `; index <ident>` clauses
  * found in `for (...)` headers, returning the rewritten code (same length
  * as the input, so every source offset is preserved) plus annotations.
+ *
+ * `delimiters` — the real string/template delimiters from
+ * `collectStringDelimiters` — is optional. Without it the scan assumes every
+ * `"`, `'` and `` ` `` opens a literal, which is wrong for JSX: an apostrophe
+ * in text (`<th>Joe Christopher's Standing</th>`) or in an attribute value
+ * opens a "string" that runs to the next quote anywhere in the file, so every
+ * `for` loop in between is invisible and its `; key …` clause reaches acorn
+ * un-blanked as `Unexpected token` at the header. Callers that can afford the
+ * tokenization pass it in; when it is unavailable the scan falls back to the
+ * plain lexical skip so mid-edit source never crashes it.
  */
-export function preprocessForClauses(source) {
+export function preprocessForClauses(source, delimiters) {
     const annotations = [];
     const chars = source.split('');
     const isString = (ch) => ch === '"' || ch === "'" || ch === '`';
@@ -38,6 +48,13 @@ export function preprocessForClauses(source) {
         const ch = source[i];
         // Skip strings and comments (nested backtick templates keep depth).
         if (isString(ch)) {
+            // A quote the tokenizer says is not a delimiter is literal JSX text (or
+            // a quote inside a literal); step over the single character instead of
+            // swallowing the rest of the file.
+            if (delimiters && !delimiters.has(i)) {
+                i++;
+                continue;
+            }
             const quote = ch;
             let j = i + 1;
             let tplDepth = 0;
@@ -503,15 +520,21 @@ export function parseGeneratedJs(source) {
 }
 export function parse(source, options = {}) {
     const ParserClass = createBaseParser();
-    const { code, annotations } = preprocessForClauses(blankGenericArrowTypeParams(blankComments(source)));
+    // Tokenize once for both delimiter detection and specifier-export filtering
+    // to avoid duplicate lexing/parsing overhead.
+    const sourceTokens = tokenizeCode(source);
+    const delimiters = sourceTokens === null
+        ? null
+        : collectStringDelimiters(source, sourceTokens);
+    const { code, annotations } = preprocessForClauses(blankGenericArrowTypeParams(blankComments(source)), delimiters);
     // Acorn validates that every `export { X }` names a real top-level binding.
     // A `.vsk` component is a registry entry, not a binding, so acorn rejects
     // `export { MyComponent }`. Remove only those specifier lists — one whose
     // local IS a real declaration keeps native module semantics — and carry the
     // pairs on the AST; the IR generator turns them into registry aliases.
     const vskSource = !options.filename || String(options.filename).endsWith('.vsk');
-    const specifierExports = vskSource
-        ? findSpecifierExports(code).filter((e) => !hasTopLevelValueDeclaration(code, e.local))
+    const specifierExports = vskSource && sourceTokens !== null
+        ? findSpecifierExports(code, sourceTokens).filter((e) => !hasTopLevelValueDeclaration(code, e.local, sourceTokens))
         : [];
     let parseable = code;
     if (specifierExports.length > 0) {

package/dist/tokens.d.ts

@@ -20,13 +20,36 @@ export interface CodeToken {
  * their span so callers can read the actual character via `code[start]`.
  */
 export declare function tokenizeCode(code: string): CodeToken[] | null;
+/**
+ * Returns the set of offsets at which a `"`, `'` or `` ` `` in `code` really
+ * opens a quoted literal, or `null` when the code cannot be tokenized.
+ *
+ * A character-level scanner cannot tell an apostrophe in JSX text
+ * (`<th>Joe Christopher's Standing</th>`) or inside an attribute value
+ * (`class="Bob's"`) from the opening quote of a JavaScript string, and the
+ * mistake is unbounded: the scanner skips to the next quote it can find — or to
+ * end of file — and everything in between is invisible to it. That is how the
+ * `; key …` clause of a `for` loop could go un-blanked and surface as
+ * `Unexpected token` at a header hundreds of lines away from the real cause.
+ *
+ * The tokenizer has no such ambiguity, so this hands the decision to it. A
+ * quote is a delimiter when a quoted-literal token starts there (ordinary JS
+ * strings and template backticks) or when no token covers it at all — which is
+ * how a JSX attribute value arrives, since acorn's JSX reader consumes the
+ * attribute's opening quote before tokenizing its content.
+ *
+ * Callers must fall back to their own lexical skip when this returns `null`;
+ * tokenizing fails on input that is mid-edit or otherwise unparsable, and the
+ * character scanners have to keep working there.
+ */
+export declare function collectStringDelimiters(code: string, tokens?: CodeToken[] | null): Set<number> | null;
 /**
  * Returns the set of identifiers in `code` that are called — an identifier
  * token immediately followed by `(` or by a `<...>` generic clause and then
  * `(`. Member accesses (`obj.fn(`, `obj?.fn(`) are excluded so methods are
  * not mistaken for imported runtime functions.
  */
-export declare function collectCalledIdentifiers(code: string): Set<string>;
+export declare function collectCalledIdentifiers(code: string, tokens?: CodeToken[] | null): Set<string>;
 /**
  * True when `code` contains the identifier `name` (as a plain identifier or
  * a JSX name), outside strings and comments.
@@ -53,7 +76,7 @@ export declare function importModuleTarget(importText: string): string | null;
  * with no binding — apart from an ordinary JS/TS declaration, whose
  * `export { name }` must keep native module semantics.
  */
-export declare function hasTopLevelValueDeclaration(code: string, name: string): boolean;
+export declare function hasTopLevelValueDeclaration(code: string, name: string, tokens?: CodeToken[] | null): boolean;
 /** One entry of a same-file `export { A, B as C }` specifier list. */
 export interface VskSpecifierExport {
     local: string;
@@ -72,7 +95,7 @@ export interface VskSpecifierExport {
  * `export const A = 1`) and re-exports (`export … from '…'`) are left alone:
  * those parse fine and are handled in the IR generator.
  */
-export declare function findSpecifierExports(code: string): VskSpecifierExport[];
+export declare function findSpecifierExports(code: string, tokens?: CodeToken[] | null): VskSpecifierExport[];
 /** `text` must be `source.slice(base, …)`; `node` is AST parsed from `source`. */
 export declare function stripTrackDeclMarkers(text: string, base: number, node: any): string;
 //# sourceMappingURL=tokens.d.ts.map

package/dist/tokens.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAaH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,IAAI,CA0B7D;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CA2BlE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAStE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAqBxE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAuC/D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOpE;AAwID;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAkB/E;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAkCvE;AA6DD,kFAAkF;AAClF,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,MAAM,CAcnF"}
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAaH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,IAAI,CA0B7D;AAKD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAerG;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CA2B/F;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAStE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAqBxE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAuC/D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOpE;AAwID;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,GAAG,OAAO,CAkB5G;AAED,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,GAAG,kBAAkB,EAAE,CAkCpG;AA6DD,kFAAkF;AAClF,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,MAAM,CAcnF"}

package/dist/tokens.js

@@ -44,26 +44,70 @@ export function tokenizeCode(code) {
         }
     });
 }
+/** Token labels whose source text is a quoted literal (string or template). */
+const QUOTED_LABELS = new Set(['string', 'template', '`', 'backQuote']);
+/**
+ * Returns the set of offsets at which a `"`, `'` or `` ` `` in `code` really
+ * opens a quoted literal, or `null` when the code cannot be tokenized.
+ *
+ * A character-level scanner cannot tell an apostrophe in JSX text
+ * (`<th>Joe Christopher's Standing</th>`) or inside an attribute value
+ * (`class="Bob's"`) from the opening quote of a JavaScript string, and the
+ * mistake is unbounded: the scanner skips to the next quote it can find — or to
+ * end of file — and everything in between is invisible to it. That is how the
+ * `; key …` clause of a `for` loop could go un-blanked and surface as
+ * `Unexpected token` at a header hundreds of lines away from the real cause.
+ *
+ * The tokenizer has no such ambiguity, so this hands the decision to it. A
+ * quote is a delimiter when a quoted-literal token starts there (ordinary JS
+ * strings and template backticks) or when no token covers it at all — which is
+ * how a JSX attribute value arrives, since acorn's JSX reader consumes the
+ * attribute's opening quote before tokenizing its content.
+ *
+ * Callers must fall back to their own lexical skip when this returns `null`;
+ * tokenizing fails on input that is mid-edit or otherwise unparsable, and the
+ * character scanners have to keep working there.
+ */
+export function collectStringDelimiters(code, tokens) {
+    const t = tokens == null ? tokenizeCode(code) : tokens;
+    if (t === null)
+        return null;
+    const starts = new Set();
+    // `t` is ordered by offset, so one pointer answers "is this offset
+    // inside a token?" for the whole source without materialising a bitmap.
+    let ti = 0;
+    for (let i = 0; i < code.length; i++) {
+        const c = code.charCodeAt(i);
+        if (c !== 34 && c !== 39 && c !== 96)
+            continue;
+        while (ti < t.length && t[ti].end <= i)
+            ti++;
+        const covered = ti < t.length && t[ti].start <= i;
+        if (!covered || QUOTED_LABELS.has(t[ti].label))
+            starts.add(i);
+    }
+    return starts;
+}
 /**
  * Returns the set of identifiers in `code` that are called — an identifier
  * token immediately followed by `(` or by a `<...>` generic clause and then
  * `(`. Member accesses (`obj.fn(`, `obj?.fn(`) are excluded so methods are
  * not mistaken for imported runtime functions.
  */
-export function collectCalledIdentifiers(code) {
-    const tokens = tokenizeCode(code);
-    if (tokens !== null) {
+export function collectCalledIdentifiers(code, tokens) {
+    const t = tokens == null ? tokenizeCode(code) : tokens;
+    if (t !== null) {
         const result = new Set();
-        for (let i = 0; i < tokens.length; i++) {
-            const t = tokens[i];
-            if (t.label !== 'name')
+        for (let i = 0; i < t.length; i++) {
+            const tok = t[i];
+            if (tok.label !== 'name')
                 continue;
-            const prev = i > 0 ? tokens[i - 1] : null;
+            const prev = i > 0 ? t[i - 1] : null;
             if (prev && (prev.label === '.' || prev.label === '?.')) {
                 // Member calls (`useFetch.text(`) surface the receiver so an
                 // auto-importable function used as a receiver is still imported.
-                const obj = i >= 2 ? tokens[i - 2] : null;
-                const next = tokens[i + 1];
+                const obj = i >= 2 ? t[i - 2] : null;
+                const next = t[i + 1];
                 if (obj && obj.label === 'name' && next) {
                     const nextCh = code[next.start];
                     if (nextCh === '(' || nextCh === '<')
@@ -71,12 +115,12 @@ export function collectCalledIdentifiers(code) {
                 }
                 continue;
             }
-            const next = tokens[i + 1];
+            const next = t[i + 1];
             if (!next)
                 continue;
             const nextCh = code[next.start];
             if (nextCh === '(' || nextCh === '<')
-                result.add(t.value);
+                result.add(tok.value);
         }
         return result;
     }
@@ -367,22 +411,22 @@ function manualExtractImportNames(importText) {
  * with no binding — apart from an ordinary JS/TS declaration, whose
  * `export { name }` must keep native module semantics.
  */
-export function hasTopLevelValueDeclaration(code, name) {
-    const tokens = tokenizeCode(code);
-    if (tokens === null)
+export function hasTopLevelValueDeclaration(code, name, tokens) {
+    const t = tokens == null ? tokenizeCode(code) : tokens;
+    if (t === null)
         return false;
     const declWords = ['const', 'let', 'var', 'function', 'class'];
-    for (let i = 1; i < tokens.length; i++) {
-        if (tokens[i].label !== 'name' || tokens[i].value !== name)
+    for (let i = 1; i < t.length; i++) {
+        if (t[i].label !== 'name' || t[i].value !== name)
             continue;
-        const prev = tokens[i - 1];
+        const prev = t[i - 1];
         // Declaration keywords carry their own token label (`const`, `var`,
         // `function`, `class`); `let` arrives as a plain name. Match on value.
         if (!prev || !declWords.includes(prev.value))
             continue;
         // `export const X` / `export function X` are preceded by `export`.
         if (i >= 2) {
-            const pp = tokens[i - 2];
+            const pp = t[i - 2];
             if (pp && pp.label === 'name' && pp.value === 'export')
                 continue;
         }
@@ -400,24 +444,24 @@ export function hasTopLevelValueDeclaration(code, name) {
  * `export const A = 1`) and re-exports (`export … from '…'`) are left alone:
  * those parse fine and are handled in the IR generator.
  */
-export function findSpecifierExports(code) {
-    const tokens = tokenizeCode(code);
-    if (tokens === null)
+export function findSpecifierExports(code, tokens) {
+    const t = tokens == null ? tokenizeCode(code) : tokens;
+    if (t === null)
         return [];
     const out = [];
-    for (let i = 0; i < tokens.length; i++) {
-        if (tokens[i].label !== 'export')
+    for (let i = 0; i < t.length; i++) {
+        if (t[i].label !== 'export')
             continue;
         // A specifier list starts with `{`. `export default`, `export const` and
         // `export *` take another route and are left for the IR generator.
-        if (!tokens[i + 1] || tokens[i + 1].label !== '{')
+        if (!t[i + 1] || t[i + 1].label !== '{')
             continue;
         // Find this statement's `}` and note any module source, which makes it a
         // re-export (`export { A } from './x.vsk'`) rather than a specifier list.
         let k = i + 2;
         let depth = 1;
-        for (; k < tokens.length; k++) {
-            const lbl = tokens[k].label;
+        for (; k < t.length; k++) {
+            const lbl = t[k].label;
             if (lbl === 'string') {
                 continue;
             }
@@ -431,8 +475,8 @@ export function findSpecifierExports(code) {
         }
         // A re-export puts `from '…'` AFTER the closing brace, so check past it.
         let isReexport = false;
-        for (let j = k + 1; j < tokens.length; j++) {
-            const lbl = tokens[j].label;
+        for (let j = k + 1; j < t.length; j++) {
+            const lbl = t[j].label;
             if (lbl === 'string') {
                 isReexport = true;
                 break;
@@ -442,12 +486,12 @@ export function findSpecifierExports(code) {
         }
         if (depth !== 0 || isReexport)
             continue;
-        const pairs = readSpecifierPairs(code, tokens[i + 1].end, tokens[k].end);
+        const pairs = readSpecifierPairs(code, t[i + 1].end, t[k].end);
         if (pairs === null)
             continue;
-        const stmtEnd = tokens[k].end;
+        const stmtEnd = t[k].end;
         for (const [local, exported] of pairs)
-            out.push({ local, exported, start: tokens[i].start, end: stmtEnd });
+            out.push({ local, exported, start: t[i].start, end: stmtEnd });
         i = k;
     }
     return out;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vesk/compiler",
-  "version": "0.2.59",
+  "version": "0.2.60",
   "type": "module",
   "description": "Vesk compiler — lexer, parser, semantic analysis, IR, codegen",
   "main": "./dist/index.js",
@@ -20,7 +20,7 @@
   },
   "dependencies": {
     "@sveltejs/acorn-typescript": "^1.0.0",
-    "@vesk/types": "^0.2.59",
+    "@vesk/types": "^0.2.60",
     "acorn": "^8.15.0",
     "esrap": "^2.3.0",
     "is-reference": "^3.0.0",