ember-estree 0.5.1 → 0.6.0

package/README.md

@@ -63,14 +63,35 @@ Both `toTree` and `parse` accept an options object as their second argument.
 
 All options are optional.
 
-| Option         | Type                                              | Description                                                                                                       |
-| -------------- | ------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
-| `filePath`     | `string`                                          | Used for language detection.                                                                                      |
-| `templateOnly` | `boolean`                                         | Parse the source as a raw Glimmer template. Use for `.hbs` files.                                                 |
-| `parser`       | `(placeholderJS: string) => { ast, ... }`         | Use a custom JS/TS parser instead of the default oxc-parser. See [Custom parser](#custom-parser).                 |
-| `visitors`     | `VisitorMap` <br /> or `(outerAst) => VisitorMap` | Callbacks fired on every node during traversal — JS/TS and Glimmer — in a single pass. See [Visitors](#visitors). |
-
-Handler signature is `(node, path) => void`, where `path = { node, parent, parentPath }` — a linked list walking back to the root.
+| Option         | Type                                              | Description                                                                                                           |
+| -------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
+| `filePath`     | `string`                                          | Used for language detection.                                                                                          |
+| `tokens`       | `boolean`                                         | Generate a flat `ast.tokens` array. Required by ESLint; skipped by default so codemods and type-checkers pay nothing. |
+| `templateOnly` | `boolean`                                         | Parse the source as a raw Glimmer template. Use for `.hbs` files.                                                     |
+| `parser`       | `(placeholderJS: string) => { ast, ... }`         | Use a custom JS/TS parser instead of the default oxc-parser. See [Custom parser](#custom-parser).                     |
+| `visitors`     | `VisitorMap` <br /> or `(outerAst) => VisitorMap` | Callbacks fired on every node during traversal — JS/TS and Glimmer — in a single pass. See [Visitors](#visitors).     |
+
+Handler signature is `(node, path) => void`, where `path = { node, parent, parentPath }` — a linked list that walks all the way back through the JS/TS root, so visitors can locate the enclosing scope or class from within a Glimmer subtree.
+
+### Token stream
+
+Pass `tokens: true` to populate `ast.tokens` with a flat, position-sorted array of lexemes spanning the full file — including Glimmer tokens spliced in place of each `<template>` region. This is what ESLint's `SourceCode` needs; omit it for codemods or type-checkers that don't use the token stream.
+
+```js
+import { toTree } from "ember-estree";
+
+const result = toTree(source, {
+  tokens: true,
+  parser: myTsParser,
+});
+// result.ast.program.tokens now contains JS + Glimmer tokens in source order
+```
+
+For `.hbs` files via `templateOnly`, pass both flags:
+
+```js
+toTree(hbsSource, { templateOnly: true, tokens: true });
+```
 
 ### Custom parser
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ember-estree",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "ESTree generator for gjs and gts file used by ember",
   "keywords": [
     "AST",

package/src/parse.js

@@ -31,6 +31,7 @@ const PLACEHOLDER_TYPES = new Set([
  * @param {string} source
  * @param {object} [options]
  * @param {string}  [options.filePath] - File path for language detection
+ * @param {boolean} [options.tokens] - Generate a flat token stream on the AST (needed by ESLint; skipped by default)
  * @param {boolean} [options.templateOnly] - Parse as raw Glimmer template content (for .hbs)
  * @param {function} [options.parser] - Custom JS/TS parser: (placeholderJS) => { ast, scopeManager?, visitorKeys?, services?, ... }
  * @param {object|function} [options.visitors] - Either a map of `{ [Type]: (node, path) => void }`
@@ -41,8 +42,13 @@ const PLACEHOLDER_TYPES = new Set([
  * @return {object}
  */
 export function toTree(source, options = {}) {
+  const generateTokens = !!options.tokens;
+
   if (options.templateOnly) {
-    return processTemplate(source, new DocumentLines(source), [0, source.length]);
+    return processTemplate(source, new DocumentLines(source), {
+      templateRange: [0, source.length],
+      tokens: generateTokens,
+    });
   }
 
   let parseResults = preprocessor.parse(source);
@@ -115,7 +121,10 @@ export function toTree(source, options = {}) {
     ];
     let fullRange = [parseResult.range.startUtf16Codepoint, parseResult.range.endUtf16Codepoint];
 
-    const { ast } = processTemplate(templateContent, codeLines, contentRange);
+    const { ast } = processTemplate(templateContent, codeLines, {
+      templateRange: contentRange,
+      tokens: generateTokens,
+    });
 
     // Fix the Template root to cover the full <template>...</template> range
     ast.range = fullRange;
@@ -126,27 +135,29 @@ export function toTree(source, options = {}) {
       end: codeLines.offsetToPosition(fullRange[1]),
     };
 
-    // Add tokens for the <template> and </template> tags
-    const openEnd = contentRange[0];
-    const closeStart = contentRange[1];
-    const openTag = source.slice(fullRange[0], openEnd);
-    const closeTag = source.slice(closeStart, fullRange[1]);
-    const makeToken = (value, range) => ({
-      type: "Punctuator",
-      value,
-      range,
-      start: range[0],
-      end: range[1],
-      loc: {
-        start: codeLines.offsetToPosition(range[0]),
-        end: codeLines.offsetToPosition(range[1]),
-      },
-    });
-    ast.tokens = [
-      makeToken(openTag, [fullRange[0], openEnd]),
-      ...(ast.tokens || []),
-      makeToken(closeTag, [closeStart, fullRange[1]]),
-    ];
+    if (generateTokens) {
+      // Add tokens for the <template> and </template> tags
+      const openEnd = contentRange[0];
+      const closeStart = contentRange[1];
+      const openTag = source.slice(fullRange[0], openEnd);
+      const closeTag = source.slice(closeStart, fullRange[1]);
+      const makeToken = (value, range) => ({
+        type: "Punctuator",
+        value,
+        range,
+        start: range[0],
+        end: range[1],
+        loc: {
+          start: codeLines.offsetToPosition(range[0]),
+          end: codeLines.offsetToPosition(range[1]),
+        },
+      });
+      ast.tokens = [
+        makeToken(openTag, [fullRange[0], openEnd]),
+        ...(ast.tokens || []),
+        makeToken(closeTag, [closeStart, fullRange[1]]),
+      ];
+    }
 
     templateInfos.push({ utf16Range: fullRange, ast });
     return ast;

package/src/tokens.js

@@ -0,0 +1,177 @@
+/**
+ * Token stream generation for Glimmer templates.
+ *
+ * Only needed by ESLint consumers — codemods, type-checkers, and formatters
+ * do not use the flat token stream and skip this entirely by omitting
+ * `{ tokens: true }` from processTemplate options.
+ *
+ * ESLint-specific note: comment tokens are placed at range[0]+1 rather than
+ * range[0] because ESLint's createIndexMap infinite-loops when a token and an
+ * ast.comments entry share range[0] — both inner loops fail the strict-less-than
+ * guard and the outer loop never advances. Tracked upstream as eslint/eslint#20492.
+ */
+
+function isAlphaNumeric(code) {
+  return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
+}
+
+/**
+ * Lex a Glimmer template into a flat token array.
+ * Tracks line/column incrementally from a single seed call to
+ * doc.offsetToPosition — O(n + log L) instead of O(n log L).
+ */
+export function tokenize(template, doc, startOffset) {
+  const tokens = [];
+  let wordStart = -1;
+
+  // Seed position from the start offset — one binary search total.
+  // Then track line/column incrementally (DocumentLines counts only \n
+  // as line separators, so this matches offsetToPosition exactly).
+  let { line: curLine, column: curCol } = doc.offsetToPosition(startOffset);
+  let wordLine = 0,
+    wordCol = 0;
+
+  for (let i = 0; i < template.length; i++) {
+    const code = template.charCodeAt(i);
+    if (isAlphaNumeric(code)) {
+      if (wordStart < 0) {
+        wordStart = i;
+        wordLine = curLine;
+        wordCol = curCol;
+      }
+      curCol++;
+    } else {
+      if (wordStart >= 0) {
+        const absStart = startOffset + wordStart;
+        const absEnd = startOffset + i;
+        tokens.push({
+          type: "word",
+          value: template.slice(wordStart, i),
+          range: [absStart, absEnd],
+          start: absStart,
+          end: absEnd,
+          loc: {
+            start: { line: wordLine, column: wordCol, index: absStart },
+            end: { line: curLine, column: curCol, index: absEnd },
+          },
+        });
+        wordStart = -1;
+      }
+      if (code === 10 /* \n */) {
+        curLine++;
+        curCol = 0;
+      } else {
+        if (code !== 32 && code !== 9 && code !== 13 && code !== 11 /* non-whitespace */) {
+          const absPos = startOffset + i;
+          tokens.push({
+            type: "Punctuator",
+            value: template[i],
+            range: [absPos, absPos + 1],
+            start: absPos,
+            end: absPos + 1,
+            loc: {
+              start: { line: curLine, column: curCol, index: absPos },
+              end: { line: curLine, column: curCol + 1, index: absPos + 1 },
+            },
+          });
+        }
+        curCol++;
+      }
+    }
+  }
+
+  if (wordStart >= 0) {
+    const absStart = startOffset + wordStart;
+    const absEnd = startOffset + template.length;
+    tokens.push({
+      type: "word",
+      value: template.slice(wordStart),
+      range: [absStart, absEnd],
+      start: absStart,
+      end: absEnd,
+      loc: {
+        start: { line: wordLine, column: wordCol, index: absStart },
+        end: { line: curLine, column: curCol, index: absEnd },
+      },
+    });
+  }
+
+  return tokens;
+}
+
+/**
+ * Merge the raw Glimmer token stream with text nodes and comment tokens,
+ * dropping raw tokens that fall inside comment or text-node intervals.
+ *
+ * All inputs are sorted by range[0] (sequential document scan), so the
+ * entire operation is a single O(n+m+k) pass with advancing pointers
+ * rather than O(n log m) per-token binary searches.
+ */
+export function buildTokenStream(rawTokens, comments, textNodes, templateContent, offset) {
+  // Comment tokens shifted by 1 to avoid the ESLint createIndexMap conflict
+  const commentTokens = comments.map((c) => {
+    const start = c.range[0] + 1;
+    const end = c.range[1];
+    return {
+      type: "Block",
+      value: templateContent.slice(start - offset, end - offset),
+      range: [start, end],
+      start,
+      end,
+      loc: c.loc,
+    };
+  });
+
+  const spliceables = linearMerge(textNodes, commentTokens);
+
+  const result = [];
+  let ri = 0; // rawTokens
+  let si = 0; // spliceables
+  let ci = 0; // comment intervals (for skip detection)
+  let ni = 0; // text-node intervals (for skip detection)
+
+  while (ri < rawTokens.length || si < spliceables.length) {
+    if (ri >= rawTokens.length) {
+      result.push(spliceables[si++]);
+      continue;
+    }
+
+    const tok = rawTokens[ri];
+
+    // Advance interval pointers past intervals that end before this token
+    while (ci < comments.length && comments[ci].range[1] <= tok.range[0]) ci++;
+    while (ni < textNodes.length && textNodes[ni].range[1] <= tok.range[0]) ni++;
+
+    // Skip raw token if it falls inside a comment or text-node interval
+    if (
+      (ci < comments.length && comments[ci].range[0] <= tok.range[0]) ||
+      (ni < textNodes.length && textNodes[ni].range[0] <= tok.range[0])
+    ) {
+      ri++;
+      continue;
+    }
+
+    // Emit the earlier of the next spliceable or this raw token
+    if (si < spliceables.length && spliceables[si].range[0] < tok.range[0]) {
+      result.push(spliceables[si++]);
+    } else {
+      result.push(tok);
+      ri++;
+    }
+  }
+
+  return result;
+}
+
+function linearMerge(a, b) {
+  const result = Array.from({ length: a.length + b.length });
+  let ai = 0,
+    bi = 0,
+    ri = 0;
+  while (ai < a.length && bi < b.length) {
+    result[ri++] = a[ai].range[0] <= b[bi].range[0] ? a[ai++] : b[bi++];
+  }
+  while (ai < a.length) result[ri++] = a[ai++];
+  while (bi < b.length) result[ri++] = b[bi++];
+  return result;
+}

package/src/transforms.js

@@ -7,6 +7,8 @@ import {
   preprocess as glimmerPreprocess,
 } from "@glimmer/syntax";
 
+import { tokenize, buildTokenStream } from "./tokens.js";
+
 /**
  * Converts between character offsets and line/column positions.
  * Lines are 1-based, columns are 0-based (matching ESTree & Glimmer conventions).
@@ -82,89 +84,6 @@ function removeFromParent(nodes) {
   }
 }
 
-function isAlphaNumeric(code) {
-  return !(!(code > 47 && code < 58) && !(code > 64 && code < 91) && !(code > 96 && code < 123));
-}
-
-function isWhiteSpaceCode(code) {
-  return code === 32 || code === 9 || code === 13 || code === 10 || code === 11;
-}
-
-function tokenize(template, doc, startOffset) {
-  const tokens = [];
-  let wordStart = -1;
-  function pushToken(value, type, range) {
-    tokens.push({
-      type,
-      value,
-      range,
-      start: range[0],
-      end: range[1],
-      loc: {
-        start: { ...doc.offsetToPosition(range[0]), index: range[0] },
-        end: { ...doc.offsetToPosition(range[1]), index: range[1] },
-      },
-    });
-  }
-  for (let i = 0; i < template.length; i++) {
-    const code = template.charCodeAt(i);
-    if (isAlphaNumeric(code)) {
-      if (wordStart < 0) wordStart = i;
-    } else {
-      if (wordStart >= 0) {
-        pushToken(template.slice(wordStart, i), "word", [startOffset + wordStart, startOffset + i]);
-        wordStart = -1;
-      }
-      if (!isWhiteSpaceCode(code)) {
-        pushToken(template[i], "Punctuator", [startOffset + i, startOffset + i + 1]);
-      }
-    }
-  }
-  if (wordStart >= 0) {
-    pushToken(template.slice(wordStart), "word", [
-      startOffset + wordStart,
-      startOffset + template.length,
-    ]);
-  }
-  return tokens;
-}
-
-function buildTokenStream(rawTokens, comments, textNodes) {
-  const commentIntervals = comments.map((c) => c.range).sort((a, b) => a[0] - b[0]);
-  const textNodeIntervals = textNodes.map((t) => t.range).sort((a, b) => a[0] - b[0]);
-
-  function isCovered(tokenRange, intervals) {
-    let lo = 0;
-    let hi = intervals.length - 1;
-    while (lo <= hi) {
-      const mid = (lo + hi) >> 1;
-      const iv = intervals[mid];
-      if (iv[0] <= tokenRange[0] && iv[1] >= tokenRange[1]) return true;
-      if (iv[0] > tokenRange[0]) hi = mid - 1;
-      else lo = mid + 1;
-    }
-    return false;
-  }
-
-  const filteredTokens = rawTokens.filter(
-    (t) => !isCovered(t.range, commentIntervals) && !isCovered(t.range, textNodeIntervals),
-  );
-
-  const sortedTextNodes = [...textNodes].sort((a, b) => a.range[0] - b.range[0]);
-  const result = [];
-  let ti = 0;
-  for (const token of filteredTokens) {
-    while (ti < sortedTextNodes.length && sortedTextNodes[ti].range[0] < token.range[0]) {
-      result.push(sortedTextNodes[ti++]);
-    }
-    result.push(token);
-  }
-  while (ti < sortedTextNodes.length) {
-    result.push(sortedTextNodes[ti++]);
-  }
-  return result;
-}
-
 /**
  * Parse and transform a Glimmer template into an ESTree-compatible AST.
  * Internal — consumed by toTree.
@@ -173,7 +92,8 @@ function buildTokenStream(rawTokens, comments, textNodes) {
  * positions, create parts/blockParamNodes, nullify empty hashes, and
  * prefix types. No separate collect-then-transform loop.
  */
-export function processTemplate(templateContent, codeLines, templateRange) {
+export function processTemplate(templateContent, codeLines, options = {}) {
+  const { templateRange, tokens: generateTokens = false } = options;
   const offset = templateRange[0];
   const docLines = offset === 0 ? codeLines : new DocumentLines(templateContent);
 
@@ -335,7 +255,15 @@ export function processTemplate(templateContent, codeLines, templateRange) {
 
   removeFromParent(emptyTextNodes);
 
-  ast.tokens = buildTokenStream(tokenize(templateContent, codeLines, offset), comments, textNodes);
+  if (generateTokens) {
+    ast.tokens = buildTokenStream(
+      tokenize(templateContent, codeLines, offset),
+      comments,
+      textNodes,
+      templateContent,
+      offset,
+    );
+  }
   ast.contents = templateContent;
 
   return { ast, comments };