@mrhenry/twig-tokenizer 0.1.0 → 0.1.2

package/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## 0.1.2 (2026-09-26)
+
+ * Reuse a single `TextEncoder` for UTF-8 byte counts (`Source#getColumn`) and
+   for `SyntaxError#addSuggestions` instead of allocating one per call.
+ * `byteLevenshtein` now uses two rolling rows instead of a full matrix, so the
+   "Did you mean …?" suggestions no longer allocate a matrix per candidate.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@mrhenry/twig-tokenizer",
-	"version": "0.1.0",
+	"version": "0.1.2",
 	"license": "MIT",
 	"type": "module",
 	"main": "src/index.js",

package/src/errors.js

@@ -194,18 +194,24 @@ export class SyntaxError extends TwigError {
 	}
 }
 
+/** Shared UTF-8 encoder: `TextEncoder` is stateless, so one instance serves all calls. */
+const UTF8_ENCODER = new TextEncoder();
+
 /**
  * @param {string} value
  * @returns {Uint8Array} The UTF-8 bytes of the string.
  */
 function toBytes(value) {
-	return new TextEncoder().encode(value);
+	return UTF8_ENCODER.encode(value);
 }
 
 /**
  * The Levenshtein distance between two byte sequences (mirrors PHP
  * `levenshtein()`, which operates on bytes).
  *
+ * Uses two rolling rows instead of a full matrix, so the working set stays
+ * O(min(a, b)) for the many candidate strings `addSuggestions` compares.
+ *
  * @param {Uint8Array} a
  * @param {Uint8Array} b
  * @returns {number}
@@ -217,24 +223,33 @@ function byteLevenshtein(a, b) {
 	if (b.length === 0) {
 		return a.length;
 	}
-	const matrix = new Uint16Array((a.length + 1) * (b.length + 1));
-	for (let i = 0; i <= a.length; i++) {
-		matrix[i * (b.length + 1)] = i;
+	// iterate over the shorter sequence so the rows are as small as possible
+	if (a.length > b.length) {
+		const swap = a;
+		a = b;
+		b = swap;
 	}
-	for (let j = 0; j <= b.length; j++) {
-		matrix[j] = j;
+	const width = b.length;
+	let previous = new Uint16Array(width + 1);
+	let current = new Uint16Array(width + 1);
+	for (let j = 0; j <= width; j++) {
+		previous[j] = j;
 	}
 	for (let i = 1; i <= a.length; i++) {
-		for (let j = 1; j <= b.length; j++) {
-			const cost = a[i - 1] === b[j - 1] ? 0 : 1;
-			matrix[i * (b.length + 1) + j] = Math.min(
-				matrix[(i - 1) * (b.length + 1) + j] + 1,
-				matrix[i * (b.length + 1) + j - 1] + 1,
-				matrix[(i - 1) * (b.length + 1) + j - 1] + cost,
-			);
+		current[0] = i;
+		const aByte = a[i - 1];
+		for (let j = 1; j <= width; j++) {
+			const cost = aByte === b[j - 1] ? 0 : 1;
+			const deletion = previous[j] + 1;
+			const insertion = current[j - 1] + 1;
+			const substitution = previous[j - 1] + cost;
+			current[j] = deletion < insertion ? (deletion < substitution ? deletion : substitution) : insertion < substitution ? insertion : substitution;
 		}
+		const swap = previous;
+		previous = current;
+		current = swap;
 	}
-	return matrix[a.length * (b.length + 1) + b.length];
+	return previous[width];
 }
 
 /**

package/src/lexer.js

@@ -95,7 +95,7 @@ export class Source {
 		const before = this.code.slice(0, offset).replace(/\r\n|\r/g, '\n');
 		const lineStart = before.lastIndexOf('\n');
 		const prefix = lineStart === -1 ? before : before.slice(lineStart + 1);
-		return new TextEncoder().encode(prefix).length + 1;
+		return UTF8_ENCODER.encode(prefix).length + 1;
 	}
 }
 
@@ -187,6 +187,9 @@ const CLOSING_BRACKETS = ['}', ')', ']'];
 /** Default punctuation characters. */
 const PUNCTUATION = '()[]{}?:.,|';
 
+/** Shared UTF-8 encoder: `TextEncoder` is stateless, so one instance serves all calls. */
+const UTF8_ENCODER = new TextEncoder();
+
 /**
  * Whether a code unit is one of the default punctuation characters.
  *