@teasel/parser 0.0.0 → 0.0.1

package/README.md

@@ -1,3 +1,139 @@
-# teasel
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="banner-dark.svg">
+  <img src="banner.svg" width="100%" alt="">
+</picture>
 
-Placeholder release. The parser lives at https://github.com/Nic-Polumeyv/teasel.
+<h1 align="center">teasel</h1>
+<p align="center">A JavaScript and TypeScript parser in Rust. It answers in ESTree.</p>
+
+<br>
+
+```js
+import { Source } from '@teasel/parser';
+
+const source = new Source(text, { typescript: true, scopes: true });
+const { node } = source.parse();
+```
+
+Every parse is `source.parse(entry, offset, { end, stopAt })`. Every answer is `{ node, end }` and what the options add.
+
+```
+source.parse()                                Program
+source.parse('expression', 7)                 the expression that starts at 7
+source.parse('pattern', 7)                    an assignment target: a name or a destructuring
+source.parse('params', 7)                     the patterns of a (a, b = 1)
+source.parse('statement', 7)                  one statement
+source.parse('typeParameters', 7)             a <T extends U>
+source.parse('program', 12, { end: 40 })      the program inside 12..40, positions of the whole
+```
+
+## Inside a larger syntax
+
+A host that embeds JavaScript in its own reads one piece at a time, from an offset, and gets back where its own syntax resumes.
+
+```
+{{ items as item, index }}
+   ▲    ▲
+   3    8
+```
+
+```js
+const { node, end } = source.parse('expression', 3, { stopAt: ['as', ','] });
+// node  Identifier items
+// end   8
+```
+
+`stopAt` lists the host's own tokens. One read outside every bracket the parse opened, where the expression could end, ends the parse: `,` does not start a sequence, `/>` is not a division. A `then` after `.` is a property name. A TypeScript `as` is the host's unless another `as` follows the assertion, so `xs as T[] as item` ends after the type.
+
+## What the options add
+
+| option | on the tree | on the answer |
+| --- | --- | --- |
+| `locations` | `loc` with line and column on every node | |
+| `comments` | `leadingComments`, `trailingComments`, `innerComments` | `comments`, every comment read |
+| `scopes` | | `scopes`, `bindings`, `references`, and the four questions below |
+| `parenthesized` | `parenthesized: true` on a node the source wraps in parens | |
+| `errorRecovery` | an `Identifier` named `''` of no width where something is missing | `errors`, instead of a throw |
+| `typescript` | TypeScript nodes | |
+| `decorators: 'legacy'` or `'proposal'` | restricts decorator syntax; unset reads both | |
+| `typescript: 'erase'` | JavaScript, the types gone | `typescript`, what could not be erased: enums, namespaces with values, parameter properties, `export =`, `import =`, decorators, accessor fields |
+| `sourceType: 'module'` | strict code, `import`, `export`, top-level `await` | |
+
+A key is on the answer exactly when its option is on. `allowReturnOutsideFunction`, `allowAwaitOutsideFunction`, `allowSuperOutsideMethod` and `allowUndeclaredExports` loosen the early errors as their names say.
+
+## Scopes
+
+The tree stays plain ESTree. The facts hang beside it, reached from a node.
+
+```js
+import { Source, scopeOf, bindingOf, referenceOf, parentOf } from '@teasel/parser';
+
+const { node } = new Source('let x = 1; function f(y) { x = y; }', { scopes: true }).parse();
+const [declaration, fn] = node.body;
+const assignment = fn.body.body[0].expression;
+
+bindingOf(declaration.declarations[0].id)   // { name: 'x', kind: 'let', scope, node, declaration }
+referenceOf(assignment.left)                // { scope, binding, write: true, read: false, mutate: false, node, writeExpr }
+scopeOf(fn)                                 // { kind: 'function', parent, node, topLevelAwait: false }
+parentOf(assignment.left)                   // the assignment
+```
+
+```
+Program ──────────────────── scope: module
+├─ let x = 1                  x  binding  let
+└─ function f(y) {            scope: function
+   │                          y  binding  param
+   └─ x = y                   x  reference  write     y  reference  read
+```
+
+A reference to a name no scope declares has `binding: null`. A copy of a node carries no facts.
+
+## Errors
+
+```js
+try {
+	new Source('x = ;').parse();
+} catch (e) {
+	e.code;    // 'unexpected_token'
+	e.pos;     // 4
+	e.end;     // 5
+	e.loc;     // { line: 1, column: 4 }
+	e.message; // 'Unexpected token'
+}
+```
+
+With `errorRecovery`, the parse comes back and the errors come with it.
+
+```
+{ f(a, }
+       ▲
+       errors  [{ code: 'unexpected_token', pos: 7, end: 7, loc }]
+       node    Identifier ''  at 7..7
+       end     7
+```
+
+## Rust and the command line
+
+```rust
+let (ast, roots, end) = teasel::parse_at(source, 0, None, Entry::Program, options, "")?;
+```
+
+```
+teasel --typescript --scopes file.ts
+```
+
+## Build
+
+```
+cargo build --release
+cd package
+bun run build          # the Node addon
+bun run build:wasm     # the WebAssembly module
+bun test.js
+```
+
+Node resolves `@teasel/parser` to the addon and everything else to the WebAssembly module, with the same API.
+
+<br>
+
+<p align="center"><sub>Named after the plant whose dried heads were used to tease apart wool fibres and raise the nap on cloth.</sub></p>

package/api.js

@@ -0,0 +1,185 @@
+import { decode, facts } from './decode.js';
+import { isIdentifierStart, isIdentifierChar } from './identifier.js';
+
+// acorn's option names, which `Request::set` of json.rs takes as they are
+const OPTIONS = new Set(['sourceType', 'typescript', 'decorators', 'comments', 'scopes', 'locations', 'parenthesized', 'allowReturnOutsideFunction', 'allowAwaitOutsideFunction', 'allowSuperOutsideMethod', 'allowUndeclaredExports', 'errorRecovery']);
+
+// the engine takes the options that are on as their names
+export function names(options) {
+	if (options === undefined) return '';
+	const on = [];
+	for (const key in options) {
+		const value = options[key];
+		if (key === 'host') {
+			if (value !== undefined && typeof value !== 'string') throw new TypeError('host must be the grammar as a string');
+			continue;
+		}
+		if (!OPTIONS.has(key)) throw new TypeError(`${key} is not an option`);
+		if (value === undefined || value === false) continue;
+		if (key === 'decorators') {
+			if (value !== 'legacy' && value !== 'proposal') throw new TypeError(`decorators must be "legacy" or "proposal", not ${JSON.stringify(value)}`);
+			on.push(`${value}Decorators`);
+		} else if (key === 'sourceType') {
+			if (value !== 'script' && value !== 'module') throw new TypeError(`sourceType must be "script" or "module", not ${JSON.stringify(value)}`);
+			if (value === 'module') on.push('module');
+		} else if (value === true) on.push(key);
+		else if (key === 'typescript' && value === 'erase') on.push('typescript', 'erase');
+		else throw new TypeError(`${key} must be a boolean, not ${JSON.stringify(value)}`);
+	}
+	return on.join(' ');
+}
+
+// `Entry` of parser/mod.rs by index
+export const ENTRY = { program: 0, expression: 1, pattern: 2, params: 3, statement: 4, typeParameters: 5 };
+
+// the engine takes the stop tokens as one string
+function stops(list) {
+	if (list === undefined) return '';
+	if (!Array.isArray(list) || !list.every((stop) => typeof stop === 'string' && stop !== '' && !/\s/.test(stop))) {
+		throw new TypeError('stopAt must be a list of words and punctuators');
+	}
+	return list.join(' ');
+}
+
+/**
+ * @typedef {ArrayBuffer | Uint32Array | string} Answer
+ * @typedef {object} Engine
+ * @property {(source: string, names: string, host: string) => any} create
+ * @property {(held: any, entry: number, offset: number, end: number | undefined, stop: string) => Answer} parse
+ * @property {(held: any) => void} [free]
+ * @property {() => string[]} constants
+ * @property {() => ArrayLike<number>} shapes
+ */
+
+// words the engine has to judge: keywords, the strict-mode reserved words and the contextual ones
+const KEYWORD = new Set('await break case catch class const continue debugger default delete do else enum export extends false finally for function if implements import in instanceof interface let new null package private protected public return static super switch this throw true try typeof var void while with yield'.split(' '));
+
+/**
+ * The offset after an identifier the host's syntax follows directly, so the answer needs no
+ * engine: a name, then optional space, then an unmatched closer, the cut, or one of `stopAt`.
+ * Anything the grammar could continue with, a dot or an operator or a TypeScript `as`, and any
+ * word the engine has to judge, is left to it.
+ * @param {string} source @param {number} at @param {number} end @param {string[] | undefined} stopAt
+ * @returns {[number, number] | null} the identifier's end and where the parse ends
+ */
+function bare(source, at, end, stopAt, typescript) {
+	let i = at;
+	const first = source.codePointAt(i);
+	if (first === undefined || !isIdentifierStart(first) || first === 0x5c) return null;
+	i += first > 0xffff ? 2 : 1;
+	while (i < end) {
+		const code = /** @type {number} */ (source.codePointAt(i));
+		if (code === 0x5c) return null;
+		if (!isIdentifierChar(code)) break;
+		i += code > 0xffff ? 2 : 1;
+	}
+	const name_end = i;
+	if (KEYWORD.has(source.slice(at, name_end))) return null;
+	while (i < end && /\s/.test(source[i])) i++;
+	if (i === end) return [name_end, name_end];
+	const c = source[i];
+	if (c === '}' || c === ')' || c === ']') return [name_end, name_end];
+	if (stopAt !== undefined) {
+		for (const stop of stopAt) {
+			if (!source.startsWith(stop, i)) continue;
+			const after = source.codePointAt(i + stop.length);
+			if (isIdentifierStart(/** @type {number} */ (stop.codePointAt(0))) && after !== undefined && isIdentifierChar(after)) continue;
+			// a stop TypeScript reads as its own is the engine's to judge
+			if (typescript && (stop === 'as' || stop === 'satisfies')) return null;
+			return [name_end, name_end];
+		}
+	}
+	return null;
+}
+
+/** @param {Engine} engine */
+export function bind(engine) {
+	const registry = engine.free && typeof FinalizationRegistry !== 'undefined' ? new FinalizationRegistry(engine.free) : null;
+
+	function result(answer, source) {
+		if (typeof answer !== 'string') return decode(answer, source, engine);
+		const { message, ...error } = JSON.parse(answer).error;
+		throw Object.assign(new SyntaxError(message), error);
+	}
+
+	return class Source {
+		#held;
+		#source;
+		#options;
+		/** @type {number[] | undefined} */
+		#lines;
+
+		constructor(source, options) {
+			this.#held = engine.create(source, names(options), options?.host ?? '');
+			this.#source = source;
+			this.#options = options ?? {};
+			registry?.register(this, this.#held, this);
+		}
+
+		/**
+		 * @param {keyof typeof ENTRY} [entry] what to read
+		 * @param {number} [offset] where it starts
+		 * @param {{ end?: number, stopAt?: string[] }} [at] where the source is cut, and the host's tokens that end the parse
+		 */
+		parse(entry = 'program', offset = 0, { end, stopAt } = {}) {
+			if (this.#held === undefined) throw new TypeError('the source is freed');
+			const index = ENTRY[entry];
+			if (index === undefined) throw new TypeError(`${JSON.stringify(entry)} is not an entry`);
+			const stop = stops(stopAt);
+			if (this.#options.host !== undefined && index === ENTRY.program) return result(engine.parse(this.#held, index, 0, undefined, ''), this.#source);
+			if ((index === ENTRY.expression || index === ENTRY.pattern) && Number.isInteger(offset) && offset >= 0) {
+				const cut = end === undefined ? this.#source.length : end;
+				const found = Number.isInteger(cut) && cut <= this.#source.length && offset <= cut ? bare(this.#source, offset, cut, index === ENTRY.pattern ? [',', '(', ':', '='] : stopAt, !!this.#options.typescript) : null;
+				if (found !== null && (index === ENTRY.expression || !this.#source.startsWith(':', found[0]))) return this.#identifier(offset, found[0], index === ENTRY.pattern);
+			}
+			return result(engine.parse(this.#held, index, offset, end, stop), this.#source);
+		}
+
+		/** The answer the engine would give for a bare identifier, built here. */
+		#identifier(start, end, pattern) {
+			const o = this.#options;
+			/** @type {any} */
+			const node = { type: 'Identifier', start, end, name: this.#source.slice(start, end) };
+			if (o.locations) node.loc = { start: this.#position(start), end: this.#position(end) };
+			/** @type {any} */
+			const answer = { node, end };
+			if (o.comments) answer.comments = [];
+			if (o.errorRecovery) answer.errors = [];
+			if (o.typescript === 'erase') answer.typescript = [];
+			if (o.scopes) {
+				const scope = { kind: 'fragment', parent: null, topLevelAwait: false, node };
+				const binding = pattern ? { name: node.name, kind: 'pattern', scope, node, declaration: null } : null;
+				const reference = pattern ? null : { scope, binding: null, write: false, read: true, mutate: false, node, writeExpr: null };
+				answer.scopes = [scope];
+				answer.bindings = binding === null ? [] : [binding];
+				answer.references = reference === null ? [] : [reference];
+				facts(node, scope, binding, reference);
+			} else {
+				facts(node, undefined, undefined, undefined);
+			}
+			return answer;
+		}
+
+		/** Line and column of an offset, from a line table built on first use. */
+		#position(offset) {
+			if (this.#lines === undefined) {
+				this.#lines = [0];
+				for (let i = this.#source.indexOf('\n'); i !== -1; i = this.#source.indexOf('\n', i + 1)) this.#lines.push(i + 1);
+			}
+			let lo = 0, hi = this.#lines.length - 1;
+			while (lo < hi) {
+				const mid = (lo + hi + 1) >> 1;
+				if (this.#lines[mid] <= offset) lo = mid;
+				else hi = mid - 1;
+			}
+			return { line: lo + 1, column: offset - this.#lines[lo] };
+		}
+
+		free() {
+			if (this.#held === undefined) return;
+			registry?.unregister(this);
+			engine.free?.(this.#held);
+			this.#held = undefined;
+		}
+	};
+}