@thomasfosterau/effect-parse 0.1.0 → 0.2.0

package/dist/Token.js

@@ -1,294 +1,360 @@
-import { advance, compileAny, countTokens, describeBytes, End, indexOf, matchLiteral, nextIf, NoMatch, scan, splitSurrogate } from "./internal/tokens.js";
-import { bounds } from "./internal/node.js";
+import * as Predicate from "effect/Predicate";
 import { described } from "./Parser.js";
+import { End, NoMatch, advance, codePointPredicate, compileAny, countTokens, describeLiteral, indexOf, matchLiteral, nextIf, scan, splitSurrogate } from "./internal/tokens.js";
+import { bounds } from "./internal/node.js";
+//#region src/Token.ts
+/**
+* Parsers for individual tokens and slices of tokens (winnow's `token`
+* module, nom's `bytes` module).
+*
+* Parsers here work on every input kind: `string` (tokens are code points),
+* `Uint8Array` (tokens are bytes) and arrays (tokens are elements). Parsers
+* whose input type cannot be inferred from their arguments take a type
+* parameter that defaults to `string`:
+*
+* ```ts
+* import { Token } from "@thomasfosterau/effect-parse"
+*
+* Token.take(4)              // Parser<string, never, never, string>
+* Token.take<Uint8Array>(4)  // Parser<Uint8Array, never, never, Uint8Array>
+* ```
+*
+* On partial input (see `Parser.parsePartial`), parsers that reach the end
+* of the available data report `Incomplete` rather than succeeding early or
+* failing.
+*
+* @since 0.1.0
+*/
+/**
+* Creates an inclusive token range (winnow's `'a'..='z'`).
+*
+* @since 0.1.0
+* @category constructors
+*/
+const range = (from, to) => ({
+	_tag: "TokenRange",
+	from,
+	to
+});
 /**
- * Creates an inclusive token range (winnow's `'a'..='z'`).
- *
- * @since 0.1.0
- * @category constructors
- */
-export const range = (from, to) => ({ _tag: "TokenRange", from, to });
+* Declares a character class as a predicate over code points, for token sets
+* on string, `Text` and byte inputs. `name` describes it in error messages.
+*
+* A plain `(char: string) => boolean` predicate receives each character as a
+* string, which costs an allocation per character tested; a `codePoints`
+* predicate receives the code point itself (a byte, on byte input).
+*
+* `ranges` optionally lists the code points the predicate accepts, as
+* inclusive `[from, to]` pairs. Compiled parsers then test the ranges
+* instead of calling the predicate, so they must agree with it exactly.
+*
+* @example
+* ```ts
+* import { Parser, Token } from "@thomasfosterau/effect-parse"
+*
+* const isNameStart = Token.codePoints(
+*   "name start",
+*   (c) => (c >= 0x61 && c <= 0x7a) || c === 0x5f || c >= 0xc0,
+*   [[0x5f, 0x5f], [0x61, 0x7a], [0xc0, 0x10ffff]]
+* )
+* const name = Token.takeWhile([isNameStart, Token.range("0", "9")], { min: 1 })
+* ```
+*
+* @since 0.2.0
+* @category constructors
+*/
+const codePoints = (name, predicate, ranges) => codePointPredicate(name, (c) => predicate(c), ranges?.map(([from, to]) => [from, to]));
 const expectedAny = ["any token"];
 const anyStep = (state) => {
-    const pos = state.pos;
-    if (pos >= state.end)
-        return state.eoi("Token", expectedAny, 1);
-    switch (state.inputKind) {
-        case 0: {
-            const s = state.input;
-            if (splitSurrogate(state, pos))
-                return state.incomplete(1, expectedAny);
-            const c = s.charCodeAt(pos);
-            if (c >= 0xd800 && c <= 0xdbff && pos + 1 < s.length && (s.charCodeAt(pos + 1) & 0xfc00) === 0xdc00) {
-                state.pos = pos + 2;
-                return s.slice(pos, pos + 2);
-            }
-            state.pos = pos + 1;
-            return s[pos];
-        }
-        case 1:
-            state.pos = pos + 1;
-            return state.input[pos];
-        default:
-            state.pos = pos + 1;
-            return state.input[pos];
-    }
+	const pos = state.pos;
+	if (pos >= state.end) return state.eoi("Token", expectedAny, 1);
+	switch (state.inputKind) {
+		case 0: {
+			const s = state.input;
+			if (splitSurrogate(state, pos)) return state.incomplete(1, expectedAny);
+			const c = s.charCodeAt(pos);
+			if (c >= 55296 && c <= 56319 && pos + 1 < s.length && (s.charCodeAt(pos + 1) & 64512) === 56320) {
+				state.pos = pos + 2;
+				return s.slice(pos, pos + 2);
+			}
+			state.pos = pos + 1;
+			return s[pos];
+		}
+		case 1:
+			state.pos = pos + 1;
+			return state.input[pos];
+		default:
+			state.pos = pos + 1;
+			return state.input[pos];
+	}
 };
 /**
- * Matches any single token: a character of a string (winnow's `any`, nom's
- * `anychar`).
- *
- * @since 0.1.0
- * @category tokens
- */
-export const any = described(anyStep, "any", { _tag: "Any", encoding: undefined });
+* Matches any single token: a character of a string (winnow's `any`, nom's
+* `anychar`).
+*
+* @since 0.1.0
+* @category tokens
+*/
+const any = /*#__PURE__*/ described(anyStep, "any", {
+	_tag: "Any",
+	encoding: void 0
+});
 /**
- * Matches any single token of an input of type `I`.
- *
- * @example
- * ```ts
- * import { Parser, Token } from "@thomasfosterau/effect-parse"
- *
- * const byte = Token.anyToken<Uint8Array>()
- * Parser.parse(byte, new Uint8Array([7])) // Result.succeed(7)
- * ```
- *
- * @since 0.1.0
- * @category tokens
- */
-export const anyToken = () => any;
+* Matches any single token of an input of type `I`.
+*
+* @example
+* ```ts
+* import { Parser, Token } from "@thomasfosterau/effect-parse"
+*
+* const byte = Token.anyToken<Uint8Array>()
+* Parser.parse(byte, new Uint8Array([7])) // Result.succeed(7)
+* ```
+*
+* @since 0.1.0
+* @category tokens
+*/
+const anyToken = () => any;
 /**
- * Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).
- * Strings are matched against string input, `Uint8Array`s against byte
- * input, and arrays element-wise (with `===`) against array input.
- *
- * @example
- * ```ts
- * import { Parser, Token } from "@thomasfosterau/effect-parse"
- *
- * const keyword = Parser.alt(Token.literal("true"), Token.literal("false"))
- * //    ^? Parser<"true" | "false">
- * ```
- *
- * @since 0.1.0
- * @category tokens
- */
-export const literal = (lit) => literalImpl(lit, false);
+* Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).
+* Strings are matched against string input, `Uint8Array`s against byte
+* input, and arrays element-wise (with `===`) against array input.
+*
+* @example
+* ```ts
+* import { Parser, Token } from "@thomasfosterau/effect-parse"
+*
+* const keyword = Parser.alt(Token.literal("true"), Token.literal("false"))
+* //    ^? Parser<"true" | "false">
+* ```
+*
+* @since 0.1.0
+* @category tokens
+*/
+const literal = (lit) => literalImpl(lit, false);
 /** @internal */
-export const literalImpl = (lit, caseless) => {
-    const description = typeof lit === "string"
-        ? JSON.stringify(lit)
-        : lit instanceof Uint8Array
-            ? describeBytes(lit)
-            : `[${lit.map((t) => JSON.stringify(t)).join(", ")}]`;
-    const expected = [caseless ? `${description} (case insensitive)` : description];
-    // A string literal may also be matched against byte input (as UTF-8)
-    let asBytes;
-    const literalKind = typeof lit === "string" ? 0 : lit instanceof Uint8Array ? 1 : 2;
-    return described((state) => {
-        const pos = state.pos;
-        const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit)) : lit;
-        const len = l.length;
-        const r = matchLiteral(state, l, caseless);
-        if (r === -1)
-            return state.backtrack("Literal", expected);
-        if (r === -2) {
-            if (state.partial)
-                return state.incomplete(len - (state.end - pos), expected);
-            return state.backtrack("Literal", expected);
-        }
-        state.pos = pos + len;
-        return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit;
-    }, description, { _tag: "Literal", literal: lit, caseless, encoding: undefined });
+const literalImpl = (lit, caseless) => {
+	const description = describeLiteral(lit);
+	const expected = [caseless ? `${description} (case insensitive)` : description];
+	let asBytes;
+	const literalKind = Predicate.isString(lit) ? 0 : lit instanceof Uint8Array ? 1 : 2;
+	return described((state) => {
+		const pos = state.pos;
+		const l = literalKind === 0 && state.inputKind === 1 ? asBytes ??= new TextEncoder().encode(lit) : lit;
+		const len = l.length;
+		const r = matchLiteral(state, l, caseless);
+		if (r === -1) return state.backtrack("Literal", expected);
+		if (r === -2) {
+			if (state.partial) return state.incomplete(len - (state.end - pos), expected);
+			return state.backtrack("Literal", expected);
+		}
+		state.pos = pos + len;
+		return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit;
+	}, description, {
+		_tag: "Literal",
+		literal: lit,
+		caseless,
+		encoding: void 0
+	});
 };
 /**
- * Matches one token belonging to a set, returning it (winnow's and nom's
- * `one_of`).
- *
- * @example
- * ```ts
- * import { Parser, Token } from "@thomasfosterau/effect-parse"
- *
- * const sign = Token.oneOf("+-")
- * const lower = Token.oneOf(Token.range("a", "z"))
- * const hexByte = Token.oneOf<Uint8Array>([Token.range(0x30, 0x39), Token.range(0x61, 0x66)])
- * ```
- *
- * @since 0.1.0
- * @category tokens
- */
-export const oneOf = (set) => {
-    const m = compileAny(set);
-    const expected = [m.description];
-    return described((state) => {
-        const t = nextIf(state, m, false);
-        if (t === End)
-            return state.eoi("OneOf", expected, 1);
-        if (t === NoMatch)
-            return state.backtrack("OneOf", expected);
-        return t;
-    }, m.description, { _tag: "OneOf", set, negate: false, encoding: undefined });
+* Matches one token belonging to a set, returning it (winnow's and nom's
+* `one_of`).
+*
+* @example
+* ```ts
+* import { Parser, Token } from "@thomasfosterau/effect-parse"
+*
+* const sign = Token.oneOf("+-")
+* const lower = Token.oneOf(Token.range("a", "z"))
+* const hexByte = Token.oneOf<Uint8Array>([Token.range(0x30, 0x39), Token.range(0x61, 0x66)])
+* ```
+*
+* @since 0.1.0
+* @category tokens
+*/
+const oneOf = (set) => {
+	const m = compileAny(set);
+	const expected = [m.description];
+	return described((state) => {
+		const t = nextIf(state, m, false);
+		if (t === End) return state.eoi("OneOf", expected, 1);
+		if (t === NoMatch) return state.backtrack("OneOf", expected);
+		return t;
+	}, m.description, {
+		_tag: "OneOf",
+		set,
+		negate: false,
+		encoding: void 0
+	});
 };
 /**
- * Matches one token *not* belonging to a set, returning it (winnow's and
- * nom's `none_of`).
- *
- * @since 0.1.0
- * @category tokens
- */
-export const noneOf = (set) => {
-    const m = compileAny(set);
-    const expected = [`anything but ${m.description}`];
-    return described((state) => {
-        const t = nextIf(state, m, true);
-        if (t === End)
-            return state.eoi("NoneOf", expected, 1);
-        if (t === NoMatch)
-            return state.backtrack("NoneOf", expected);
-        return t;
-    }, expected[0], { _tag: "OneOf", set, negate: true, encoding: undefined });
+* Matches one token *not* belonging to a set, returning it (winnow's and
+* nom's `none_of`).
+*
+* @since 0.1.0
+* @category tokens
+*/
+const noneOf = (set) => {
+	const m = compileAny(set);
+	const expected = [`anything but ${m.description}`];
+	return described((state) => {
+		const t = nextIf(state, m, true);
+		if (t === End) return state.eoi("NoneOf", expected, 1);
+		if (t === NoMatch) return state.backtrack("NoneOf", expected);
+		return t;
+	}, expected[0], {
+		_tag: "OneOf",
+		set,
+		negate: true,
+		encoding: void 0
+	});
 };
 /**
- * Takes exactly `count` tokens (winnow's and nom's `take`).
- *
- * @since 0.1.0
- * @category slices
- */
-export const take = (count) => {
-    const expected = [count === 1 ? "1 token" : `${count} tokens`];
-    return described((state) => {
-        const start = state.pos;
-        const { count: got, offset } = advance(state, start, count);
-        if (got < count)
-            return state.eoi("Take", expected, count - got);
-        state.pos = offset;
-        return state.slice(start, offset);
-    }, "take", { _tag: "Take", count, encoding: undefined });
+* Takes exactly `count` tokens (winnow's and nom's `take`).
+*
+* @since 0.1.0
+* @category slices
+*/
+const take = (count) => {
+	const expected = [count === 1 ? "1 token" : `${count} tokens`];
+	return described((state) => {
+		const start = state.pos;
+		const { count: got, offset } = advance(state, start, count);
+		if (got < count) return state.eoi("Take", expected, count - got);
+		state.pos = offset;
+		return state.slice(start, offset);
+	}, "take", {
+		_tag: "Take",
+		count,
+		encoding: void 0
+	});
 };
 const normalize = (r) => {
-    if (r === undefined)
-        return [0, Infinity];
-    if (typeof r === "number")
-        return [r, r];
-    return [r.min ?? 0, r.max ?? Infinity];
+	if (r === void 0) return [0, Infinity];
+	if (Predicate.isNumber(r)) return [r, r];
+	return [r.min ?? 0, r.max ?? Infinity];
 };
 /** @internal */
-export const takeWhileImpl = (set, r, negate, name, description, kind = "TakeWhile", hint = false) => {
-    const m = compileAny(set);
-    const [min, max] = normalize(r);
-    const expected = [description ?? (negate ? `anything but ${m.description}` : m.description)];
-    return described((state) => {
-        const start = state.pos;
-        const { count, hitEnd, offset } = scan(state, m, negate, max);
-        if (hitEnd && state.partial)
-            return state.incomplete(1, expected);
-        if (count < min)
-            return state.backtrack(kind, expected, offset);
-        // More matching tokens would have been accepted where we stopped
-        if (hint && count < max)
-            state.setHint(offset, expected);
-        state.pos = offset;
-        return state.slice(start, offset);
-    }, name, { _tag: "TakeWhile", set, negate, min, max, encoding: undefined });
+const takeWhileImpl = (set, r, negate, name, description, kind = "TakeWhile", hint = false) => {
+	const m = compileAny(set);
+	const [min, max] = normalize(r);
+	const expected = [description ?? (negate ? `anything but ${m.description}` : m.description)];
+	return described((state) => {
+		const start = state.pos;
+		const { count, hitEnd, offset } = scan(state, m, negate, max);
+		if (hitEnd && state.partial) return state.incomplete(1, expected);
+		if (count < min) return state.backtrack(kind, expected, offset);
+		if (hint && count < max) state.setHint(offset, expected);
+		state.pos = offset;
+		return state.slice(start, offset);
+	}, name, {
+		_tag: "TakeWhile",
+		set,
+		negate,
+		min,
+		max,
+		encoding: void 0
+	});
 };
 /**
- * Takes the longest run of tokens belonging to a set, with the number of
- * tokens constrained by `range` (default: zero or more). This is winnow's
- * `take_while(range, set)`, and nom's `take_while`, `take_while1` and
- * `take_while_m_n`.
- *
- * @example
- * ```ts
- * import { Parser, Token } from "@thomasfosterau/effect-parse"
- *
- * const identifier = Token.takeWhile(
- *   [Token.range("a", "z"), Token.range("A", "Z"), "_"],
- *   { min: 1 }
- * )
- * ```
- *
- * @since 0.1.0
- * @category slices
- */
-export const takeWhile = (set, range) => takeWhileImpl(set, range, false, "takeWhile");
+* Takes the longest run of tokens belonging to a set, with the number of
+* tokens constrained by `range` (default: zero or more). This is winnow's
+* `take_while(range, set)`, and nom's `take_while`, `take_while1` and
+* `take_while_m_n`.
+*
+* @example
+* ```ts
+* import { Parser, Token } from "@thomasfosterau/effect-parse"
+*
+* const identifier = Token.takeWhile(
+*   [Token.range("a", "z"), Token.range("A", "Z"), "_"],
+*   { min: 1 }
+* )
+* ```
+*
+* @since 0.1.0
+* @category slices
+*/
+const takeWhile = (set, range) => takeWhileImpl(set, range, false, "takeWhile");
 /**
- * Takes the longest run of tokens *not* belonging to a set (winnow's
- * `take_till`, nom's `take_till` and `take_till1`).
- *
- * @since 0.1.0
- * @category slices
- */
-export const takeTill = (set, range) => takeWhileImpl(set, range, true, "takeTill");
+* Takes the longest run of tokens *not* belonging to a set (winnow's
+* `take_till`, nom's `take_till` and `take_till1`).
+*
+* @since 0.1.0
+* @category slices
+*/
+const takeTill = (set, range) => takeWhileImpl(set, range, true, "takeTill");
 /**
- * Takes tokens up to (not including) the first occurrence of a literal
- * (winnow's `take_until`, nom's `take_until`). The literal is not
- * consumed. `range` constrains the number of tokens taken.
- *
- * @since 0.1.0
- * @category slices
- */
-export const takeUntil = (lit, r) => {
-    const [min, max] = normalize(r);
-    const description = typeof lit === "string"
-        ? JSON.stringify(lit)
-        : lit instanceof Uint8Array
-            ? describeBytes(lit)
-            : `[${lit.map((t) => JSON.stringify(t)).join(", ")}]`;
-    const expected = [description];
-    return described((state) => {
-        const start = state.pos;
-        const idx = indexOf(state, lit, start);
-        if (idx < 0) {
-            if (state.partial)
-                return state.incomplete(0, expected);
-            return state.backtrack("TakeUntil", expected, state.end);
-        }
-        // As in winnow, the first occurrence must lie within the range
-        if (min > 0 || max !== Infinity) {
-            const count = countTokens(state, start, idx);
-            if (count < min)
-                return state.backtrack("TakeUntil", expected, idx);
-            if (count > max)
-                return state.backtrack("TakeUntil", expected);
-        }
-        state.pos = idx;
-        return state.slice(start, idx);
-    }, "takeUntil", { _tag: "TakeUntil", literal: lit, ...bounds(r), encoding: undefined });
+* Takes tokens up to (not including) the first occurrence of a literal
+* (winnow's `take_until`, nom's `take_until`). The literal is not
+* consumed. `range` constrains the number of tokens taken.
+*
+* @since 0.1.0
+* @category slices
+*/
+const takeUntil = (lit, r) => {
+	const [min, max] = normalize(r);
+	const expected = [describeLiteral(lit)];
+	return described((state) => {
+		const start = state.pos;
+		const idx = indexOf(state, lit, start);
+		if (idx < 0) {
+			if (state.partial) return state.incomplete(0, expected);
+			return state.backtrack("TakeUntil", expected, state.end);
+		}
+		state.syntax?.read(idx + lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1));
+		if (min > 0 || max !== Infinity) {
+			const count = countTokens(state, start, idx);
+			if (count < min) return state.backtrack("TakeUntil", expected, idx);
+			if (count > max) return state.backtrack("TakeUntil", expected);
+		}
+		state.pos = idx;
+		return state.slice(start, idx);
+	}, "takeUntil", {
+		_tag: "TakeUntil",
+		literal: lit,
+		...bounds(r),
+		encoding: void 0
+	});
 };
 /**
- * Takes the rest of the input (winnow's and nom's `rest`).
- *
- * @since 0.1.0
- * @category slices
- */
-export const rest = described((state) => {
-    const start = state.pos;
-    state.pos = state.end;
-    return state.slice(start, state.end);
-}, "rest", { _tag: "Rest", encoding: undefined });
+* Takes the rest of the input (winnow's and nom's `rest`).
+*
+* @since 0.1.0
+* @category slices
+*/
+const rest = /*#__PURE__*/ described((state) => {
+	const start = state.pos;
+	state.pos = state.end;
+	return state.slice(start, state.end);
+}, "rest", {
+	_tag: "Rest",
+	encoding: void 0
+});
 /**
- * Takes the rest of an input of type `I`.
- *
- * @since 0.1.0
- * @category slices
- */
-export const restOf = () => rest;
+* Takes the rest of an input of type `I`.
+*
+* @since 0.1.0
+* @category slices
+*/
+const restOf = () => rest;
 /**
- * Returns the length of the remaining input without consuming it (winnow's
- * and nom's `rest_len`).
- *
- * @since 0.1.0
- * @category slices
- */
-export const restLen = described((state) => state.end - state.pos, "restLen", {
-    _tag: "RestLen"
-});
+* Returns the length of the remaining input without consuming it (winnow's
+* and nom's `rest_len`).
+*
+* @since 0.1.0
+* @category slices
+*/
+const restLen = /*#__PURE__*/ described((state) => state.end - state.pos, "restLen", { _tag: "RestLen" });
 /**
- * Matches a token satisfying a predicate (nom's `satisfy`). Equivalent to
- * `Token.oneOf(predicate)`.
- *
- * @since 0.1.0
- * @category tokens
- */
-export const satisfy = (predicate) => oneOf(predicate);
+* Matches a token satisfying a predicate (nom's `satisfy`). Equivalent to
+* `Token.oneOf(predicate)`.
+*
+* @since 0.1.0
+* @category tokens
+*/
+const satisfy = (predicate) => oneOf(predicate);
+//#endregion
+export { any, anyToken, codePoints, literal, literalImpl, noneOf, oneOf, range, rest, restLen, restOf, satisfy, take, takeTill, takeUntil, takeWhile, takeWhileImpl };
+
 //# sourceMappingURL=Token.js.map