@thomasfosterau/effect-parse 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/README.md +56 -14
- package/dist/Ascii.d.ts +11 -4
- package/dist/Ascii.d.ts.map +1 -1
- package/dist/Ascii.js +12 -4
- package/dist/Ascii.js.map +1 -1
- package/dist/Binary.d.ts +5 -2
- package/dist/Binary.d.ts.map +1 -1
- package/dist/Binary.js +83 -37
- package/dist/Binary.js.map +1 -1
- package/dist/Bits.d.ts.map +1 -1
- package/dist/Bits.js +11 -4
- package/dist/Bits.js.map +1 -1
- package/dist/Compiler.d.ts +74 -2
- package/dist/Compiler.d.ts.map +1 -1
- package/dist/Compiler.js +78 -1
- package/dist/Compiler.js.map +1 -1
- package/dist/Expression.d.ts.map +1 -1
- package/dist/Expression.js +86 -74
- package/dist/Expression.js.map +1 -1
- package/dist/Input.d.ts +56 -5
- package/dist/Input.d.ts.map +1 -1
- package/dist/Input.js +70 -1
- package/dist/Input.js.map +1 -1
- package/dist/Nom.d.ts +5 -15
- package/dist/Nom.d.ts.map +1 -1
- package/dist/Nom.js.map +1 -1
- package/dist/ParseError.d.ts +69 -11
- package/dist/ParseError.d.ts.map +1 -1
- package/dist/ParseError.js +129 -28
- package/dist/ParseError.js.map +1 -1
- package/dist/Parser.d.ts +375 -29
- package/dist/Parser.d.ts.map +1 -1
- package/dist/Parser.js +394 -37
- package/dist/Parser.js.map +1 -1
- package/dist/ParserSchema.d.ts.map +1 -1
- package/dist/ParserSchema.js +2 -3
- package/dist/ParserSchema.js.map +1 -1
- package/dist/ParserStream.d.ts +122 -14
- package/dist/ParserStream.d.ts.map +1 -1
- package/dist/ParserStream.js +293 -62
- package/dist/ParserStream.js.map +1 -1
- package/dist/Syntax.d.ts +23 -5
- package/dist/Syntax.d.ts.map +1 -1
- package/dist/Syntax.js +1 -1
- package/dist/Syntax.js.map +1 -1
- package/dist/Text.d.ts.map +1 -1
- package/dist/Text.js +21 -15
- package/dist/Text.js.map +1 -1
- package/dist/Token.d.ts +65 -11
- package/dist/Token.d.ts.map +1 -1
- package/dist/Token.js +73 -26
- package/dist/Token.js.map +1 -1
- package/dist/internal/compiler/closure.d.ts +1 -1
- package/dist/internal/compiler/closure.d.ts.map +1 -1
- package/dist/internal/compiler/closure.js +263 -113
- package/dist/internal/compiler/closure.js.map +1 -1
- package/dist/internal/compiler/entries.d.ts +3 -1
- package/dist/internal/compiler/entries.d.ts.map +1 -1
- package/dist/internal/compiler/entries.js +3 -2
- package/dist/internal/compiler/entries.js.map +1 -1
- package/dist/internal/compiler/first.d.ts.map +1 -1
- package/dist/internal/compiler/first.js +9 -2
- package/dist/internal/compiler/first.js.map +1 -1
- package/dist/internal/compiler/graph.d.ts +39 -0
- package/dist/internal/compiler/graph.d.ts.map +1 -0
- package/dist/internal/compiler/graph.js +78 -0
- package/dist/internal/compiler/graph.js.map +1 -0
- package/dist/internal/compiler/jit.d.ts +5 -0
- package/dist/internal/compiler/jit.d.ts.map +1 -1
- package/dist/internal/compiler/jit.js +208 -106
- package/dist/internal/compiler/jit.js.map +1 -1
- package/dist/internal/compiler/registry.d.ts +11 -3
- package/dist/internal/compiler/registry.d.ts.map +1 -1
- package/dist/internal/compiler/registry.js.map +1 -1
- package/dist/internal/compiler/search.d.ts +32 -0
- package/dist/internal/compiler/search.d.ts.map +1 -0
- package/dist/internal/compiler/search.js +68 -0
- package/dist/internal/compiler/search.js.map +1 -0
- package/dist/internal/compiler/sets.d.ts.map +1 -1
- package/dist/internal/compiler/sets.js +1 -1
- package/dist/internal/compiler/sets.js.map +1 -1
- package/dist/internal/compiler/wasm.d.ts +8 -0
- package/dist/internal/compiler/wasm.d.ts.map +1 -0
- package/dist/internal/compiler/wasm.js +2840 -0
- package/dist/internal/compiler/wasm.js.map +1 -0
- package/dist/internal/compiler/wasmAsm.d.ts +199 -0
- package/dist/internal/compiler/wasmAsm.d.ts.map +1 -0
- package/dist/internal/compiler/wasmAsm.js +427 -0
- package/dist/internal/compiler/wasmAsm.js.map +1 -0
- package/dist/internal/entries.d.ts +3 -1
- package/dist/internal/entries.d.ts.map +1 -1
- package/dist/internal/entries.js.map +1 -1
- package/dist/internal/node.d.ts +77 -3
- package/dist/internal/node.d.ts.map +1 -1
- package/dist/internal/node.js +54 -10
- package/dist/internal/node.js.map +1 -1
- package/dist/internal/numbers.d.ts.map +1 -1
- package/dist/internal/numbers.js.map +1 -1
- package/dist/internal/run.d.ts +14 -0
- package/dist/internal/run.d.ts.map +1 -1
- package/dist/internal/run.js +53 -11
- package/dist/internal/run.js.map +1 -1
- package/dist/internal/stack.d.ts +14 -0
- package/dist/internal/stack.d.ts.map +1 -1
- package/dist/internal/stack.js +24 -2
- package/dist/internal/stack.js.map +1 -1
- package/dist/internal/syntax.d.ts +4 -4
- package/dist/internal/syntax.d.ts.map +1 -1
- package/dist/internal/syntax.js +18 -10
- package/dist/internal/syntax.js.map +1 -1
- package/dist/internal/text.d.ts +32 -0
- package/dist/internal/text.d.ts.map +1 -1
- package/dist/internal/text.js +68 -14
- package/dist/internal/text.js.map +1 -1
- package/dist/internal/tokens.d.ts +20 -0
- package/dist/internal/tokens.d.ts.map +1 -1
- package/dist/internal/tokens.js +72 -39
- package/dist/internal/tokens.js.map +1 -1
- package/dist/internal/tuple.d.ts +6 -0
- package/dist/internal/tuple.d.ts.map +1 -0
- package/dist/internal/tuple.js +170 -0
- package/dist/internal/tuple.js.map +1 -0
- package/package.json +18 -10
- package/src/Ascii.ts +47 -12
- package/src/Binary.ts +174 -73
- package/src/Bits.ts +16 -6
- package/src/Compiler.ts +101 -10
- package/src/Expression.ts +94 -81
- package/src/Input.ts +99 -7
- package/src/Nom.ts +15 -5
- package/src/ParseError.ts +209 -52
- package/src/Parser.ts +684 -143
- package/src/ParserSchema.ts +1 -2
- package/src/ParserStream.ts +361 -68
- package/src/Syntax.ts +60 -36
- package/src/Text.ts +32 -27
- package/src/Token.ts +152 -34
- package/src/internal/compiler/closure.ts +371 -153
- package/src/internal/compiler/entries.ts +4 -2
- package/src/internal/compiler/first.ts +16 -0
- package/src/internal/compiler/graph.ts +96 -0
- package/src/internal/compiler/jit.ts +319 -169
- package/src/internal/compiler/registry.ts +10 -2
- package/src/internal/compiler/search.ts +68 -0
- package/src/internal/compiler/sets.ts +7 -3
- package/src/internal/compiler/wasm.ts +3276 -0
- package/src/internal/compiler/wasmAsm.ts +484 -0
- package/src/internal/entries.ts +8 -2
- package/src/internal/node.ts +103 -7
- package/src/internal/numbers.ts +3 -1
- package/src/internal/run.ts +73 -12
- package/src/internal/stack.ts +25 -1
- package/src/internal/syntax.ts +39 -18
- package/src/internal/text.ts +97 -9
- package/src/internal/tokens.ts +106 -14
- package/src/internal/tuple.ts +141 -0
package/dist/Token.d.ts
CHANGED
|
@@ -109,6 +109,17 @@ export declare const any: Parser<string>;
|
|
|
109
109
|
* @category tokens
|
|
110
110
|
*/
|
|
111
111
|
export declare const anyToken: <I extends Input = string>() => Parser<Token<NoInfer<I>>, never, never, NoInfer<I>>;
|
|
112
|
+
/**
|
|
113
|
+
* The overloads of {@link literal}.
|
|
114
|
+
*
|
|
115
|
+
* @since 0.5.0
|
|
116
|
+
* @category tokens
|
|
117
|
+
*/
|
|
118
|
+
export interface LiteralFn {
|
|
119
|
+
<const L extends string>(literal: L): Parser<L, never, never, string>;
|
|
120
|
+
(literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>;
|
|
121
|
+
<const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>;
|
|
122
|
+
}
|
|
112
123
|
/**
|
|
113
124
|
* Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).
|
|
114
125
|
* Strings are matched against string input, `Uint8Array`s against byte
|
|
@@ -125,11 +136,7 @@ export declare const anyToken: <I extends Input = string>() => Parser<Token<NoIn
|
|
|
125
136
|
* @since 0.1.0
|
|
126
137
|
* @category tokens
|
|
127
138
|
*/
|
|
128
|
-
export declare const literal:
|
|
129
|
-
<const L extends string>(literal: L): Parser<L, never, never, string>;
|
|
130
|
-
(literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>;
|
|
131
|
-
<const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>;
|
|
132
|
-
};
|
|
139
|
+
export declare const literal: LiteralFn;
|
|
133
140
|
/** @internal */
|
|
134
141
|
export declare const literalImpl: (lit: Input, caseless: boolean) => Parser<any, never, never, any>;
|
|
135
142
|
/**
|
|
@@ -158,7 +165,16 @@ export declare const oneOf: <I extends Input = string>(set: TokenSet<I>) => Pars
|
|
|
158
165
|
*/
|
|
159
166
|
export declare const noneOf: <I extends Input = string>(set: TokenSet<I>) => Parser<Token<NoInfer<I>>, never, never, NoInfer<I>>;
|
|
160
167
|
/**
|
|
161
|
-
* Takes exactly `count` tokens (winnow's and nom's `take`).
|
|
168
|
+
* Takes exactly `count` tokens (winnow's and nom's `take`). `count` must be
|
|
169
|
+
* a non-negative integer: anything else throws a `RangeError` here, when
|
|
170
|
+
* the parser is built.
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
* ```ts
|
|
174
|
+
* import { Parser, Token } from "@thomasfosterau/effect-parse"
|
|
175
|
+
*
|
|
176
|
+
* Parser.parsePeek(Token.take(2), "abc") // Result.succeed(["c", "ab"])
|
|
177
|
+
* ```
|
|
162
178
|
*
|
|
163
179
|
* @since 0.1.0
|
|
164
180
|
* @category slices
|
|
@@ -194,6 +210,17 @@ export declare const takeWhile: <I extends Input = string>(set: TokenSet<I>, ran
|
|
|
194
210
|
* @category slices
|
|
195
211
|
*/
|
|
196
212
|
export declare const takeTill: <I extends Input = string>(set: TokenSet<I>, range?: Range) => Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>>;
|
|
213
|
+
/**
|
|
214
|
+
* The overloads of {@link takeUntil}.
|
|
215
|
+
*
|
|
216
|
+
* @since 0.5.0
|
|
217
|
+
* @category slices
|
|
218
|
+
*/
|
|
219
|
+
export interface TakeUntilFn {
|
|
220
|
+
(literal: string, range?: Range): Parser<string, never, never, string>;
|
|
221
|
+
(literal: Uint8Array, range?: Range): Parser<Uint8Array, never, never, Uint8Array>;
|
|
222
|
+
<T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>;
|
|
223
|
+
}
|
|
197
224
|
/**
|
|
198
225
|
* Takes tokens up to (not including) the first occurrence of a literal
|
|
199
226
|
* (winnow's `take_until`, nom's `take_until`). The literal is not
|
|
@@ -202,11 +229,38 @@ export declare const takeTill: <I extends Input = string>(set: TokenSet<I>, rang
|
|
|
202
229
|
* @since 0.1.0
|
|
203
230
|
* @category slices
|
|
204
231
|
*/
|
|
205
|
-
export declare const takeUntil:
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
232
|
+
export declare const takeUntil: TakeUntilFn;
|
|
233
|
+
/**
|
|
234
|
+
* The overloads of {@link takeUntilAny}.
|
|
235
|
+
*
|
|
236
|
+
* @since 0.5.0
|
|
237
|
+
* @category slices
|
|
238
|
+
*/
|
|
239
|
+
export interface TakeUntilAnyFn {
|
|
240
|
+
(literals: ReadonlyArray<string>, range?: Range): Parser<string, never, never, string>;
|
|
241
|
+
(literals: ReadonlyArray<Uint8Array>, range?: Range): Parser<Uint8Array, never, never, Uint8Array>;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Takes tokens up to (not including) the first occurrence of any of
|
|
245
|
+
* several literals: {@link takeUntil} for a set of them, the search a
|
|
246
|
+
* multi-pattern prefilter makes. No literal is consumed. `range` constrains
|
|
247
|
+
* the number of tokens taken. On bytes, `Compiler.wasm` searches with SIMD
|
|
248
|
+
* (Teddy, from ripgrep), sixteen bytes at a time.
|
|
249
|
+
*
|
|
250
|
+
* @example
|
|
251
|
+
* ```ts
|
|
252
|
+
* import { Parser, Token } from "@thomasfosterau/effect-parse"
|
|
253
|
+
* import * as Result from "effect/Result"
|
|
254
|
+
*
|
|
255
|
+
* const upTo = Parser.seq(Token.takeUntilAny(["Holmes", "Watson"]), Token.rest)
|
|
256
|
+
* console.log(Result.getOrThrow(Parser.parseSync(upTo, "Dr. Watson and Holmes")))
|
|
257
|
+
* // ["Dr. ", "Watson and Holmes"]
|
|
258
|
+
* ```
|
|
259
|
+
*
|
|
260
|
+
* @since 0.5.0
|
|
261
|
+
* @category slices
|
|
262
|
+
*/
|
|
263
|
+
export declare const takeUntilAny: TakeUntilAnyFn;
|
|
210
264
|
/**
|
|
211
265
|
* Takes the rest of the input (winnow's and nom's `rest`).
|
|
212
266
|
*
|
package/dist/Token.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Token.d.ts","names":[],"sources":["../src/Token.ts"],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"Token.d.ts","names":[],"sources":["../src/Token.ts"],"mappings":";;;;;;;;;iBA0DiB,WAAW;WACjB;WACA,MAAM;WACN,IAAI;;;;;;;;qBASF,QAAS,2BAA2B,MAAM,GAAG,IAAI,MAAI,WAAW;cAE/D;;;;;;;;;iBAUG;GACd;YACS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBA8BC,aACX,cACA,YAAY,+BACZ,SAAS,6CACR;;;;;;;;;;YAkBS,0BAER,qBACA,eACE,4BACF,cAAc;;;;;;;;YASN,iCAGR,aACA,qBACA,eACE,4BACF,cAAc;;;;;;;;YASN,SAAS,KAAK,mBACtB,iBACA,UAAU,aACR,eACA,UAAU,oBAAoB,OAC1B,OAAO,iBAAiB,cAAc;;;;;;;;qBA6CnC,KAAK;;;;;;;;;;;;;;;qBAkBL,WAAY,UAAU,qBAAmB,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;iBAUrF;SACR,kBAAkB,SAAS,IAAI,OAAO;GAC5C,SAAS,aAAa,OAAO,0BAA0B;SACjD,GAAG,SAAS,cAAc,KAAK,OAAO,cAAc,kBAAkB,cAAc;;;;;;;;;;;;;;;;;;qBAmBhF,SAAS;;qBAGT,cAAe,KAAK,OAAO,sBAAoB;;;;;;;;;;;;;;;;;qBA0C/C,QAAS,UAAU,gBAC9B,KAAK,SAAS,OACb,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;;qBAuBtC,SAAU,UAAU,gBAC/B,KAAK,SAAS,OACb,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;;;;;;;;;;qBA+BtC,OAAQ,UAAU,gBAAgB,kBAAgB,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;qBAsBlG,gBAAiB,UAAU,OACtC,cACA,GAAG,mBACH,iBACA,cACA,sBACA,OAAM,WACN,mBACC,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;;;;;;;;;;;;;;qBAuCtC,YAAa,UAAU,gBAClC,KAAK,SAAS,IACd,QAAQ,UACP,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;;qBAStC,WAAY,UAAU,gBACjC,KAAK,SAAS,IACd,QAAQ,UACP,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;iBAQlC;GACd,iBAAiB,QAAQ,QAAQ;GACjC,SAAS,YAAY,QAAQ,QAAQ,OAAO,0BAA0B;GACtE,GAAG,SAAS,cAAc,IAAI,QAAQ,QAAQ,OAAO,cAAc,kBAAkB,cAAc;;;;;;;;;;qBAWzF,WAAW;;;;;;;iBAqDP;GACd,UAAU,uBAAuB,QAAQ,QAAQ;GACjD,UAAU,cAAc,aAAa,QAAQ,QAAQ,OAAO,0BAA0B;;;;;;;;;;;;;;;;;;;;;;qBAuB5E,cAAc;;;;;;;qBAiCd,MAAM;;;;;;;qBAUN,SAAU,UAAU,qBAAmB,OAAO,MAAM,QAAQ,mBAAmB,QAAQ;;;;;;;;qBAWvF,SAAS,6BAA6B;;;;;;;;qBAetC,UAAW,UAAU,gBAChC,YAAY,OAAO,MAAM,mBACxB,OAAO,MAAM,QAAQ,mBAAmB,QAAQ"}
|
package/dist/Token.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as Predicate from "effect/Predicate";
|
|
2
2
|
import { described } from "./Parser.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { bounds, checkCount } from "./internal/node.js";
|
|
4
|
+
import { End, NoMatch, advance, codePointPredicate, compileAny, countTokens, describeLiteral, indexOf, indexOfAny, matchLiteral, nextIf, scan, splitSurrogate } from "./internal/tokens.js";
|
|
5
5
|
// src/Token.ts
|
|
6
6
|
/**
|
|
7
7
|
* Parsers for individual tokens and slices of tokens (winnow's `token`
|
|
@@ -88,6 +88,11 @@ const anyStep = (state) => {
|
|
|
88
88
|
return state.input[pos];
|
|
89
89
|
}
|
|
90
90
|
};
|
|
91
|
+
/** The parser behind `any` and `anyToken`, on every input kind. */
|
|
92
|
+
const anyInput = /*#__PURE__*/ described(anyStep, "any", {
|
|
93
|
+
_tag: "Any",
|
|
94
|
+
encoding: void 0
|
|
95
|
+
});
|
|
91
96
|
/**
|
|
92
97
|
* Matches any single token: a character of a string (winnow's `any`, nom's
|
|
93
98
|
* `anychar`).
|
|
@@ -95,10 +100,7 @@ const anyStep = (state) => {
|
|
|
95
100
|
* @since 0.1.0
|
|
96
101
|
* @category tokens
|
|
97
102
|
*/
|
|
98
|
-
const any =
|
|
99
|
-
_tag: "Any",
|
|
100
|
-
encoding: void 0
|
|
101
|
-
});
|
|
103
|
+
const any = anyInput;
|
|
102
104
|
/**
|
|
103
105
|
* Matches any single token of an input of type `I`.
|
|
104
106
|
*
|
|
@@ -113,7 +115,7 @@ const any = /*#__PURE__*/ described(anyStep, "any", {
|
|
|
113
115
|
* @since 0.1.0
|
|
114
116
|
* @category tokens
|
|
115
117
|
*/
|
|
116
|
-
const anyToken = () =>
|
|
118
|
+
const anyToken = () => anyInput;
|
|
117
119
|
/**
|
|
118
120
|
* Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).
|
|
119
121
|
* Strings are matched against string input, `Uint8Array`s against byte
|
|
@@ -210,12 +212,22 @@ const noneOf = (set) => {
|
|
|
210
212
|
});
|
|
211
213
|
};
|
|
212
214
|
/**
|
|
213
|
-
* Takes exactly `count` tokens (winnow's and nom's `take`).
|
|
215
|
+
* Takes exactly `count` tokens (winnow's and nom's `take`). `count` must be
|
|
216
|
+
* a non-negative integer: anything else throws a `RangeError` here, when
|
|
217
|
+
* the parser is built.
|
|
218
|
+
*
|
|
219
|
+
* @example
|
|
220
|
+
* ```ts
|
|
221
|
+
* import { Parser, Token } from "@thomasfosterau/effect-parse"
|
|
222
|
+
*
|
|
223
|
+
* Parser.parsePeek(Token.take(2), "abc") // Result.succeed(["c", "ab"])
|
|
224
|
+
* ```
|
|
214
225
|
*
|
|
215
226
|
* @since 0.1.0
|
|
216
227
|
* @category slices
|
|
217
228
|
*/
|
|
218
229
|
const take = (count) => {
|
|
230
|
+
checkCount(count);
|
|
219
231
|
const expected = [count === 1 ? "1 token" : `${count} tokens`];
|
|
220
232
|
return described((state) => {
|
|
221
233
|
const start = state.pos;
|
|
@@ -230,9 +242,8 @@ const take = (count) => {
|
|
|
230
242
|
});
|
|
231
243
|
};
|
|
232
244
|
const normalize = (r) => {
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
return [r.min ?? 0, r.max ?? Infinity];
|
|
245
|
+
const { max, min } = bounds(r);
|
|
246
|
+
return [min, max];
|
|
236
247
|
};
|
|
237
248
|
/** @internal */
|
|
238
249
|
const takeWhileImpl = (set, r, negate, name, description, kind = "TakeWhile", hint = false) => {
|
|
@@ -292,17 +303,27 @@ const takeTill = (set, range) => takeWhileImpl(set, range, true, "takeTill");
|
|
|
292
303
|
* @since 0.1.0
|
|
293
304
|
* @category slices
|
|
294
305
|
*/
|
|
295
|
-
const takeUntil = (lit, r) => {
|
|
306
|
+
const takeUntil = (lit, r) => untilParser((state, from) => indexOf(state, lit, from), (state) => lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1), [describeLiteral(lit)], r, "takeUntil", {
|
|
307
|
+
_tag: "TakeUntil",
|
|
308
|
+
literal: lit,
|
|
309
|
+
...bounds(r),
|
|
310
|
+
encoding: void 0
|
|
311
|
+
});
|
|
312
|
+
/**
|
|
313
|
+
* The parser of `takeUntil` and `takeUntilAny`: `search` finds the first
|
|
314
|
+
* occurrence, which is not consumed (`read` is how far past it the search
|
|
315
|
+
* read). As in winnow, the first occurrence must lie within the range.
|
|
316
|
+
*/
|
|
317
|
+
const untilParser = (search, read, expected, r, name, node) => {
|
|
296
318
|
const [min, max] = normalize(r);
|
|
297
|
-
const expected = [describeLiteral(lit)];
|
|
298
319
|
return described((state) => {
|
|
299
320
|
const start = state.pos;
|
|
300
|
-
const idx =
|
|
321
|
+
const idx = search(state, start);
|
|
301
322
|
if (idx < 0) {
|
|
302
323
|
if (state.partial) return state.incomplete(0, expected);
|
|
303
324
|
return state.backtrack("TakeUntil", expected, state.end);
|
|
304
325
|
}
|
|
305
|
-
state.syntax?.read(idx +
|
|
326
|
+
state.syntax?.read(idx + read(state));
|
|
306
327
|
if (min > 0 || max !== Infinity) {
|
|
307
328
|
const count = countTokens(state, start, idx);
|
|
308
329
|
if (count < min) return state.backtrack("TakeUntil", expected, idx);
|
|
@@ -310,20 +331,39 @@ const takeUntil = (lit, r) => {
|
|
|
310
331
|
}
|
|
311
332
|
state.pos = idx;
|
|
312
333
|
return state.slice(start, idx);
|
|
313
|
-
},
|
|
314
|
-
_tag: "TakeUntil",
|
|
315
|
-
literal: lit,
|
|
316
|
-
...bounds(r),
|
|
317
|
-
encoding: void 0
|
|
318
|
-
});
|
|
334
|
+
}, name, node);
|
|
319
335
|
};
|
|
320
336
|
/**
|
|
321
|
-
* Takes
|
|
337
|
+
* Takes tokens up to (not including) the first occurrence of any of
|
|
338
|
+
* several literals: {@link takeUntil} for a set of them, the search a
|
|
339
|
+
* multi-pattern prefilter makes. No literal is consumed. `range` constrains
|
|
340
|
+
* the number of tokens taken. On bytes, `Compiler.wasm` searches with SIMD
|
|
341
|
+
* (Teddy, from ripgrep), sixteen bytes at a time.
|
|
322
342
|
*
|
|
323
|
-
* @
|
|
343
|
+
* @example
|
|
344
|
+
* ```ts
|
|
345
|
+
* import { Parser, Token } from "@thomasfosterau/effect-parse"
|
|
346
|
+
* import * as Result from "effect/Result"
|
|
347
|
+
*
|
|
348
|
+
* const upTo = Parser.seq(Token.takeUntilAny(["Holmes", "Watson"]), Token.rest)
|
|
349
|
+
* console.log(Result.getOrThrow(Parser.parseSync(upTo, "Dr. Watson and Holmes")))
|
|
350
|
+
* // ["Dr. ", "Watson and Holmes"]
|
|
351
|
+
* ```
|
|
352
|
+
*
|
|
353
|
+
* @since 0.5.0
|
|
324
354
|
* @category slices
|
|
325
355
|
*/
|
|
326
|
-
const
|
|
356
|
+
const takeUntilAny = (literals, r) => {
|
|
357
|
+
const longest = Math.max(0, ...literals.map((lit) => lit.length));
|
|
358
|
+
return untilParser((state, from) => indexOfAny(state, literals, from), (state) => longest * (state.inputKind === 1 ? 3 : 1), literals.map(describeLiteral), r, "takeUntilAny", {
|
|
359
|
+
_tag: "TakeUntilAny",
|
|
360
|
+
literals,
|
|
361
|
+
...bounds(r),
|
|
362
|
+
encoding: void 0
|
|
363
|
+
});
|
|
364
|
+
};
|
|
365
|
+
/** The parser behind `rest` and `restOf`, on every input kind. */
|
|
366
|
+
const restInput = /*#__PURE__*/ described((state) => {
|
|
327
367
|
const start = state.pos;
|
|
328
368
|
state.pos = state.end;
|
|
329
369
|
return state.slice(start, state.end);
|
|
@@ -332,12 +372,19 @@ const rest = /*#__PURE__*/ described((state) => {
|
|
|
332
372
|
encoding: void 0
|
|
333
373
|
});
|
|
334
374
|
/**
|
|
375
|
+
* Takes the rest of the input (winnow's and nom's `rest`).
|
|
376
|
+
*
|
|
377
|
+
* @since 0.1.0
|
|
378
|
+
* @category slices
|
|
379
|
+
*/
|
|
380
|
+
const rest = restInput;
|
|
381
|
+
/**
|
|
335
382
|
* Takes the rest of an input of type `I`.
|
|
336
383
|
*
|
|
337
384
|
* @since 0.1.0
|
|
338
385
|
* @category slices
|
|
339
386
|
*/
|
|
340
|
-
const restOf = () =>
|
|
387
|
+
const restOf = () => restInput;
|
|
341
388
|
/**
|
|
342
389
|
* Returns the length of the remaining input without consuming it (winnow's
|
|
343
390
|
* and nom's `rest_len`).
|
|
@@ -354,6 +401,6 @@ const restLen = /*#__PURE__*/ described((state) => state.end - state.pos, "restL
|
|
|
354
401
|
* @category tokens
|
|
355
402
|
*/
|
|
356
403
|
const satisfy = (predicate) => oneOf(predicate);
|
|
357
|
-
export { any, anyToken, codePoints, literal, literalImpl, noneOf, oneOf, range, rest, restLen, restOf, satisfy, take, takeTill, takeUntil, takeWhile, takeWhileImpl };
|
|
404
|
+
export { any, anyToken, codePoints, literal, literalImpl, noneOf, oneOf, range, rest, restLen, restOf, satisfy, take, takeTill, takeUntil, takeUntilAny, takeWhile, takeWhileImpl };
|
|
358
405
|
|
|
359
406
|
//# sourceMappingURL=Token.js.map
|
package/dist/Token.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Token.js","names":[],"sources":["../src/Token.ts"],"sourcesContent":["/**\n * Parsers for individual tokens and slices of tokens (winnow's `token`\n * module, nom's `bytes` module).\n *\n * Parsers here work on every input kind: `string` (tokens are code points),\n * `Uint8Array` (tokens are bytes) and arrays (tokens are elements). Parsers\n * whose input type cannot be inferred from their arguments take a type\n * parameter that defaults to `string`:\n *\n * ```ts\n * import { Token } from \"@thomasfosterau/effect-parse\"\n *\n * Token.take(4) // Parser<string, never, never, string>\n * Token.take<Uint8Array>(4) // Parser<Uint8Array, never, never, Uint8Array>\n * ```\n *\n * On partial input (see `Parser.parsePartial`), parsers that reach the end\n * of the available data report `Incomplete` rather than succeeding early or\n * failing.\n *\n * @since 0.1.0\n */\nimport * as Predicate from \"effect/Predicate\"\nimport type { Input, Slice, State, Token } from \"./Input.js\"\nimport {\n advance,\n codePointPredicate,\n compileAny,\n countTokens,\n describeLiteral,\n End,\n indexOf,\n matchLiteral,\n nextIf,\n NoMatch,\n scan,\n splitSurrogate\n} from \"./internal/tokens.js\"\nimport type { ErrorKind } from \"./ParseError.js\"\nimport { bounds } from \"./internal/node.js\"\nimport type { Parser, Range } from \"./Parser.js\"\nimport { described } from \"./Parser.js\"\n\nexport type {\n /**\n * @since 0.1.0\n * @category models\n */\n Token\n}\n\n/**\n * An inclusive range of tokens, e.g. `Token.range(\"a\", \"z\")`.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface TokenRange<T> {\n readonly _tag: \"TokenRange\"\n readonly from: T\n readonly to: T\n}\n\n/**\n * Creates an inclusive token range (winnow's `'a'..='z'`).\n *\n * @since 0.1.0\n * @category constructors\n */\nexport const range = <T extends string | number>(from: T, to: T): TokenRange<T> => ({ _tag: \"TokenRange\", from, to })\n\ndeclare const CodePointsBrand: unique symbol\n\n/**\n * A predicate over code points, made with {@link codePoints}. In a token set\n * it receives each character as a number, so testing a character allocates\n * nothing.\n *\n * @since 0.2.0\n * @category models\n */\nexport interface CodePoints {\n (codePoint: number): boolean\n readonly [CodePointsBrand]: true\n}\n\n/**\n * Declares a character class as a predicate over code points, for token sets\n * on string, `Text` and byte inputs. `name` describes it in error messages.\n *\n * A plain `(char: string) => boolean` predicate receives each character as a\n * string, which costs an allocation per character tested; a `codePoints`\n * predicate receives the code point itself (a byte, on byte input).\n *\n * `ranges` optionally lists the code points the predicate accepts, as\n * inclusive `[from, to]` pairs. Compiled parsers then test the ranges\n * instead of calling the predicate, so they must agree with it exactly.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const isNameStart = Token.codePoints(\n * \"name start\",\n * (c) => (c >= 0x61 && c <= 0x7a) || c === 0x5f || c >= 0xc0,\n * [[0x5f, 0x5f], [0x61, 0x7a], [0xc0, 0x10ffff]]\n * )\n * const name = Token.takeWhile([isNameStart, Token.range(\"0\", \"9\")], { min: 1 })\n * ```\n *\n * @since 0.2.0\n * @category constructors\n */\nexport const codePoints = (\n name: string,\n predicate: (codePoint: number) => boolean,\n ranges?: ReadonlyArray<readonly [number, number]>\n): CodePoints =>\n // Wrapped, so the caller's function is left unmarked\n codePointPredicate(\n name,\n (c: number): boolean => predicate(c),\n ranges?.map(([from, to]) => [from, to] as const)\n ) as unknown as CodePoints\n\n/**\n * A set of string tokens: the characters of a string, a range, a predicate,\n * or an array of these. Strings, ranges, {@link codePoints} predicates and\n * the `Ascii` and `Text` predicates test characters without allocating; a\n * `(char: string) => boolean` predicate allocates a string per character.\n *\n * @since 0.1.0\n * @category models\n */\nexport type StringTokenSet =\n | string\n | TokenRange<string>\n | CodePoints\n | ((char: string) => boolean)\n | ReadonlyArray<StringTokenSet>\n\n/**\n * A set of byte tokens: a byte, the (ASCII) characters of a string, a\n * `Uint8Array` of bytes, a range, a predicate, or an array of these.\n *\n * @since 0.1.0\n * @category models\n */\nexport type ByteTokenSet =\n | number\n | string\n | Uint8Array\n | TokenRange<number>\n | CodePoints\n | ((byte: number) => boolean)\n | ReadonlyArray<ByteTokenSet>\n\n/**\n * A set of tokens (winnow's `ContainsToken`). For array inputs, a set is a\n * predicate or an array of allowed elements (compared with `===`).\n *\n * @since 0.1.0\n * @category models\n */\nexport type TokenSet<I> = I extends string\n ? StringTokenSet\n : I extends Uint8Array\n ? ByteTokenSet\n : I extends ReadonlyArray<infer T>\n ? ((token: T) => boolean) | ReadonlyArray<T>\n : never\n\nconst expectedAny = [\"any token\"]\n\nconst anyStep = (state: State<Input>): unknown => {\n const pos = state.pos\n if (pos >= state.end) return state.eoi(\"Token\", expectedAny, 1)\n switch (state.inputKind) {\n case 0: {\n const s = state.input as string\n if (splitSurrogate(state, pos)) return state.incomplete(1, expectedAny)\n const c = s.charCodeAt(pos)\n if (c >= 0xd800 && c <= 0xdbff && pos + 1 < s.length && (s.charCodeAt(pos + 1) & 0xfc00) === 0xdc00) {\n state.pos = pos + 2\n return s.slice(pos, pos + 2)\n }\n state.pos = pos + 1\n return s[pos]\n }\n case 1:\n state.pos = pos + 1\n return (state.input as Uint8Array)[pos]\n default:\n state.pos = pos + 1\n return (state.input as ReadonlyArray<unknown>)[pos]\n }\n}\n\n/**\n * Matches any single token: a character of a string (winnow's `any`, nom's\n * `anychar`).\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const any: Parser<string> = /*#__PURE__*/ described(anyStep as any, \"any\", { _tag: \"Any\", encoding: undefined })\n\n/**\n * Matches any single token of an input of type `I`.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const byte = Token.anyToken<Uint8Array>()\n * Parser.parse(byte, new Uint8Array([7])) // Result.succeed(7)\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const anyToken = <I extends Input = string>(): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => any as any\n\n/**\n * Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).\n * Strings are matched against string input, `Uint8Array`s against byte\n * input, and arrays element-wise (with `===`) against array input.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const keyword = Parser.alt(Token.literal(\"true\"), Token.literal(\"false\"))\n * // ^? Parser<\"true\" | \"false\">\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const literal: {\n <const L extends string>(literal: L): Parser<L, never, never, string>\n (literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>\n <const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>\n} = (lit: Input): Parser<any, never, never, any> => literalImpl(lit, false)\n\n/** @internal */\nexport const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, never, any> => {\n const description = describeLiteral(lit)\n const expected = [caseless ? `${description} (case insensitive)` : description]\n // A string literal may also be matched against byte input (as UTF-8)\n let asBytes: Uint8Array | undefined\n const literalKind = Predicate.isString(lit) ? 0 : lit instanceof Uint8Array ? 1 : 2\n return described(\n (state: State<Input>) => {\n const pos = state.pos\n const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit as string)) : lit\n const len = l.length\n const r = matchLiteral(state, l, caseless)\n if (r === -1) return state.backtrack(\"Literal\", expected)\n if (r === -2) {\n if (state.partial) return state.incomplete(len - (state.end - pos), expected)\n return state.backtrack(\"Literal\", expected)\n }\n state.pos = pos + len\n return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit\n },\n description,\n { _tag: \"Literal\", literal: lit, caseless, encoding: undefined }\n )\n}\n\n/**\n * Matches one token belonging to a set, returning it (winnow's and nom's\n * `one_of`).\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const sign = Token.oneOf(\"+-\")\n * const lower = Token.oneOf(Token.range(\"a\", \"z\"))\n * const hexByte = Token.oneOf<Uint8Array>([Token.range(0x30, 0x39), Token.range(0x61, 0x66)])\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const oneOf = <I extends Input = string>(\n set: TokenSet<I>\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const expected = [m.description]\n return described(\n (state) => {\n const t = nextIf(state, m, false)\n if (t === End) return state.eoi(\"OneOf\", expected, 1)\n if (t === NoMatch) return state.backtrack(\"OneOf\", expected)\n return t as Token<I>\n },\n m.description,\n { _tag: \"OneOf\", set, negate: false, encoding: undefined }\n )\n}\n\n/**\n * Matches one token *not* belonging to a set, returning it (winnow's and\n * nom's `none_of`).\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const noneOf = <I extends Input = string>(\n set: TokenSet<I>\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const expected = [`anything but ${m.description}`]\n return described(\n (state) => {\n const t = nextIf(state, m, true)\n if (t === End) return state.eoi(\"NoneOf\", expected, 1)\n if (t === NoMatch) return state.backtrack(\"NoneOf\", expected)\n return t as Token<I>\n },\n expected[0],\n { _tag: \"OneOf\", set, negate: true, encoding: undefined }\n )\n}\n\n/**\n * Takes exactly `count` tokens (winnow's and nom's `take`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const take = <I extends Input = string>(count: number): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {\n const expected = [count === 1 ? \"1 token\" : `${count} tokens`]\n return described(\n (state) => {\n const start = state.pos\n const { count: got, offset } = advance(state, start, count)\n if (got < count) return state.eoi(\"Take\", expected, count - got)\n state.pos = offset\n return state.slice(start, offset)\n },\n \"take\",\n { _tag: \"Take\", count, encoding: undefined }\n )\n}\n\nconst normalize = (r: Range | undefined): [number, number] => {\n if (r === undefined) return [0, Infinity]\n if (Predicate.isNumber(r)) return [r, r]\n return [r.min ?? 0, r.max ?? Infinity]\n}\n\n/** @internal */\nexport const takeWhileImpl = <I extends Input>(\n set: unknown,\n r: Range | undefined,\n negate: boolean,\n name: string,\n description?: string,\n kind: ErrorKind = \"TakeWhile\",\n hint = false\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const [min, max] = normalize(r)\n const expected = [description ?? (negate ? `anything but ${m.description}` : m.description)]\n return described(\n (state) => {\n const start = state.pos\n const { count, hitEnd, offset } = scan(state, m, negate, max)\n if (hitEnd && state.partial) return state.incomplete(1, expected)\n if (count < min) return state.backtrack(kind, expected, offset)\n // More matching tokens would have been accepted where we stopped\n if (hint && count < max) state.setHint(offset, expected)\n state.pos = offset\n return state.slice(start, offset)\n },\n name,\n { _tag: \"TakeWhile\", set, negate, min, max, encoding: undefined }\n )\n}\n\n/**\n * Takes the longest run of tokens belonging to a set, with the number of\n * tokens constrained by `range` (default: zero or more). This is winnow's\n * `take_while(range, set)`, and nom's `take_while`, `take_while1` and\n * `take_while_m_n`.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const identifier = Token.takeWhile(\n * [Token.range(\"a\", \"z\"), Token.range(\"A\", \"Z\"), \"_\"],\n * { min: 1 }\n * )\n * ```\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeWhile = <I extends Input = string>(\n set: TokenSet<I>,\n range?: Range\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => takeWhileImpl(set, range, false, \"takeWhile\")\n\n/**\n * Takes the longest run of tokens *not* belonging to a set (winnow's\n * `take_till`, nom's `take_till` and `take_till1`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeTill = <I extends Input = string>(\n set: TokenSet<I>,\n range?: Range\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => takeWhileImpl(set, range, true, \"takeTill\")\n\n/**\n * Takes tokens up to (not including) the first occurrence of a literal\n * (winnow's `take_until`, nom's `take_until`). The literal is not\n * consumed. `range` constrains the number of tokens taken.\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeUntil: {\n (literal: string, range?: Range): Parser<string, never, never, string>\n (literal: Uint8Array, range?: Range): Parser<Uint8Array, never, never, Uint8Array>\n <T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>\n} = (lit: Input, r?: Range): Parser<any, never, never, any> => {\n const [min, max] = normalize(r)\n const description = describeLiteral(lit)\n const expected = [description]\n return described(\n (state: State<Input>) => {\n const start = state.pos\n const idx = indexOf(state, lit, start)\n if (idx < 0) {\n if (state.partial) return state.incomplete(0, expected)\n return state.backtrack(\"TakeUntil\", expected, state.end)\n }\n // The literal was read but is not consumed (a string literal is up to three bytes per unit of UTF-8)\n state.syntax?.read(idx + lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1))\n // As in winnow, the first occurrence must lie within the range\n if (min > 0 || max !== Infinity) {\n const count = countTokens(state, start, idx)\n if (count < min) return state.backtrack(\"TakeUntil\", expected, idx)\n if (count > max) return state.backtrack(\"TakeUntil\", expected)\n }\n state.pos = idx\n return state.slice(start, idx)\n },\n \"takeUntil\",\n { _tag: \"TakeUntil\", literal: lit, ...bounds(r), encoding: undefined }\n )\n}\n\n/**\n * Takes the rest of the input (winnow's and nom's `rest`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const rest: Parser<string> = /*#__PURE__*/ described(\n (state: State<any>) => {\n const start = state.pos\n state.pos = state.end\n return state.slice(start, state.end) as string\n },\n \"rest\",\n { _tag: \"Rest\", encoding: undefined }\n)\n\n/**\n * Takes the rest of an input of type `I`.\n *\n * @since 0.1.0\n * @category slices\n */\nexport const restOf = <I extends Input = string>(): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => rest as any\n\n/**\n * Returns the length of the remaining input without consuming it (winnow's\n * and nom's `rest_len`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const restLen: Parser<number, never, never, Input> = /*#__PURE__*/ described(\n (state) => state.end - state.pos,\n \"restLen\",\n {\n _tag: \"RestLen\"\n }\n)\n\n/**\n * Matches a token satisfying a predicate (nom's `satisfy`). Equivalent to\n * `Token.oneOf(predicate)`.\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const satisfy = <I extends Input = string>(\n predicate: (token: Token<I>) => boolean\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => oneOf<I>(predicate as any)\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,MAAa,SAAoC,MAAS,QAA0B;CAAE,MAAM;CAAc;CAAM;AAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4CnH,MAAa,cACX,MACA,WACA,WAGA,mBACE,OACC,MAAuB,UAAU,CAAC,GACnC,QAAQ,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAU,CACjD;AAiDF,MAAM,cAAc,CAAC,WAAW;AAEhC,MAAM,WAAW,UAAiC;CAChD,MAAM,MAAM,MAAM;CAClB,IAAI,OAAO,MAAM,KAAK,OAAO,MAAM,IAAI,SAAS,aAAa,CAAC;CAC9D,QAAQ,MAAM,WAAd;EACE,KAAK,GAAG;GACN,MAAM,IAAI,MAAM;GAChB,IAAI,eAAe,OAAO,GAAG,GAAG,OAAO,MAAM,WAAW,GAAG,WAAW;GACtE,MAAM,IAAI,EAAE,WAAW,GAAG;GAC1B,IAAI,KAAK,SAAU,KAAK,SAAU,MAAM,IAAI,EAAE,WAAW,EAAE,WAAW,MAAM,CAAC,IAAI,WAAY,OAAQ;IACnG,MAAM,MAAM,MAAM;IAClB,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;GAC7B;GACA,MAAM,MAAM,MAAM;GAClB,OAAO,EAAE;EACX;EACA,KAAK;GACH,MAAM,MAAM,MAAM;GAClB,OAAQ,MAAM,MAAqB;EACrC;GACE,MAAM,MAAM,MAAM;GAClB,OAAQ,MAAM,MAAiC;CACnD;AACF;;;;;;;;AASA,MAAa,MAAoC,wBAAU,SAAgB,OAAO;CAAE,MAAM;CAAO,UAAU,KAAA;AAAU,CAAC;;;;;;;;;;;;;;;AAgBtH,MAAa,iBAAgG;;;;;;;;;;;;;;;;;AAkB7G,MAAa,WAIR,QAA+C,YAAY,KAAK,KAAK;;AAG1E,MAAa,eAAe,KAAY,aAAsD;CAC5F,MAAM,cAAc,gBAAgB,GAAG;CACvC,MAAM,WAAW,CAAC,WAAW,GAAG,YAAY,uBAAuB,WAAW;CAE9E,IAAI;CACJ,MAAM,cAAc,UAAU,SAAS,GAAG,IAAI,IAAI,eAAe,aAAa,IAAI;CAClF,OAAO,WACJ,UAAwB;EACvB,MAAM,MAAM,MAAM;EAClB,MAAM,IAAI,gBAAgB,KAAK,MAAM,cAAc,IAAK,YAAY,IAAI,YAAY,CAAC,CAAC,OAAO,GAAa,IAAK;EAC/G,MAAM,MAAM,EAAE;EACd,MAAM,IAAI,aAAa,OAAO,GAAG,QAAQ;EACzC,IAAI,MAAM,IAAI,OAAO,MAAM,UAAU,WAAW,QAAQ;EACxD,IAAI,MAAM,IAAI;GACZ,IAAI,MAAM,SAAS,OAAO,MAAM,WAAW,OAAO,MAAM,MAAM,MAAM,QAAQ;GAC5E,OAAO,MAAM,UAAU,WAAW,QAAQ;EAC5C;EACA,MAAM,MAAM,MAAM;EAClB,OAAO,YAAY,MAAM,cAAc,cAAc,MAAM,MAAM,KAAK,MAAM,GAAG,IAAI;CACrF,GACA,aACA;EAAE,MAAM;EAAW,SAAS;EAAK;EAAU,UAAU,KAAA;CAAU,CACjE;AACF;;;;;;;;;;;;;;;;;AAkBA,MAAa,SACX,QACwD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,WAAW,CAAC,EAAE,WAAW;CAC/B,OAAO,WACJ,UAAU;EACT,MAAM,IAAI,OAAO,OAAO,GAAG,KAAK;EAChC,IAAI,MAAM,KAAK,OAAO,MAAM,IAAI,SAAS,UAAU,CAAC;EACpD,IAAI,MAAM,SAAS,OAAO,MAAM,UAAU,SAAS,QAAQ;EAC3D,OAAO;CACT,GACA,EAAE,aACF;EAAE,MAAM;EAAS;EAAK,QAAQ;EAAO,UAAU,KAAA;CAAU,CAC3D;AACF;;;;;;;;AASA,MAAa,UACX,QACwD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,WAAW,CAAC,gBAAgB,EAAE,aAAa;CACjD,OAAO,WACJ,UAAU;EACT,MAAM,IAAI,OAAO,OAAO,GAAG,IAAI;EAC/B,IAAI,MAAM,KAAK,OAAO,MAAM,IAAI,UAAU,UAAU,CAAC;EACrD,IAAI,MAAM,SAAS,OAAO,MAAM,UAAU,UAAU,QAAQ;EAC5D,OAAO;CACT,GACA,SAAS,IACT;EAAE,MAAM;EAAS;EAAK,QAAQ;EAAM,UAAU,KAAA;CAAU,CAC1D;AACF;;;;;;;AAQA,MAAa,QAAkC,UAAuE;CACpH,MAAM,WAAW,CAAC,UAAU,IAAI,YAAY,GAAG,MAAM,QAAQ;CAC7D,OAAO,WACJ,UAAU;EACT,MAAM,QAAQ,MAAM;EACpB,MAAM,EAAE,OAAO,KAAK,WAAW,QAAQ,OAAO,OAAO,KAAK;EAC1D,IAAI,MAAM,OAAO,OAAO,MAAM,IAAI,QAAQ,UAAU,QAAQ,GAAG;EAC/D,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,MAAM;CAClC,GACA,QACA;EAAE,MAAM;EAAQ;EAAO,UAAU,KAAA;CAAU,CAC7C;AACF;AAEA,MAAM,aAAa,MAA2C;CAC5D,IAAI,MAAM,KAAA,GAAW,OAAO,CAAC,GAAG,QAAQ;CACxC,IAAI,UAAU,SAAS,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC;CACvC,OAAO,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,QAAQ;AACvC;;AAGA,MAAa,iBACX,KACA,GACA,QACA,MACA,aACA,OAAkB,aAClB,OAAO,UACiD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,CAAC,KAAK,OAAO,UAAU,CAAC;CAC9B,MAAM,WAAW,CAAC,gBAAgB,SAAS,gBAAgB,EAAE,gBAAgB,EAAE,YAAY;CAC3F,OAAO,WACJ,UAAU;EACT,MAAM,QAAQ,MAAM;EACpB,MAAM,EAAE,OAAO,QAAQ,WAAW,KAAK,OAAO,GAAG,QAAQ,GAAG;EAC5D,IAAI,UAAU,MAAM,SAAS,OAAO,MAAM,WAAW,GAAG,QAAQ;EAChE,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,MAAM,UAAU,MAAM;EAE9D,IAAI,QAAQ,QAAQ,KAAK,MAAM,QAAQ,QAAQ,QAAQ;EACvD,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,MAAM;CAClC,GACA,MACA;EAAE,MAAM;EAAa;EAAK;EAAQ;EAAK;EAAK,UAAU,KAAA;CAAU,CAClE;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,aACX,KACA,UACwD,cAAc,KAAK,OAAO,OAAO,WAAW;;;;;;;;AAStG,MAAa,YACX,KACA,UACwD,cAAc,KAAK,OAAO,MAAM,UAAU;;;;;;;;;AAUpG,MAAa,aAIR,KAAY,MAA8C;CAC7D,MAAM,CAAC,KAAK,OAAO,UAAU,CAAC;CAE9B,MAAM,WAAW,CADG,gBAAgB,GACR,CAAC;CAC7B,OAAO,WACJ,UAAwB;EACvB,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,QAAQ,OAAO,KAAK,KAAK;EACrC,IAAI,MAAM,GAAG;GACX,IAAI,MAAM,SAAS,OAAO,MAAM,WAAW,GAAG,QAAQ;GACtD,OAAO,MAAM,UAAU,aAAa,UAAU,MAAM,GAAG;EACzD;EAEA,MAAM,QAAQ,KAAK,MAAM,IAAI,UAAU,UAAU,SAAS,GAAG,KAAK,MAAM,cAAc,IAAI,IAAI,EAAE;EAEhG,IAAI,MAAM,KAAK,QAAQ,UAAU;GAC/B,MAAM,QAAQ,YAAY,OAAO,OAAO,GAAG;GAC3C,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,aAAa,UAAU,GAAG;GAClE,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,aAAa,QAAQ;EAC/D;EACA,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,GAAG;CAC/B,GACA,aACA;EAAE,MAAM;EAAa,SAAS;EAAK,GAAG,OAAO,CAAC;EAAG,UAAU,KAAA;CAAU,CACvE;AACF;;;;;;;AAQA,MAAa,OAAqC,yBAC/C,UAAsB;CACrB,MAAM,QAAQ,MAAM;CACpB,MAAM,MAAM,MAAM;CAClB,OAAO,MAAM,MAAM,OAAO,MAAM,GAAG;AACrC,GACA,QACA;CAAE,MAAM;CAAQ,UAAU,KAAA;AAAU,CACtC;;;;;;;AAQA,MAAa,eAA8F;;;;;;;;AAS3G,MAAa,UAA6D,yBACvE,UAAU,MAAM,MAAM,MAAM,KAC7B,WACA,EACE,MAAM,UACR,CACF;;;;;;;;AASA,MAAa,WACX,cACwD,MAAS,SAAgB"}
|
|
1
|
+
{"version":3,"file":"Token.js","names":[],"sources":["../src/Token.ts"],"sourcesContent":["/**\n * Parsers for individual tokens and slices of tokens (winnow's `token`\n * module, nom's `bytes` module).\n *\n * Parsers here work on every input kind: `string` (tokens are code points),\n * `Uint8Array` (tokens are bytes) and arrays (tokens are elements). Parsers\n * whose input type cannot be inferred from their arguments take a type\n * parameter that defaults to `string`:\n *\n * ```ts\n * import { Token } from \"@thomasfosterau/effect-parse\"\n *\n * Token.take(4) // Parser<string, never, never, string>\n * Token.take<Uint8Array>(4) // Parser<Uint8Array, never, never, Uint8Array>\n * ```\n *\n * On partial input (see `Parser.parsePartial`), parsers that reach the end\n * of the available data report `Incomplete` rather than succeeding early or\n * failing.\n *\n * @since 0.1.0\n */\nimport * as Predicate from \"effect/Predicate\"\nimport type { Input, Slice, State, Token } from \"./Input.js\"\nimport {\n advance,\n codePointPredicate,\n compileAny,\n countTokens,\n describeLiteral,\n End,\n indexOf,\n indexOfAny,\n matchLiteral,\n nextIf,\n NoMatch,\n scan,\n splitSurrogate\n} from \"./internal/tokens.js\"\nimport type { ErrorKind } from \"./ParseError.js\"\nimport { bounds, checkCount } from \"./internal/node.js\"\nimport type { Parser, Range } from \"./Parser.js\"\nimport { described } from \"./Parser.js\"\n\nexport type {\n /**\n * @since 0.1.0\n * @category models\n */\n Token\n}\n\n/**\n * An inclusive range of tokens, e.g. `Token.range(\"a\", \"z\")`.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface TokenRange<T> {\n readonly _tag: \"TokenRange\"\n readonly from: T\n readonly to: T\n}\n\n/**\n * Creates an inclusive token range (winnow's `'a'..='z'`).\n *\n * @since 0.1.0\n * @category constructors\n */\nexport const range = <T extends string | number>(from: T, to: T): TokenRange<T> => ({ _tag: \"TokenRange\", from, to })\n\ndeclare const CodePointsBrand: unique symbol\n\n/**\n * A predicate over code points, made with {@link codePoints}. In a token set\n * it receives each character as a number, so testing a character allocates\n * nothing.\n *\n * @since 0.2.0\n * @category models\n */\nexport interface CodePoints {\n (codePoint: number): boolean\n readonly [CodePointsBrand]: true\n}\n\n/**\n * Declares a character class as a predicate over code points, for token sets\n * on string, `Text` and byte inputs. `name` describes it in error messages.\n *\n * A plain `(char: string) => boolean` predicate receives each character as a\n * string, which costs an allocation per character tested; a `codePoints`\n * predicate receives the code point itself (a byte, on byte input).\n *\n * `ranges` optionally lists the code points the predicate accepts, as\n * inclusive `[from, to]` pairs. Compiled parsers then test the ranges\n * instead of calling the predicate, so they must agree with it exactly.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const isNameStart = Token.codePoints(\n * \"name start\",\n * (c) => (c >= 0x61 && c <= 0x7a) || c === 0x5f || c >= 0xc0,\n * [[0x5f, 0x5f], [0x61, 0x7a], [0xc0, 0x10ffff]]\n * )\n * const name = Token.takeWhile([isNameStart, Token.range(\"0\", \"9\")], { min: 1 })\n * ```\n *\n * @since 0.2.0\n * @category constructors\n */\nexport const codePoints = (\n name: string,\n predicate: (codePoint: number) => boolean,\n ranges?: ReadonlyArray<readonly [number, number]>\n): CodePoints =>\n // Wrapped, so the caller's function is left unmarked.\n // SAFETY: `CodePointsBrand` is a declared, phantom key, so the marked function is all a `CodePoints` is at runtime\n codePointPredicate(\n name,\n (c: number): boolean => predicate(c),\n ranges?.map(([from, to]) => [from, to] as const)\n ) as CodePoints\n\n/**\n * A set of string tokens: the characters of a string, a range, a predicate,\n * or an array of these. Strings, ranges, {@link codePoints} predicates and\n * the `Ascii` and `Text` predicates test characters without allocating; a\n * `(char: string) => boolean` predicate allocates a string per character.\n *\n * @since 0.1.0\n * @category models\n */\nexport type StringTokenSet =\n | string\n | TokenRange<string>\n | CodePoints\n | ((char: string) => boolean)\n | ReadonlyArray<StringTokenSet>\n\n/**\n * A set of byte tokens: a byte, the (ASCII) characters of a string, a\n * `Uint8Array` of bytes, a range, a predicate, or an array of these.\n *\n * @since 0.1.0\n * @category models\n */\nexport type ByteTokenSet =\n | number\n | string\n | Uint8Array\n | TokenRange<number>\n | CodePoints\n | ((byte: number) => boolean)\n | ReadonlyArray<ByteTokenSet>\n\n/**\n * A set of tokens (winnow's `ContainsToken`). For array inputs, a set is a\n * predicate or an array of allowed elements (compared with `===`).\n *\n * @since 0.1.0\n * @category models\n */\nexport type TokenSet<I> = I extends string\n ? StringTokenSet\n : I extends Uint8Array\n ? ByteTokenSet\n : I extends ReadonlyArray<infer T>\n ? ((token: T) => boolean) | ReadonlyArray<T>\n : never\n\nconst expectedAny = [\"any token\"]\n\nconst anyStep = (state: State<Input>): unknown => {\n const pos = state.pos\n if (pos >= state.end) return state.eoi(\"Token\", expectedAny, 1)\n switch (state.inputKind) {\n case 0: {\n // SAFETY: `inputKind` 0 is set only for a string input\n const s = state.input as string\n if (splitSurrogate(state, pos)) return state.incomplete(1, expectedAny)\n const c = s.charCodeAt(pos)\n if (c >= 0xd800 && c <= 0xdbff && pos + 1 < s.length && (s.charCodeAt(pos + 1) & 0xfc00) === 0xdc00) {\n state.pos = pos + 2\n return s.slice(pos, pos + 2)\n }\n state.pos = pos + 1\n return s[pos]\n }\n case 1:\n state.pos = pos + 1\n // SAFETY: `inputKind` 1 is set only for a `Uint8Array` input\n return (state.input as Uint8Array)[pos]\n default:\n state.pos = pos + 1\n // SAFETY: `inputKind` 2 is set for every input that is neither a string nor a `Uint8Array`, an array\n return (state.input as ReadonlyArray<unknown>)[pos]\n }\n}\n\n/** The parser behind `any` and `anyToken`, on every input kind. */\nconst anyInput: Parser<unknown, never, never, Input> = /*#__PURE__*/ described(anyStep, \"any\", {\n _tag: \"Any\",\n encoding: undefined\n})\n\n/**\n * Matches any single token: a character of a string (winnow's `any`, nom's\n * `anychar`).\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const any: Parser<string> =\n // SAFETY: on string input, `anyStep` returns the character (one or two code units) at the position, a string\n anyInput as Parser<string>\n\n/**\n * Matches any single token of an input of type `I`.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const byte = Token.anyToken<Uint8Array>()\n * Parser.parse(byte, new Uint8Array([7])) // Result.succeed(7)\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const anyToken = <I extends Input = string>(): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> =>\n // SAFETY: `anyStep` handles every input kind, returning the token at the position (a character, byte or element)\n anyInput as Parser<Token<NoInfer<I>>, never, never, NoInfer<I>>\n\n/**\n * The overloads of {@link literal}.\n *\n * @since 0.5.0\n * @category tokens\n */\nexport interface LiteralFn {\n <const L extends string>(literal: L): Parser<L, never, never, string>\n (literal: Uint8Array): Parser<Uint8Array, never, never, Uint8Array>\n <const T>(literal: ReadonlyArray<T>): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>\n}\n\n/**\n * Matches a literal sequence of tokens (winnow's `literal`, nom's `tag`).\n * Strings are matched against string input, `Uint8Array`s against byte\n * input, and arrays element-wise (with `===`) against array input.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const keyword = Parser.alt(Token.literal(\"true\"), Token.literal(\"false\"))\n * // ^? Parser<\"true\" | \"false\">\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const literal: LiteralFn = (lit: Input): Parser<any, never, never, any> => literalImpl(lit, false)\n\n/** @internal */\nexport const literalImpl = (lit: Input, caseless: boolean): Parser<any, never, never, any> => {\n const description = describeLiteral(lit)\n const expected = [caseless ? `${description} (case insensitive)` : description]\n // A string literal may also be matched against byte input (as UTF-8)\n let asBytes: Uint8Array | undefined\n const literalKind = Predicate.isString(lit) ? 0 : lit instanceof Uint8Array ? 1 : 2\n return described(\n (state: State<Input>) => {\n const pos = state.pos\n // SAFETY: `literalKind` 0 is set only for a string literal\n const l = literalKind === 0 && state.inputKind === 1 ? (asBytes ??= new TextEncoder().encode(lit as string)) : lit\n const len = l.length\n const r = matchLiteral(state, l, caseless)\n if (r === -1) return state.backtrack(\"Literal\", expected)\n if (r === -2) {\n if (state.partial) return state.incomplete(len - (state.end - pos), expected)\n return state.backtrack(\"Literal\", expected)\n }\n state.pos = pos + len\n return caseless || state.inputKind !== literalKind ? state.slice(pos, pos + len) : lit\n },\n description,\n { _tag: \"Literal\", literal: lit, caseless, encoding: undefined }\n )\n}\n\n/**\n * Matches one token belonging to a set, returning it (winnow's and nom's\n * `one_of`).\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const sign = Token.oneOf(\"+-\")\n * const lower = Token.oneOf(Token.range(\"a\", \"z\"))\n * const hexByte = Token.oneOf<Uint8Array>([Token.range(0x30, 0x39), Token.range(0x61, 0x66)])\n * ```\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const oneOf = <I extends Input = string>(\n set: TokenSet<I>\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const expected = [m.description]\n return described(\n (state) => {\n const t = nextIf(state, m, false)\n if (t === End) return state.eoi(\"OneOf\", expected, 1)\n if (t === NoMatch) return state.backtrack(\"OneOf\", expected)\n // SAFETY: past `End` and `NoMatch`, `nextIf` returns the token it consumed from the parser's input `I`\n return t as Token<I>\n },\n m.description,\n { _tag: \"OneOf\", set, negate: false, encoding: undefined }\n )\n}\n\n/**\n * Matches one token *not* belonging to a set, returning it (winnow's and\n * nom's `none_of`).\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const noneOf = <I extends Input = string>(\n set: TokenSet<I>\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const expected = [`anything but ${m.description}`]\n return described(\n (state) => {\n const t = nextIf(state, m, true)\n if (t === End) return state.eoi(\"NoneOf\", expected, 1)\n if (t === NoMatch) return state.backtrack(\"NoneOf\", expected)\n // SAFETY: past `End` and `NoMatch`, `nextIf` returns the token it consumed from the parser's input `I`\n return t as Token<I>\n },\n expected[0],\n { _tag: \"OneOf\", set, negate: true, encoding: undefined }\n )\n}\n\n/**\n * Takes exactly `count` tokens (winnow's and nom's `take`). `count` must be\n * a non-negative integer: anything else throws a `RangeError` here, when\n * the parser is built.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * Parser.parsePeek(Token.take(2), \"abc\") // Result.succeed([\"c\", \"ab\"])\n * ```\n *\n * @since 0.1.0\n * @category slices\n */\nexport const take = <I extends Input = string>(count: number): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {\n checkCount(count)\n const expected = [count === 1 ? \"1 token\" : `${count} tokens`]\n return described(\n (state) => {\n const start = state.pos\n const { count: got, offset } = advance(state, start, count)\n if (got < count) return state.eoi(\"Take\", expected, count - got)\n state.pos = offset\n return state.slice(start, offset)\n },\n \"take\",\n { _tag: \"Take\", count, encoding: undefined }\n )\n}\n\nconst normalize = (r: Range | undefined): [number, number] => {\n const { max, min } = bounds(r)\n return [min, max]\n}\n\n/** @internal */\nexport const takeWhileImpl = <I extends Input>(\n set: unknown,\n r: Range | undefined,\n negate: boolean,\n name: string,\n description?: string,\n kind: ErrorKind = \"TakeWhile\",\n hint = false\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => {\n const m = compileAny(set)\n const [min, max] = normalize(r)\n const expected = [description ?? (negate ? `anything but ${m.description}` : m.description)]\n return described(\n (state) => {\n const start = state.pos\n const { count, hitEnd, offset } = scan(state, m, negate, max)\n if (hitEnd && state.partial) return state.incomplete(1, expected)\n if (count < min) return state.backtrack(kind, expected, offset)\n // More matching tokens would have been accepted where we stopped\n if (hint && count < max) state.setHint(offset, expected)\n state.pos = offset\n return state.slice(start, offset)\n },\n name,\n { _tag: \"TakeWhile\", set, negate, min, max, encoding: undefined }\n )\n}\n\n/**\n * Takes the longest run of tokens belonging to a set, with the number of\n * tokens constrained by `range` (default: zero or more). This is winnow's\n * `take_while(range, set)`, and nom's `take_while`, `take_while1` and\n * `take_while_m_n`.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n *\n * const identifier = Token.takeWhile(\n * [Token.range(\"a\", \"z\"), Token.range(\"A\", \"Z\"), \"_\"],\n * { min: 1 }\n * )\n * ```\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeWhile = <I extends Input = string>(\n set: TokenSet<I>,\n range?: Range\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => takeWhileImpl(set, range, false, \"takeWhile\")\n\n/**\n * Takes the longest run of tokens *not* belonging to a set (winnow's\n * `take_till`, nom's `take_till` and `take_till1`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeTill = <I extends Input = string>(\n set: TokenSet<I>,\n range?: Range\n): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> => takeWhileImpl(set, range, true, \"takeTill\")\n\n/**\n * The overloads of {@link takeUntil}.\n *\n * @since 0.5.0\n * @category slices\n */\nexport interface TakeUntilFn {\n (literal: string, range?: Range): Parser<string, never, never, string>\n (literal: Uint8Array, range?: Range): Parser<Uint8Array, never, never, Uint8Array>\n <T>(literal: ReadonlyArray<T>, range?: Range): Parser<ReadonlyArray<T>, never, never, ReadonlyArray<T>>\n}\n\n/**\n * Takes tokens up to (not including) the first occurrence of a literal\n * (winnow's `take_until`, nom's `take_until`). The literal is not\n * consumed. `range` constrains the number of tokens taken.\n *\n * @since 0.1.0\n * @category slices\n */\nexport const takeUntil: TakeUntilFn = (lit: Input, r?: Range): Parser<any, never, never, any> =>\n untilParser(\n (state, from) => indexOf(state, lit, from),\n // A string literal is up to three bytes per unit of UTF-8\n (state) => lit.length * (Predicate.isString(lit) && state.inputKind === 1 ? 3 : 1),\n [describeLiteral(lit)],\n r,\n \"takeUntil\",\n { _tag: \"TakeUntil\", literal: lit, ...bounds(r), encoding: undefined }\n )\n\n/**\n * The parser of `takeUntil` and `takeUntilAny`: `search` finds the first\n * occurrence, which is not consumed (`read` is how far past it the search\n * read). As in winnow, the first occurrence must lie within the range.\n */\nconst untilParser = (\n search: (state: State<Input>, from: number) => number,\n read: (state: State<Input>) => number,\n expected: ReadonlyArray<string>,\n r: Range | undefined,\n name: string,\n node: Parameters<typeof described>[2]\n): Parser<any, never, never, any> => {\n const [min, max] = normalize(r)\n return described(\n (state: State<Input>) => {\n const start = state.pos\n const idx = search(state, start)\n if (idx < 0) {\n if (state.partial) return state.incomplete(0, expected)\n return state.backtrack(\"TakeUntil\", expected, state.end)\n }\n state.syntax?.read(idx + read(state))\n if (min > 0 || max !== Infinity) {\n const count = countTokens(state, start, idx)\n if (count < min) return state.backtrack(\"TakeUntil\", expected, idx)\n if (count > max) return state.backtrack(\"TakeUntil\", expected)\n }\n state.pos = idx\n return state.slice(start, idx)\n },\n name,\n node\n )\n}\n\n/**\n * The overloads of {@link takeUntilAny}.\n *\n * @since 0.5.0\n * @category slices\n */\nexport interface TakeUntilAnyFn {\n (literals: ReadonlyArray<string>, range?: Range): Parser<string, never, never, string>\n (literals: ReadonlyArray<Uint8Array>, range?: Range): Parser<Uint8Array, never, never, Uint8Array>\n}\n\n/**\n * Takes tokens up to (not including) the first occurrence of any of\n * several literals: {@link takeUntil} for a set of them, the search a\n * multi-pattern prefilter makes. No literal is consumed. `range` constrains\n * the number of tokens taken. On bytes, `Compiler.wasm` searches with SIMD\n * (Teddy, from ripgrep), sixteen bytes at a time.\n *\n * @example\n * ```ts\n * import { Parser, Token } from \"@thomasfosterau/effect-parse\"\n * import * as Result from \"effect/Result\"\n *\n * const upTo = Parser.seq(Token.takeUntilAny([\"Holmes\", \"Watson\"]), Token.rest)\n * console.log(Result.getOrThrow(Parser.parseSync(upTo, \"Dr. Watson and Holmes\")))\n * // [\"Dr. \", \"Watson and Holmes\"]\n * ```\n *\n * @since 0.5.0\n * @category slices\n */\nexport const takeUntilAny: TakeUntilAnyFn = (\n literals: ReadonlyArray<Input>,\n r?: Range\n): Parser<any, never, never, any> => {\n const longest = Math.max(0, ...literals.map((lit) => lit.length))\n return untilParser(\n (state, from) => indexOfAny(state, literals, from),\n // Every literal was read up to where it would end (a string literal is up to three bytes per unit of UTF-8)\n (state) => longest * (state.inputKind === 1 ? 3 : 1),\n literals.map(describeLiteral),\n r,\n \"takeUntilAny\",\n { _tag: \"TakeUntilAny\", literals, ...bounds(r), encoding: undefined }\n )\n}\n\n/** The parser behind `rest` and `restOf`, on every input kind. */\nconst restInput: Parser<unknown, never, never, Input> = /*#__PURE__*/ described(\n (state) => {\n const start = state.pos\n state.pos = state.end\n return state.slice(start, state.end)\n },\n \"rest\",\n { _tag: \"Rest\", encoding: undefined }\n)\n\n/**\n * Takes the rest of the input (winnow's and nom's `rest`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const rest: Parser<string> =\n // SAFETY: on string input, `State.slice` returns a string\n restInput as Parser<string>\n\n/**\n * Takes the rest of an input of type `I`.\n *\n * @since 0.1.0\n * @category slices\n */\nexport const restOf = <I extends Input = string>(): Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>> =>\n // SAFETY: `State.slice` returns the input's own slice type for every input kind, so `restInput` returns `Slice<I>`\n restInput as Parser<Slice<NoInfer<I>>, never, never, NoInfer<I>>\n\n/**\n * Returns the length of the remaining input without consuming it (winnow's\n * and nom's `rest_len`).\n *\n * @since 0.1.0\n * @category slices\n */\nexport const restLen: Parser<number, never, never, Input> = /*#__PURE__*/ described(\n (state) => state.end - state.pos,\n \"restLen\",\n {\n _tag: \"RestLen\"\n }\n)\n\n/**\n * Matches a token satisfying a predicate (nom's `satisfy`). Equivalent to\n * `Token.oneOf(predicate)`.\n *\n * @since 0.1.0\n * @category tokens\n */\nexport const satisfy = <I extends Input = string>(\n predicate: (token: Token<I>) => boolean\n): Parser<Token<NoInfer<I>>, never, never, NoInfer<I>> =>\n // SAFETY: a predicate over the input's tokens is a member of `TokenSet<I>` for each input kind\n oneOf<I>(predicate as TokenSet<I>)\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsEA,MAAa,SAAoC,MAAS,QAA0B;CAAE,MAAM;CAAc;CAAM;AAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4CnH,MAAa,cACX,MACA,WACA,WAIA,mBACE,OACC,MAAuB,UAAU,CAAC,GACnC,QAAQ,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAU,CACjD;AAiDF,MAAM,cAAc,CAAC,WAAW;AAEhC,MAAM,WAAW,UAAiC;CAChD,MAAM,MAAM,MAAM;CAClB,IAAI,OAAO,MAAM,KAAK,OAAO,MAAM,IAAI,SAAS,aAAa,CAAC;CAC9D,QAAQ,MAAM,WAAd;EACE,KAAK,GAAG;GAEN,MAAM,IAAI,MAAM;GAChB,IAAI,eAAe,OAAO,GAAG,GAAG,OAAO,MAAM,WAAW,GAAG,WAAW;GACtE,MAAM,IAAI,EAAE,WAAW,GAAG;GAC1B,IAAI,KAAK,SAAU,KAAK,SAAU,MAAM,IAAI,EAAE,WAAW,EAAE,WAAW,MAAM,CAAC,IAAI,WAAY,OAAQ;IACnG,MAAM,MAAM,MAAM;IAClB,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;GAC7B;GACA,MAAM,MAAM,MAAM;GAClB,OAAO,EAAE;EACX;EACA,KAAK;GACH,MAAM,MAAM,MAAM;GAElB,OAAQ,MAAM,MAAqB;EACrC;GACE,MAAM,MAAM,MAAM;GAElB,OAAQ,MAAM,MAAiC;CACnD;AACF;;AAGA,MAAM,WAA+D,wBAAU,SAAS,OAAO;CAC7F,MAAM;CACN,UAAU,KAAA;AACZ,CAAC;;;;;;;;AASD,MAAa,MAEX;;;;;;;;;;;;;;;AAgBF,MAAa,iBAEX;;;;;;;;;;;;;;;;;AA8BF,MAAa,WAAsB,QAA+C,YAAY,KAAK,KAAK;;AAGxG,MAAa,eAAe,KAAY,aAAsD;CAC5F,MAAM,cAAc,gBAAgB,GAAG;CACvC,MAAM,WAAW,CAAC,WAAW,GAAG,YAAY,uBAAuB,WAAW;CAE9E,IAAI;CACJ,MAAM,cAAc,UAAU,SAAS,GAAG,IAAI,IAAI,eAAe,aAAa,IAAI;CAClF,OAAO,WACJ,UAAwB;EACvB,MAAM,MAAM,MAAM;EAElB,MAAM,IAAI,gBAAgB,KAAK,MAAM,cAAc,IAAK,YAAY,IAAI,YAAY,CAAC,CAAC,OAAO,GAAa,IAAK;EAC/G,MAAM,MAAM,EAAE;EACd,MAAM,IAAI,aAAa,OAAO,GAAG,QAAQ;EACzC,IAAI,MAAM,IAAI,OAAO,MAAM,UAAU,WAAW,QAAQ;EACxD,IAAI,MAAM,IAAI;GACZ,IAAI,MAAM,SAAS,OAAO,MAAM,WAAW,OAAO,MAAM,MAAM,MAAM,QAAQ;GAC5E,OAAO,MAAM,UAAU,WAAW,QAAQ;EAC5C;EACA,MAAM,MAAM,MAAM;EAClB,OAAO,YAAY,MAAM,cAAc,cAAc,MAAM,MAAM,KAAK,MAAM,GAAG,IAAI;CACrF,GACA,aACA;EAAE,MAAM;EAAW,SAAS;EAAK;EAAU,UAAU,KAAA;CAAU,CACjE;AACF;;;;;;;;;;;;;;;;;AAkBA,MAAa,SACX,QACwD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,WAAW,CAAC,EAAE,WAAW;CAC/B,OAAO,WACJ,UAAU;EACT,MAAM,IAAI,OAAO,OAAO,GAAG,KAAK;EAChC,IAAI,MAAM,KAAK,OAAO,MAAM,IAAI,SAAS,UAAU,CAAC;EACpD,IAAI,MAAM,SAAS,OAAO,MAAM,UAAU,SAAS,QAAQ;EAE3D,OAAO;CACT,GACA,EAAE,aACF;EAAE,MAAM;EAAS;EAAK,QAAQ;EAAO,UAAU,KAAA;CAAU,CAC3D;AACF;;;;;;;;AASA,MAAa,UACX,QACwD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,WAAW,CAAC,gBAAgB,EAAE,aAAa;CACjD,OAAO,WACJ,UAAU;EACT,MAAM,IAAI,OAAO,OAAO,GAAG,IAAI;EAC/B,IAAI,MAAM,KAAK,OAAO,MAAM,IAAI,UAAU,UAAU,CAAC;EACrD,IAAI,MAAM,SAAS,OAAO,MAAM,UAAU,UAAU,QAAQ;EAE5D,OAAO;CACT,GACA,SAAS,IACT;EAAE,MAAM;EAAS;EAAK,QAAQ;EAAM,UAAU,KAAA;CAAU,CAC1D;AACF;;;;;;;;;;;;;;;;AAiBA,MAAa,QAAkC,UAAuE;CACpH,WAAW,KAAK;CAChB,MAAM,WAAW,CAAC,UAAU,IAAI,YAAY,GAAG,MAAM,QAAQ;CAC7D,OAAO,WACJ,UAAU;EACT,MAAM,QAAQ,MAAM;EACpB,MAAM,EAAE,OAAO,KAAK,WAAW,QAAQ,OAAO,OAAO,KAAK;EAC1D,IAAI,MAAM,OAAO,OAAO,MAAM,IAAI,QAAQ,UAAU,QAAQ,GAAG;EAC/D,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,MAAM;CAClC,GACA,QACA;EAAE,MAAM;EAAQ;EAAO,UAAU,KAAA;CAAU,CAC7C;AACF;AAEA,MAAM,aAAa,MAA2C;CAC5D,MAAM,EAAE,KAAK,QAAQ,OAAO,CAAC;CAC7B,OAAO,CAAC,KAAK,GAAG;AAClB;;AAGA,MAAa,iBACX,KACA,GACA,QACA,MACA,aACA,OAAkB,aAClB,OAAO,UACiD;CACxD,MAAM,IAAI,WAAW,GAAG;CACxB,MAAM,CAAC,KAAK,OAAO,UAAU,CAAC;CAC9B,MAAM,WAAW,CAAC,gBAAgB,SAAS,gBAAgB,EAAE,gBAAgB,EAAE,YAAY;CAC3F,OAAO,WACJ,UAAU;EACT,MAAM,QAAQ,MAAM;EACpB,MAAM,EAAE,OAAO,QAAQ,WAAW,KAAK,OAAO,GAAG,QAAQ,GAAG;EAC5D,IAAI,UAAU,MAAM,SAAS,OAAO,MAAM,WAAW,GAAG,QAAQ;EAChE,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,MAAM,UAAU,MAAM;EAE9D,IAAI,QAAQ,QAAQ,KAAK,MAAM,QAAQ,QAAQ,QAAQ;EACvD,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,MAAM;CAClC,GACA,MACA;EAAE,MAAM;EAAa;EAAK;EAAQ;EAAK;EAAK,UAAU,KAAA;CAAU,CAClE;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,MAAa,aACX,KACA,UACwD,cAAc,KAAK,OAAO,OAAO,WAAW;;;;;;;;AAStG,MAAa,YACX,KACA,UACwD,cAAc,KAAK,OAAO,MAAM,UAAU;;;;;;;;;AAsBpG,MAAa,aAA0B,KAAY,MACjD,aACG,OAAO,SAAS,QAAQ,OAAO,KAAK,IAAI,IAExC,UAAU,IAAI,UAAU,UAAU,SAAS,GAAG,KAAK,MAAM,cAAc,IAAI,IAAI,IAChF,CAAC,gBAAgB,GAAG,CAAC,GACrB,GACA,aACA;CAAE,MAAM;CAAa,SAAS;CAAK,GAAG,OAAO,CAAC;CAAG,UAAU,KAAA;AAAU,CACvE;;;;;;AAOF,MAAM,eACJ,QACA,MACA,UACA,GACA,MACA,SACmC;CACnC,MAAM,CAAC,KAAK,OAAO,UAAU,CAAC;CAC9B,OAAO,WACJ,UAAwB;EACvB,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,OAAO,OAAO,KAAK;EAC/B,IAAI,MAAM,GAAG;GACX,IAAI,MAAM,SAAS,OAAO,MAAM,WAAW,GAAG,QAAQ;GACtD,OAAO,MAAM,UAAU,aAAa,UAAU,MAAM,GAAG;EACzD;EACA,MAAM,QAAQ,KAAK,MAAM,KAAK,KAAK,CAAC;EACpC,IAAI,MAAM,KAAK,QAAQ,UAAU;GAC/B,MAAM,QAAQ,YAAY,OAAO,OAAO,GAAG;GAC3C,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,aAAa,UAAU,GAAG;GAClE,IAAI,QAAQ,KAAK,OAAO,MAAM,UAAU,aAAa,QAAQ;EAC/D;EACA,MAAM,MAAM;EACZ,OAAO,MAAM,MAAM,OAAO,GAAG;CAC/B,GACA,MACA,IACF;AACF;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAa,gBACX,UACA,MACmC;CACnC,MAAM,UAAU,KAAK,IAAI,GAAG,GAAG,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC;CAChE,OAAO,aACJ,OAAO,SAAS,WAAW,OAAO,UAAU,IAAI,IAEhD,UAAU,WAAW,MAAM,cAAc,IAAI,IAAI,IAClD,SAAS,IAAI,eAAe,GAC5B,GACA,gBACA;EAAE,MAAM;EAAgB;EAAU,GAAG,OAAO,CAAC;EAAG,UAAU,KAAA;CAAU,CACtE;AACF;;AAGA,MAAM,YAAgE,yBACnE,UAAU;CACT,MAAM,QAAQ,MAAM;CACpB,MAAM,MAAM,MAAM;CAClB,OAAO,MAAM,MAAM,OAAO,MAAM,GAAG;AACrC,GACA,QACA;CAAE,MAAM;CAAQ,UAAU,KAAA;AAAU,CACtC;;;;;;;AAQA,MAAa,OAEX;;;;;;;AAQF,MAAa,eAEX;;;;;;;;AASF,MAAa,UAA6D,yBACvE,UAAU,MAAM,MAAM,MAAM,KAC7B,WACA,EACE,MAAM,UACR,CACF;;;;;;;;AASA,MAAa,WACX,cAGA,MAAS,SAAwB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { State } from "../../Input.js";
|
|
2
|
-
import { Backend } from "./registry.js";
|
|
3
2
|
import { Node, ParserLike } from "../node.js";
|
|
3
|
+
import { Backend } from "./registry.js";
|
|
4
4
|
import { FirstSets } from "./first.js";
|
|
5
5
|
export type Step = (state: State<any>) => any;
|
|
6
6
|
/** A parser that the compiler left to its interpreted step. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"closure.d.ts","names":[],"sources":["../../../src/internal/compiler/closure.ts"],"mappings":";;;;
|
|
1
|
+
{"version":3,"file":"closure.d.ts","names":[],"sources":["../../../src/internal/compiler/closure.ts"],"mappings":";;;;YAgCY,QAAQ,OAAO;;iBAGV;WACN;WACA,KAAK;WACL;;iBA+zDM;WACN,UAAU,GAAG,YAAY,kBAAkB;WAC3C,OAAO,GAAG,YAAY,MAAM,MAAM;WAClC,QAAQ;WACR,WAAW,cAAc;;qBAGvB,wBAAyB,iBAAiB,wBAAoB;;iBAW1D;WACN,MAAM;WACN,WAAW,cAAc;;;qBAWvB,SAAS"}
|