functionalscript 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/fs/asserts/module.f.d.ts +1 -1
- package/fs/asserts/module.f.js +4 -1
- package/fs/base64/proof.f.d.ts +1 -0
- package/fs/base64/proof.f.js +25 -56
- package/fs/base_n/module.f.js +11 -5
- package/fs/base_n/proof.f.d.ts +1 -0
- package/fs/base_n/proof.f.js +12 -1
- package/fs/bnf/data/module.f.d.ts +0 -103
- package/fs/bnf/data/module.f.js +10 -246
- package/fs/bnf/data/proof.f.d.ts +0 -10
- package/fs/bnf/data/proof.f.js +2 -642
- package/fs/bnf/descent/module.f.d.ts +56 -0
- package/fs/bnf/descent/module.f.js +119 -0
- package/fs/bnf/descent/proof.f.d.ts +5 -0
- package/fs/bnf/descent/proof.f.js +296 -0
- package/fs/bnf/ll1/module.f.d.ts +72 -0
- package/fs/bnf/ll1/module.f.js +153 -0
- package/fs/bnf/ll1/proof.f.d.ts +9 -0
- package/fs/bnf/ll1/proof.f.js +358 -0
- package/fs/cas/cli/module.f.d.ts +6 -0
- package/fs/cas/cli/module.f.js +56 -0
- package/fs/cas/cli/proof.f.d.ts +14 -0
- package/fs/cas/cli/proof.f.js +149 -0
- package/fs/cas/mcp/module.f.d.ts +9 -17
- package/fs/cas/mcp/module.f.js +123 -96
- package/fs/cas/mcp/proof.f.d.ts +12 -1
- package/fs/cas/mcp/proof.f.js +199 -32
- package/fs/cas/module.f.d.ts +35 -15
- package/fs/cas/module.f.js +168 -131
- package/fs/cas/proof.f.d.ts +16 -13
- package/fs/cas/proof.f.js +255 -121
- package/fs/ci/config/module.f.d.ts +14 -14
- package/fs/ci/config/module.f.js +14 -14
- package/fs/djs/tokenizer-new/module.f.js +1 -1
- package/fs/djs/tokenizer-new/proof.f.d.ts +2 -0
- package/fs/djs/tokenizer-new/proof.f.js +357 -229
- package/fs/effects/list/module.f.d.ts +31 -0
- package/fs/effects/list/module.f.js +16 -0
- package/fs/effects/module.f.d.ts +11 -5
- package/fs/effects/module.f.js +16 -6
- package/fs/effects/node/module.f.d.ts +29 -3
- package/fs/effects/node/module.f.js +35 -1
- package/fs/effects/node/module.js +32 -18
- package/fs/effects/node/virtual/module.f.js +57 -0
- package/fs/effects/proof.f.d.ts +4 -0
- package/fs/effects/proof.f.js +18 -1
- package/fs/fjs/module.f.js +2 -8
- package/fs/html/module.f.js +6 -8
- package/fs/mcp/module.f.d.ts +1 -1
- package/fs/mcp/stdio/module.f.d.ts +8 -0
- package/fs/mcp/stdio/module.f.js +33 -6
- package/fs/mcp/stdio/proof.f.d.ts +3 -0
- package/fs/mcp/stdio/proof.f.js +52 -1
- package/fs/mime/module.f.d.ts +101 -5
- package/fs/mime/module.f.js +159 -6
- package/fs/mime/proof.f.d.ts +25 -0
- package/fs/mime/proof.f.js +193 -3
- package/fs/text/code_point/module.f.d.ts +39 -2
- package/fs/text/code_point/module.f.js +74 -2
- package/fs/text/code_point/proof.f.d.ts +8 -0
- package/fs/text/code_point/proof.f.js +72 -0
- package/fs/text/module.f.d.ts +11 -1
- package/fs/text/module.f.js +13 -3
- package/fs/text/proof.f.d.ts +6 -0
- package/fs/text/proof.f.js +25 -2
- package/fs/text/utf16/module.f.js +4 -26
- package/fs/text/utf8/module.f.d.ts +27 -0
- package/fs/text/utf8/module.f.js +8 -14
- package/fs/types/bit_vec/module.f.d.ts +20 -2
- package/fs/types/bit_vec/module.f.js +77 -27
- package/fs/types/bit_vec/proof.f.d.ts +8 -0
- package/fs/types/bit_vec/proof.f.js +24 -6
- package/fs/types/function/operator/module.f.d.ts +26 -0
- package/fs/types/list/module.f.d.ts +3 -0
- package/fs/types/nullable/module.f.d.ts +9 -5
- package/fs/types/nullable/module.f.js +19 -0
- package/fs/types/result/module.d.ts +2 -4
- package/fs/types/result/module.js +9 -3
- package/package.json +3 -3
- package/issues/demo/data/data.f.js +0 -12
- package/issues/demo/data/shared.f.js +0 -3
- package/issues/demo/fs/app.js +0 -4
- package/issues/demo/fs/math.f.js +0 -4
- package/issues/demo/sample/proof.f.js +0 -13
- /package/{issues → todo}/031-json.f.d.ts +0 -0
- /package/{issues → todo}/031-json.f.js +0 -0
- /package/{issues → todo}/proof.f.d.ts +0 -0
- /package/{issues → todo}/proof.f.js +0 -0
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared Unicode code-point contract for the UTF-8 and UTF-16 decoders: the
|
|
3
|
-
* error-tag mask used to flag invalid sequences,
|
|
3
|
+
* error-tag mask used to flag invalid sequences, the streaming `decoder`
|
|
4
4
|
* factory that wraps a per-unit step and an end-of-input step into a single
|
|
5
|
-
* `List`-to-`List` conversion
|
|
5
|
+
* `List`-to-`List` conversion, and the code-point classification predicates
|
|
6
|
+
* (BMP / surrogate / supplementary-plane / overall validity) that both codecs
|
|
7
|
+
* share.
|
|
6
8
|
*
|
|
7
9
|
* @module
|
|
8
10
|
*/
|
|
@@ -26,3 +28,38 @@ export declare const errorMask = 2147483648;
|
|
|
26
28
|
* @returns A function converting a list of code units into a list of code points.
|
|
27
29
|
*/
|
|
28
30
|
export declare const decoder: <Unit, S, Cp>(byteOp: StateScan<Unit, S | null, List<Cp>>, eofOp: (state: S | null) => readonly [List<Cp>, S | null]) => (input: List<Unit>) => List<Cp>;
|
|
31
|
+
/**
|
|
32
|
+
* Checks whether the 16-bit word (U16) is a high surrogate.
|
|
33
|
+
* Range: 0xD800 - 0xDBFF.
|
|
34
|
+
*/
|
|
35
|
+
export declare const isHighSurrogate: (i: number) => boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Checks whether the 16-bit word (U16) is a low surrogate.
|
|
38
|
+
* Range: 0xDC00 - 0xDFFF.
|
|
39
|
+
*/
|
|
40
|
+
export declare const isLowSurrogate: (i: number) => boolean;
|
|
41
|
+
/**
|
|
42
|
+
* Checks whether the code point is in the BMP range.
|
|
43
|
+
* BMP is the main Unicode plane that covers code points 0x0000 - 0xFFFF, except
|
|
44
|
+
* for the range of surrogates.
|
|
45
|
+
*/
|
|
46
|
+
export declare const isBmpCodePoint: (codePoint: number) => boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Checks whether the code point belongs to a supplementary (additional) Unicode
|
|
49
|
+
* plane. Supplementary planes cover code points from 0x010000 to 0x10FFFF.
|
|
50
|
+
*/
|
|
51
|
+
export declare const isSupplementaryPlane: (i: number) => boolean;
|
|
52
|
+
/**
|
|
53
|
+
* Checks whether the code point is a valid scalar value: within the assignable
|
|
54
|
+
* Unicode range (0x0000 - 0x10FFFF) and not a surrogate.
|
|
55
|
+
*/
|
|
56
|
+
export declare const isValidCodePoint: (c: number) => boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Checks whether the code point is a *text* code point — distinct from
|
|
59
|
+
* {@link isValidCodePoint}, which gates well-formedness for decoding. A code
|
|
60
|
+
* point is text unless it is a control character: the controls are 0x0000 -
|
|
61
|
+
* 0x001F, 0x007F (DEL), and 0x0080 - 0x009F (C1), minus the whitespace block
|
|
62
|
+
* 0x0009 - 0x000D (TAB, LF, VT, FF, CR), which is legitimate in text. Every
|
|
63
|
+
* code point at or above 0x0020 that is not DEL or a C1 control is text.
|
|
64
|
+
*/
|
|
65
|
+
export declare const isTextCodePoint: (c: number) => boolean;
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared Unicode code-point contract for the UTF-8 and UTF-16 decoders: the
|
|
3
|
-
* error-tag mask used to flag invalid sequences,
|
|
3
|
+
* error-tag mask used to flag invalid sequences, the streaming `decoder`
|
|
4
4
|
* factory that wraps a per-unit step and an end-of-input step into a single
|
|
5
|
-
* `List`-to-`List` conversion
|
|
5
|
+
* `List`-to-`List` conversion, and the code-point classification predicates
|
|
6
|
+
* (BMP / surrogate / supplementary-plane / overall validity) that both codecs
|
|
7
|
+
* share.
|
|
6
8
|
*
|
|
7
9
|
* @module
|
|
8
10
|
*/
|
|
9
11
|
import { flat, stateScan } from "../../types/list/module.f.js";
|
|
12
|
+
import { contains } from "../../types/range/module.f.js";
|
|
10
13
|
/**
|
|
11
14
|
* Error mask used to tag invalid code points or encoding errors. A decoded
|
|
12
15
|
* value with this bit set represents a malformed unit rather than a valid
|
|
@@ -29,3 +32,72 @@ export const decoder = (byteOp, eofOp) => {
|
|
|
29
32
|
const run = stateScan(op)(null);
|
|
30
33
|
return input => flat(run(flat([input, [null]])));
|
|
31
34
|
};
|
|
35
|
+
/**
|
|
36
|
+
* Unicode code-point classification boundaries. The surrogate block
|
|
37
|
+
* (`0xD800`–`0xDFFF`) splits into a high half (`0xD800`–`0xDBFF`) and a low half
|
|
38
|
+
* (`0xDC00`–`0xDFFF`); the BMP ends at `0xFFFF` and `maxCodePoint` (`0x10FFFF`)
|
|
39
|
+
* is the largest assignable code point. Every predicate below is derived from
|
|
40
|
+
* these constants so the surrogate bounds and the maximum appear exactly once.
|
|
41
|
+
*/
|
|
42
|
+
const surrogateMin = 0xd800;
|
|
43
|
+
const lowSurrogateMin = 0xdc00;
|
|
44
|
+
const surrogateMax = 0xdfff;
|
|
45
|
+
const bmpMax = 0xffff;
|
|
46
|
+
const maxCodePoint = 0x10_ffff;
|
|
47
|
+
/**
|
|
48
|
+
* Checks whether the 16-bit word (U16) is a high surrogate.
|
|
49
|
+
* Range: 0xD800 - 0xDBFF.
|
|
50
|
+
*/
|
|
51
|
+
export const isHighSurrogate = contains([surrogateMin, lowSurrogateMin - 1]);
|
|
52
|
+
/**
|
|
53
|
+
* Checks whether the 16-bit word (U16) is a low surrogate.
|
|
54
|
+
* Range: 0xDC00 - 0xDFFF.
|
|
55
|
+
*/
|
|
56
|
+
export const isLowSurrogate = contains([lowSurrogateMin, surrogateMax]);
|
|
57
|
+
/**
|
|
58
|
+
* Ranges of code points for the lower (Low) and higher (High) parts of the BMP
|
|
59
|
+
* (Basic Multilingual Plane), i.e. the BMP with the surrogate block removed.
|
|
60
|
+
*/
|
|
61
|
+
const lowBmp = contains([0x0000, surrogateMin - 1]);
|
|
62
|
+
const highBmp = contains([surrogateMax + 1, bmpMax]);
|
|
63
|
+
/**
|
|
64
|
+
* Checks whether the code point is in the BMP range.
|
|
65
|
+
* BMP is the main Unicode plane that covers code points 0x0000 - 0xFFFF, except
|
|
66
|
+
* for the range of surrogates.
|
|
67
|
+
*/
|
|
68
|
+
export const isBmpCodePoint = (codePoint) => lowBmp(codePoint) || highBmp(codePoint);
|
|
69
|
+
/**
|
|
70
|
+
* Checks whether the code point belongs to a supplementary (additional) Unicode
|
|
71
|
+
* plane. Supplementary planes cover code points from 0x010000 to 0x10FFFF.
|
|
72
|
+
*/
|
|
73
|
+
export const isSupplementaryPlane = contains([bmpMax + 1, maxCodePoint]);
|
|
74
|
+
/**
|
|
75
|
+
* The full assignable code-point range and the surrogate block, used to gate
|
|
76
|
+
* overall validity below.
|
|
77
|
+
*/
|
|
78
|
+
const validRange = contains([0, maxCodePoint]);
|
|
79
|
+
const isSurrogate = contains([surrogateMin, surrogateMax]);
|
|
80
|
+
/**
|
|
81
|
+
* Checks whether the code point is a valid scalar value: within the assignable
|
|
82
|
+
* Unicode range (0x0000 - 0x10FFFF) and not a surrogate.
|
|
83
|
+
*/
|
|
84
|
+
export const isValidCodePoint = (c) => validRange(c) && !isSurrogate(c);
|
|
85
|
+
/**
|
|
86
|
+
* The control-character blocks and the whitespace exception used to separate
|
|
87
|
+
* *text* code points from binary control bytes. C0 controls span 0x0000 -
|
|
88
|
+
* 0x001F; 0x007F (DEL) and the C1 controls (0x0080 - 0x009F) form one
|
|
89
|
+
* contiguous run; the whitespace block 0x0009 - 0x000D (TAB, LF, VT, FF, CR) is
|
|
90
|
+
* the only control range legitimate in text.
|
|
91
|
+
*/
|
|
92
|
+
const c0Control = contains([0x00, 0x1f]);
|
|
93
|
+
const delAndC1Control = contains([0x7f, 0x9f]);
|
|
94
|
+
const textWhitespace = contains([0x09, 0x0d]);
|
|
95
|
+
/**
|
|
96
|
+
* Checks whether the code point is a *text* code point — distinct from
|
|
97
|
+
* {@link isValidCodePoint}, which gates well-formedness for decoding. A code
|
|
98
|
+
* point is text unless it is a control character: the controls are 0x0000 -
|
|
99
|
+
* 0x001F, 0x007F (DEL), and 0x0080 - 0x009F (C1), minus the whitespace block
|
|
100
|
+
* 0x0009 - 0x000D (TAB, LF, VT, FF, CR), which is legitimate in text. Every
|
|
101
|
+
* code point at or above 0x0020 that is not DEL or a C1 control is text.
|
|
102
|
+
*/
|
|
103
|
+
export const isTextCodePoint = (c) => textWhitespace(c) || !(c0Control(c) || delAndC1Control(c));
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { isBmpCodePoint, isHighSurrogate, isLowSurrogate, isSupplementaryPlane, isTextCodePoint, isValidCodePoint, } from "./module.f.js";
|
|
2
|
+
const check = (actual, expected) => {
|
|
3
|
+
if (actual !== expected) {
|
|
4
|
+
throw `${actual} !== ${expected}`;
|
|
5
|
+
}
|
|
6
|
+
};
|
|
7
|
+
export const proof = {
|
|
8
|
+
isHighSurrogate: [
|
|
9
|
+
() => check(isHighSurrogate(0xd800), true),
|
|
10
|
+
() => check(isHighSurrogate(0xdbff), true),
|
|
11
|
+
() => check(isHighSurrogate(0xd7ff), false),
|
|
12
|
+
() => check(isHighSurrogate(0xdc00), false),
|
|
13
|
+
],
|
|
14
|
+
isLowSurrogate: [
|
|
15
|
+
() => check(isLowSurrogate(0xdc00), true),
|
|
16
|
+
() => check(isLowSurrogate(0xdfff), true),
|
|
17
|
+
() => check(isLowSurrogate(0xdbff), false),
|
|
18
|
+
() => check(isLowSurrogate(0xe000), false),
|
|
19
|
+
],
|
|
20
|
+
isBmpCodePoint: [
|
|
21
|
+
// lowBmp branch true
|
|
22
|
+
() => check(isBmpCodePoint(0x0000), true),
|
|
23
|
+
() => check(isBmpCodePoint(0xd7ff), true),
|
|
24
|
+
// lowBmp false, highBmp true
|
|
25
|
+
() => check(isBmpCodePoint(0xe000), true),
|
|
26
|
+
() => check(isBmpCodePoint(0xffff), true),
|
|
27
|
+
// both false: surrogate and supplementary
|
|
28
|
+
() => check(isBmpCodePoint(0xd800), false),
|
|
29
|
+
() => check(isBmpCodePoint(0x10000), false),
|
|
30
|
+
],
|
|
31
|
+
isSupplementaryPlane: [
|
|
32
|
+
() => check(isSupplementaryPlane(0x10000), true),
|
|
33
|
+
() => check(isSupplementaryPlane(0x10ffff), true),
|
|
34
|
+
() => check(isSupplementaryPlane(0xffff), false),
|
|
35
|
+
() => check(isSupplementaryPlane(0x110000), false),
|
|
36
|
+
],
|
|
37
|
+
isValidCodePoint: [
|
|
38
|
+
// in range, not surrogate
|
|
39
|
+
() => check(isValidCodePoint(0x0000), true),
|
|
40
|
+
() => check(isValidCodePoint(0x10ffff), true),
|
|
41
|
+
// in range, surrogate -> invalid
|
|
42
|
+
() => check(isValidCodePoint(0xd800), false),
|
|
43
|
+
() => check(isValidCodePoint(0xdfff), false),
|
|
44
|
+
// out of range -> validRange short-circuits false
|
|
45
|
+
() => check(isValidCodePoint(-1), false),
|
|
46
|
+
() => check(isValidCodePoint(0x110000), false),
|
|
47
|
+
],
|
|
48
|
+
isTextCodePoint: [
|
|
49
|
+
// C0 controls are binary...
|
|
50
|
+
() => check(isTextCodePoint(0x00), false), // NUL
|
|
51
|
+
() => check(isTextCodePoint(0x08), false), // BS
|
|
52
|
+
() => check(isTextCodePoint(0x1b), false), // ESC
|
|
53
|
+
() => check(isTextCodePoint(0x1f), false), // US
|
|
54
|
+
// ...except the whitespace block 0x09 - 0x0D
|
|
55
|
+
() => check(isTextCodePoint(0x09), true), // TAB
|
|
56
|
+
() => check(isTextCodePoint(0x0a), true), // LF
|
|
57
|
+
() => check(isTextCodePoint(0x0b), true), // VT
|
|
58
|
+
() => check(isTextCodePoint(0x0c), true), // FF
|
|
59
|
+
() => check(isTextCodePoint(0x0d), true), // CR
|
|
60
|
+
// printable ASCII is text
|
|
61
|
+
() => check(isTextCodePoint(0x20), true), // space
|
|
62
|
+
() => check(isTextCodePoint(0x41), true), // 'A'
|
|
63
|
+
() => check(isTextCodePoint(0x7e), true), // '~'
|
|
64
|
+
// DEL and the C1 controls are binary
|
|
65
|
+
() => check(isTextCodePoint(0x7f), false), // DEL
|
|
66
|
+
() => check(isTextCodePoint(0x80), false), // C1 start
|
|
67
|
+
() => check(isTextCodePoint(0x9f), false), // C1 end
|
|
68
|
+
// above C1 is text again
|
|
69
|
+
() => check(isTextCodePoint(0xa0), true), // NBSP
|
|
70
|
+
() => check(isTextCodePoint(0x10ffff), true),
|
|
71
|
+
],
|
|
72
|
+
};
|
package/fs/text/module.f.d.ts
CHANGED
|
@@ -7,19 +7,29 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { type Vec } from '../types/bit_vec/module.f.ts';
|
|
9
9
|
import { type List } from '../types/list/module.f.ts';
|
|
10
|
+
import { type Nullable } from '../types/nullable/module.f.ts';
|
|
10
11
|
export type Block = ItemThunk | ItemArray;
|
|
11
12
|
type ItemArray = readonly Item[];
|
|
12
13
|
type ItemThunk = () => List<Item>;
|
|
13
14
|
export type Item = string | ItemArray | ItemThunk;
|
|
14
15
|
export declare const flat: (indent: string) => (text: Block) => List<string>;
|
|
15
16
|
export type Utf8 = Vec;
|
|
17
|
+
/**
|
|
18
|
+
* Converts a string to an UTF-8, represented as an MSB first bit vector,
|
|
19
|
+
* returning `null` instead of throwing if the result would exceed
|
|
20
|
+
* `maxLength`.
|
|
21
|
+
*
|
|
22
|
+
* @param s The input string to be converted.
|
|
23
|
+
* @returns The resulting UTF-8 bit vector, MSB first, or `null` on overflow.
|
|
24
|
+
*/
|
|
25
|
+
export declare const tryUtf8: (s: string) => Nullable<Utf8>;
|
|
16
26
|
/**
|
|
17
27
|
* Converts a string to an UTF-8, represented as an MSB first bit vector.
|
|
18
28
|
*
|
|
19
29
|
* @param s The input string to be converted.
|
|
20
30
|
* @returns The resulting UTF-8 bit vector, MSB first.
|
|
21
31
|
*/
|
|
22
|
-
export declare const utf8: (
|
|
32
|
+
export declare const utf8: import("../types/function/module.f.ts").Func<string, Vec>;
|
|
23
33
|
/**
|
|
24
34
|
* Converts a UTF-8 bit vector with MSB first encoding to a string.
|
|
25
35
|
*
|
package/fs/text/module.f.js
CHANGED
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @module
|
|
7
7
|
*/
|
|
8
|
-
import { msb,
|
|
8
|
+
import { msb, tryU8ListToVec, u8List } from "../types/bit_vec/module.f.js";
|
|
9
9
|
import { flatMap } from "../types/list/module.f.js";
|
|
10
10
|
import { fromCodePointList, toCodePointList } from "./utf8/module.f.js";
|
|
11
11
|
import { stringToCodePointList, codePointListToString } from "./utf16/module.f.js";
|
|
12
|
+
import { mapUnwrap } from "../types/nullable/module.f.js";
|
|
12
13
|
export const flat = (indent) => {
|
|
13
14
|
const f = (prefix) => {
|
|
14
15
|
const g = (item) => typeof (item) === 'string' ? [`${prefix}${item}`] : f(`${prefix}${indent}`)(item);
|
|
@@ -16,14 +17,23 @@ export const flat = (indent) => {
|
|
|
16
17
|
};
|
|
17
18
|
return f('');
|
|
18
19
|
};
|
|
19
|
-
const
|
|
20
|
+
const tryU8ListToVecMsb = tryU8ListToVec(msb);
|
|
21
|
+
/**
|
|
22
|
+
* Converts a string to an UTF-8, represented as an MSB first bit vector,
|
|
23
|
+
* returning `null` instead of throwing if the result would exceed
|
|
24
|
+
* `maxLength`.
|
|
25
|
+
*
|
|
26
|
+
* @param s The input string to be converted.
|
|
27
|
+
* @returns The resulting UTF-8 bit vector, MSB first, or `null` on overflow.
|
|
28
|
+
*/
|
|
29
|
+
export const tryUtf8 = (s) => tryU8ListToVecMsb(fromCodePointList(stringToCodePointList(s)));
|
|
20
30
|
/**
|
|
21
31
|
* Converts a string to an UTF-8, represented as an MSB first bit vector.
|
|
22
32
|
*
|
|
23
33
|
* @param s The input string to be converted.
|
|
24
34
|
* @returns The resulting UTF-8 bit vector, MSB first.
|
|
25
35
|
*/
|
|
26
|
-
export const utf8 = (
|
|
36
|
+
export const utf8 = mapUnwrap(tryUtf8);
|
|
27
37
|
/**
|
|
28
38
|
* Converts a UTF-8 bit vector with MSB first encoding to a string.
|
|
29
39
|
*
|
package/fs/text/proof.f.d.ts
CHANGED
package/fs/text/proof.f.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { assert, assertEq } from "../asserts/module.f.js";
|
|
2
|
+
import { flat, utf8, utf8ToString, tryUtf8 } from "./module.f.js";
|
|
2
3
|
import { join } from "../types/string/module.f.js";
|
|
4
|
+
import { empty, maxLengthBytes } from "../types/bit_vec/module.f.js";
|
|
5
|
+
const overflowStr = 'a'.repeat(Number(maxLengthBytes) + 1);
|
|
3
6
|
export const proof = {
|
|
4
7
|
block: () => {
|
|
5
8
|
const text = [
|
|
@@ -22,5 +25,25 @@ export const proof = {
|
|
|
22
25
|
if (r !== 'Hello world!') {
|
|
23
26
|
throw r;
|
|
24
27
|
}
|
|
25
|
-
}
|
|
28
|
+
},
|
|
29
|
+
tryUtf8RoundTrip: () => {
|
|
30
|
+
const v = tryUtf8('Hello world!');
|
|
31
|
+
assert(v !== null);
|
|
32
|
+
assertEq(v, utf8('Hello world!'));
|
|
33
|
+
assertEq(utf8ToString(v), 'Hello world!');
|
|
34
|
+
},
|
|
35
|
+
tryUtf8Empty: () => {
|
|
36
|
+
assertEq(tryUtf8(''), empty);
|
|
37
|
+
},
|
|
38
|
+
tryUtf8Overflow: {
|
|
39
|
+
// One byte past `maxLengthBytes`; `tryUtf8` should report `null`
|
|
40
|
+
// instead of building an oversized `bigint`, and the throwing `utf8`
|
|
41
|
+
// wrapper should raise on the same input.
|
|
42
|
+
try: () => {
|
|
43
|
+
assertEq(tryUtf8(overflowStr), null);
|
|
44
|
+
},
|
|
45
|
+
throw: () => {
|
|
46
|
+
utf8(overflowStr);
|
|
47
|
+
},
|
|
48
|
+
},
|
|
26
49
|
};
|
|
@@ -9,33 +9,11 @@ import { map, reduce, flatMap, empty, } from "../../types/list/module.f.js";
|
|
|
9
9
|
import { concat } from "../../types/function/operator/module.f.js";
|
|
10
10
|
import { contains } from "../../types/range/module.f.js";
|
|
11
11
|
import { fn } from "../../types/function/module.f.js";
|
|
12
|
-
import { decoder, errorMask } from "../code_point/module.f.js";
|
|
13
|
-
/**
|
|
14
|
-
* Ranges of code points for the lower (Low) and higher (High) parts of the BMP (Basic Multilingual Plane) plane.
|
|
15
|
-
*/
|
|
16
|
-
const lowBmp = contains([0x0000, 0xd7ff]);
|
|
17
|
-
const highBmp = contains([0xe000, 0xffff]);
|
|
18
|
-
/**
|
|
19
|
-
* Checks whether the code point is in the BMP range.
|
|
20
|
-
* BMP is the main multi-plane Unicode plane that covers code points 0x0000 - 0xFFFF, except for the range of surrogates.
|
|
21
|
-
*/
|
|
22
|
-
const isBmpCodePoint = (codePoint) => lowBmp(codePoint) || highBmp(codePoint);
|
|
23
|
-
/**
|
|
24
|
-
* Checks whether the 16-bit word (U16) is a surrogate of the high part.
|
|
25
|
-
* Range: 0xD800 - 0xDBFF.
|
|
26
|
-
*/
|
|
27
|
-
const isHighSurrogate = contains([0xd800, 0xdbff]);
|
|
28
|
-
/**
|
|
29
|
-
* Checks whether the 16-bit word (U16) is a substitute for the low part.
|
|
30
|
-
* Range: 0xDC00 – 0xDFFF.
|
|
31
|
-
*/
|
|
32
|
-
const isLowSurrogate = contains([0xdc00, 0xdfff]);
|
|
33
|
-
/**
|
|
34
|
-
* Checks whether the code point belongs to the additional (Supplementary) plane of Unicode.
|
|
35
|
-
* Additional planes include code points from 0x010000 to 0x10FFFF.
|
|
36
|
-
*/
|
|
37
|
-
const isSupplementaryPlane = contains([0x01_0000, 0x10_ffff]);
|
|
12
|
+
import { decoder, errorMask, isBmpCodePoint, isHighSurrogate, isLowSurrogate, isSupplementaryPlane, } from "../code_point/module.f.js";
|
|
38
13
|
/**
|
|
14
|
+
* The BMP / surrogate / supplementary-plane predicates used below live in
|
|
15
|
+
* `code_point`, the shared Unicode contract; see that module for their ranges.
|
|
16
|
+
*
|
|
39
17
|
* Converts a Unicode code point to its corresponding UTF-16 representation.
|
|
40
18
|
*
|
|
41
19
|
* This function handles:
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* @module
|
|
5
5
|
*/
|
|
6
6
|
import { type List, type Thunk } from '../../types/list/module.f.ts';
|
|
7
|
+
import type { StateScan } from '../../types/function/operator/module.f.ts';
|
|
7
8
|
import type { Array1, Array2, Array3 } from '../../types/array/module.f.ts';
|
|
8
9
|
import { type Vec } from '../../types/bit_vec/module.f.ts';
|
|
9
10
|
/**
|
|
@@ -35,6 +36,32 @@ export type Utf8State = null | Utf8NonEmptyState;
|
|
|
35
36
|
* @returns A thunk that lazily produces a sequence of UTF-8 bytes.
|
|
36
37
|
*/
|
|
37
38
|
export declare const fromCodePointList: (input: List<number>) => Thunk<U8>;
|
|
39
|
+
/**
|
|
40
|
+
* Converts a non-empty UTF-8 decoding state to an error code.
|
|
41
|
+
*
|
|
42
|
+
* @param state - A non-empty UTF-8 decoding state.
|
|
43
|
+
* @returns An I32 error code derived from the invalid UTF-8 state.
|
|
44
|
+
*/
|
|
45
|
+
export declare const utf8StateToError: (state: Utf8NonEmptyState) => I32;
|
|
46
|
+
/**
|
|
47
|
+
* Decodes a byte into a Unicode code point, using a given UTF-8 state.
|
|
48
|
+
*
|
|
49
|
+
* @param state - The current UTF-8 decoding state.
|
|
50
|
+
* @param byte - A single byte to decode.
|
|
51
|
+
* @returns A tuple containing:
|
|
52
|
+
* - A list of decoded Unicode code points or error codes.
|
|
53
|
+
* - The updated UTF-8 state.
|
|
54
|
+
*/
|
|
55
|
+
export declare const utf8ByteToCodePointOp: StateScan<number, Utf8State, readonly I32[]>;
|
|
56
|
+
/**
|
|
57
|
+
* Handles the end-of-file (EOF) case for UTF-8 decoding.
|
|
58
|
+
*
|
|
59
|
+
* @param state - The current UTF-8 decoding state.
|
|
60
|
+
* @returns A tuple containing:
|
|
61
|
+
* - A list of decoded Unicode code points or error codes.
|
|
62
|
+
* - The reset UTF-8 state (`null`).
|
|
63
|
+
*/
|
|
64
|
+
export declare const utf8EofToCodePointOp: (state: Utf8State) => readonly [List<I32>, Utf8State];
|
|
38
65
|
/**
|
|
39
66
|
* Converts a list of UTF-8 bytes into a list of Unicode code points.
|
|
40
67
|
*
|
package/fs/text/utf8/module.f.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* @module
|
|
5
5
|
*/
|
|
6
6
|
import { flatMap, toArray } from "../../types/list/module.f.js";
|
|
7
|
-
import { decoder, errorMask } from "../code_point/module.f.js";
|
|
7
|
+
import { decoder, errorMask, isValidCodePoint } from "../code_point/module.f.js";
|
|
8
8
|
import { msb, u8List, length } from "../../types/bit_vec/module.f.js";
|
|
9
9
|
import { codePointListToString } from "../utf16/module.f.js";
|
|
10
10
|
/**
|
|
@@ -106,7 +106,7 @@ export const fromCodePointList = flatMap(codePointToUtf8);
|
|
|
106
106
|
* @param state - A non-empty UTF-8 decoding state.
|
|
107
107
|
* @returns An I32 error code derived from the invalid UTF-8 state.
|
|
108
108
|
*/
|
|
109
|
-
const utf8StateToError = (state) => {
|
|
109
|
+
export const utf8StateToError = (state) => {
|
|
110
110
|
let x;
|
|
111
111
|
switch (state.length) {
|
|
112
112
|
case 1: {
|
|
@@ -127,8 +127,8 @@ const utf8StateToError = (state) => {
|
|
|
127
127
|
contPayload(s2) + 0b1000_0000_0000_0000;
|
|
128
128
|
break;
|
|
129
129
|
}
|
|
130
|
-
default:
|
|
131
|
-
|
|
130
|
+
//default:
|
|
131
|
+
// throw 'invalid state'
|
|
132
132
|
}
|
|
133
133
|
return x | errorMask;
|
|
134
134
|
};
|
|
@@ -141,7 +141,7 @@ const utf8StateToError = (state) => {
|
|
|
141
141
|
* - A list of decoded Unicode code points or error codes.
|
|
142
142
|
* - The updated UTF-8 state.
|
|
143
143
|
*/
|
|
144
|
-
const utf8ByteToCodePointOp = (byte, state) => {
|
|
144
|
+
export const utf8ByteToCodePointOp = (byte, state) => {
|
|
145
145
|
if (byte < 0x00 || byte > 0xff) {
|
|
146
146
|
return [[errorMask], state];
|
|
147
147
|
}
|
|
@@ -199,7 +199,7 @@ const utf8ByteToCodePointOp = (byte, state) => {
|
|
|
199
199
|
* - A list of decoded Unicode code points or error codes.
|
|
200
200
|
* - The reset UTF-8 state (`null`).
|
|
201
201
|
*/
|
|
202
|
-
const utf8EofToCodePointOp = (state) => [
|
|
202
|
+
export const utf8EofToCodePointOp = (state) => [
|
|
203
203
|
state === null ? null : [utf8StateToError(state)],
|
|
204
204
|
null,
|
|
205
205
|
];
|
|
@@ -216,18 +216,12 @@ export const toCodePointList = decoder(utf8ByteToCodePointOp, utf8EofToCodePoint
|
|
|
216
216
|
* code points.
|
|
217
217
|
*/
|
|
218
218
|
export const fromVec = (v) => {
|
|
219
|
-
if (length(v)
|
|
219
|
+
if ((length(v) & 7n) !== 0n) {
|
|
220
220
|
return null;
|
|
221
221
|
}
|
|
222
222
|
const arr = toArray(toCodePointList(u8List(msb)(v)));
|
|
223
223
|
for (const cp of arr) {
|
|
224
|
-
if (cp
|
|
225
|
-
return null;
|
|
226
|
-
}
|
|
227
|
-
if (cp > 0x10FFFF) {
|
|
228
|
-
return null;
|
|
229
|
-
}
|
|
230
|
-
if (cp >= 0xD800 && cp <= 0xDFFF) {
|
|
224
|
+
if (!isValidCodePoint(cp)) {
|
|
231
225
|
return null;
|
|
232
226
|
}
|
|
233
227
|
}
|
|
@@ -26,6 +26,7 @@ import type { Binary, Fold, Reduce as OpReduce } from '../function/operator/modu
|
|
|
26
26
|
import { type List, type Thunk } from '../list/module.f.ts';
|
|
27
27
|
import { type Nominal } from '../nominal/module.f.ts';
|
|
28
28
|
import { type Sign } from '../function/compare/module.f.ts';
|
|
29
|
+
import { type Nullable } from '../nullable/module.f.ts';
|
|
29
30
|
/**
|
|
30
31
|
* A vector of bits represented as a signed `bigint`.
|
|
31
32
|
*/
|
|
@@ -195,6 +196,12 @@ export type BitOrder = {
|
|
|
195
196
|
* ```
|
|
196
197
|
*/
|
|
197
198
|
readonly concat: Reduce;
|
|
199
|
+
/**
|
|
200
|
+
* Folds a list of vectors into a single vector in this bit order, like
|
|
201
|
+
* `listToVec`, but returns `null` instead of throwing when the combined
|
|
202
|
+
* length would exceed `maxLength`.
|
|
203
|
+
*/
|
|
204
|
+
readonly tryListToVec: (list: List<Vec>) => Nullable<Vec>;
|
|
198
205
|
/**
|
|
199
206
|
* Folds a list of vectors into a single vector in this bit order.
|
|
200
207
|
*
|
|
@@ -221,9 +228,10 @@ export type BitOrder = {
|
|
|
221
228
|
*/
|
|
222
229
|
readonly cmp: (a: Vec) => (b: Vec) => Sign;
|
|
223
230
|
readonly unpackSplit: (len: bigint) => (u: Unpacked) => readonly [bigint, bigint];
|
|
224
|
-
readonly unpackConcat:
|
|
231
|
+
readonly unpackConcat: UnpackConcat;
|
|
225
232
|
readonly startsWith: (prefix: Vec) => (v: Vec) => boolean;
|
|
226
233
|
};
|
|
234
|
+
type UnpackConcat = (a: Unpacked) => (b: Unpacked) => Unpacked;
|
|
227
235
|
/**
|
|
228
236
|
* Implements operations for handling vectors in a least-significant-bit (LSb) first order.
|
|
229
237
|
*
|
|
@@ -240,6 +248,16 @@ export declare const lsb: BitOrder;
|
|
|
240
248
|
* Usually associated with Big-Endian (BE) byte order.
|
|
241
249
|
*/
|
|
242
250
|
export declare const msb: BitOrder;
|
|
251
|
+
/**
|
|
252
|
+
* Converts a list of unsigned 8-bit integers to a bit vector using the provided
|
|
253
|
+
* bit order, like `u8ListToVec`, but returns `null` instead of throwing when the
|
|
254
|
+
* result would exceed `maxLength`.
|
|
255
|
+
*
|
|
256
|
+
* @param bo The bit order for the conversion
|
|
257
|
+
* @param list The list of unsigned 8-bit integers to be converted.
|
|
258
|
+
* @returns The resulting vector, or `null` if it would exceed `maxLength`.
|
|
259
|
+
*/
|
|
260
|
+
export declare const tryU8ListToVec: ({ unpackConcat }: BitOrder) => (list: List<number>) => Nullable<Vec>;
|
|
243
261
|
/**
|
|
244
262
|
* Converts a list of unsigned 8-bit integers to a bit vector using the provided bit order.
|
|
245
263
|
*
|
|
@@ -247,7 +265,7 @@ export declare const msb: BitOrder;
|
|
|
247
265
|
* @param list The list of unsigned 8-bit integers to be converted.
|
|
248
266
|
* @returns The resulting vector based on the provided bit order.
|
|
249
267
|
*/
|
|
250
|
-
export declare const u8ListToVec: (
|
|
268
|
+
export declare const u8ListToVec: (bo: BitOrder) => import("../function/module.f.ts").Func<List<number>, Vec>;
|
|
251
269
|
/**
|
|
252
270
|
* Chunks an unpacked vector into fixed-size pieces of `n` bits using the provided bit order,
|
|
253
271
|
* returning each chunk as an unsigned integer.
|