functionalscript 0.0.405 → 0.0.408

package/doc/fa.md

@@ -0,0 +1,158 @@
+# FA
+
+F ::= A 'hello'
+F ::= A 'help'
+
+## Classic FA
+
+S0 ::= A 'h'
+S1 ::= S0 'e'
+S2 ::= S1 'l'
+S3 ::= S2 'l'
+F ::= S3 'o'
+
+X0 ::= A 'h'
+X1 ::= X0 'e'
+X2 ::= X1 'l'
+F ::= X2 'p'
+
+## DFA
+
+{S0,X0} = A 'h'
+{S1,X1} ::= {S0,X0} 'e'
+{S2,X2} ::= {S1,X1} 'l'
+S3 ::= {S2,X2} 'l'
+F ::= {S2,X2} 'p'
+F ::= S3 'o'
+
+P0 = A 'h'
+P1 ::= P0 'e'
+P2 ::= P1 'l'
+S3 ::= P2 'l'
+F ::= P2 'p'
+F ::= S3 'o'
+
+## Tokenizer FA
+
+T ::= I 'true'  // T0, T1, T2
+F ::= I 'false' // F0, F2, F2, F3
+N ::= I 'null'  // N0, N1, N2
+Id ::= I letter
+Id ::= Id letter
+Id ::= Id digit
+
+## Tokenizer DFA
+
+{T0,Id} = I 't'
+{T1,Id} = {T0,Id} 'r'
+Id = {T0,Id} letter(except 'r')
+Id = {T0,Id} digit
+
+{a..b}{c..d}{e..f}
+
+```js
+const t0 = [[init, one('t')]]
+const t1 = [[t0, one('r')]]
+const t2 = [[t1, one('u')]]
+const t = [[t2, one('e')]]
+
+const id = [
+    [init, letter]
+    [() => id, letter]
+    [() => id, digit]
+]
+
+const dfa = ([t, f, n, id]) => ?
+```
+
+## Set
+
+```js
+const letter = byteSet(['_', '$', ['a', 'z'], ['A', 'Z']])
+```
+
+Letters and digits
+
+|      |            |
+|------|------------|
+|`$`   |`0x24`      |
+|`0..9`|`0x30..0x39`|
+|`A..Z`|`0x41..0x5A`|
+|`_`   |`0x5F`      |
+|`a..z`|`0x61..0x7A`|
+
+## Bit set
+
+For a byte, it is an array of 8 uint32, bigint (0..2^256-1), or string of 16 characters.
+
+### 16 bit set.
+
+It can use an intermediate state.
+
+|   |       |        |
+|---|-------|--------|
+|`2`|`4`    |`$`     |
+|`3`|`..9`  |`0..9`  |
+|`4`|`1..`  |`A..O`  |
+|`5`|`..A,F`|`P..Z,_`|
+|`6`|`1..`  |`a..o`  |
+|`7`|`..A`  |`p..z`  |
+
+```js
+const init = [
+   _,  _, i2,  _,
+  i4, i5, i4, i7,
+   _,  _,  _,  _,
+   _,  _,  _,  _]
+
+const i2 = [
+   _,  _,  _,  _,
+  id,  _,  _,  _,
+   _,  _,  _,  _,
+   _,  _,  _,  _]
+
+const i3 = [
+  id, id, id, id,
+  id, id, id, id,
+  id, id,  _,  _,
+   _,  _,  _,  _]
+
+const i4 = [
+   _, id, id, id,
+  id, id, id, id,
+  id, id, id, id,
+  id, id, id, id]
+
+const i5 = [
+  id, id, id, id,
+  id, id, id, id,
+  id, id, id,  _,
+   _,  _,  _, id]
+
+const i6 = [
+  id, id, id, id,
+  id, id, id, id,
+  id, id, id,  _,
+   _,  _,  _,  _]
+```
+
+```js
+const init =
+  000_000_001_000
+  010_011_100_101
+  000_000_000_000
+  000_000_000_000
+
+const i = [
+  // 1
+  0000_1000_0000_0000,
+  // 2
+  1111_1111_1100_0000,
+  // 3
+  0111_1111_1111_1111,
+  // 4
+  1111_1111_1110_0001,
+  // 5
+  1111_1111_1110_0000,
+]
+```

package/html/module.f.cjs

@@ -3,8 +3,8 @@ const { map, flatMap, flat, concat: listConcat } = list
 const { concat: stringConcat } = require('../types/string/module.f.cjs')
 const object = require('../types/object/module.f.cjs')
 const { compose } = require('../types/function/module.f.cjs')
-const encoding = require('../text/encoding/module.f.cjs');
-const { stringToUtf16List } = encoding
+const encoding = require('../text/utf16/module.f.cjs');
+const { stringToList } = encoding
 
 const { fromCharCode } = String
 const { entries } = Object
@@ -80,7 +80,7 @@ const escapeCharCode = code => {
     }
 }
 
-const escape = compose(stringToUtf16List)(map(escapeCharCode))
+const escape = compose(stringToList)(map(escapeCharCode))
 
 /** @type {(n: Node) => list.List<string>} */
 const node = n => typeof n === 'string' ? escape(n) : element(n)

package/json/tokenizer/test.f.cjs

@@ -2,10 +2,10 @@ const tokenizer = require('./module.f.cjs')
 const list = require('../../types/list/module.f.cjs')
 const json = require('../module.f.cjs')
 const { sort } = require('../../types/object/module.f.cjs')
-const encoding = require('../../text/encoding/module.f.cjs');
+const encoding = require('../../text/utf16/module.f.cjs');
 
 /** @type {(s: string) => readonly tokenizer.JsonToken[]} */
-const tokenizeString = s => list.toArray(tokenizer.tokenize(encoding.stringToUtf16List(s)))
+const tokenizeString = s => list.toArray(tokenizer.tokenize(encoding.stringToList(s)))
 
 const stringify = json.stringify(sort)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "functionalscript",
-  "version": "0.0.405",
+  "version": "0.0.408",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "module.f.cjs",
   "scripts": {

package/test.f.cjs

@@ -22,7 +22,8 @@ require('./commonjs/build/test.f.cjs')
 require('./types/range/test.f.cjs')
 require('./html/test.f.cjs')
 require('./text/test.f.cjs')
-require('./text/encoding/test.f.cjs')
+require('./text/utf8/test.f.cjs')
+require('./text/utf16/test.f.cjs')
 require('./com/cs/test.f.cjs')
 require('./com/cpp/test.f.cjs')
 require('./nodejs/version/test.f.cjs')

package/text/{encoding/README.md → README.md}

@@ -34,12 +34,14 @@ Total error states:
 - 34_432
 - < 2^16
 
+### utf8/module.f.cjs
+
 ```js
 /** @type {(input: List<u8|undefined>) => List<i32>} */
-const utf8ToCodePoint
+const toCodePointList
 
 /** @type {(input: List<i32>) => List<u8>} */
-const codePointToUtf8
+const fromCodePointList
 ```
 
 ## UTF-16
@@ -67,18 +69,20 @@ Requirement: no loss for UTF16 => codepoint => UTF16
 
 Total error states: 11 bit
 
+### utf16/module.f.cjs
+
 ```js
 /** @type {(input: List<u16|undefined>) => List<i32>} */
-const utf16ListToCodePointList
+const toCodePointList
 
 /** @type {(input: List<i32>) => List<u16>} */
-const codePointListToUtf16List
+const fromCodePointList
 
 /** @type {(input: string) => List<u16>} */
-const stringToUtf16List
+const stringToList
 
 /** @type {(input: List<u16>) => string} */
-const utf16ListToString
+const listToString
 ```
 
 UTF-16 => CP => UTF-8 => CP = UTF-16

package/text/utf16/module.f.cjs

@@ -0,0 +1,90 @@
+const list = require('../../types/list/module.f.cjs')
+const operator = require('../../types/function/operator/module.f.cjs')
+const array = require('../../types/array/module.f.cjs')
+const { contains } = require('../../types/range/module.f.cjs')
+const { compose } = require('../../types/function/module.f.cjs')
+const { map, flat, stateScan, concat, reduce, flatMap } = list
+
+/** @typedef {u16|undefined} WordOrEof */
+
+/** @typedef {undefined|number} Utf16State */
+
+/** @typedef {number} u16 */
+
+/** @typedef {number} i32 */
+
+/** @type {(a:number) => boolean} */
+const isBmpCodePoint = a => a >= 0x0000 && a <= 0xd7ff || a >= 0xe000 && a <= 0xffff
+
+const isHighSurrogate = contains([0xd800, 0xdbff])
+
+/** @type {(a:number) => boolean} */
+const isLowSurrogate = contains([0xdc00, 0xdfff])
+
+const errorMask = 0b1000_0000_0000_0000_0000_0000_0000_0000
+
+/** @type {(input:i32) => list.List<u16>} */
+const codePointToUtf16 = input =>
+{
+    if (isBmpCodePoint(input)) { return [input] }
+    if (input >= 0x010000 && input <= 0x10ffff) {
+        const high = ((input - 0x10000) >> 10) + 0xd800
+        const low = ((input - 0x10000) & 0b0011_1111_1111) + 0xdc00
+        return [high, low]
+    }
+    return [input & 0xffff]
+}
+
+const fromCodePointList = flatMap(codePointToUtf16)
+
+/** @type {operator.StateScan<u16, Utf16State, list.List<i32>>} */
+const utf16ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xffff) {
+        return [[0xffffffff], state]
+    }
+    if (state === undefined) {
+        if (isBmpCodePoint(byte)) { return [[byte], undefined] }
+        if (isHighSurrogate(byte)) { return [[], byte] }
+        return [[byte | errorMask], undefined]
+    }
+    if (isLowSurrogate(byte)) {
+        const high = state - 0xd800
+        const low = byte - 0xdc00
+        return [[(high << 10) + low + 0x10000], undefined]
+    }
+    if (isBmpCodePoint(byte)) { return [[state | errorMask, byte], undefined] }
+    if (isHighSurrogate(byte)) { return [[state | errorMask], byte] }
+    return [[state | errorMask, byte | errorMask], undefined]
+}
+
+/** @type {(state: Utf16State) => readonly[list.List<i32>, Utf16State]} */
+const utf16EofToCodePointOp = state => [state === undefined ? undefined : [state | errorMask],  undefined]
+
+/** @type {operator.StateScan<WordOrEof, Utf16State, list.List<i32>>} */
+const utf16ByteOrEofToCodePointOp = state => input => input === undefined ? utf16EofToCodePointOp(state) : utf16ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<u16>) => list.List<i32>} */
+const toCodePointList = input => flat(stateScan(utf16ByteOrEofToCodePointOp)(undefined)(concat(/** @type {list.List<WordOrEof>} */(input))([undefined])))
+
+/** @type {(s: string) => list.List<u16>} */
+const stringToList = s => {
+    /** @type {(i: number) => list.Result<number>} */
+    const at = i => {
+        const first = s.charCodeAt(i)
+        return isNaN(first) ? undefined : { first, tail: () => at(i + 1) }
+    }
+    return at(0)
+}
+
+const listToString = compose(map(String.fromCharCode))(reduce(operator.concat)(''))
+
+module.exports = {
+    /** @readonly */
+    fromCodePointList,
+    /** @readonly */
+    toCodePointList,
+    /** @readonly */
+    stringToList,
+    /** @readonly */
+    listToString
+}

package/text/utf16/test.f.cjs

@@ -0,0 +1,118 @@
+const encoding = require('./module.f.cjs')
+const json = require('../../json/module.f.cjs')
+const { sort } = require('../../types/object/module.f.cjs')
+const { list } = require('../../types/module.f.cjs')
+
+/** @type {(a: readonly json.Unknown[]) => string} */
+const stringify = a => json.stringify(sort)(a)
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([-1, 65536])))
+    if (result !== '[4294967295,4294967295]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([0, 36, 8364, 55295, 57344, 65535])))
+    if (result !== '[0,36,8364,55295,57344,65535]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([56320, 57343])))
+    if (result !== '[-2147427328,-2147426305]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([55296, 56320, 55297, 56375, 55378, 57186, 56319, 57343])))
+    if (result !== '[65536,66615,150370,1114111]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([55296, 55296])))
+    if (result !== '[-2147428352,-2147428352]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([55296, 0])))
+    if (result !== '[-2147428352,0]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([56320])))
+    if (result !== '[-2147427328]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([56320, 0])))
+    if (result !== '[-2147427328,0]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0])))
+    if (result !== '[0]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x24])))
+    if (result !== '[36]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x20AC])))
+    if (result !== '[8364]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0xd7ff])))
+    if (result !== '[55295]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0xe000])))
+    if (result !== '[57344]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0xffff])))
+    if (result !== '[65535]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10000])))
+    if (result !== '[55296,56320]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10437])))
+    if (result !== '[55297,56375]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x24B62])))
+    if (result !== '[55378,57186]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10ffff])))
+    if (result !== '[56319,57343]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([-1, 0xd800, 0xdfff, 0x110000])))
+    if (result !== '[65535,55296,57343,0]') { throw result }
+}
+
+{
+    const utf16List = encoding.stringToList("Hello world!😂🚜🚲")
+    const result = encoding.listToString(utf16List)
+    if (result !== "Hello world!😂🚜🚲") { throw result }
+}
+
+{
+    const a = encoding.stringToList("Hello world!😂🚜🚲")
+    const b = encoding.toCodePointList(a)
+    const c = encoding.fromCodePointList(b)
+    const result = encoding.listToString(c)
+    if (result !== "Hello world!😂🚜🚲") { throw result }
+}
+
+module.exports = {}

package/text/utf8/module.f.cjs

@@ -0,0 +1,96 @@
+const list = require('../../types/list/module.f.cjs')
+const operator = require('../../types/function/operator/module.f.cjs')
+const array = require('../../types/array/module.f.cjs')
+const { flatMap } = list
+
+/** @typedef {u8|undefined} ByteOrEof */
+
+/** @typedef {array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf8NonEmptyState */
+
+/** @typedef {undefined|Utf8NonEmptyState} Utf8State */
+
+/** @typedef {number} u8 */
+
+/** @typedef {number} i32 */
+
+const errorMask = 0b1000_0000_0000_0000_0000_0000_0000_0000
+
+/** @type {(input:number) => list.List<u8>} */
+const codePointToUtf8 = input =>
+{
+    if (input >= 0x0000 && input <= 0x007f) { return [input & 0b01111_1111] }
+    if (input >= 0x0080 && input <= 0x07ff) { return [input >> 6 | 0b1100_0000, input & 0b0011_1111 | 0b1000_0000] }
+    if (input >= 0x0800 && input <= 0xffff) { return [input >> 12 | 0b1110_0000, input >> 6 & 0b0011_1111 | 0b1000_0000, input & 0b0011_1111 | 0b1000_0000] }
+    if (input >= 0x10000 && input <= 0x10ffff) { return [input >> 18 | 0b1111_0000, input >> 12 & 0b0011_1111 | 0b1000_0000, input >> 6 & 0b0011_1111 | 0b1000_0000, input & 0b0011_1111 | 0b1000_0000] }
+    if ((input & errorMask) !== 0) {
+        if ((input & 0b1000_0000_0000_0000) !== 0) { return [input >> 12 & 0b0000_0111 | 0b1111_0000, input >> 6 & 0b0011_1111 | 0b1000_0000, input & 0b0011_1111 | 0b1000_0000] }
+        if ((input & 0b0000_0100_0000_0000) !== 0) { return [input >> 6 & 0b0000_1111 | 0b1110_0000, input & 0b0011_1111 | 0b1000_0000] }
+        if ((input & 0b0000_0010_0000_0000) !== 0) { return [input >> 6 & 0b0000_0111 | 0b1111_0000, input & 0b0011_1111 | 0b1000_0000] }
+        if ((input & 0b0000_0000_1000_0000) !== 0) { return [input & 0b1111_1111] }
+    }
+    return [errorMask]
+}
+
+const fromCodePointList = flatMap(codePointToUtf8)
+
+/** @type {(state: Utf8NonEmptyState) => i32}*/
+const utf8StateToError = state => {
+    switch(state.length) {
+        case 1:
+             return state[0] | errorMask
+        case 2:
+            if (state[0] < 0b1111_0000) return (((state[0] & 0b0000_1111) << 6) + (state[1] & 0b0011_1111) + 0b0000_0100_0000_0000) | errorMask
+            return (((state[0] & 0b0000_0111) << 6) + (state[1] & 0b0011_1111) + 0b0000_0010_0000_0000) | errorMask
+        case 3:
+            return (((state[0] & 0b0000_0111) << 12) + ((state[1] & 0b0011_1111) << 6) + (state[2] & 0b0011_1111) + 0b1000_0000_0000_0000) | errorMask
+    }
+}
+
+/** @type {operator.StateScan<number, Utf8State, list.List<i32>>} */
+const utf8ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[errorMask], state]
+    }
+    if (state == undefined) {
+        if (byte < 0b1000_0000) { return [[byte], undefined] }
+        if (byte >= 0b1100_0010 && byte <= 0b1111_0100) { return [[], [byte]] }
+        return [[byte | errorMask], undefined]
+    }
+    if (byte >= 0b1000_0000 && byte < 0b1100_0000) {
+        switch(state.length) {
+            case 1:
+                if (state[0] < 0b1110_0000) { return [[((state[0] & 0b0001_1111) << 6) + (byte & 0b0011_1111)], undefined] }
+                if (state[0] < 0b1111_1000) { return [[], [state[0], byte]] }
+                break
+            case 2:
+                if (state[0] < 0b1111_0000) { return [[((state[0] & 0b0000_1111) << 12) + ((state[1] & 0b0011_1111) << 6) + (byte & 0b0011_1111)], undefined] }
+                if (state[0] < 0b1111_1000) { return [[], [state[0], state[1], byte]] }
+                break
+            case 3:
+                return [[((state[0] & 0b0000_0111) << 18) + ((state[1] & 0b0011_1111) << 12) + ((state[2] & 0b0011_1111) << 6) + (byte & 0b0011_1111)], undefined]
+        }
+    }
+    const error = utf8StateToError(state)
+    if (byte < 0b1000_0000) { return [[error, byte], undefined] }
+    if (byte >= 0b1100_0010 && byte <= 0b1111_0100) { return [[error], [byte]] }
+    return [[error, byte | errorMask], undefined]
+}
+
+/** @type {(state: Utf8State) => readonly[list.List<i32>, Utf8State]} */
+const utf8EofToCodePointOp = state => {
+    if (state === undefined) { return [undefined, undefined] }
+    return [[utf8StateToError(state)], undefined]
+}
+
+/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<i32>>} */
+const utf8ByteOrEofToCodePointOp = state => input => input === undefined ? utf8EofToCodePointOp(state) : utf8ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<u8>) => list.List<i32>} */
+const toCodePointList = input => list.flat(list.stateScan(utf8ByteOrEofToCodePointOp)(undefined)(list.concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
+
+module.exports = {
+    /** @readonly */
+    fromCodePointList,
+    /** @readonly */
+    toCodePointList
+}

package/text/utf8/test.f.cjs

@@ -0,0 +1,144 @@
+const encoding = require('./module.f.cjs')
+const json = require('../../json/module.f.cjs')
+const { sort } = require('../../types/object/module.f.cjs')
+const { list } = require('../../types/module.f.cjs')
+
+/** @type {(a: readonly json.Unknown[]) => string} */
+const stringify = a => json.stringify(sort)(a)
+
+{
+    const result = stringify(list.toArray(encoding.toCodePointList([-1, 256])))
+    if (result !== '[2147483648,2147483648]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([128, 193, 245, 255])))
+   if (result !== '[-2147483520,-2147483455,-2147483403,-2147483393]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([0, 1, 127])))
+   if (result !== '[0,1,127]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([194, 128, 194, 169, 223, 191])))
+   if (result !== '[128,169,2047]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([194, 194, 127, 194, 192, 194])))
+   if (result !== '[-2147483454,-2147483454,127,-2147483454,-2147483456,-2147483454]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([224, 160, 128, 224, 160, 129, 239, 191, 191])))
+   if (result !== '[2048,2049,65535]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([224, 224, 160, 127, 239, 191])))
+   if (result !== '[-2147483424,-2147482592,127,-2147481601]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([240, 144, 128, 128, 240, 144, 128, 129, 244, 143, 191, 191])))
+   if (result !== '[65536,65537,1114111]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([240, 240, 160, 127, 244, 191])))
+   if (result !== '[-2147483408,-2147483104,127,-2147482817]') { throw result }
+}
+
+{
+   const result = stringify(list.toArray(encoding.toCodePointList([240, 160, 160, 244, 160, 160])))
+   if (result !== '[-2147448800,-2147432416]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0,1,0x7F])))
+    if (result !== '[0,1,127]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x80])))
+    if (result !== '[194,128]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0xa9])))
+    if (result !== '[194,169]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x7ff])))
+    if (result !== '[223,191]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x800])))
+    if (result !== '[224,160,128]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x801])))
+    if (result !== '[224,160,129]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0xffff])))
+    if (result !== '[239,191,191]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10000])))
+    if (result !== '[240,144,128,128]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10001])))
+    if (result !== '[240,144,128,129]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x10FFFF])))
+    if (result !== '[244,143,191,191]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.fromCodePointList([0x110000,2147483648])))
+    if (result !== '[2147483648,2147483648]') { throw result }
+}
+
+{
+    const codePointList = encoding.toCodePointList([128, 193, 245, 255])
+    const result = stringify(list.toArray(encoding.fromCodePointList(codePointList)))
+    if (result !== '[128,193,245,255]') { throw result }
+}
+
+{
+    const codePointList = encoding.toCodePointList([194, 194, 127, 194, 192, 194])
+    const result = stringify(list.toArray(encoding.fromCodePointList(codePointList)))
+    if (result !== '[194,194,127,194,192,194]') { throw result }
+}
+
+{
+    const codePointList = encoding.toCodePointList([224, 224, 160, 127, 239, 191])
+    const result = stringify(list.toArray(encoding.fromCodePointList(codePointList)))
+    if (result !== '[224,224,160,127,239,191]') { throw result }
+}
+
+{
+    const codePointList = encoding.toCodePointList([240, 240, 160, 127, 244, 191])
+    const result = stringify(list.toArray(encoding.fromCodePointList(codePointList)))
+    if (result !== '[240,240,160,127,244,191]') { throw result }
+}
+
+{
+    const codePointList = encoding.toCodePointList([240, 160, 160, 244, 160, 160])
+    const result = stringify(list.toArray(encoding.fromCodePointList(codePointList)))
+    if (result !== '[240,160,160,244,160,160]') { throw result }
+}
+
+module.exports = {}

package/text/encoding/module.f.cjs

@@ -1,154 +0,0 @@
-const result = require('../../types/result/module.f.cjs')
-const list = require('../../types/list/module.f.cjs')
-const operator = require('../../types/function/operator/module.f.cjs')
-const array = require('../../types/array/module.f.cjs')
-const { contains } = require('../../types/range/module.f.cjs')
-const { compose } = require('../../types/function/module.f.cjs')
-const { map, flat, stateScan, concat, reduce, toArray, flatMap } = list
-const { ok, error } = result
-
-/** @typedef {result.Result<number,number>} ByteResult */
-
-/** @typedef {result.Result<number,readonly number[]>} CodePointResult */
-
-/** @typedef {number|undefined} ByteOrEof */
-
-/** @typedef {u16|undefined} WordOrEof */
-
-/** @typedef {undefined|array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf8State */
-
-/** @typedef {undefined|number} Utf16State */
-
-/** @typedef {number} u16 */
-
-/** @typedef {number} i32 */
-
-/** @type {(a:number) => boolean} */
-const isBmpCodePoint = a => a >= 0x0000 && a <= 0xd7ff || a >= 0xe000 && a <= 0xffff
-
-const isHighSurrogate = contains([0xd800, 0xdbff])
-
-/** @type {(a:number) => boolean} */
-const isLowSurrogate = contains([0xdc00, 0xdfff])
-
-const errorMask = 0b1000_0000_0000_0000_0000_0000_0000_0000
-
-/** @type {(input:number) => list.List<ByteResult>} */
-const codePointToUtf8 = input =>
-{
-    if (input >= 0x0000 && input <= 0x007f) { return [ok(input & 0x7f)] }
-    if (input >= 0x0080 && input <= 0x07ff) { return [ok(input >> 6 | 0xc0), ok(input & 0x3f | 0x80)] }
-    if (input >= 0x0800 && input <= 0xffff) { return [ok(input >> 12 | 0xe0), ok(input >> 6 & 0x3f | 0x80), ok(input & 0x3f | 0x80)] }
-    if (input >= 0x10000 && input <= 0x10ffff) { return [ok(input >> 18 | 0xf0), ok(input >> 12 & 0x3f | 0x80), ok(input >> 6 & 0x3f | 0x80), ok(input & 0x3f | 0x80)] }
-    return [error(input)]
-}
-
-/** @type {(input:i32) => list.List<u16>} */
-const codePointToUtf16 = input =>
-{
-    if (isBmpCodePoint(input)) { return [input] }
-    if (input >= 0x010000 && input <= 0x10ffff) {
-        const high = ((input - 0x10000) >> 10) + 0xd800
-        const low = ((input - 0x10000) & 0b0011_1111_1111) + 0xdc00
-        return [high, low]
-    }
-    return [input & 0xffff]
-}
-
-const codePointListToUtf8List = flatMap(codePointToUtf8)
-
-const codePointListToUtf16List = flatMap(codePointToUtf16)
-
-/** @type {operator.StateScan<number, Utf8State, list.List<CodePointResult>>} */
-const utf8ByteToCodePointOp = state => byte => {
-    if (byte < 0x00 || byte > 0xff) {
-        return [[error([byte])], state]
-    }
-    if (state == undefined) {
-        if (byte < 0x80) { return [[ok(byte)], undefined] }
-        if (byte >= 0xc2 && byte <= 0xf4) { return [[], [byte]] }
-        return [[error([byte])], undefined]
-    }
-    if (byte >= 0x80 && byte < 0xc0)
-    {
-        switch(state.length)
-        {
-            case 1:
-                if (state[0] < 0xe0) { return [[ok(((state[0] & 0x1f) << 6) + (byte & 0x3f))], undefined] }
-                if (state[0] < 0xf8) { return [[], [state[0], byte]] }
-                break
-            case 2:
-                if (state[0] < 0xf0) { return [[ok(((state[0] & 0x0f) << 12) + ((state[1] & 0x3f) << 6) + (byte & 0x3f))], undefined] }
-                if (state[0] < 0xf8) { return [[], [state[0], state[1], byte]] }
-                break
-            case 3:
-                return [[ok(((state[0] & 0x07) << 18) + ((state[1] & 0x3f) << 12) + ((state[2] & 0x3f) << 6) + (byte & 0x3f))], undefined]
-        }
-    }
-    return [[error(toArray(concat(state)([byte])))], undefined]
-}
-
-/** @type {(state: Utf8State) => readonly[list.List<CodePointResult>, Utf8State]} */
-const utf8EofToCodePointOp = state => [state === undefined ? undefined : [error(state)],  undefined]
-
-/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<CodePointResult>>} */
-const utf8ByteOrEofToCodePointOp = state => input => input === undefined ? utf8EofToCodePointOp(state) : utf8ByteToCodePointOp(state)(input)
-
-/** @type {(input: list.List<number>) => list.List<CodePointResult>} */
-const utf8ListToCodePointList = input => flat(stateScan(utf8ByteOrEofToCodePointOp)(undefined)(concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
-
-/** @type {operator.StateScan<u16, Utf16State, list.List<i32>>} */
-const utf16ByteToCodePointOp = state => byte => {
-    if (byte < 0x00 || byte > 0xffff) {
-        return [[0xffffffff], state]
-    }
-    if (state === undefined) {
-        if (isBmpCodePoint(byte)) { return [[byte], undefined] }
-        if (isHighSurrogate(byte)) { return [[], byte] }
-        return [[byte | errorMask], undefined]
-    }
-    if (isLowSurrogate(byte)) {
-        const high = state - 0xd800
-        const low = byte - 0xdc00
-        return [[(high << 10) + low + 0x10000], undefined]
-    }
-    if (isBmpCodePoint(byte)) { return [[state | errorMask, byte], undefined] }
-    if (isHighSurrogate(byte)) { return [[state | errorMask], byte] }
-    return [[state | errorMask, byte | errorMask], undefined]
-}
-
-/** @type {(state: Utf16State) => readonly[list.List<i32>, Utf16State]} */
-const utf16EofToCodePointOp = state => [state === undefined ? undefined : [state | errorMask],  undefined]
-
-/** @type {operator.StateScan<WordOrEof, Utf16State, list.List<i32>>} */
-const utf16ByteOrEofToCodePointOp = state => input => input === undefined ? utf16EofToCodePointOp(state) : utf16ByteToCodePointOp(state)(input)
-
-/** @type {(input: list.List<u16>) => list.List<i32>} */
-const utf16ListToCodePointList = input => flat(stateScan(utf16ByteOrEofToCodePointOp)(undefined)(concat(/** @type {list.List<WordOrEof>} */(input))([undefined])))
-
-/** @type {(s: string) => list.List<u16>} */
-const stringToUtf16List = s => {
-    /** @type {(i: number) => list.Result<number>} */
-    const at = i => {
-        const first = s.charCodeAt(i)
-        return isNaN(first) ? undefined : { first, tail: () => at(i + 1) }
-    }
-    return at(0)
-}
-
-const utf16ListToString = compose(map(String.fromCharCode))(reduce(operator.concat)(''))
-
-module.exports = {
-    /** @readonly */
-    codePointListToUtf8List,
-    /** @readonly */
-    codePointListToUtf16List,
-    /** @readonly */
-    utf8ListToCodePointList,
-    /** @readonly */
-    utf16ListToCodePointList,
-    /** @readonly */
-    stringToUtf16List,
-    /** @readonly */
-    utf16ListToString
-}

package/text/encoding/test.f.cjs

@@ -1,223 +0,0 @@
-const encoding = require('./module.f.cjs')
-const json = require('../../json/module.f.cjs')
-const { sort } = require('../../types/object/module.f.cjs')
-const { list } = require('../../types/module.f.cjs')
-
-/** @type {(a: readonly json.Unknown[]) => string} */
-const stringify = a => json.stringify(sort)(a)
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0,1,0x7F])))
-    if (result !== '[["ok",0],["ok",1],["ok",127]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x80])))
-    if (result !== '[["ok",194],["ok",128]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0xa9])))
-    if (result !== '[["ok",194],["ok",169]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x7ff])))
-    if (result !== '[["ok",223],["ok",191]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x800])))
-    if (result !== '[["ok",224],["ok",160],["ok",128]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x801])))
-    if (result !== '[["ok",224],["ok",160],["ok",129]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0xffff])))
-    if (result !== '[["ok",239],["ok",191],["ok",191]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x10000])))
-    if (result !== '[["ok",240],["ok",144],["ok",128],["ok",128]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x10001])))
-    if (result !== '[["ok",240],["ok",144],["ok",128],["ok",129]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([0x10FFFF])))
-    if (result !== '[["ok",244],["ok",143],["ok",191],["ok",191]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf8List([-1,0x110000])))
-    if (result !== '[["error",-1],["error",1114112]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0])))
-    if (result !== '[0]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x24])))
-    if (result !== '[36]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x20AC])))
-    if (result !== '[8364]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0xd7ff])))
-    if (result !== '[55295]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0xe000])))
-    if (result !== '[57344]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0xffff])))
-    if (result !== '[65535]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x10000])))
-    if (result !== '[55296,56320]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x10437])))
-    if (result !== '[55297,56375]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x24B62])))
-    if (result !== '[55378,57186]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([0x10ffff])))
-    if (result !== '[56319,57343]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.codePointListToUtf16List([-1, 0xd800, 0xdfff, 0x110000])))
-    if (result !== '[65535,55296,57343,0]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([-1, 256])))
-    if (result !== '[["error",[-1]],["error",[256]]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([128, 193, 245, 255])))
-    if (result !== '[["error",[128]],["error",[193]],["error",[245]],["error",[255]]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([0, 1, 127])))
-    if (result !== '[["ok",0],["ok",1],["ok",127]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([194, 128, 194, 169, 223, 191])))
-    if (result !== '[["ok",128],["ok",169],["ok",2047]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([194, 127, 194, 192, 194])))
-    if (result !== '[["error",[194,127]],["error",[194,192]],["error",[194]]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([224, 160, 128, 224, 160, 129, 239, 191, 191])))
-    if (result !== '[["ok",2048],["ok",2049],["ok",65535]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([224, 160, 127, 224, 160, 192, 224, 160])))
-    if (result !== '[["error",[224,160,127]],["error",[224,160,192]],["error",[224,160]]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([240, 144, 128, 128, 240, 144, 128, 129, 244, 143, 191, 191])))
-    if (result !== '[["ok",65536],["ok",65537],["ok",1114111]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([240, 144, 128, 127, 240, 144, 128, 192, 240, 144, 128])))
-    if (result !== '[["error",[240,144,128,127]],["error",[240,144,128,192]],["error",[240,144,128]]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf8ListToCodePointList([194, -1, 128])))
-    if (result !== '[["error",[-1]],["ok",128]]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([-1, 65536])))
-    if (result !== '[4294967295,4294967295]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([0, 36, 8364, 55295, 57344, 65535])))
-    if (result !== '[0,36,8364,55295,57344,65535]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([56320, 57343])))
-    if (result !== '[-2147427328,-2147426305]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([55296, 56320, 55297, 56375, 55378, 57186, 56319, 57343])))
-    if (result !== '[65536,66615,150370,1114111]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([55296, 55296])))
-    if (result !== '[-2147428352,-2147428352]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([55296, 0])))
-    if (result !== '[-2147428352,0]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([56320])))
-    if (result !== '[-2147427328]') { throw result }
-}
-
-{
-    const result = stringify(list.toArray(encoding.utf16ListToCodePointList([56320, 0])))
-    if (result !== '[-2147427328,0]') { throw result }
-}
-
-{
-    const utf16List = encoding.stringToUtf16List("Hello world!😂🚜🚲")
-    const result = encoding.utf16ListToString(utf16List)
-    if (result !== "Hello world!😂🚜🚲") { throw result }
-}
-
-{
-    const a = encoding.stringToUtf16List("Hello world!😂🚜🚲")
-    const b = encoding.utf16ListToCodePointList(a)
-    const c = encoding.codePointListToUtf16List(b)
-    const result = encoding.utf16ListToString(c)
-    if (result !== "Hello world!😂🚜🚲") { throw result }
-}
-
-module.exports = {}