functionalscript 0.0.373 → 0.0.376

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "functionalscript",
-  "version": "0.0.373",
+  "version": "0.0.376",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "module.f.cjs",
   "scripts": {

package/text/encoding/README.md

@@ -0,0 +1,90 @@
+# UNICODE
+
+## UTF-8
+
+Requirement: no loss for UTF8 => codepoint => UTF8
+
+|utf8     |codepoint                              |size     |
+|---------|---------------------------------------|---------|
+|[a]      |0xxx_xxxx                              |7 bit    |
+|[b,a]    |110x_xxxx 10xx_xxxx                    |11 bit   |
+|[c,b,a]  |1110_xxxx 10xx_xxxx 10xx_xxxx          |16 bit   |
+|[d,c,b,a]|1111_0xxx 10xx_xxxx 10xx_xxxx 10xx_xxxx|21 bit   |
+
+|utf8 error|codepoint                    |size  |
+|----------|-----------------------------|------|
+|[e]       |10xx_xxxx                    |6 bit |
+|[e]       |1111_1xxx                    |3 bit |
+|[b,]      |110x_xxxx                    |5 bit |
+|[c,]      |1110_xxxx                    |4 bit |
+|[c,b,]    |1110_xxxx 10xx_xxxx          |10 bit|
+|[d,]      |1111_0xxx                    |3 bit |
+|[d,c,]    |1111_0xxx 10xx_xxxx          |9 bit |
+|[d,c,b]   |1111_0xxx 10xx_xxxx 10xx_xxxx|15 bit|
+
+Total error states:
+
+- 2^6 + 2^3 + 2^5 + 2^4 + 2^10 + 2^3 + + 2^9 + 2^15
+- 2^4 + 2^6 + 2^5 + 2^4 + 2^10 + 2^9 + 2^15
+- 2^5 + 2^6 + 2^5 + 2^10 + 2^9 + 2^15
+- 2^6 + 2^6 + 2^10 + 2^9 + 2^15
+- 2^7 + 2^9 + 2^10 + 2^15
+- < 2^16
+
+|utf8 error|codepoint                    |size  |map                |
+|----------|-----------------------------|------|-------------------|
+|[e]       |1111_1xxx                    | 3 bit|                   |
+|[d,]      |1111_0xxx                    | 3 bit|                   |
+|[c,]      |1110_xxxx                    | 4 bit|                   |
+|[b,]      |110x_xxxx                    | 5 bit|                   |
+|[e]       |10xx_xxxx                    | 6 bit|1111_1111 1xxx_xxxx|
+|[d,c,]    |1111_0xxx 10xx_xxxx          | 9 bit|1111_0xxx 10xx_xxxx|
+|[c,b,]    |1110_xxxx 10xx_xxxx          |10 bit|1110_xxxx 10xx_xxxx|
+|[d,c,b]   |1111_0xxx 10xx_xxxx 10xx_xxxx|15 bit|0xxx_xxxx xxxx_xxxx|
+
+```js
+/** @type {(input: List<u8|undefined>) => List<i32>} */
+const utf8ToCodePoint
+
+/** @type {(input: List<i32>) => List<u8>} */
+const codePointToUtf8
+```
+
+## UTF-16
+
+Requirement: no loss for UTF16 => codepoint => UTF16
+
+0xD800..0xDFFF
+0b_1101_1000_0000_0000
+0b_1101_1111_1111_1111
+
+0b_1101_1xxx_xxxx_xxxx : 11 bits
+
+- first : 0xD800: 0b_1101_10xx_xxxx_xxxx : 10 bit
+- second: 0xDC00: 0b_1101_11xx_xxxx_xxxx : 10 bit
+
+|utf16    |codepoint                              |size  |
+|---------|---------------------------------------|------|
+|[a]      |xxxx_xxxx_xxxx_xxxx                    |16 bit|
+|[b,a]    |1101_10xx_xxxx_xxxx 1101_11xx_xxxx_xxxx|20 bit|
+
+|utf16 error|codepoint          |size  |
+|-----------|-------------------|------|
+|[e]        |1101_11xx_xxxx_xxxx|10 bit|
+|[b,]       |1101_10xx_xxxx_xxxx|10 bit|
+
+Total error states: 11 bit
+
+```js
+/** @type {(input: List<u16|undefined>) => List<i32>} */
+const utf16ToCodePoint
+
+/** @type {(input: List<i32>) => List<u16>} */
+const codePointToUtf16
+
+/** @type {(input: string) => List<u16> */
+const stringToUtf16
+
+/** @type {(input: List<u16>) => string} */
+const utf16ToString
+```

package/text/encoding/module.f.cjs

@@ -1,10 +1,29 @@
 const result = require('../../types/result/module.f.cjs')
 const list = require('../../types/list/module.f.cjs')
+const operator = require('../../types/function/operator/module.f.cjs')
+const array = require('../../types/array/module.f.cjs')
+const { contains } = require('../../types/range/module.f.cjs')
 const { ok, error } = result
 
-/** @typedef {result.Result<number,number>} Utf8Result */
+/** @typedef {result.Result<number,number>} ByteResult */
 
-/** @type {(input:number) => list.List<Utf8Result>} */
+/** @typedef {result.Result<number,readonly number[]>} CodePointResult */
+
+/** @typedef {number|undefined} ByteOrEof */
+
+/** @typedef {undefined|array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf8State */
+
+/** @typedef {undefined|array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf16State */
+
+/** @type {(a:number) => boolean} */
+const isBmpCodePoint = a => a >= 0x0000 && a <= 0xd7ff || a >= 0xe000 && a <= 0xffff
+
+const isHighSurrogate = contains([0xd800, 0xdbff])
+
+/** @type {(a:number) => boolean} */
+const isLowSurrogate = contains([0xdc00, 0xdfff])
+
+/** @type {(input:number) => list.List<ByteResult>} */
 const codePointToUtf8 = input =>
 {
     if (input >= 0x0000 && input <= 0x007f) { return [ok(input & 0x7f)] }
@@ -14,10 +33,10 @@ const codePointToUtf8 = input =>
     return [error(input)]
 }
 
-/** @type {(input:number) => list.List<Utf8Result>} */
+/** @type {(input:number) => list.List<ByteResult>} */
 const codePointToUtf16 = input =>
 {
-    if (input >= 0x0000 && input <= 0xd7ff || input >= 0xe000 && input <= 0xffff) { return [ok(input >> 8), ok(input & 0xff)] }
+    if (isBmpCodePoint(input)) { return [ok(input >> 8), ok(input & 0xff)] }
     if (input >= 0x010000 && input <= 0x10ffff) {
         const high = ((input - 0x10000) >> 10) + 0xd800
         const low = ((input - 0x10000) & 0x3ff) + 0xdc00
@@ -26,15 +45,94 @@ const codePointToUtf16 = input =>
     return [error(input)]
 }
 
-/** @type {(input: list.List<number>) => list.List<Utf8Result>} */
+/** @type {(input: list.List<number>) => list.List<ByteResult>} */
 const codePointListToUtf8 = list.flatMap(codePointToUtf8)
 
-/** @type {(input: list.List<number>) => list.List<Utf8Result>} */
+/** @type {(input: list.List<number>) => list.List<ByteResult>} */
 const codePointListToUtf16 = list.flatMap(codePointToUtf16)
 
+/** @type {operator.StateScan<number, Utf8State, list.List<CodePointResult>>} */
+const utf8ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[error([byte])], state]
+    }    
+    if (state == undefined) {
+        if (byte < 0x80) { return [[ok(byte)], undefined] }
+        if (byte >= 0xc2 && byte <= 0xf4) { return [[], [byte]] }
+        return [[error([byte])], undefined]
+    }
+    if (byte >= 0x80 && byte < 0xc0)
+    {
+        switch(state.length)
+        {
+            case 1:
+                if (state[0] < 0xe0) { return [[ok(((state[0] & 0x1f) << 6) + (byte & 0x3f))], undefined] }
+                if (state[0] < 0xf8) { return [[], [state[0], byte]] }
+                break         
+            case 2:
+                if (state[0] < 0xf0) { return [[ok(((state[0] & 0x0f) << 12) + ((state[1] & 0x3f) << 6) + (byte & 0x3f))], undefined] }
+                if (state[0] < 0xf8) { return [[], [state[0], state[1], byte]] }
+                break
+            case 3: 
+                return [[ok(((state[0] & 0x07) << 18) + ((state[1] & 0x3f) << 12) + ((state[2] & 0x3f) << 6) + (byte & 0x3f))], undefined]
+        }
+    }    
+    return [[error(list.toArray(list.concat(state)([byte])))], undefined]
+}
+
+/** @type {(state: Utf8State) => readonly[list.List<CodePointResult>, Utf8State]} */
+const utf8EofToCodePointOp = state => [state == undefined ? undefined : [error(state)],  undefined]
+
+/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<CodePointResult>>} */
+const utf8ByteOrEofToCodePointOp = state => input => input === undefined ? utf8EofToCodePointOp(state) : utf8ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<number>) => list.List<CodePointResult>} */
+const utf8ListToCodePoint = input => list.flat(list.stateScan(utf8ByteOrEofToCodePointOp)(undefined)(list.concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
+
+/** @type {operator.StateScan<number, Utf16State, list.List<CodePointResult>>} */
+const utf16ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[error([byte])], state]
+    }
+    if (state == undefined) {
+        return [[], [byte]]
+    }
+    switch(state.length)
+    {
+        case 1:
+            const codeUnit = (state[0] << 8) + byte
+            if (isBmpCodePoint(codeUnit)) { return [[ok(codeUnit)], undefined] }
+            if (isHighSurrogate(codeUnit)) { return [[], [state[0], byte]] }
+            break
+        case 2:
+            return [[], [state[0], state[1], byte]]
+        case 3:             
+            if (isLowSurrogate((state[2] << 8) + byte)) {
+                const high = (state[0] << 8) + state[1] - 0xd800
+                const low = (state[2] << 8) + byte - 0xdc00
+                return [[ok((high << 10) + low + 0x10000)], undefined]
+            } 
+            break
+    }
+    return [[error(list.toArray(list.concat(state)([byte])))], undefined]
+}
+
+/** @type {(state: Utf8State) => readonly[list.List<CodePointResult>, Utf16State]} */
+const utf16EofToCodePointOp = state => [state == undefined ? undefined : [error(state)],  undefined]
+
+/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<CodePointResult>>} */
+const utf16ByteOrEofToCodePointOp = state => input => input === undefined ? utf16EofToCodePointOp(state) : utf16ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<number>) => list.List<CodePointResult>} */
+const utf16ListToCodePoint = input => list.flat(list.stateScan(utf16ByteOrEofToCodePointOp)(undefined)(list.concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
+
 module.exports = {
     /** @readonly */
     codePointListToUtf8,
     /** @readonly */
     codePointListToUtf16,
+    /** @readonly */
+    utf8ListToCodePoint,
+    /** @readonly */
+    utf16ListToCodePoint
 }

package/text/encoding/test.f.cjs

@@ -116,4 +116,84 @@ const stringify = a => json.stringify(sort)(a)
     if (result !== '[["error",-1],["error",55296],["error",57343],["error",1114112]]') { throw result }
 }
 
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([-1, 256])))
+    if (result !== '[["error",[-1]],["error",[256]]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([128, 193, 245, 255])))
+    if (result !== '[["error",[128]],["error",[193]],["error",[245]],["error",[255]]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([0, 1, 127])))
+    if (result !== '[["ok",0],["ok",1],["ok",127]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, 128, 194, 169, 223, 191])))
+    if (result !== '[["ok",128],["ok",169],["ok",2047]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, 127, 194, 192, 194])))
+    if (result !== '[["error",[194,127]],["error",[194,192]],["error",[194]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([224, 160, 128, 224, 160, 129, 239, 191, 191])))
+    if (result !== '[["ok",2048],["ok",2049],["ok",65535]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([224, 160, 127, 224, 160, 192, 224, 160])))
+    if (result !== '[["error",[224,160,127]],["error",[224,160,192]],["error",[224,160]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([240, 144, 128, 128, 240, 144, 128, 129, 244, 143, 191, 191])))
+    if (result !== '[["ok",65536],["ok",65537],["ok",1114111]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([240, 144, 128, 127, 240, 144, 128, 192, 240, 144, 128])))
+    if (result !== '[["error",[240,144,128,127]],["error",[240,144,128,192]],["error",[240,144,128]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, -1, 128])))
+    if (result !== '[["error",[-1]],["ok",128]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([-1, 256,])))
+    if (result !== '[["error",[-1]],["error",[256]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([0, 0, 0, 36, 32, 172, 215, 255, 224, 0, 255, 255])))
+    if (result !== '[["ok",0],["ok",36],["ok",8364],["ok",55295],["ok",57344],["ok",65535]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([220, 0, 223, 255])))
+    if (result !== '[["error",[220,0]],["error",[223,255]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 220, 0, 216, 1, 220, 55, 216, 82, 223, 98, 219, 255, 223, 255])))
+    if (result !== '[["ok",65536],["ok",66615],["ok",150370],["ok",1114111]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 216, 0])))
+    if (result !== '[["error",[216,0,216,0]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 0, 0])))
+    if (result !== '[["error",[216,0,0,0]]]') { throw result }
+}
+
 module.exports = {}