functionalscript 0.0.372 → 0.0.375

package/com/cpp/test/build.cjs

@@ -1,12 +1,14 @@
 const fs = require('node:fs')
 const cp = require('node:child_process')
-const cpp = require('../test.f.cjs').result
 const os = require('node:os');
+const cpp = require('../test.f.cjs').result
+const list = require('../../../types/list/module.f.cjs')
 
 fs.writeFileSync('_result.hpp', cpp)
 try {
-    const flags = os.platform() === 'win32' ? '' : ' -std=c++11 -lc++'
-    console.log(cp.execSync(`clang${flags} main.cpp`).toString())
+    const flags = os.platform() === 'win32' ? [] : ['-std=c++11', '-lc++']
+    const line = list.join(' ')(list.flat([['clang'], flags, ['main.cpp']]))
+    console.log(cp.execSync(line).toString())
 } catch (e) {
     // @ts-ignore
     console.error(e.output.toString())

package/{io/commonjs → commonjs}/module.cjs

@@ -1,6 +1,6 @@
-const { tryCatch } = require('../result/module.cjs')
-const { unwrap } = require('../../types/result/module.f.cjs')
-const moduleFunction = require('../../commonjs/module/function/module.f.cjs')
+const { tryCatch } = require('../types/result/module.cjs')
+const { unwrap } = require('../types/result/module.f.cjs')
+const moduleFunction = require('./module/function/module.f.cjs')
 
 /** @type {(f: Function) => moduleFunction.Function_} */
 const build = f => immutableRequire => mutableData => {

package/{io/commonjs → commonjs}/test.cjs

@@ -1,5 +1,5 @@
 const _ = require('./module.cjs')
-const run = require('../../commonjs/module/function/module.f.cjs')
+const run = require('./module/function/module.f.cjs')
 
 // ok:
 {

package/{io/nodejs → nodejs}/version/main.cjs

@@ -1,6 +1,6 @@
 const cp = require('child_process')
 const fs = require('fs')
-const package_json = require('../../../package.json')
+const package_json = require('../../package.json')
 
 const b =cp.execSync('git log --oneline')
 

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "functionalscript",
-  "version": "0.0.372",
+  "version": "0.0.375",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "module.f.cjs",
   "scripts": {
     "tsc": "tsc",
     "test": "tsc && npm run test-only",
-    "version": "node ./io/nodejs/version/main.cjs",
+    "version": "node ./nodejs/version/main.cjs",
     "test-only": "node --trace-uncaught ./test.f.cjs"
   },
   "repository": {

package/test.f.cjs

@@ -7,7 +7,7 @@ require('./sha2/test.f.cjs')
 require('./json/test.f.cjs')
 require('./json/tokenizer/test.f.cjs')
 require('./types/object/test.f.cjs')
-require('./io/commonjs/test.cjs')
+require('./commonjs/test.cjs')
 require('./commonjs/package/dependencies/test.f.cjs')
 require('./commonjs/package/test.f.cjs')
 require('./commonjs/path/test.f.cjs')

package/text/encoding/README.md

@@ -0,0 +1,42 @@
+# UNICODE
+
+## UTF-8
+
+Requirement: no loss for UTF8 => codepoint => UTF8
+
+|utf8     |codepoint|
+|---------|---------|
+|[a]      |+        |
+|[a,b]    |+        |
+|[a,b,c]  |+        |
+|[a,b,c,d]|+        |
+|[e]      |-        |
+|[a,e]    |-        |
+|[a,b,e]  |-        |
+|[a,b,c,e]|-        |
+|[a,]     |-        |
+|[a,b,]   |-        |
+|[a,b,c,] |-        |
+
+## UTF-16
+
+Requirement: no loss for UTF16 => codepoint => UTF16
+
+|utf16    |codepoint|
+|---------|---------|
+|[a]      |+        |
+|[a,b]    |+        |
+|[e]      |-        |
+|[a,e]    |-        |
+|[a,]     |-        |
+
+- UTF8 => codepoint => UTF16 => codepoint => UTF8 ?
+- UTF16 => codepoint => UTF8 => codepoint => UTF16 ?
+
+```js
+/** @typedef {number} u8 */
+/** @typedef {number} i8 */
+/** @typedef {number} u16 */
+/** @typedef {number} i16 */
+/** @typedef {number} i32 */
+```

package/text/encoding/module.f.cjs

@@ -1,10 +1,29 @@
 const result = require('../../types/result/module.f.cjs')
 const list = require('../../types/list/module.f.cjs')
+const operator = require('../../types/function/operator/module.f.cjs')
+const array = require('../../types/array/module.f.cjs')
+const { contains } = require('../../types/range/module.f.cjs')
 const { ok, error } = result
 
-/** @typedef {result.Result<number,number>} Utf8Result */
+/** @typedef {result.Result<number,number>} ByteResult */
 
-/** @type {(input:number) => list.List<Utf8Result>} */
+/** @typedef {result.Result<number,readonly number[]>} CodePointResult */
+
+/** @typedef {number|undefined} ByteOrEof */
+
+/** @typedef {undefined|array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf8State */
+
+/** @typedef {undefined|array.Array1<number>|array.Array2<number>|array.Array3<number>} Utf16State */
+
+/** @type {(a:number) => boolean} */
+const isBmpCodePoint = a => a >= 0x0000 && a <= 0xd7ff || a >= 0xe000 && a <= 0xffff
+
+const isHighSurrogate = contains([0xd800, 0xdbff])
+
+/** @type {(a:number) => boolean} */
+const isLowSurrogate = contains([0xdc00, 0xdfff])
+
+/** @type {(input:number) => list.List<ByteResult>} */
 const codePointToUtf8 = input =>
 {
     if (input >= 0x0000 && input <= 0x007f) { return [ok(input & 0x7f)] }
@@ -14,10 +33,10 @@ const codePointToUtf8 = input =>
     return [error(input)]
 }
 
-/** @type {(input:number) => list.List<Utf8Result>} */
+/** @type {(input:number) => list.List<ByteResult>} */
 const codePointToUtf16 = input =>
 {
-    if (input >= 0x0000 && input <= 0xd7ff || input >= 0xe000 && input <= 0xffff) { return [ok(input >> 8), ok(input & 0xff)] }
+    if (isBmpCodePoint(input)) { return [ok(input >> 8), ok(input & 0xff)] }
     if (input >= 0x010000 && input <= 0x10ffff) {
         const high = ((input - 0x10000) >> 10) + 0xd800
         const low = ((input - 0x10000) & 0x3ff) + 0xdc00
@@ -26,15 +45,94 @@ const codePointToUtf16 = input =>
     return [error(input)]
 }
 
-/** @type {(input: list.List<number>) => list.List<Utf8Result>} */
+/** @type {(input: list.List<number>) => list.List<ByteResult>} */
 const codePointListToUtf8 = list.flatMap(codePointToUtf8)
 
-/** @type {(input: list.List<number>) => list.List<Utf8Result>} */
+/** @type {(input: list.List<number>) => list.List<ByteResult>} */
 const codePointListToUtf16 = list.flatMap(codePointToUtf16)
 
+/** @type {operator.StateScan<number, Utf8State, list.List<CodePointResult>>} */
+const utf8ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[error([byte])], state]
+    }    
+    if (state == undefined) {
+        if (byte < 0x80) { return [[ok(byte)], undefined] }
+        if (byte >= 0xc2 && byte <= 0xf4) { return [[], [byte]] }
+        return [[error([byte])], undefined]
+    }
+    if (byte >= 0x80 && byte < 0xc0)
+    {
+        switch(state.length)
+        {
+            case 1:
+                if (state[0] < 0xe0) { return [[ok(((state[0] & 0x1f) << 6) + (byte & 0x3f))], undefined] }
+                if (state[0] < 0xf8) { return [[], [state[0], byte]] }
+                break         
+            case 2:
+                if (state[0] < 0xf0) { return [[ok(((state[0] & 0x0f) << 12) + ((state[1] & 0x3f) << 6) + (byte & 0x3f))], undefined] }
+                if (state[0] < 0xf8) { return [[], [state[0], state[1], byte]] }
+                break
+            case 3: 
+                return [[ok(((state[0] & 0x07) << 18) + ((state[1] & 0x3f) << 12) + ((state[2] & 0x3f) << 6) + (byte & 0x3f))], undefined]
+        }
+    }    
+    return [[error(list.toArray(list.concat(state)([byte])))], undefined]
+}
+
+/** @type {(state: Utf8State) => readonly[list.List<CodePointResult>, Utf8State]} */
+const utf8EofToCodePointOp = state => [state == undefined ? undefined : [error(state)],  undefined]
+
+/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<CodePointResult>>} */
+const utf8ByteOrEofToCodePointOp = state => input => input === undefined ? utf8EofToCodePointOp(state) : utf8ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<number>) => list.List<CodePointResult>} */
+const utf8ListToCodePoint = input => list.flat(list.stateScan(utf8ByteOrEofToCodePointOp)(undefined)(list.concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
+
+/** @type {operator.StateScan<number, Utf16State, list.List<CodePointResult>>} */
+const utf16ByteToCodePointOp = state => byte => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[error([byte])], state]
+    }
+    if (state == undefined) {
+        return [[], [byte]]
+    }
+    switch(state.length)
+    {
+        case 1:
+            const codeUnit = (state[0] << 8) + byte
+            if (isBmpCodePoint(codeUnit)) { return [[ok(codeUnit)], undefined] }
+            if (isHighSurrogate(codeUnit)) { return [[], [state[0], byte]] }
+            break
+        case 2:
+            return [[], [state[0], state[1], byte]]
+        case 3:             
+            if (isLowSurrogate((state[2] << 8) + byte)) {
+                const high = (state[0] << 8) + state[1] - 0xd800
+                const low = (state[2] << 8) + byte - 0xdc00
+                return [[ok((high << 10) + low + 0x10000)], undefined]
+            } 
+            break
+    }
+    return [[error(list.toArray(list.concat(state)([byte])))], undefined]
+}
+
+/** @type {(state: Utf8State) => readonly[list.List<CodePointResult>, Utf16State]} */
+const utf16EofToCodePointOp = state => [state == undefined ? undefined : [error(state)],  undefined]
+
+/** @type {operator.StateScan<ByteOrEof, Utf8State, list.List<CodePointResult>>} */
+const utf16ByteOrEofToCodePointOp = state => input => input === undefined ? utf16EofToCodePointOp(state) : utf16ByteToCodePointOp(state)(input)
+
+/** @type {(input: list.List<number>) => list.List<CodePointResult>} */
+const utf16ListToCodePoint = input => list.flat(list.stateScan(utf16ByteOrEofToCodePointOp)(undefined)(list.concat(/** @type {list.List<ByteOrEof>} */(input))([undefined])))
+
 module.exports = {
     /** @readonly */
     codePointListToUtf8,
     /** @readonly */
     codePointListToUtf16,
+    /** @readonly */
+    utf8ListToCodePoint,
+    /** @readonly */
+    utf16ListToCodePoint
 }

package/text/encoding/test.f.cjs

@@ -116,4 +116,84 @@ const stringify = a => json.stringify(sort)(a)
     if (result !== '[["error",-1],["error",55296],["error",57343],["error",1114112]]') { throw result }
 }
 
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([-1, 256])))
+    if (result !== '[["error",[-1]],["error",[256]]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([128, 193, 245, 255])))
+    if (result !== '[["error",[128]],["error",[193]],["error",[245]],["error",[255]]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([0, 1, 127])))
+    if (result !== '[["ok",0],["ok",1],["ok",127]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, 128, 194, 169, 223, 191])))
+    if (result !== '[["ok",128],["ok",169],["ok",2047]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, 127, 194, 192, 194])))
+    if (result !== '[["error",[194,127]],["error",[194,192]],["error",[194]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([224, 160, 128, 224, 160, 129, 239, 191, 191])))
+    if (result !== '[["ok",2048],["ok",2049],["ok",65535]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([224, 160, 127, 224, 160, 192, 224, 160])))
+    if (result !== '[["error",[224,160,127]],["error",[224,160,192]],["error",[224,160]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([240, 144, 128, 128, 240, 144, 128, 129, 244, 143, 191, 191])))
+    if (result !== '[["ok",65536],["ok",65537],["ok",1114111]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([240, 144, 128, 127, 240, 144, 128, 192, 240, 144, 128])))
+    if (result !== '[["error",[240,144,128,127]],["error",[240,144,128,192]],["error",[240,144,128]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf8ListToCodePoint([194, -1, 128])))
+    if (result !== '[["error",[-1]],["ok",128]]') { throw result }
+}
+
+{    
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([-1, 256,])))
+    if (result !== '[["error",[-1]],["error",[256]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([0, 0, 0, 36, 32, 172, 215, 255, 224, 0, 255, 255])))
+    if (result !== '[["ok",0],["ok",36],["ok",8364],["ok",55295],["ok",57344],["ok",65535]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([220, 0, 223, 255])))
+    if (result !== '[["error",[220,0]],["error",[223,255]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 220, 0, 216, 1, 220, 55, 216, 82, 223, 98, 219, 255, 223, 255])))
+    if (result !== '[["ok",65536],["ok",66615],["ok",150370],["ok",1114111]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 216, 0])))
+    if (result !== '[["error",[216,0,216,0]]]') { throw result }
+}
+
+{
+    const result = stringify(list.toArray(encoding.utf16ListToCodePoint([216, 0, 0, 0])))
+    if (result !== '[["error",[216,0,0,0]]]') { throw result }
+}
+
 module.exports = {}

package/{io → types}/result/module.cjs

@@ -1,4 +1,4 @@
-const result = require('../../types/result/module.f.cjs')
+const result = require('./module.f.cjs')
 
 /** @type {<T>(f: () => T) => result.Result<T, unknown>} */
 const tryCatch = f => {