functionalscript 0.0.532 → 0.0.534

package/fjson/README.md

@@ -10,7 +10,7 @@
 ## Next steps
 
 - [ ] rename `fjson` to `djs` (data javascript), File extensions: `.d.cjs`, `.d.mjs`, `.d.js`.
-- [ ] use JS tokenizer 
+- [x] use JS tokenizer 
 - [x] identifiers `{a:5}`
 - [x] big int
 - [ ] `module.exports = ...`

package/fjson/parser/module.f.cjs

@@ -0,0 +1,229 @@
+const result = require('../../types/result/module.f.cjs')
+const list = require('../../types/list/module.f.cjs')
+const { fold, first, drop, toArray } = list
+const operator = require('../../types/function/operator/module.f.cjs')
+const tokenizer = require('../tokenizer/module.f.cjs')
+const map = require('../../types/map/module.f.cjs')
+const { setReplace } = map
+const fjson = require('../module.f.cjs')
+const { fromMap } = require('../../types/object/module.f.cjs')
+
+/**
+ * @typedef {{
+* readonly kind: 'object'
+* readonly values: map.Map<fjson.Unknown>
+* readonly key: string
+* }} JsonObject
+* */
+
+/**
+ * @typedef {{
+* readonly kind: 'array'
+* readonly values: list.List<fjson.Unknown>
+* }} JsonArray
+* */
+
+/**
+ * @typedef {|
+* JsonObject |
+* JsonArray
+* } JsonStackElement
+*/
+
+/** @typedef {list.List<JsonStackElement>} JsonStack */
+
+/**
+ * @typedef {{
+*  readonly status: '' | '[' | '[v' | '[,' | '{' | '{k' | '{:' | '{v' | '{,'
+*  readonly top: JsonStackElement | null
+*  readonly stack: JsonStack
+* }} StateParse
+*/
+
+/**
+ * @typedef {{
+*  readonly status: 'result'
+*  readonly value: fjson.Unknown
+* }} StateResult
+*/
+
+/**
+ * @typedef {{
+*  readonly status: 'error'
+*  readonly message: string
+* }} StateError
+*/
+
+/**
+ * @typedef {|
+* StateParse |
+* StateResult |
+* StateError
+* } FjsonState
+*/
+
+/** @type {(obj: JsonObject) => (key: string) => JsonObject} */
+const addKeyToObject = obj => key => ({ kind: 'object', values: obj.values, key: key })
+
+/** @type {(obj: JsonObject) => (value: fjson.Unknown) => JsonObject} */
+const addValueToObject = obj => value => ({ kind: 'object', values: setReplace(obj.key)(value)(obj.values), key: '' })
+
+/** @type {(array: JsonArray) => (value: fjson.Unknown) => JsonArray} */
+const addToArray = array => value => ({ kind: 'array', values: list.concat(array.values)([value]) })
+
+/** @type {(state: StateParse) => (key: string) => FjsonState} */
+const pushKey = state => value => {
+    if (state.top?.kind === 'object') { return { status: '{k', top: addKeyToObject(state.top)(value), stack: state.stack } }
+    return { status: 'error', message: 'error' }
+}
+
+/** @type {(state: StateParse) => (value: fjson.Unknown) => FjsonState} */
+const pushValue = state => value => {
+    if (state.top === null) { return { status: 'result', value: value } }
+    if (state.top.kind === 'array') { return { status: '[v', top: addToArray(state.top)(value), stack: state.stack } }
+    return { status: '{v', top: addValueToObject(state.top)(value), stack: state.stack }
+}
+
+/** @type {(state: StateParse) => FjsonState} */
+const startArray = state => {
+    const newStack = state.top === null ? null : { first: state.top, tail: state.stack }
+    return { status: '[', top: { kind: 'array', values: null }, stack: newStack }
+}
+
+/** @type {(state: StateParse) => FjsonState} */
+const endArray = state => {
+    const array = state.top !== null ? toArray(state.top.values) : null
+    /** @type {StateParse} */
+    const newState = { status: '', top: first(null)(state.stack), stack: drop(1)(state.stack) }
+    return pushValue(newState)(array)
+}
+
+/** @type {(state: StateParse) => FjsonState} */
+const startObject = state => {
+    const newStack = state.top === null ? null : { first: state.top, tail: state.stack }
+    return { status: '{', top: { kind: 'object', values: null, key: '' }, stack: newStack }
+}
+
+/** @type {(state: StateParse) => FjsonState} */
+const endObject = state => {
+    const obj = state.top?.kind === 'object' ? fromMap(state.top.values) : null
+    /** @type {StateParse} */
+    const newState = { status: '', top: first(null)(state.stack), stack: drop(1)(state.stack) }
+    return pushValue(newState)(obj)
+}
+
+/** @type {(token: tokenizer.FjsonToken) => fjson.Unknown} */
+const tokenToValue = token => {
+    switch (token.kind) {
+        case 'null': return null
+        case 'false': return false
+        case 'true': return true
+        case 'number': return parseFloat(token.value)
+        case 'string': return token.value
+        case 'bigint': return token.value
+        default: return null
+    }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => boolean} */
+const isValueToken = token => {
+    switch (token.kind) {
+        case 'null':
+        case 'false':
+        case 'true':
+        case 'number':
+        case 'string':
+        case 'bigint': return true
+        default: return false
+    }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseValueOp = token => state => {
+    if (isValueToken(token)) { return pushValue(state)(tokenToValue(token)) }
+    if (token.kind === '[') { return startArray(state) }
+    if (token.kind === '{') { return startObject(state) }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseArrayStartOp = token => state => {
+    if (isValueToken(token)) { return pushValue(state)(tokenToValue(token)) }
+    if (token.kind === '[') { return startArray(state) }
+    if (token.kind === ']') { return endArray(state) }
+    if (token.kind === '{') { return startObject(state) }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseArrayValueOp = token => state => {
+    if (token.kind === ']') { return endArray(state) }
+    if (token.kind === ',') { return { status: '[,', top: state.top, stack: state.stack } }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseObjectStartOp = token => state => {
+    if (token.kind === 'string') { return pushKey(state)(token.value) }
+    if (token.kind === '}') { return endObject(state) }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseObjectKeyOp = token => state => {
+    if (token.kind === ':') { return { status: '{:', top: state.top, stack: state.stack } }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseObjectColonOp = token => state => {
+    if (isValueToken(token)) { return pushValue(state)(tokenToValue(token)) }
+    if (token.kind === '[') { return startArray(state) }
+    if (token.kind === '{') { return startObject(state) }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseObjectNextOp = token => state => {
+    if (token.kind === '}') { return endObject(state) }
+    if (token.kind === ',') { return { status: '{,', top: state.top, stack: state.stack } }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {(token: tokenizer.FjsonToken) => (state: StateParse) => FjsonState}} */
+const parseObjectCommaOp = token => state => {
+    if (token.kind === 'string') { return pushKey(state)(token.value) }
+    return { status: 'error', message: 'unexpected token' }
+}
+
+/** @type {operator.Fold<tokenizer.FjsonToken, FjsonState>} */
+const foldOp = token => state => {
+    switch (state.status) {
+        case 'result': return { status: 'error', message: 'unexpected token' }
+        case 'error': return { status: 'error', message: state.message }
+        case '': return parseValueOp(token)(state)
+        case '[': return parseArrayStartOp(token)(state)
+        case '[v': return parseArrayValueOp(token)(state)
+        case '[,': return parseValueOp(token)(state)
+        case '{': return parseObjectStartOp(token)(state)
+        case '{k': return parseObjectKeyOp(token)(state)
+        case '{:': return parseObjectColonOp(token)(state)
+        case '{v': return parseObjectNextOp(token)(state)
+        case '{,': return parseObjectCommaOp(token)(state)
+    }
+}
+
+/** @type {(tokenList: list.List<tokenizer.FjsonToken>) => result.Result<fjson.Unknown, string>} */
+const parse = tokenList => {
+    const state = fold(foldOp)({ status: '', top: null, stack: null })(tokenList)
+    switch (state.status) {
+        case 'result': return result.ok(state.value)
+        case 'error': return result.error(state.message)
+        default: return result.error('unexpected end')
+    }
+}
+
+module.exports = {
+    /** @readonly */
+    parse
+}

package/fjson/parser/test.f.cjs

@@ -0,0 +1,258 @@
+const parser = require('./module.f.cjs')
+const tokenizer = require('../tokenizer/module.f.cjs')
+const { toArray } = require('../../types/list/module.f.cjs')
+const fjson = require('../module.f.cjs')
+const { sort } = require('../../types/object/module.f.cjs')
+const encoding = require('../../text/utf16/module.f.cjs');
+
+/** @type {(s: string) => readonly tokenizer.FjsonToken[]} */
+const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s)))
+
+const stringify = fjson.stringify(sort)
+
+module.exports = {
+    valid: [
+        () => {
+            const tokenList = tokenizeString('null')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",null]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('true')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",true]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('false')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",false]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('0.1')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",0.1]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('1.1e+2')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",110]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('"abc"')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok","abc"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[]]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[1]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[1]]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[[]]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[[]]]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[0,[1,[2,[]]],3]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[0,[1,[2,[]]],3]]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",{}]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[{}]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[{}]]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"a":true,"b":false,"c":null}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",{"a":true,"b":false,"c":null}]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"a":{"b":{"c":["d"]}}}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",{"a":{"b":{"c":["d"]}}}]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('1234567890n')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",1234567890n]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[1234567890n]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["ok",[1234567890n]]') { throw result }
+        }
+    ],
+    invalid: [
+        () => {
+            const tokenList = tokenizeString('')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected end"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('"123')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[,]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[1 2]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[1,,2]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[]]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('["a"')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected end"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[1,]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[,1]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[:]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString(']')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{,}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{1:2}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"1"2}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"1"::2}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"1":2,,"3":4')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{}}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"1":2')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected end"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{"1":2,}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{,"1":2}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('[{]}')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('{[}]')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+        () => {
+            const tokenList = tokenizeString('10-5')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
+    ]
+}

package/json/parser/test.f.cjs

@@ -11,7 +11,7 @@ const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s))
 const stringify = json.stringify(sort)
 
 module.exports = {
-    valud: [
+    valid: [
         () => {
             const tokenList = tokenizeString('null')
             const obj = parser.parse(tokenList)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "functionalscript",
-  "version": "0.0.532",
+  "version": "0.0.534",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "module.f.cjs",
   "scripts": {