functionalscript 0.0.525 → 0.0.527

package/fjson/README.md

@@ -6,3 +6,21 @@
 
 - can serialize/deserialize without reading source code
   - no function serialization/deserialization
+
+## Next steps
+
+- [x] identifiers
+- [x] big int
+- [ ] `module.exports = ...`
+- [ ] constants `const a = [3];module.exports = { a: a, b: a }`. Serialization `const _0=[3];module.exports={a:_0,b:_0}`
+- [ ] import `const a = require('c.fon.js');module.exports = { a: a, b: a}`
+
+Optional, for fun, syntax sugar:
+
+- [ ] comments. Ignore them. Not an error.
+- [ ] double/single quote strings
+
+## Decidable Language
+  
+- [ ] using operator and functions `const a = 2+2+Math.abs(5); module.exports = { a: a };`
+- [ ] decidable functions `const f = a => b => a + b; module.exports = f(1)(2)`?

package/fjson/tokenizer/module.f.cjs

@@ -1,9 +1,13 @@
+const operator = require('../../types/function/operator/module.f.cjs')
 const list = require('../../types/list/module.f.cjs')
+const { empty, flat, stateScan } = list
+const { multiply } = require('../../types/bigfloat/module.f.cjs')
 const jsTokenizer = require('../../js/tokenizer/module.f.cjs')
 
 /**
  * @typedef {|
-* jsTokenizer.SimpleToken |
+* {readonly kind: 'true' | 'false' | 'null'} |
+* {readonly kind: '{' | '}' | ':' | ',' | '[' | ']' } |
 * jsTokenizer.StringToken |
 * jsTokenizer.NumberToken |
 * jsTokenizer.ErrorToken |
@@ -12,31 +16,82 @@ const jsTokenizer = require('../../js/tokenizer/module.f.cjs')
 * } FjsonToken
 */
 
-/** @type {(input: jsTokenizer.JsToken) => FjsonToken} */
+/**
+ * @typedef {|
+* {readonly kind: 'def' | '-' }
+* } ScanState
+*/
+
+/**
+ * @typedef {|
+* jsTokenizer.JsToken | null
+* } ScanInput
+*/
+
+/** @type {(input: jsTokenizer.JsToken) => list.List<FjsonToken>} */
 const mapToken = input =>
 {
     switch(input.kind)
     {
-        case "id":
-        case "bigint":
-        case "{":
-        case "}":
-        case ":":
-        case ",":
-        case "[":
-        case "]":
-        case "true":
-        case "false":
-        case "null":
-        case "string":
-        case "number":
-        case "error": return input
-        default: return { kind: "error", message: "invalid token" }
+        case 'id':
+        case 'bigint':
+        case '{':
+        case '}':
+        case ':':
+        case ',':
+        case '[':
+        case ']':
+        case 'true':
+        case 'false':
+        case 'null':
+        case 'string':
+        case 'number':
+        case 'error': return [input]
+        case 'ws': return empty
+        default: return jsTokenizer.isKeywordToken(input) ? [{ kind: 'id', value: input.kind }] : [{ kind: 'error', message: 'invalid token' }]
+    }
+}
+
+/** @type {(input: ScanInput) => readonly [list.List<FjsonToken>, ScanState]} */
+const parseDefaultState = input =>
+{
+    if (input === null) return [empty, { kind: 'def'}]
+    switch(input.kind)
+    {
+        case '-': return [empty, { kind: '-'}]
+        default: return [mapToken(input),  { kind: 'def'}]
+    }
+}
+
+/** @type {(input: ScanInput) => readonly [list.List<FjsonToken>, ScanState]} */
+const parseMinusState = input =>
+{
+    if (input === null) return [[{ kind: 'error', message: 'invalid token' }], { kind: 'def'}]
+    switch(input.kind)
+    {
+        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-'}]
+        case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def'}]
+        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def'}]
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input)},  { kind: 'def'}]
+    }
+}
+
+/** @type {operator.StateScan<ScanInput, ScanState, list.List<FjsonToken>>} */
+const scanToken = state => input => {
+    switch(state.kind)
+    {
+        case '-': return parseMinusState(input)
+        default: return parseDefaultState(input)
     }
 }
 
 /** @type {(input: list.List<number>) => list.List<FjsonToken>} */
-const tokenize = input => list.map(mapToken)(jsTokenizer.tokenize(input))
+const tokenize = input =>
+{
+    /** @type {list.List<ScanInput>} */
+    const jsTokens = jsTokenizer.tokenize(input)
+    return flat(stateScan(scanToken)({ kind: 'def' })(list.concat(jsTokens)([null])))
+}
 
 module.exports = {
     /** @readonly */

package/fjson/tokenizer/test.f.cjs

@@ -43,38 +43,10 @@ module.exports = {
             const result = stringify(tokenizeString('ᄑ'))
             if (result !== '[{"kind":"error","message":"unexpected character"}]') { throw result }
         },
-        () => {
-            const result = stringify(tokenizeString('err'))
-            if (result !== '[{"kind":"id","value":"err"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('{e}'))
-            if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') { throw result }
-        },
         () => {
             const result = stringify(tokenizeString('{ \t\n\r}'))
             if (result !== '[{"kind":"{"},{"kind":"}"}]') { throw result }
         },
-        () => {
-            const result = stringify(tokenizeString('true'))
-            if (result !== '[{"kind":"true"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('tru'))
-            if (result !== '[{"kind":"id","value":"tru"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('false'))
-            if (result !== '[{"kind":"false"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('null'))
-            if (result !== '[{"kind":"null"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('[null]'))
-            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') { throw result }
-        },
         () => {
             const result = stringify(tokenizeString('""'))
             if (result !== '[{"kind":"string","value":""}]') { throw result }
@@ -165,7 +137,7 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('10-0'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('9a:'))
@@ -175,17 +147,29 @@ module.exports = {
             const result = stringify(tokenizeString('-10'))
             if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"}]') { throw result }
         },
+        () => {
+            const result = stringify(tokenizeString('-'))
+            if (result !== '[{"kind":"error","message":"invalid token"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('--'))
+            if (result !== '[{"kind":"error","message":"invalid token"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('---'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"}]') { throw result }
+        },
         () => {
             const result = stringify(tokenizeString('-0'))
             if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-00'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-.123'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"bf":[123n,0],"kind":"number","value":"123"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('0.01'))
@@ -197,11 +181,11 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-0.'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-0.]'))
-            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('12.34'))
@@ -213,7 +197,7 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-12.'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('12.]'))
@@ -303,5 +287,41 @@ module.exports = {
             const result = stringify(tokenizeString('1234567890nn'))
             if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"id","value":"n"}]') { throw result }
         },
+    ],
+    id: [
+        () => {
+            const result = stringify(tokenizeString('err'))
+            if (result !== '[{"kind":"id","value":"err"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{e}'))
+            if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('tru'))
+            if (result !== '[{"kind":"id","value":"tru"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('break'))
+            if (result !== '[{"kind":"id","value":"break"}]') { throw result }
+        },
+    ],
+    keywords: [
+        () => {
+            const result = stringify(tokenizeString('true'))
+            if (result !== '[{"kind":"true"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('false'))
+            if (result !== '[{"kind":"false"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('null'))
+            if (result !== '[{"kind":"null"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('[null]'))
+            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') { throw result }
+        },
     ]
 }