functionalscript 0.0.524 → 0.0.526

package/js/tokenizer/test.f.cjs

@@ -43,38 +43,11 @@ module.exports = {
             const result = stringify(tokenizeString('ᄑ'))
             if (result !== '[{"kind":"error","message":"unexpected character"}]') { throw result }
         },
-        () => {
-            const result = stringify(tokenizeString('err'))
-            if (result !== '[{"kind":"id","value":"err"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('{e}'))
-            if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') { throw result }
-        },
         () => {
             const result = stringify(tokenizeString('{ \t\n\r}'))
-            if (result !== '[{"kind":"{"},{"kind":"}"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('true'))
-            if (result !== '[{"kind":"true"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('tru'))
-            if (result !== '[{"kind":"id","value":"tru"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('false'))
-            if (result !== '[{"kind":"false"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('null'))
-            if (result !== '[{"kind":"null"}]') { throw result }
-        },
-        () => {
-            const result = stringify(tokenizeString('[null]'))
-            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') { throw result }
+            if (result !== '[{"kind":"{"},{"kind":"ws"},{"kind":"}"}]') { throw result }
         },
+
         () => {
             const result = stringify(tokenizeString('""'))
             if (result !== '[{"kind":"string","value":""}]') { throw result }
@@ -89,7 +62,7 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('"value1" "value2"'))
-            if (result !== '[{"kind":"string","value":"value1"},{"kind":"string","value":"value2"}]') { throw result }
+            if (result !== '[{"kind":"string","value":"value1"},{"kind":"ws"},{"kind":"string","value":"value2"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('"'))
@@ -157,15 +130,15 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('1 2'))
-            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"bf":[2n,0],"kind":"number","value":"2"}]') { throw result }
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('0. 2'))
-            if (result !== '[{"kind":"error","message":"invalid number"},{"bf":[2n,0],"kind":"number","value":"2"}]') { throw result }
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('10-0'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('9a:'))
@@ -173,19 +146,19 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-10'))
-            if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[10n,0],"kind":"number","value":"10"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-0'))
-            if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-00'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-.123'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('0.01'))
@@ -193,15 +166,15 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-0.9'))
-            if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[9n,-1],"kind":"number","value":"0.9"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-0.'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-0.]'))
-            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"},{"kind":"]"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('12.34'))
@@ -209,11 +182,11 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-12.00'))
-            if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[1200n,-2],"kind":"number","value":"12.00"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-12.'))
-            if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"kind":"error","message":"invalid number"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('12.]'))
@@ -237,11 +210,11 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('-12e-0001'))
-            if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[12n,-1],"kind":"number","value":"12e-0001"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('-12.34e1234'))
-            if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"}]') { throw result }
+            if (result !== '[{"kind":"-"},{"bf":[1234n,1232],"kind":"number","value":"12.34e1234"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('0e'))
@@ -261,11 +234,11 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('123 _123'))
-            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"id","value":"_123"}]') { throw result }
+            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('123 $123'))
-            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"id","value":"$123"}]') { throw result }
+            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('123_123'))
@@ -285,7 +258,7 @@ module.exports = {
         },
         () => {
             const result = stringify(tokenizeString('[-1234567890n]'))
-            if (result !== '[{"kind":"["},{"kind":"bigint","value":-1234567890n},{"kind":"]"}]') { throw result }
+            if (result !== '[{"kind":"["},{"kind":"-"},{"kind":"bigint","value":1234567890n},{"kind":"]"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('123.456n'))
@@ -318,37 +291,279 @@ module.exports = {
             const result = stringify(tokenizeString('-'))
             if (result !== '[{"kind":"-"}]') { throw result }
         },
+        () => {
+            const result = stringify(tokenizeString('1*2'))
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"*"},{"bf":[2n,0],"kind":"number","value":"2"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('( )'))
+            if (result !== '[{"kind":"("},{"kind":"ws"},{"kind":")"}]') { throw result }
+        },
         () => {
             const result = stringify(tokenizeString('== != === !== > >= < <='))
-            if (result !== '[{"kind":"=="},{"kind":"!="},{"kind":"==="},{"kind":"!=="},{"kind":">"},{"kind":">="},{"kind":"<"},{"kind":"<="}]') { throw result }
+            if (result !== '[{"kind":"=="},{"kind":"ws"},{"kind":"!="},{"kind":"ws"},{"kind":"==="},{"kind":"ws"},{"kind":"!=="},{"kind":"ws"},{"kind":">"},{"kind":"ws"},{"kind":">="},{"kind":"ws"},{"kind":"<"},{"kind":"ws"},{"kind":"<="}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('+ - * / % ++ -- **'))
-            if (result !== '[{"kind":"+"},{"kind":"-"},{"kind":"*"},{"kind":"/"},{"kind":"%"},{"kind":"++"},{"kind":"--"},{"kind":"**"}]') { throw result }
+            if (result !== '[{"kind":"+"},{"kind":"ws"},{"kind":"-"},{"kind":"ws"},{"kind":"*"},{"kind":"ws"},{"kind":"/"},{"kind":"ws"},{"kind":"%"},{"kind":"ws"},{"kind":"++"},{"kind":"ws"},{"kind":"--"},{"kind":"ws"},{"kind":"**"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('= += -= *= /= %= **='))
-            if (result !== '[{"kind":"="},{"kind":"+="},{"kind":"-="},{"kind":"*="},{"kind":"/="},{"kind":"%="},{"kind":"**="}]') { throw result }
+            if (result !== '[{"kind":"="},{"kind":"ws"},{"kind":"+="},{"kind":"ws"},{"kind":"-="},{"kind":"ws"},{"kind":"*="},{"kind":"ws"},{"kind":"/="},{"kind":"ws"},{"kind":"%="},{"kind":"ws"},{"kind":"**="}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('& | ^ ~ << >> >>>'))
-            if (result !== '[{"kind":"&"},{"kind":"|"},{"kind":"^"},{"kind":"~"},{"kind":"<<"},{"kind":">>"},{"kind":">>>"}]') { throw result }
+            if (result !== '[{"kind":"&"},{"kind":"ws"},{"kind":"|"},{"kind":"ws"},{"kind":"^"},{"kind":"ws"},{"kind":"~"},{"kind":"ws"},{"kind":"<<"},{"kind":"ws"},{"kind":">>"},{"kind":"ws"},{"kind":">>>"}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('&= |= ^= <<= >>= >>>='))
-            if (result !== '[{"kind":"&="},{"kind":"|="},{"kind":"^="},{"kind":"<<="},{"kind":">>="},{"kind":">>>="}]') { throw result }
+            if (result !== '[{"kind":"&="},{"kind":"ws"},{"kind":"|="},{"kind":"ws"},{"kind":"^="},{"kind":"ws"},{"kind":"<<="},{"kind":"ws"},{"kind":">>="},{"kind":"ws"},{"kind":">>>="}]') { throw result }
         },
         () => {
-            const result = stringify(tokenizeString('&& || !'))
-            if (result !== '[{"kind":"&&"},{"kind":"||"},{"kind":"!"}]') { throw result }
+            const result = stringify(tokenizeString('&& || ! ??'))
+            if (result !== '[{"kind":"&&"},{"kind":"ws"},{"kind":"||"},{"kind":"ws"},{"kind":"!"},{"kind":"ws"},{"kind":"??"}]') { throw result }
         },
         () => {
-            const result = stringify(tokenizeString('&&= ||= ?? ??='))
-            if (result !== '[{"kind":"&&="},{"kind":"||="},{"kind":"error","message":"invalid token"},{"kind":"??="}]') { throw result }
+            const result = stringify(tokenizeString('&&= ||= ??='))
+            if (result !== '[{"kind":"&&="},{"kind":"ws"},{"kind":"||="},{"kind":"ws"},{"kind":"??="}]') { throw result }
         },
         () => {
             const result = stringify(tokenizeString('? ?. . =>'))
-            if (result !== '[{"kind":"?"},{"kind":"?."},{"kind":"."},{"kind":"=>"}]') { throw result }
+            if (result !== '[{"kind":"?"},{"kind":"ws"},{"kind":"?."},{"kind":"ws"},{"kind":"."},{"kind":"ws"},{"kind":"=>"}]') { throw result }
+        },
+    ],
+    ws: [
+        () => {
+            const result = stringify(tokenizeString(' '))
+            if (result !== '[{"kind":"ws"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('\t'))
+            if (result !== '[{"kind":"ws"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('\n'))
+            if (result !== '[{"kind":"ws"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('\r'))
+            if (result !== '[{"kind":"ws"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString(' \t\n\r'))
+            if (result !== '[{"kind":"ws"}]') { throw result }
+        },
+    ],
+    id: [
+        () => {
+            const result = stringify(tokenizeString('err'))
+            if (result !== '[{"kind":"id","value":"err"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{e}'))
+            if (result !== '[{"kind":"{"},{"kind":"id","value":"e"},{"kind":"}"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('tru'))
+            if (result !== '[{"kind":"id","value":"tru"}]') { throw result }
+        },
+    ],
+    keywords: [
+        () => {
+            const result = stringify(tokenizeString('true'))
+            if (result !== '[{"kind":"true"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('false'))
+            if (result !== '[{"kind":"false"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('null'))
+            if (result !== '[{"kind":"null"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('[null]'))
+            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('arguments'))
+            if (result !== '[{"kind":"arguments"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('await'))
+            if (result !== '[{"kind":"await"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('break'))
+            if (result !== '[{"kind":"break"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('case'))
+            if (result !== '[{"kind":"case"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('catch'))
+            if (result !== '[{"kind":"catch"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('class'))
+            if (result !== '[{"kind":"class"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('const'))
+            if (result !== '[{"kind":"const"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('continue'))
+            if (result !== '[{"kind":"continue"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('debugger'))
+            if (result !== '[{"kind":"debugger"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('default'))
+            if (result !== '[{"kind":"default"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('delete'))
+            if (result !== '[{"kind":"delete"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('do'))
+            if (result !== '[{"kind":"do"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('else'))
+            if (result !== '[{"kind":"else"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('enum'))
+            if (result !== '[{"kind":"enum"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('eval'))
+            if (result !== '[{"kind":"eval"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('export'))
+            if (result !== '[{"kind":"export"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('extends'))
+            if (result !== '[{"kind":"extends"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('finally'))
+            if (result !== '[{"kind":"finally"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('for'))
+            if (result !== '[{"kind":"for"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('function'))
+            if (result !== '[{"kind":"function"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('if'))
+            if (result !== '[{"kind":"if"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('implements'))
+            if (result !== '[{"kind":"implements"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('import'))
+            if (result !== '[{"kind":"import"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('in'))
+            if (result !== '[{"kind":"in"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('instanceof'))
+            if (result !== '[{"kind":"instanceof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('interface'))
+            if (result !== '[{"kind":"interface"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('let'))
+            if (result !== '[{"kind":"let"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('new'))
+            if (result !== '[{"kind":"new"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('package'))
+            if (result !== '[{"kind":"package"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('private'))
+            if (result !== '[{"kind":"private"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('protected'))
+            if (result !== '[{"kind":"protected"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('public'))
+            if (result !== '[{"kind":"public"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('return'))
+            if (result !== '[{"kind":"return"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('static'))
+            if (result !== '[{"kind":"static"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('super'))
+            if (result !== '[{"kind":"super"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('switch'))
+            if (result !== '[{"kind":"switch"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('this'))
+            if (result !== '[{"kind":"this"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('throw'))
+            if (result !== '[{"kind":"throw"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('try'))
+            if (result !== '[{"kind":"try"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('typeof'))
+            if (result !== '[{"kind":"typeof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('var'))
+            if (result !== '[{"kind":"var"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('void'))
+            if (result !== '[{"kind":"void"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('while'))
+            if (result !== '[{"kind":"while"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('with'))
+            if (result !== '[{"kind":"with"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('yield'))
+            if (result !== '[{"kind":"yield"}]') { throw result }
         },
     ]
 }

package/json/parser/test.f.cjs

@@ -95,7 +95,7 @@ module.exports = {
             const obj = parser.parse(tokenList)
             const result = stringify(obj)
             if (result !== '["ok",{"a":{"b":{"c":["d"]}}}]') { throw result }
-        },
+        }
     ],
     invalid: [
         () => {
@@ -236,5 +236,11 @@ module.exports = {
             const result = stringify(obj)
             if (result !== '["error","unexpected token"]') { throw result }
         },
+        () => {
+            const tokenList = tokenizeString('10-5')
+            const obj = parser.parse(tokenList)
+            const result = stringify(obj)
+            if (result !== '["error","unexpected token"]') { throw result }
+        },
     ]
 }

package/json/tokenizer/module.f.cjs

@@ -1,38 +1,92 @@
+const operator = require('../../types/function/operator/module.f.cjs')
 const list = require('../../types/list/module.f.cjs')
+const { empty, flat, stateScan } = list
+const { multiply } = require('../../types/bigfloat/module.f.cjs')
 const jsTokenizer = require('../../js/tokenizer/module.f.cjs')
 
 /**
  * @typedef {|
-* jsTokenizer.SimpleToken |
+* {readonly kind: 'true' | 'false' | 'null' } |
+* {readonly kind: '{' | '}' | ':' | ',' | '[' | ']' } |
 * jsTokenizer.StringToken |
 * jsTokenizer.NumberToken |
 * jsTokenizer.ErrorToken
 * } JsonToken
 */
 
-/** @type {(input: jsTokenizer.JsToken) => JsonToken} */
+/**
+ * @typedef {|
+* {readonly kind: 'def' | '-' }
+* } ScanState
+*/
+
+/**
+ * @typedef {|
+* jsTokenizer.JsToken | null
+* } ScanInput
+*/
+
+/** @type {(input: jsTokenizer.JsToken) => list.List<JsonToken>} */
 const mapToken = input =>
 {
     switch(input.kind)
     {
-        case "{":
-        case "}":
-        case ":":
-        case ",":
-        case "[":
-        case "]":
-        case "true":
-        case "false":
-        case "null":
-        case "string":
-        case "number":
-        case "error": return input
-        default: return { kind: "error", message: "invalid token" }
+        case '{':
+        case '}':
+        case ':':
+        case ',':
+        case '[':
+        case ']':
+        case 'true':
+        case 'false':
+        case 'null':
+        case 'string':
+        case 'number':
+        case 'error': return [input]
+        case 'ws': return empty
+        default: return [{ kind: 'error', message: 'invalid token' }]
+    }
+}
+
+/** @type {(input: ScanInput) => readonly [list.List<JsonToken>, ScanState]} */
+const parseDefaultState = input =>
+{
+    if (input === null) return [empty, { kind: 'def'}]
+    switch(input.kind)
+    {
+        case '-': return [empty, { kind: '-'}]
+        default: return [mapToken(input),  { kind: 'def'}]
+    }
+}
+
+/** @type {(input: ScanInput) => readonly [list.List<JsonToken>, ScanState]} */
+const parseMinusState = input =>
+{
+    if (input === null) return [[{ kind: 'error', message: 'invalid token' }], { kind: 'def'}]
+    switch(input.kind)
+    {
+        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-'}]
+        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def'}]
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input)},  { kind: 'def'}]
+    }
+}
+
+/** @type {operator.StateScan<ScanInput, ScanState, list.List<JsonToken>>} */
+const scanToken = state => input => {
+    switch(state.kind)
+    {
+        case '-': return parseMinusState(input)
+        default: return parseDefaultState(input)
     }
 }
 
 /** @type {(input: list.List<number>) => list.List<JsonToken>} */
-const tokenize = input => list.map(mapToken)(jsTokenizer.tokenize(input))
+const tokenize = input =>
+{
+    /** @type {list.List<ScanInput>} */
+    const jsTokens = jsTokenizer.tokenize(input)
+    return flat(stateScan(scanToken)({ kind: 'def' })(list.concat(jsTokens)([null])))
+}
 
 module.exports = {
     /** @readonly */