functionalscript 0.0.336 → 0.0.339

package/json/tokenizer/index.js

@@ -1,5 +1,6 @@
 const operator = require('../../types/function/operator/index.js')
 const list = require('../../types/list/index.js')
+const range = require('../../types/range/index.js')
 
 /**
  * @typedef {{
@@ -67,6 +68,12 @@ const letterT = 0x74
 const letterU = 0x75
 const letterZ = 0x7a
 
+const containsDigit = range.contains([digit0, digit9])
+const containsDigitOneNine = range.contains([digit1, digit9])
+const containsSmallAF = range.contains([letterA, letterF])
+const containsCapitalAF = range.contains([capitalLetterA, capitalLetterF])
+const containsSmallLetter = range.contains([letterA, letterZ])
+
 /**
  * @typedef {|
  * InitialState |
@@ -116,22 +123,23 @@ const letterZ = 0x7a
 
 /** @typedef {number|undefined} CharCodeOrEof */
 
-/** @type {(old: string) => (input: CharCodeOrEof) => string} */
-const appendChar = old => input => input === undefined ? old : operator.concat(charToString(input))(old)
-
-/** @type {(input: CharCodeOrEof) => string} */
-const charToString = input => input === undefined ? '' : String.fromCharCode(input)
+/** @type {(old: string) => (input: number) => string} */
+const appendChar = old => input => `${old}${String.fromCharCode(input)}`
 
 /** @type {(state: InitialState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const initialStateOp = initialState => input =>
 {
-    if (input >= digit1 && input <= digit9)
+    if (containsDigitOneNine(input))
+    {
+        return [undefined, { kind: 'number', value: String.fromCharCode(input), numberKind: 'int'}]
+    }
+    if (containsSmallLetter(input))
     {
-        return [undefined, { kind: 'number', value: charToString(input), numberKind: 'int'}]
+        return [undefined, { kind: 'keyword', value: String.fromCharCode(input)}]
     }
-    if (input >= letterA && input <= letterZ)
+    if (isWhiteSpace(input))
     {
-        return [undefined, { kind: 'keyword', value: charToString(input)}]
+        return [undefined, initialState]
     }
     switch(input)
     {
@@ -142,12 +150,8 @@ const initialStateOp = initialState => input =>
         case leftBracket: return [[{kind: '['}], initialState]
         case rightBracket: return [[{kind: ']'}], initialState]
         case quotationMark: return[undefined, {kind: 'string', value: ''}]
-        case digit0: return [undefined, { kind: 'number', value: charToString(input), numberKind: '0'}]
-        case signMinus: return [undefined, { kind: 'number', value: charToString(input), numberKind: '-'}]
-        case horizontalTab:
-        case newLine:
-        case carriageReturn:
-        case space: return[undefined, initialState]
+        case digit0: return [undefined, { kind: 'number', value: String.fromCharCode(input), numberKind: '0'}]
+        case signMinus: return [undefined, { kind: 'number', value: String.fromCharCode(input), numberKind: '-'}]
         default: return [[{kind: 'error', message: 'unexpected character'}], initialState]
     }
 }
@@ -177,7 +181,7 @@ const parseNumberStateOp = state => input =>
             default: return [undefined, {kind:'number', value: appendChar(state.value)(input), numberKind: state.numberKind}]
         }
     }
-    if (input >= digit1 && input <= digit9)
+    if (containsDigitOneNine(input))
     {
         switch (state.numberKind)
         {
@@ -242,6 +246,10 @@ const parseNumberStateOp = state => input =>
 /** @type {(char: number) => boolean} */
 const isTerminalForNumber = char =>
 {
+    if (isWhiteSpace(char))
+    {
+        return true;
+    }
     switch (char)
     {
         case quotationMark:
@@ -255,6 +263,19 @@ const isTerminalForNumber = char =>
     }
 }
 
+/** @type {(char: number) => boolean} */
+const isWhiteSpace = char =>
+{
+    switch (char)
+    {
+        case horizontalTab:
+        case newLine:
+        case carriageReturn:
+        case space: return true
+        default: return false
+    }
+}
+
 /** @type {(state: InvalidNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const invalidNumberStateOp = state => input =>
 {
@@ -301,9 +322,9 @@ const parseEscapeCharStateOp = state => input =>
 /** @type {(hex: number) => number|undefined} */
 const hexDigitToNumber = hex =>
 {
-    if (hex >= digit0 && hex <= digit9) { return hex - digit0 }
-    if (hex >= capitalLetterA && hex <= capitalLetterF) { return hex - capitalLetterA + 10 }
-    if (hex >= letterA && hex <= letterF) { return hex - letterA + 10 }
+    if (containsDigit(hex)) { return hex - digit0 }
+    if (containsCapitalAF(hex)) { return hex - capitalLetterA + 10 }
+    if (containsSmallAF(hex)) { return hex - letterA + 10 }
 }
 
 /** @type {(state: ParseUnicodeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
@@ -336,7 +357,7 @@ const stringToKeywordToken = s =>
 /** @type {(state: ParseKeywordState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const parseKeyWordStateOp = state => input =>
 {
-    if (input >= letterA && input <= letterZ)
+    if (containsSmallLetter(input))
     {
         return [undefined, {kind: 'keyword', value: appendChar(state.value)(input)}]
     }
@@ -390,10 +411,12 @@ const tokenizeEofOp = state => {
 /** @type {operator.StateScan<CharCodeOrEof, TokenizerState, list.List<JsonToken>>} */
 const tokenizeOp = state => input => input === undefined ? tokenizeEofOp(state) : tokenizeCharCodeOp(state)(input)
 
-/** @type {(input: list.List<CharCodeOrEof>) => list.List<JsonToken>} */
-const tokenize = input => list.flat(list.stateScan(tokenizeOp)({kind: 'initial'})(input))
+const initial = list.stateScan(tokenizeOp)({ kind: 'initial' })
+
+/** @type {(input: list.List<number>) => list.List<JsonToken>} */
+const tokenize = input => list.flat(initial(list.concat(/** @type {list.List<CharCodeOrEof>} */(input))([undefined])))
 
 module.exports = {
     /** @readonly */
-    tokenize,
+    tokenize
 }

package/json/tokenizer/test.js

@@ -3,23 +3,10 @@ const list = require('../../types/list/index.js')
 const json = require('../index.js')
 const { sort } = require('../../types/object/index.js')
 
-/** @type {(s: string) => list.List<tokenizer.CharCodeOrEof>} */
-const toCharacters = s =>
-{
-    /** @type {list.List<tokenizer.CharCodeOrEof>} */
-    const charCodes = list.toCharCodes(s)
-    return list.concat(charCodes)([undefined])
-}
-
 /** @type {(s: string) => readonly tokenizer.JsonToken[]} */
-const tokenizeString = s =>
-{
-    const characters = toCharacters(s)
-    return list.toArray(tokenizer.tokenize(characters))
-}
+const tokenizeString = s => list.toArray(tokenizer.tokenize(list.toCharCodes(s)))
 
-/** @type {(a: readonly tokenizer.JsonToken[]) => string} */
-const stringify = a => json.stringify(sort)(a)
+const stringify = json.stringify(sort)
 
 {
     const result = tokenizeString('')
@@ -201,6 +188,16 @@ const stringify = a => json.stringify(sort)(a)
     if (result !== '[{"kind":"{"},{"kind":"number","value":"90"},{"kind":"}"}]') { throw result }
 }
 
+{
+    const result = stringify(tokenizeString('1 2'))
+    if (result !== '[{"kind":"number","value":"1"},{"kind":"number","value":"2"}]') { throw result }
+}
+
+{
+    const result = stringify(tokenizeString('0. 2'))
+    if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"number","value":"2"}]') { throw result }
+}
+
 {
     const result = stringify(tokenizeString('10-0'))
     if (result !== '[{"kind":"error","message":"invalid number"}]') { throw result }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "functionalscript",
-  "version": "0.0.336",
+  "version": "0.0.339",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "index.js",
   "scripts": {