functionalscript 0.0.326 → 0.0.330

package/json/tokenizer/index.js

@@ -3,53 +3,29 @@ const operator = require('../../types/function/operator')
 const { concat } = require('../../types/list')
 const list = require('../../types/list')
 
-/** @typedef {{readonly kind: '{'}} LeftBraceToken */
-
-/** @typedef {{readonly kind: '}'}} RightBraceToken */
-
-/** @typedef {{readonly kind: ':'}} ColonToken */
-
-/** @typedef {{readonly kind: ','}} CommaToken */
-
-/** @typedef {{readonly kind: '['}} LeftBracketToken */
-
-/** @typedef {{readonly kind: ']'}} RightBracketToken */
-
-/** 
+/**
  * @typedef {{
  * readonly kind: 'string'
  * readonly value: string
- * }} StringToken 
+ * }} StringToken
  * */
 
-/** 
+/**
  * @typedef {{
  * readonly kind: 'number'
  * readonly value: string
- * }} NumberToken 
+ * }} NumberToken
  * */
 
-/** @typedef {{readonly kind: 'true'}} TrueToken */
-
-/** @typedef {{readonly kind: 'false'}} FalseToken */
-
-/** @typedef {{readonly kind: 'null'}} NullToken */
-
 /** @typedef {{readonly kind: 'error', message: ErrorMessage}} ErrorToken */
 
-/** 
+/** @typedef {{readonly kind: '{' | '}' | ':' | ',' | '[' | ']' | 'true' | 'false' | 'null'}} SimpleToken */
+
+/**
  * @typedef {|
- * LeftBraceToken |
- * RightBraceToken |
- * ColonToken |
- * CommaToken |
- * LeftBracketToken |
- * RightBracketToken |
+ * SimpleToken |
  * StringToken |
  * NumberToken |
- * TrueToken |
- * FalseToken |
- * NullToken |
  * ErrorToken
  * } JsonToken
  */
@@ -74,7 +50,7 @@ const newLine = 0x0a
 const carriageReturn = 0x0d
 const space = 0x20
 
-const backslach = 0x5c
+const backslash = 0x5c
 const slash = 0x2f
 const backspace = 0x08
 const formfeed = 0x0c
@@ -93,7 +69,7 @@ const letterT = 0x74
 const letterU = 0x75
 const letterZ = 0x7a
 
-/** 
+/**
  * @typedef {|
  * InitialState |
  * ParseKeywordState |
@@ -103,10 +79,10 @@ const letterZ = 0x7a
  * ParseNumberState |
  * InvalidNumberState |
  * EofState
- * } TokenizerState 
+ * } TokenizerState
  */
 
-/** 
+/**
  * @typedef {|
  * 'invalid keyword' |
  * '" are missing' |
@@ -115,7 +91,7 @@ const letterZ = 0x7a
  * 'unexpected character' |
  * 'invalid number' |
  * 'eof'
- * } ErrorMessage 
+ * } ErrorMessage
  */
 
 /** @typedef {{ readonly kind: 'initial'}} InitialState */
@@ -129,7 +105,7 @@ const letterZ = 0x7a
 /** @typedef {{ readonly kind: 'unicodeChar', readonly value: string, readonly unicode: number, readonly hexIndex: number}} ParseUnicodeCharState */
 
 /**
- *  @typedef {{ 
+ *  @typedef {{
  * readonly kind: 'number',
  * readonly numberKind: '0' | '-' | 'int' | '.' | 'fractional' | 'e' | 'e+' | 'e-' | 'expDigits'
  * readonly value: string
@@ -140,21 +116,17 @@ const letterZ = 0x7a
 
 /** @typedef {{ readonly kind: 'eof'}} EofState */
 
-/** @typedef {number|undefined} JsonCharacter */
+/** @typedef {number|undefined} CharCodeOrEof */
 
-/** @type {(old: string) => (input: JsonCharacter) => string} */
+/** @type {(old: string) => (input: CharCodeOrEof) => string} */
 const appendChar = old => input => input === undefined ? old : operator.concat(charToString(input))(old)
 
-/** @type {(input: JsonCharacter) => string} */
+/** @type {(input: CharCodeOrEof) => string} */
 const charToString = input => input === undefined ? '' : String.fromCharCode(input)
 
-/** @type {(state: InitialState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
-const initialStateOp = initialState => input => 
+/** @type {(state: InitialState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
+const initialStateOp = initialState => input =>
 {
-    if (input === undefined) 
-    {
-         return[undefined, {kind: 'eof'}] 
-    }
     if (input >= digit1 && input <= digit9)
     {
         return [undefined, { kind: 'number', value: charToString(input), numberKind: 'int'}]
@@ -182,21 +154,9 @@ const initialStateOp = initialState => input =>
     }
 }
 
-/** @type {(state: ParseNumberState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: ParseNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const parseNumberStateOp = state => input =>
 {
-    if (input === undefined)
-    {
-        switch (state.numberKind)
-        {
-            case '-':
-            case '.':
-            case 'e':
-            case 'e+':
-            case 'e-': return [[{kind: 'error', message: 'invalid number'}], {kind: 'invalidNumber', }] 
-            default: return [[{kind: 'number', value: state.value}], {kind: 'eof'}]
-        }
-    }
     if (input === decimalPoint)
     {
         switch (state.numberKind)
@@ -266,12 +226,12 @@ const parseNumberStateOp = state => input =>
             case '.':
             case 'e':
             case 'e+':
-            case 'e-': 
+            case 'e-':
             {
                 const next = tokenizeOp({kind: 'initial'})(input)
                 return [{first: {kind: 'error', message: 'invalid number'}, tail: next[0]}, next[1]]
             }
-            default: 
+            default:
             {
                 const next = tokenizeOp({kind: 'initial'})(input)
                 return [{first: {kind: 'number', value: state.value}, tail: next[0]}, next[1]]
@@ -297,14 +257,10 @@ const isTerminalForNumber = char =>
     }
 }
 
-/** @type {(state: InvalidNumberState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: InvalidNumberState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const invalidNumberStateOp = state => input =>
 {
-    if (input === undefined)
-    {
-        return [[{kind: 'error', message: 'invalid number'}], {kind: 'eof'}]
-    }
-    if (isTerminalForNumber(input)) 
+    if (isTerminalForNumber(input))
     {
         const next = tokenizeOp({kind: 'initial'})(input)
         return [{first: {kind: 'error', message: 'invalid number'}, tail: next[0]}, next[1]]
@@ -312,25 +268,24 @@ const invalidNumberStateOp = state => input =>
     return [undefined, {kind: 'invalidNumber'}]
 }
 
-/** @type {(state: ParseStringState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: ParseStringState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const parseStringStateOp = state => input =>
 {
     switch(input)
     {
         case quotationMark: return[[{kind: 'string', value: state.value}], {kind: 'initial'}]
-        case backslach: return [undefined, {kind:'escapeChar', value: state.value}]
-        case undefined: return [[{kind: 'error', message: '" are missing'}], {kind: 'eof'}]
+        case backslash: return [undefined, {kind:'escapeChar', value: state.value}]
         default: return [undefined, {kind:'string', value: appendChar(state.value)(input)}]
     }
 }
 
-/** @type {(state: ParseEscapeCharState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: ParseEscapeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const parseEscapeCharStateOp = state => input =>
 {
     switch(input)
     {
         case quotationMark:
-        case backslach:
+        case backslash:
         case slash: return [undefined, {kind: 'string', value: appendChar(state.value)(input)}]
         case letterB: return [undefined, {kind: 'string', value: appendChar(state.value)(backspace)}]
         case letterF: return [undefined, {kind: 'string', value: appendChar(state.value)(formfeed)}]
@@ -338,7 +293,6 @@ const parseEscapeCharStateOp = state => input =>
         case letterR: return [undefined, {kind: 'string', value: appendChar(state.value)(carriageReturn)}]
         case letterT: return [undefined, {kind: 'string', value: appendChar(state.value)(horizontalTab)}]
         case letterU: return [undefined, {kind: 'unicodeChar', value: state.value, unicode: 0, hexIndex: 0}]
-        case undefined: return [[{kind: 'error', message: '" are missing'}], {kind: 'eof'}]
         default: {
             const next = tokenizeOp({kind: 'string', value: state.value})(input)
             return [{first: {kind: 'error', message: 'unescaped character'}, tail: next[0]}, next[1]]
@@ -354,13 +308,9 @@ const hexDigitToNumber = hex =>
     if (hex >= letterA && hex <= letterF) { return hex - letterA + 10 }
 }
 
-/** @type {(state: ParseUnicodeCharState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: ParseUnicodeCharState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const parseUnicodeCharStateOp = state => input =>
 {
-    if (input === undefined)
-    {
-        return [[{kind: 'error', message: '" are missing'}], {kind: 'eof'}]
-    }
     const hexValue = hexDigitToNumber(input)
     if (hexValue === undefined)
     {
@@ -385,14 +335,9 @@ const stringToKeywordToken = s =>
     }
 }
 
-/** @type {(state: ParseKeywordState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
-const parseKeyWordStateOp = state => input => 
+/** @type {(state: ParseKeywordState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
+const parseKeyWordStateOp = state => input =>
 {
-    if (input === undefined)
-    {
-        const keyWordToken = stringToKeywordToken(state.value)
-        return [[keyWordToken], {kind: 'eof'}]
-    }
     if (input >= letterA && input <= letterZ)
     {
         return [undefined, {kind: 'keyword', value: appendChar(state.value)(input)}]
@@ -402,26 +347,52 @@ const parseKeyWordStateOp = state => input =>
     return [{first: keyWordToken, tail: next[0]}, next[1]]
 }
 
-/** @type {(state: EofState) => (input: JsonCharacter) => readonly[list.List<JsonToken>, TokenizerState]} */
+/** @type {(state: EofState) => (input: number) => readonly[list.List<JsonToken>, TokenizerState]} */
 const eofStateOp = state => input => [[{kind: 'error', message: 'eof'}], state]
 
-/** @type {operator.StateScan<JsonCharacter, TokenizerState, list.List<JsonToken>>} */
-const tokenizeOp = state => input =>
-{
+/** @type {operator.StateScan<number, TokenizerState, list.List<JsonToken>>} */
+const tokenizeCharCodeOp = state => {
+    switch(state.kind)
+    {
+        case 'initial': return initialStateOp(state)
+        case 'keyword': return parseKeyWordStateOp(state)
+        case 'string': return parseStringStateOp(state)
+        case 'escapeChar': return parseEscapeCharStateOp(state)
+        case 'unicodeChar': return parseUnicodeCharStateOp(state)
+        case 'invalidNumber': return invalidNumberStateOp(state)
+        case 'number': return parseNumberStateOp(state)
+        case 'eof': return eofStateOp(state)
+    }
+}
+
+/** @type {(state: TokenizerState) => readonly[list.List<JsonToken>, TokenizerState]} */
+const tokenizeEofOp = state => {
     switch(state.kind)
     {
-        case 'initial': return initialStateOp(state)(input)
-        case 'keyword': return parseKeyWordStateOp(state)(input)
-        case 'string': return parseStringStateOp(state)(input)
-        case 'escapeChar': return parseEscapeCharStateOp(state)(input)
-        case 'unicodeChar': return parseUnicodeCharStateOp(state)(input)
-        case 'invalidNumber': return invalidNumberStateOp(state)(input)
-        case 'number' : return parseNumberStateOp(state)(input)
-        case 'eof': return eofStateOp(state)(input)
+        case 'initial': return[undefined, {kind: 'eof'}]
+        case 'keyword': return [[stringToKeywordToken(state.value)], {kind: 'eof'}]
+        case 'string':
+        case 'escapeChar':
+        case 'unicodeChar': return [[{kind: 'error', message: '" are missing'}], {kind: 'eof'}]
+        case 'invalidNumber': return [[{kind: 'error', message: 'invalid number'}], {kind: 'eof'}]
+        case 'number':
+            switch (state.numberKind)
+            {
+                case '-':
+                case '.':
+                case 'e':
+                case 'e+':
+                case 'e-': return [[{kind: 'error', message: 'invalid number'}], {kind: 'invalidNumber', }]
+                default: return [[{kind: 'number', value: state.value}], {kind: 'eof'}]
+            }
+        case 'eof': return [[{kind: 'error', message: 'eof'}], state]
     }
 }
 
-/** @type {(input: list.List<JsonCharacter>) => list.List<JsonToken>} */
+/** @type {operator.StateScan<CharCodeOrEof, TokenizerState, list.List<JsonToken>>} */
+const tokenizeOp = state => input => input === undefined ? tokenizeEofOp(state) : tokenizeCharCodeOp(state)(input)
+
+/** @type {(input: list.List<CharCodeOrEof>) => list.List<JsonToken>} */
 const tokenize = input => list.flat(list.stateScan(tokenizeOp)({kind: 'initial'})(input))
 
 module.exports = {

package/json/tokenizer/test.js

@@ -3,17 +3,17 @@ const list = require('../../types/list')
 const json = require('..')
 const { sort } = require('../../types/object')
 
-/** @type {(s: string) => list.List<tokenizer.JsonCharacter>} */
+/** @type {(s: string) => list.List<tokenizer.CharCodeOrEof>} */
 const toCharacters = s =>
 {
-    /** @type {list.List<tokenizer.JsonCharacter>} */   
+    /** @type {list.List<tokenizer.CharCodeOrEof>} */
     const charCodes = list.toCharCodes(s)
     return list.concat(charCodes)([undefined])
-} 
+}
 
 /** @type {(s: string) => readonly tokenizer.JsonToken[]} */
 const tokenizeString = s =>
-{   
+{
     const characters = toCharacters(s)
     return list.toArray(tokenizer.tokenize(characters))
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "functionalscript",
-  "version": "0.0.326",
+  "version": "0.0.330",
   "description": "FunctionalScript is a functional subset of JavaScript",
   "main": "index.js",
   "scripts": {