functionalscript 0.6.8 → 0.6.10

package/djs/parser/test.f.js

@@ -1,10 +1,12 @@
 import * as parser from "./module.f.js";
 import * as tokenizer from "../tokenizer/module.f.js";
-import { toArray } from "../../types/list/module.f.js";
+import * as list from "../../types/list/module.f.js";
+const { toArray } = list;
 import { sort } from "../../types/object/module.f.js";
 import * as encoding from "../../text/utf16/module.f.js";
 import { stringifyAsTree } from "../serializer/module.f.js";
-const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s)));
+import { stringify } from "../../json/module.f.js";
+const tokenizeString = s => toArray(tokenizer.tokenize(encoding.stringToList(s))(''));
 const stringifyDjsModule = stringifyAsTree(sort);
 export default {
     valid: [
@@ -173,6 +175,17 @@ export default {
                 throw result;
             }
         },
+        () => {
+            const tokenList = tokenizeString('export default {a: 1}');
+            const obj = parser.parseFromTokens(tokenList);
+            if (obj[0] !== 'ok') {
+                throw obj;
+            }
+            const result = stringifyDjsModule(obj[1]);
+            if (result !== '[[],[{"a":1}]]') {
+                throw result;
+            }
+        },
         () => {
             const tokenList = tokenizeString('export default 1234567890n');
             const obj = parser.parseFromTokens(tokenList);
@@ -194,6 +207,28 @@ export default {
             if (result !== '[[],[["array",[1234567890n]]]]') {
                 throw result;
             }
+        },
+        () => {
+            const tokenList = tokenizeString('export default [1,]');
+            const obj = parser.parseFromTokens(tokenList);
+            if (obj[0] !== 'ok') {
+                throw obj;
+            }
+            const result = stringifyDjsModule(obj[1]);
+            if (result !== '[[],[["array",[1]]]]') {
+                throw result;
+            }
+        },
+        () => {
+            const tokenList = tokenizeString('export default {"a":1,}');
+            const obj = parser.parseFromTokens(tokenList);
+            if (obj[0] !== 'ok') {
+                throw obj;
+            }
+            const result = stringifyDjsModule(obj[1]);
+            if (result !== '[[],[{"a":1}]]') {
+                throw result;
+            }
         }
     ],
     invalid: [
@@ -203,8 +238,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected end') {
-                throw obj;
+            if (obj[1].message !== 'unexpected end') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -213,8 +248,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -223,8 +258,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -233,8 +268,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -243,8 +278,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -253,8 +288,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -263,18 +298,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected end') {
-                throw obj;
-            }
-        },
-        () => {
-            const tokenList = tokenizeString('export default [1,]');
-            const obj = parser.parseFromTokens(tokenList);
-            if (obj[0] !== 'error') {
-                throw obj;
-            }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected end') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -283,8 +308,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -293,8 +318,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -303,8 +328,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -313,8 +338,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -323,8 +348,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -333,8 +358,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -343,8 +368,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -353,8 +378,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -363,8 +388,8 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
@@ -373,68 +398,71 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected end') {
-                throw obj;
+            if (obj[1].message !== 'unexpected end') {
+                throw obj[1].message;
             }
         },
         () => {
-            const tokenList = tokenizeString('export default {"1":2,}');
+            const tokenList = tokenizeString('export default {,"1":2}');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
-            const tokenList = tokenizeString('export default {,"1":2}');
+            const tokenList = tokenizeString('export default }');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
-            const tokenList = tokenizeString('export default }');
+            const tokenList = tokenizeString('export default [{]}');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
-            const tokenList = tokenizeString('export default [{]}');
+            const tokenList = tokenizeString('export default {[}]');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
         () => {
-            const tokenList = tokenizeString('export default {[}]');
+            const tokenList = tokenizeString('export default 10-5');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            if (obj[1].message !== 'unexpected token') {
+                throw obj[1].message;
             }
         },
+    ],
+    errorMetadata: [
         () => {
-            const tokenList = tokenizeString('export default 10-5');
+            const tokenList = tokenizeString('export default [,]');
             const obj = parser.parseFromTokens(tokenList);
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
-                throw obj;
+            const errorString = stringify(sort)(obj[1]);
+            if (errorString !== '{"message":"unexpected token","metadata":{"column":18,"line":1,"path":""}}') {
+                throw errorString;
             }
         },
     ],
@@ -537,7 +565,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'const not found') {
+            if (obj[1].message !== 'const not found') {
                 throw obj;
             }
         },
@@ -547,7 +575,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -557,7 +585,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -615,7 +643,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -625,7 +653,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -635,7 +663,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'duplicate id') {
+            if (obj[1].message !== 'duplicate id') {
                 throw obj;
             }
         },
@@ -682,7 +710,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -692,7 +720,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -702,7 +730,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -712,7 +740,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'unexpected token') {
+            if (obj[1].message !== 'unexpected token') {
                 throw obj;
             }
         },
@@ -722,7 +750,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'duplicate id') {
+            if (obj[1].message !== 'duplicate id') {
                 throw obj;
             }
         },
@@ -732,7 +760,7 @@ export default {
             if (obj[0] !== 'error') {
                 throw obj;
             }
-            if (obj[1] !== 'duplicate id') {
+            if (obj[1].message !== 'duplicate id') {
                 throw obj;
             }
         },

package/djs/tokenizer/module.f.d.ts

@@ -4,5 +4,9 @@ export type DjsToken = {
     readonly kind: 'true' | 'false' | 'null' | 'undefined';
 } | {
     readonly kind: '{' | '}' | ':' | ',' | '[' | ']' | '.' | '=';
-} | jsTokenizer.StringToken | jsTokenizer.NumberToken | jsTokenizer.ErrorToken | jsTokenizer.IdToken | jsTokenizer.BigIntToken | jsTokenizer.WhitespaceToken | jsTokenizer.NewLineToken | jsTokenizer.CommentToken;
-export declare const tokenize: (input: list.List<number>) => list.List<DjsToken>;
+} | jsTokenizer.StringToken | jsTokenizer.NumberToken | jsTokenizer.ErrorToken | jsTokenizer.IdToken | jsTokenizer.BigIntToken | jsTokenizer.WhitespaceToken | jsTokenizer.NewLineToken | jsTokenizer.CommentToken | jsTokenizer.EofToken;
+export type DjsTokenWithMetadata = {
+    readonly token: DjsToken;
+    readonly metadata: jsTokenizer.TokenMetadata;
+};
+export declare const tokenize: (input: list.List<number>) => (path: string) => list.List<DjsTokenWithMetadata>;

package/djs/tokenizer/module.f.js

@@ -25,22 +25,21 @@ const mapToken = input => {
         case 'undefined':
         case '//':
         case '/*':
+        case 'eof':
         case 'error': return [input];
         default: return jsTokenizer.isKeywordToken(input) ? [{ kind: 'id', value: input.kind }] : [{ kind: 'error', message: 'invalid token' }];
     }
 };
 const parseDefaultState = input => {
-    if (input === null)
-        return [empty, { kind: 'def' }];
     switch (input.kind) {
+        case 'eof': return [[{ kind: 'eof' }], { kind: 'def' }];
         case '-': return [empty, { kind: '-' }];
         default: return [mapToken(input), { kind: 'def' }];
     }
 };
 const parseMinusState = input => {
-    if (input === null)
-        return [[{ kind: 'error', message: 'invalid token' }], { kind: 'def' }];
     switch (input.kind) {
+        case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }];
         case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }];
         case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }];
         case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }];
@@ -53,7 +52,13 @@ const scanToken = state => input => {
         default: return parseDefaultState(input);
     }
 };
-export const tokenize = input => {
-    const jsTokens = jsTokenizer.tokenize(input);
-    return flat(stateScan(scanToken)({ kind: 'def' })(list.concat(jsTokens)([null])));
+const mapTokenWithMetadata = metadata => token => { return { token, metadata }; };
+const scanTokenWithMetadata = state => (input) => {
+    const [djsTokens, newState] = scanToken(state)(input.token);
+    const djsTokensWithMetadata = list.map(mapTokenWithMetadata(input.metadata))(djsTokens);
+    return [djsTokensWithMetadata, newState];
+};
+export const tokenize = input => path => {
+    const jsTokens = jsTokenizer.tokenize(input)(path);
+    return flat(stateScan(scanTokenWithMetadata)({ kind: 'def' })(jsTokens));
 };

package/djs/tokenizer/test.f.d.ts

@@ -3,5 +3,6 @@ declare const _default: {
     id: (() => void)[];
     keywords: (() => void)[];
     comments: (() => void)[];
+    metadata: (() => void)[];
 };
 export default _default;