funcity 0.3.0 → 0.4.0

package/dist/index.cjs

@@ -302,16 +302,8 @@ const tokenizeIdentity = (context) => {
     range: { start, end: context.cursor.location("end") }
   };
 };
-const tokenizeCodeBlock = (context) => {
-  const openStart = context.cursor.location("start");
-  context.cursor.skip(2);
-  const tokens = [
-    {
-      kind: "open",
-      symbol: "{{",
-      range: { start: openStart, end: context.cursor.location("end") }
-    }
-  ];
+const tokenizeCodeTokens = (context, stopOnClose, finalizeUnknownOnEot) => {
+  const tokens = [];
   let unknownStartLocation;
   const finalizeUnknown = () => {
     if (unknownStartLocation) {
@@ -327,20 +319,21 @@ const tokenizeCodeBlock = (context) => {
     }
   };
   while (!context.cursor.eot()) {
-    if (context.cursor.assert("}}")) {
+    if (stopOnClose && context.cursor.assert("}}")) {
       finalizeUnknown();
-      const location = context.cursor.location("start");
-      context.cursor.skip(2);
-      tokens.push({
-        kind: "close",
-        symbol: "}}",
-        range: { start: location, end: context.cursor.location("end") }
-      });
-      return tokens;
+      return { tokens, closed: true };
     }
     const ch = context.cursor.getChar();
     if (ch === "\\") {
       const next = context.cursor.getChar(1);
+      if (next === "\n") {
+        context.cursor.skip(2);
+        continue;
+      }
+      if (next === "\r" && context.cursor.getChar(2) === "\n") {
+        context.cursor.skip(3);
+        continue;
+      }
       if (next === "{" || next === "}") {
         context.cursor.skip(2);
         continue;
@@ -405,12 +398,41 @@ const tokenizeCodeBlock = (context) => {
     } else if (ch === " ") {
       finalizeUnknown();
       context.cursor.skip(1);
-    } else if (!unknownStartLocation) {
-      unknownStartLocation = context.cursor.location("start");
+    } else {
+      if (!unknownStartLocation) {
+        unknownStartLocation = context.cursor.location("start");
+      }
       context.cursor.skip(1);
     }
     context.cursor.skipChars(" ");
   }
+  if (finalizeUnknownOnEot) {
+    finalizeUnknown();
+  }
+  return { tokens, closed: false };
+};
+const tokenizeCodeBlock = (context) => {
+  const openStart = context.cursor.location("start");
+  context.cursor.skip(2);
+  const tokens = [
+    {
+      kind: "open",
+      symbol: "{{",
+      range: { start: openStart, end: context.cursor.location("end") }
+    }
+  ];
+  const result = tokenizeCodeTokens(context, true, false);
+  tokens.push(...result.tokens);
+  if (result.closed) {
+    const location = context.cursor.location("start");
+    context.cursor.skip(2);
+    tokens.push({
+      kind: "close",
+      symbol: "}}",
+      range: { start: location, end: context.cursor.location("end") }
+    });
+    return tokens;
+  }
   const causeLocation = context.cursor.location("start");
   context.errors.push({
     type: "error",
@@ -435,9 +457,12 @@ const createTokenizerCursor = (script) => {
       const ch = script[currentIndex];
       if (ch === "\r") {
         rawColumn = 0;
-      } else if (ch === "\n" && lastch !== "\r") {
-        rawColumn = 0;
         rawLine++;
+      } else if (ch === "\n") {
+        rawColumn = 0;
+        if (lastch !== "\r") {
+          rawLine++;
+        }
       } else {
         rawColumn++;
       }
@@ -514,6 +539,14 @@ const createTokenizerCursor = (script) => {
     location
   };
 };
+const runCodeTokenizer = (script, errors) => {
+  const context = {
+    cursor: createTokenizerCursor(script),
+    errors
+  };
+  const result = tokenizeCodeTokens(context, false, true);
+  return result.tokens;
+};
 const runTokenizer = (script, errors) => {
   const context = {
     cursor: createTokenizerCursor(script),
@@ -645,6 +678,9 @@ const parseLambdaExpression = (cursor, errors) => {
 };
 const parsePartialExpression = (cursor, errors) => {
   const token = cursor.peekToken();
+  if (!token) {
+    return void 0;
+  }
   switch (token.kind) {
     case "number": {
       const node = parseNumber(cursor);
@@ -1029,14 +1065,15 @@ const extractParameterArguments = (namesNode, errors) => {
     }
   }
 };
-const parseBlock = (cursor, errors) => {
+const parseBlockCore = (cursor, errors, mode) => {
   const rootState = {
     kind: "root",
     startRange: emptyRange,
     branch: createBranchState()
   };
   const statementStates = [rootState];
-  let isInExpressionBlock = false;
+  const isCodeMode = mode === "code";
+  let isInExpressionBlock = isCodeMode;
   while (true) {
     const token = drainEndOfLineAndPeek(cursor);
     if (!token) {
@@ -1063,14 +1100,16 @@ const parseBlock = (cursor, errors) => {
       case "open": {
         if (token.symbol === "{{") {
           cursor.skipToken();
-          if (isInExpressionBlock) {
-            errors.push({
-              type: "error",
-              description: `Already opened expression block`,
-              range: token.range
-            });
+          if (!isCodeMode) {
+            if (isInExpressionBlock) {
+              errors.push({
+                type: "error",
+                description: `Already opened expression block`,
+                range: token.range
+              });
+            }
+            isInExpressionBlock = true;
           }
-          isInExpressionBlock = true;
         } else {
           const node = parseExpression(cursor, errors);
           if (node) {
@@ -1081,6 +1120,10 @@ const parseBlock = (cursor, errors) => {
       }
       case "close": {
         cursor.skipToken();
+        if (token.symbol === "}}" && isCodeMode) {
+          flushCurrentBranch(statementStates);
+          break;
+        }
         if (!isInExpressionBlock) {
           errors.push({
             type: "error",
@@ -1089,6 +1132,14 @@ const parseBlock = (cursor, errors) => {
           });
           break;
         }
+        if (isCodeMode) {
+          errors.push({
+            type: "error",
+            description: `Mismatched close bracket`,
+            range: token.range
+          });
+          break;
+        }
         flushCurrentBranch(statementStates);
         isInExpressionBlock = false;
         break;
@@ -1335,6 +1386,9 @@ const parseBlock = (cursor, errors) => {
   }
   return rootState.branch.blocks;
 };
+const parseBlock = (cursor, errors) => {
+  return parseBlockCore(cursor, errors, "script");
+};
 const createParserCursor = (tokens) => {
   let index = 0;
   const peekToken = () => {
@@ -1360,6 +1414,10 @@ const createParserCursor = (tokens) => {
     skipToken
   };
 };
+const parseExpressions = (tokens, errors) => {
+  const cursor = createParserCursor(tokens);
+  return parseBlockCore(cursor, errors, "code");
+};
 const runParser = (tokens, errors) => {
   const cursor = createParserCursor(tokens);
   const blockNodes = parseBlock(cursor, errors);
@@ -2137,8 +2195,10 @@ exports.makeFunCityFunction = makeFunCityFunction;
 exports.outputErrors = outputErrors;
 exports.parseBlock = parseBlock;
 exports.parseExpression = parseExpression;
+exports.parseExpressions = parseExpressions;
 exports.reduceExpressionNode = reduceExpressionNode;
 exports.reduceNode = reduceNode;
+exports.runCodeTokenizer = runCodeTokenizer;
 exports.runParser = runParser;
 exports.runReducer = runReducer;
 exports.runScriptOnce = runScriptOnce;