@bikky/replication 1.0.1

package/Expressions/Compiler/Parser.js

@@ -0,0 +1,314 @@
+import { ExpressionGrammar } from "./ExpressionGrammar.js";
+import { getEASTTokenValues, isToken, TokenizeExpression, TokenType } from "./Tokenizer.js";
+import { GetVersionIndex } from "../../Constants/Versions.js";
+let debug = false;
+export function CheckRuleDone(grammar, currentRuleIndex) {
+    var _a, _b;
+    //Ran out of input to iterate over but still need to match more parts of this rule.
+    for (let remainingRule = currentRuleIndex; remainingRule < grammar.rules.length; remainingRule++) {
+        if (!grammar.rules[remainingRule].optional || ((_b = (_a = grammar.rules[remainingRule].repeat) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : 1) > 0) {
+            return false;
+        }
+    }
+    //Ran out of input but finished matching or everything else is optional.
+    return true;
+}
+export function IsRuleSkippable(rule, previousRuleMatched) {
+    let skippable = !!rule.optional || (rule.repeat && rule.repeat[0] == 0);
+    //This must exist if the previous rule existed but must not match if the previous rule didn't exist.
+    if (rule.optional == "previous") {
+        if (!previousRuleMatched) {
+            return true;
+        }
+        else {
+            skippable = false;
+        }
+    }
+    return !!skippable;
+}
+export function ComputeValidTypes(rule, extraTypes) {
+    let validTypes = Array.isArray(rule.type) ? [...rule.type] : [rule.type];
+    if (validTypes.includes(ExpressionGrammar.SpecialType.INPUT_TYPES)) {
+        console.debug(`* adding ${extraTypes.map((e) => ExpressionGrammar.NodeType[e])} to ${ExpressionGrammar.NodeType[rule.type]}`);
+        validTypes.splice(validTypes.indexOf(ExpressionGrammar.SpecialType.INPUT_TYPES), 1, ...extraTypes);
+    }
+    return validTypes;
+}
+export function ComputeChildInputs(rule, extraTypes) {
+    var _a;
+    let childInputs = (_a = rule.input) !== null && _a !== void 0 ? _a : [];
+    if (childInputs.includes(ExpressionGrammar.SpecialType.INPUT_TYPES)) {
+        console.debug(`** sending ${extraTypes.map((e) => ExpressionGrammar.NodeType[e])} to child`);
+        childInputs.splice(childInputs.indexOf(ExpressionGrammar.SpecialType.INPUT_TYPES), 1, ...extraTypes);
+    }
+    return childInputs;
+}
+export function TokenDataMatchesRule(version, gram, token) {
+    var _a;
+    if (gram.value) {
+        return gram.value == token.data;
+    }
+    if (gram.values) {
+        if (Array.isArray(gram.values)) {
+            return gram.values.includes(token.data);
+        }
+        else {
+            let vals = getEASTTokenValues(version, gram.values);
+            return vals.has(token.data);
+        }
+    }
+    return !((_a = gram.excludeValues) === null || _a === void 0 ? void 0 : _a.includes(token.data));
+}
+export function PrintNodeName(value) {
+    if (isToken(value)) {
+        return TokenType[value.type];
+    }
+    return ExpressionGrammar.NodeType[value.type];
+}
+export function PrintTokenText(token) {
+    if (token.data.length > 20) {
+        return token.data.slice(0, 20) + "...";
+    }
+    return token.data;
+}
+export function MakeNewNodeFrom(details, value) {
+    //If it's a token, make a copy and add a file reference to convert it to a node.
+    if (isToken(value)) {
+        return Object.assign(Object.assign({}, value), { file: details.parser.file });
+    }
+    //Otherwise add it as-is.
+    else {
+        return value;
+    }
+}
+//Sets the node's values object to equal the values specified by the rule.
+export function AddToPartialEASTNode(details, rule, progress, value) {
+    //Set the values to a particular name.
+    if (rule.name) {
+        console.debug(`${details.debugTabs}Adding ${PrintNodeName(value)} ${rule.repeat ? "to" : "as"} ${rule.name}`);
+        //Repeated rules are stored as arrays
+        if (rule.repeat) {
+            if (rule.unroll) {
+                throw new Error(`Can't unroll repeated grammar rule.`);
+            }
+            //Create the array if it doesn't exist.
+            if (!progress.values[rule.name]) {
+                progress.values[rule.name] = [];
+            }
+            //Ensure the array is an array.
+            if (!Array.isArray(progress.values[rule.name])) {
+                throw new Error(`Node ${PrintNodeName(progress)}'s ${rule.name} should be an array but isn't.`);
+            }
+            progress.values[rule.name]
+                .push(MakeNewNodeFrom(details, value));
+        }
+        else {
+            progress.values[rule.name] = MakeNewNodeFrom(details, value);
+        }
+    }
+    else {
+        if (rule.unroll) {
+            if (!("values" in value)) {
+                throw new Error("Can't unroll tokens.");
+            }
+            console.debug(`${details.debugTabs} - Unrolling ${Object.keys(value.values)} to `
+                + `${PrintNodeName(progress)}'s ${Object.keys(progress.values)}`);
+            progress.values = Object.assign(Object.assign({}, value.values), progress.values);
+        }
+        // Skip any rule that isn't given a name or unroll, since it's not meant to be stored.
+    }
+    //Only increment token position if token.
+    if (isToken(value)) {
+        details.parser.consumeToken();
+    }
+}
+export function MatchRule(details, rule, progress, historicalMatches) {
+    var _a, _b;
+    let validTypes = ComputeValidTypes(rule, details.extraTypes);
+    let childInputs = ComputeChildInputs(rule, details.extraTypes);
+    for (let type of validTypes) {
+        //Check if type is the current valid type.
+        if (type == details.parser.token.type) {
+            console.debug(`${details.debugTabs}  - Matched a ${TokenType[type]} ${PrintTokenText(details.parser.token)}`);
+            //Token type is correct but value is incorrect, so abort.
+            if (!TokenDataMatchesRule(details.parser.version, rule, details.parser.token)) {
+                console.debug(`${details.debugTabs}  - Token data ${PrintTokenText(details.parser.token)} is undesirable, skipping type.`);
+                continue;
+            }
+            AddToPartialEASTNode(details, rule, progress, details.parser.token);
+            return progress;
+        }
+        //Or if we're currently looking at a compound node type, we'll see if we can create it!
+        else if (type in ExpressionGrammar.NodeType) {
+            console.debug(`${details.debugTabs} - Trying to make a ${ExpressionGrammar.NodeType[type]}`);
+            //Try make a node.
+            //Run through all of the rules of a possible compound rule.
+            //If the rules match, then make a new node.
+            let newNode = MatchGrammar(Object.assign(Object.assign({}, details), { extraTypes: childInputs }), type, historicalMatches);
+            //Node succeeded, continue.
+            if (newNode) {
+                console.debug(`${details.debugTabs}Made a ${ExpressionGrammar.NodeType[type]}`);
+                AddToPartialEASTNode(details, rule, progress, newNode);
+                return progress;
+            }
+            //Node failed, try the next type.
+            else {
+                continue;
+            }
+        }
+    }
+    console.debug(`${details.debugTabs}Failed to create a ${PrintNodeName(progress)}, '${details.parser.token.data}' != '${(_b = (_a = rule.debug_name) !== null && _a !== void 0 ? _a : rule.value) !== null && _b !== void 0 ? _b : rule.values}'`);
+    return null;
+}
+export function MatchRepeatingRule(details, rule, progress, historicalMatches) {
+    var _a, _b, _c, _d;
+    let ruleCurrentRepeatNumber = 0;
+    let continueRepeating = true;
+    do {
+        //For each type that may be next, there is an early exit if repeating but no more tokens.
+        let valid = details.parser.done ? null : MatchRule(details, rule, progress, historicalMatches);
+        if (valid == null) {
+            //Repeat too much, so force continue.
+            if (ruleCurrentRepeatNumber + 1 >= rule.repeat[1]) {
+                return progress;
+            }
+            //Didn't repeat enough, so die in a fire!
+            else if (ruleCurrentRepeatNumber < rule.repeat[0]) {
+                console.debug(`${details.debugTabs}Repeated ${ruleCurrentRepeatNumber} times, which is less than ${rule.repeat[0]}!`);
+                if (!details.parser.done) {
+                    console.debug(`${details.debugTabs}Failed to create a repeating`
+                        + ` ${PrintNodeName(progress)}, '${details.parser.token.data}' != '${(_b = (_a = rule.debug_name) !== null && _a !== void 0 ? _a : rule.value) !== null && _b !== void 0 ? _b : rule.values}'`);
+                }
+                else {
+                    console.debug(`${details.debugTabs}Ran out of tokens to repeat ${(_d = (_c = rule.debug_name) !== null && _c !== void 0 ? _c : rule.value) !== null && _d !== void 0 ? _d : rule.values}`);
+                }
+                return null;
+            }
+            //Repeated enough, but didn't match.
+            else {
+                return progress;
+            }
+        }
+        //Consume repeat token if present!
+        if (rule.repeatSeparator) {
+            if (rule.repeatSeparator.optional == "previous") {
+                throw new Error(`Repeat separator's optionality can't be based on previous rule.`);
+            }
+            if (details.parser.token.type == rule.repeatSeparator.type
+                && TokenDataMatchesRule(details.parser.version, rule.repeatSeparator, details.parser.token)) {
+                console.debug(`${details.debugTabs}Skipping ${PrintNodeName(details.parser.token)} ${details.parser.token.data}`);
+                details.parser.consumeToken();
+            }
+            else {
+                continueRepeating = !!rule.repeatSeparator.optional;
+            }
+        }
+        ruleCurrentRepeatNumber++;
+    } while (continueRepeating);
+    return progress;
+}
+export function MatchGrammar(details, grammarType, historicalMatches) {
+    let grammar = ExpressionGrammar.getRule(details.parser.versionIndex, grammarType);
+    let tokenIndexAtStartOfNode = details.parser.currentNodeIndex;
+    let previousRuleMatched = true;
+    let start = details.parser.token.position;
+    let column = details.parser.token.column;
+    let line = details.parser.token.line;
+    // We've hit a loop - we've already tried to evaluate this grammar type for this node.
+    // We should only need to do this for the first rule in a grammar, because it's fine for
+    // rules to be recursive after the first entry (e.g. having multiple parenthesised expressions
+    // starting with multiple '('s ).
+    if (historicalMatches.includes(grammar.ID)) {
+        return null;
+    }
+    let node = { type: grammarType, values: {}, debugName: PrintNodeName({ type: grammarType }) };
+    let childDetails = Object.assign(Object.assign({}, details), { debugTabs: details.debugTabs + "\t" });
+    for (let grammarRuleIndex = 0; grammarRuleIndex < grammar.rules.length; grammarRuleIndex++) {
+        if (details.parser.done) {
+            if (CheckRuleDone(grammar, grammarRuleIndex)) {
+                break;
+            }
+            else {
+                details.parser.resetToken(tokenIndexAtStartOfNode);
+                return null;
+            }
+        }
+        let rule = grammar.rules[grammarRuleIndex];
+        let ruleSkippable = IsRuleSkippable(rule, previousRuleMatched);
+        let ruleRepeat = !!rule.repeat;
+        let history = grammarRuleIndex == 0 ? [...historicalMatches, grammar.ID] : [];
+        let result = ruleRepeat ? MatchRepeatingRule(childDetails, rule, node, history) : MatchRule(childDetails, rule, node, history);
+        if (!result) {
+            if (ruleSkippable) {
+                continue;
+            }
+            else {
+                details.parser.resetToken(tokenIndexAtStartOfNode);
+                return null;
+            }
+        }
+    }
+    if (!node) {
+        details.parser.resetToken(tokenIndexAtStartOfNode);
+        return null;
+    }
+    let lastToken = details.parser.token ? details.parser.token : details.parser.tokens[details.parser.tokens.length - 1];
+    return Object.assign(Object.assign({}, node), { file: details.parser.file, position: start, version: details.parser.version, column: column, line: line, length: lastToken.position + lastToken.length - start });
+}
+export class ExpressionParser {
+    constructor(version, lines, sourceLocation) {
+        this.tokens = TokenizeExpression(lines, version);
+        this.versionIndex = GetVersionIndex(version);
+        this.version = version;
+        this.sourceLocation = sourceLocation;
+        this.currentNodeIndex = 0;
+        this.done = false;
+        this.token = this.tokens[0];
+        if (this.token.type == TokenType.WHITESPACE)
+            this.consumeToken();
+        this.file = {
+            text: lines,
+            sourceLocation: sourceLocation
+        };
+    }
+    consumeToken() {
+        var _a;
+        this.currentNodeIndex++;
+        while (((_a = this.tokens[this.currentNodeIndex]) === null || _a === void 0 ? void 0 : _a.type) == TokenType.WHITESPACE) {
+            this.currentNodeIndex++;
+        }
+        this.token = this.tokens[this.currentNodeIndex];
+        if (this.token) {
+            debug &&
+                console.log(`[New token is ${this.currentNodeIndex}: ${TokenType[this.token.type]} '${this.token.data.slice(0, Math.min(this.token.data.length, 20))}']`);
+        }
+        this.done = this.currentNodeIndex >= this.tokens.length;
+    }
+    //We reset if we fail to generate a node.
+    resetToken(index) {
+        this.currentNodeIndex = index;
+        this.token = this.tokens[this.currentNodeIndex];
+        console.debug(`[Reset token to ${this.currentNodeIndex}: ${TokenType[this.token.type]} '${this.token.data.slice(0, Math.min(this.token.data.length, 20))}']`);
+        this.done = this.currentNodeIndex >= this.tokens.length;
+    }
+    run() {
+        if (this.token.type == TokenType.WHITESPACE)
+            this.consumeToken();
+        //Start with a file :]
+        let node = MatchGrammar({
+            parser: this,
+            extraTypes: [],
+            debugTabs: ""
+        }, ExpressionGrammar.NodeType.FILE, []);
+        if (node == null)
+            return null;
+        let prop;
+        for (prop in node) {
+            this.file[prop] = node[prop];
+        }
+        return this.file;
+    }
+}
+export function CreateASTFromString(version, lines, sourceLocation) {
+    return new ExpressionParser(version, lines, sourceLocation).run();
+}

package/Expressions/Compiler/Tokenizer.d.ts

@@ -0,0 +1,52 @@
+export declare enum TokenType {
+    BLOCK_COMMENT = 0,
+    LINE_COMMENT = 1,
+    PREPROCESSOR = 2,
+    OPERATOR = 3,
+    INTEGER = 4,
+    FLOAT = 5,
+    VAR_TYPE = 6,
+    VAR_QUAL = 7,
+    IDENTIFIER = 8,
+    BUILTIN = 9,
+    KEYWORD = 10,
+    WHITESPACE = 11,
+    HEX = 12
+}
+export interface TokenCategory {
+    versions: [Set<string> | RegExp];
+    type: number;
+    debug_name: string;
+    precedence: number;
+}
+export declare enum Precedence {
+    Important = 0,
+    Block = 1,
+    Line = 2,
+    Intricate_Builtins = 4,
+    Standard_Builtins = 6,
+    Operators = 8,
+    User_Identifiers = 10,
+    LeastImportant = 11
+}
+export declare const EAST_Tokens: {
+    [category: string]: TokenCategory;
+};
+export interface Token {
+    type: TokenType;
+    data: string;
+    position: number;
+    length: number;
+    line: number;
+    column: number;
+}
+export declare function isToken(object: any): object is Token;
+export declare function getEASTTokenValues(fileVersion: string, category: TokenCategory): Set<string>;
+/**
+ *
+ * @param expression The whole string
+ * @param version The version of the expression language this was compiled against
+ * @param offset The offset from the start of the file that this expression occurred within.
+ * @constructor
+ */
+export declare function TokenizeExpression(expression: string, version: string, offset?: number): Token[];

package/Expressions/Compiler/Tokenizer.js

@@ -0,0 +1,222 @@
+import { GetVersionIndex } from "../../Constants/Versions.js";
+let debug = false;
+export var TokenType;
+(function (TokenType) {
+    TokenType[TokenType["BLOCK_COMMENT"] = 0] = "BLOCK_COMMENT";
+    TokenType[TokenType["LINE_COMMENT"] = 1] = "LINE_COMMENT";
+    TokenType[TokenType["PREPROCESSOR"] = 2] = "PREPROCESSOR";
+    TokenType[TokenType["OPERATOR"] = 3] = "OPERATOR";
+    TokenType[TokenType["INTEGER"] = 4] = "INTEGER";
+    TokenType[TokenType["FLOAT"] = 5] = "FLOAT";
+    TokenType[TokenType["VAR_TYPE"] = 6] = "VAR_TYPE";
+    TokenType[TokenType["VAR_QUAL"] = 7] = "VAR_QUAL";
+    TokenType[TokenType["IDENTIFIER"] = 8] = "IDENTIFIER";
+    TokenType[TokenType["BUILTIN"] = 9] = "BUILTIN";
+    TokenType[TokenType["KEYWORD"] = 10] = "KEYWORD";
+    TokenType[TokenType["WHITESPACE"] = 11] = "WHITESPACE";
+    TokenType[TokenType["HEX"] = 12] = "HEX";
+})(TokenType || (TokenType = {}));
+export var Precedence;
+(function (Precedence) {
+    Precedence[Precedence["Important"] = 0] = "Important";
+    Precedence[Precedence["Block"] = 1] = "Block";
+    Precedence[Precedence["Line"] = 2] = "Line";
+    Precedence[Precedence["Intricate_Builtins"] = 4] = "Intricate_Builtins";
+    Precedence[Precedence["Standard_Builtins"] = 6] = "Standard_Builtins";
+    Precedence[Precedence["Operators"] = 8] = "Operators";
+    Precedence[Precedence["User_Identifiers"] = 10] = "User_Identifiers";
+    Precedence[Precedence["LeastImportant"] = 11] = "LeastImportant";
+})(Precedence || (Precedence = {}));
+export const EAST_Tokens = {
+    Builtin_BlockComment: {
+        type: TokenType.BLOCK_COMMENT,
+        versions: [/\/\*(.|\n|\r)*?\*\//g],
+        debug_name: "Block Comment",
+        precedence: Precedence.Block
+    },
+    Builtin_LineComment: {
+        type: TokenType.LINE_COMMENT,
+        versions: [/\/\/.*/g],
+        debug_name: "Line Comment",
+        precedence: Precedence.Line
+    },
+    Builtin_Preprocessor: {
+        type: TokenType.PREPROCESSOR,
+        versions: [/#.*/gm],
+        debug_name: "Preprocessor Directive",
+        precedence: Precedence.Line
+    },
+    //Precedence 3 so that it is a lower priority than number (because of '.').
+    Builtin_Hex: {
+        type: TokenType.HEX,
+        versions: [/\b0x[0-9abcdef]+\b/g],
+        debug_name: "Hex Number",
+        precedence: Precedence.Intricate_Builtins
+    },
+    //Precedence 2 so that it is a lower priority than float and hex.
+    Builtin_Integer: {
+        type: TokenType.INTEGER,
+        versions: [/\b[0-9]+\b/g],
+        debug_name: "Integer",
+        precedence: Precedence.Standard_Builtins
+    },
+    Builtin_Float: {
+        type: TokenType.FLOAT,
+        versions: [/\b([0-9]+[.][0-9]+([eE][0-9]+)?)|([0-9]+[fF])\b/g],
+        debug_name: "Float",
+        precedence: Precedence.Intricate_Builtins
+    },
+    //Lower priority than builtin, keyword and hex.
+    Builtin_Identifier: {
+        type: TokenType.IDENTIFIER,
+        versions: [/\b[a-zA-Z_][a-zA-Z0-9_]+\b/g],
+        debug_name: "Identifier",
+        precedence: Precedence.User_Identifiers
+    },
+    //The regex for this might be a bit limited, it should probably be revisited.
+    Builtin_Whitespace: {
+        type: TokenType.WHITESPACE,
+        versions: [/[ \t\r\n]+/g],
+        debug_name: "Whitespace",
+        precedence: Precedence.Intricate_Builtins
+    },
+    Operator: {
+        versions: [new Set([".", "(", ")", ",", "=", ":"])],
+        type: TokenType.OPERATOR,
+        debug_name: "Operator",
+        precedence: Precedence.Operators
+    }
+};
+export function isToken(object) {
+    return "type" in object && typeof TokenType[object.type] !== "undefined";
+}
+export function getEASTTokenValues(fileVersion, category) {
+    let index = GetVersionIndex(fileVersion);
+    if (index < 0)
+        throw new Error(`Can't parse file using version that isn't supported: ${fileVersion}`);
+    let vals = new Set();
+    for (; index >= 0; index--) {
+        let thisVersionVals = category.versions[index];
+        if (thisVersionVals instanceof RegExp) {
+            if (vals.size == 0) {
+                throw new Error(`Token is regexp not values.`);
+            }
+            else {
+                break;
+            }
+        }
+        for (let val of thisVersionVals) {
+            vals.add(val);
+        }
+    }
+    return vals;
+}
+function getEASTTokenRegex(fileVersion, category) {
+    let index = GetVersionIndex(fileVersion);
+    if (index < 0)
+        throw new Error(`Can't parse file using version that isn't supported: ${fileVersion}`);
+    let vals = new Set();
+    for (; index >= 0; index--) {
+        let thisVersionVals = category.versions[index];
+        //Regular expression overrides all previously defined syntax.
+        if (thisVersionVals instanceof RegExp) {
+            if (vals.size == 0) {
+                return thisVersionVals;
+            }
+            else {
+                break;
+            }
+        }
+        else if (thisVersionVals) {
+            for (let val of thisVersionVals) {
+                vals.add(val);
+            }
+        }
+    }
+    return new RegExp([...vals].map((e) => {
+        return "[.*+?^${}()|[]\\]".includes(e) ? "\\" + e : e;
+    }).join("|"), "g");
+}
+function makeVersionParser(version) {
+    let parser = [];
+    for (let category of Object.values(EAST_Tokens)) {
+        parser.push({
+            type: category.type,
+            regex: getEASTTokenRegex(version, category),
+            precedence: category.precedence,
+            name: category.debug_name
+        });
+    }
+    return parser;
+}
+let Parsers = new Map();
+/**
+ *
+ * @param expression The whole string
+ * @param version The version of the expression language this was compiled against
+ * @param offset The offset from the start of the file that this expression occurred within.
+ * @constructor
+ */
+export function TokenizeExpression(expression, version, offset = 0) {
+    let rawTokens = [];
+    if (!Parsers.has(version))
+        Parsers.set(version, makeVersionParser(version));
+    let parser = Parsers.get(version);
+    for (let category of parser) {
+        let matches = expression.matchAll(category.regex);
+        debug && console.log(TokenType[category.type], category.precedence);
+        for (let match of matches) {
+            rawTokens.push({
+                data: match[0],
+                position: match.index,
+                length: match[0].length,
+                precedence: category.precedence,
+                type: category.type
+            });
+        }
+    }
+    //Newlines are stored based on the last character in that line, once a line's position exceeds the node's position we know we're
+    // at a line that contains this given node.
+    let newlines = [...expression.matchAll(/\r\n|\n|\r/g)].map((e) => e.index);
+    newlines.unshift(0);
+    //Add one more line for anything left-over.
+    newlines.push(expression.length);
+    /**
+     * The first token takes precedence, but if multiple start at the same time then we sort
+     *  by precedence.
+     */
+    rawTokens = rawTokens.sort((a, b) => {
+        let offset = a.position - b.position;
+        if (offset == 0) {
+            return a.precedence - b.precedence;
+        }
+        return offset;
+    });
+    let lastToken;
+    let tokens = [];
+    for (let rtoken of rawTokens) {
+        if (lastToken && lastToken.position + lastToken.length > rtoken.position) {
+            //We are in the middle of a different kind of identifier, so skip
+            // this token.
+            //console.log("Discarding", TokenType[token.type], token.data, token.precedence, "because of", TokenType[lastToken.type], lastToken.data, token.precedence);
+            continue;
+        }
+        let line;
+        for (let i = 0; i < newlines.length; i++) {
+            if (newlines[i] > rtoken.position + offset) {
+                line = i;
+            }
+        }
+        tokens.push({
+            data: rtoken.data,
+            type: rtoken.type,
+            //Add an offset, since multiple expressions can exist in the same string.
+            position: rtoken.position + offset,
+            length: rtoken.length,
+            line: line,
+            column: rtoken.position + offset - newlines[line - 1]
+        });
+        lastToken = rtoken;
+    }
+    return tokens;
+}

package/Expressions/Compiler/__tests__/Replicable.Expressions.Parser.test.d.ts

@@ -0,0 +1 @@
+import "../BuiltinGrammar.js";