@playcanvas/attribute-parser 1.0.0

package/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  unit-test-node:
+    name: Unit Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Setup Node.js 18.x
+      uses: actions/setup-node@v4
+      with:
+        node-version: 18.x
+        cache: 'npm'
+
+    - name: Install dependencies
+      run: npm clean-install --progress=false --no-fund
+
+    - name: Run unit tests
+      run: npm test

package/README.md

@@ -0,0 +1,76 @@
+# PlayCanvas Attribute Parser
+
+This the JSDoc attribute parser used in the PlayCanvas editor. It's used to collect metadata from a script via it's jsdoc tags.
+
+It takes a script like this...
+
+```javascript
+class Rotator extends ScriptType {
+  /**
+   * @attribute
+   * Speed determines how fast to rotate things"
+   */
+   speed = new Vec3()
+
+   /**
+    * @attribute
+    *  An array of Entities to rotate
+    * 
+    * @type {Entity[]}
+    */
+   thingsToRotate
+}
+```
+
+and turns it into this...
+
+
+```json
+{
+    "rotator": {
+        "attributes": {
+            "speed": {
+                "type": "vec3",
+                "name": "speed",
+                "array": false,
+                "description": "Speed determines how fast to rotate things\"",
+                "default": [
+                    0,
+                    0,
+                    0
+                ]
+            },
+            "thingsToRotate": {
+                "type": "entity",
+                "name": "thingsToRotate",
+                "array": true,
+                "description": "An array of Entities to rotate",
+                "default": null
+            }
+        },
+        "errors": []
+    }
+}
+```
+
+It takes the JSDocs and values and outputs the metadata in a serializable format.
+
+### Usage 
+
+```javascript
+// Initialise the parser
+const parser = new JSDocParser();
+await parser.init();
+
+// fetch your program source {[filename: string, contents: string][]}
+const scripts = await fetchScripts([...paths, './playcanvas.d.ts']);
+
+// Parse the program, starting from the first path
+const [attribute, errors] parser.parseAttributes(paths[0], scripts);
+```
+
+### Attribute structure
+
+You can learn more about the special tags [available here ](https://github.com/playcanvas/attribute-parser/tree/main/test/fixtures)
+
+

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@playcanvas/attribute-parser",
+  "type": "module",
+  "main": "src/index.js",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "version": "1.0.0",
+  "dependencies": {
+    "@microsoft/tsdoc": "^0.15.0",
+    "@typescript/vfs": "^1.6.0",
+    "typescript": "^5.5.3"
+  },
+  "devDependencies": {
+    "chai": "^5.1.1",
+    "mocha": "^10.6.0"
+  },
+  "scripts": {
+    "test": "mocha test/**/*.test.js --timeout 10000"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/index.js

@@ -0,0 +1,226 @@
+import ts from 'typescript';
+import { createSystem, createDefaultMapFromNodeModules, createVirtualTypeScriptEnvironment } from '@typescript/vfs';
+
+import { ScriptParser } from './parsers/script-parser.js';
+import { createDefaultMapFromCDN, flatMapAnyNodes, getExportedNodes, getType, inheritsFrom, isAliasedClassDeclaration } from './utils/ts-utils.js';
+
+const toLowerCamelCase = str => str[0].toLowerCase() + str.substring(1);
+
+const COMPILER_OPTIONS = {
+    strictPropertyInitialization: false, // Allow uninitialized properties
+    target: ts.ScriptTarget.ES2022, // If this version changes, the types must be updated in the /rollup.config.mjs
+    module: ts.ModuleKind.CommonJS,
+    checkJs: true, // Enable JSDoc parsing
+    allowJs: true,
+    noImplicitAny: false, // Allow implicit any
+    baseUrl: './',
+    paths: {
+        "playcanvas": ["/playcanvas.js"]
+    }
+};
+
+export class JSDocParser {
+    /**
+     * @type {import('@typescript/vfs').VirtualTypeScriptEnvironment}
+     * @private
+     */
+    _env;
+
+    /**
+     * An internal array of program errors
+     * @private
+     */
+    _errors = [];
+
+    async init(basePath = '') {
+        if (this._env) {
+            return;
+        }
+
+        let fsMap;
+        if (basePath) {
+            fsMap = await createDefaultMapFromCDN({ target: ts.ScriptTarget.ES2022 }, basePath, ts);
+        } else {
+            fsMap = await createDefaultMapFromNodeModules(COMPILER_OPTIONS, ts);
+        }
+
+        // Set up the virtual file system and environment
+        const system = createSystem(fsMap);
+        this._env = createVirtualTypeScriptEnvironment(system, Array.from(fsMap.keys()), ts, COMPILER_OPTIONS);
+    }
+
+    /**
+     * Creates program from fileName and scriptsContents
+     *
+     * @param {[string, string][]} newFiles - An array of {[path]: [content]} which represent the files in the program
+     * @param {string[]} removedFiles - An array of file paths that should be removed from the program
+     * @returns {{ parser: ScriptParser, program: ts.Program,
+     * errors: import('./parsers/script-parser.js').ValidationError[]}} - The extracted attributes
+     */
+    updateProgram(newFiles = [], removedFiles = []) {
+        const errors = [];
+
+        // Add custom files to the map
+        newFiles.forEach(([filePath, fileContent]) => {
+            if (this._env.getSourceFile(filePath)) {
+                this._env.updateFile(filePath, fileContent);
+            } else {
+                this._env.createFile(filePath, fileContent);
+            }
+        });
+
+        // Remove deleted Files
+        removedFiles.forEach((filePath) => {
+            this._env.deleteFile(filePath);
+        });
+
+
+        this.parser = new ScriptParser(this._env);
+        this.program = this._env.languageService.getProgram();
+
+        // Check for compiler errors
+        this.parser.validateProgram(errors);
+
+        this._errors = errors;
+
+        return errors;
+    }
+
+    /**
+     * Returns all the valid ESM Scripts within a file
+     * @param fileName - The file name in the program to check
+     * @returns {import('typescript').Node[]} - An array of any exported ESM Script nodes within the file
+     */
+    getAllEsmScripts(fileName) {
+
+        const typeChecker = this.program.getTypeChecker();
+
+        // Find the Script class in the PlayCanvas namespace
+        const pcTypes = this.program.getSourceFile('/playcanvas.d.ts');
+
+        if (!pcTypes) {
+            throw new Error(`PlayCanvas Types must be supplied`);
+        }
+        
+        // Parse the source file and pc types
+        const sourceFile = this.program.getSourceFile(fileName);
+        
+        if (!sourceFile) {
+            throw new Error(`Source file ${fileName} not found`);
+        }
+        
+        // Extract all exported nodes
+        const nodes = getExportedNodes(this.program, sourceFile);
+
+        const esmScriptClass = pcTypes.statements.find(node => node.kind === ts.SyntaxKind.ClassDeclaration && node.name.text === 'Script')?.symbol;
+
+        // Check if the file exports a class that inherits from `Script`
+        return Array.from(nodes).filter(node => isAliasedClassDeclaration(node, typeChecker) && inheritsFrom(node, typeChecker, esmScriptClass));
+
+    }
+
+    /**
+     * Analyzes the specified TypeScript file and extracts metadata from JSDoc comments.
+     *
+     * @param {string} fileName - The name of the file in the program to run
+     * @returns {[results: object, errors: import('./parsers/script-parser.js').ValidationError[]]} - The extracted attributes
+     */
+    parseAttributes(fileName) {
+        const results = {};
+        const errors = [...this._errors];
+
+        if(errors.length > 0 ){
+            return [results, errors];
+        }
+
+        // Extract all exported nodes
+        const nodes = this.getAllEsmScripts(fileName);
+
+        // Extract attributes from each script
+        nodes.forEach((node) => {
+            const name = toLowerCamelCase(node.name.text);
+            const opts = results[name] = { attributes: {}, errors: [] };
+            this.parser.extractAttributes(node, opts);
+        });
+
+        return [results, errors];
+    }
+
+    /**
+     * Analyzes the specified TypeScript file and extracts metadata from JSDoc comments.
+     * @param {string} fileName - The name of the file to run
+     * @returns {[results: object, errors: import('./parsers/script-parser.js').ValidationError[]]} - The extracted attributes
+     */
+    getAttributes(fileName) {
+        const results = {};
+        const errors = [...this._errors];
+
+        if(errors.length > 0 ){
+            return [results, errors];
+        }
+
+        const typeChecker = this.program.getTypeChecker();
+
+        // Find the Script class in the PlayCanvas namespace
+        const pcTypes = this.program.getSourceFile('/playcanvas.d.ts');
+        const esmScriptClass = pcTypes.statements.find(node => node.kind === ts.SyntaxKind.ClassDeclaration && node.name.text === 'Script')?.symbol;
+
+        // Parse the source file and pc types
+        const sourceFile = this.program.getSourceFile(fileName);
+
+        if (!sourceFile) {
+            throw new Error(`Source file ${fileName} not found`);
+        }
+
+        // Filter out classes that do not inherit from `Script` or are interfaces
+        const isInterface = node => node.jsDoc.some(jsdoc => jsdoc.tags?.some(tag => tag.tagName.getText() === 'interface'));
+
+        const nodes = flatMapAnyNodes(sourceFile, (node) => {
+            if (!ts.isClassDeclaration(node)) {
+                return false;
+            }
+
+            return inheritsFrom(node, typeChecker, esmScriptClass) || isInterface(node);
+        });
+
+        for (const node of nodes) {
+            const name = toLowerCamelCase(node.name.text);
+            const members = [];
+
+            for (const member of node.members) {
+                if (member.kind !== ts.SyntaxKind.PropertyDeclaration) {
+                    continue;
+                }
+
+                // Extract JSDoc tags
+                const tags = member.jsDoc.map(jsdoc => jsdoc.tags?.map(tag => tag.tagName.getText()) ?? []).flat();
+
+                const name = member.name.getText();
+                const type = getType(member, typeChecker);
+                if (!type) {
+                    continue;
+                }
+
+                const namePos = ts.getLineAndCharacterOfPosition(member.getSourceFile(), member.name.getStart());
+
+                const jsdocNode = member.jsDoc[member.jsDoc.length - 1];
+                const jsdocPos = jsdocNode ? ts.getLineAndCharacterOfPosition(member.getSourceFile(), jsdocNode.getStart()) : null;
+
+                const data = {
+                    name,
+                    type: type.name + (type.array ? '[]' : ''),
+                    isAttribute: tags.includes('attribute'),
+                    start: jsdocNode ? { line: jsdocPos.line + 1, column: jsdocPos.character + 1 } :
+                        { line: namePos.line + 1, column: namePos.character + 1 },
+                    end: { line: namePos.line + 1, column: namePos.character + name.length + 1 }
+                };
+
+                members.push(data);
+            }
+
+            results[name] = members;
+        }
+
+        return [results, errors];
+    }
+}

package/src/parsers/attribute-parser.js

@@ -0,0 +1,280 @@
+import ts from 'typescript';
+import { TSDocConfiguration, TSDocTagDefinition, TSDocTagSyntaxKind, TextRange, TSDocParser } from "@microsoft/tsdoc";
+import { extractTextFromDocNode, getLeadingBlockCommentRanges, getType } from '../utils/ts-utils.js';
+
+import { ParsingError } from "./parsing-error.js";
+import { hasTag } from "../utils/attribute-utils.js";
+import { parseTag, validateTag } from "../utils/tag-utils.js";
+
+/**
+ * A class to parse JSDoc comments and extract attribute metadata.
+ */
+export class AttributeParser {
+    /**
+     * Creates a new instance of the ScriptParser class.
+     *
+     * @param {Map<string, string>} tags - An set of tag definitions and their validators to add to the parser
+     * @param {object} env - The TypeScript environment to use
+     * @param {Map<string, Function>} typeSerializerMap - A map of custom serializers
+     */
+    constructor(tags = new Map(), env, typeSerializerMap) {
+
+        this.env = env;
+        this.tagTypeAnnotations = tags;
+        this.typeSerializerMap = typeSerializerMap;
+        this.program = env.languageService.getProgram();
+        this.typeChecker = this.program.getTypeChecker();
+
+        this.attributeTag = new TSDocTagDefinition({
+            tagName: '@attribute', syntaxKind: TSDocTagSyntaxKind.ModifierTag
+        });
+
+        this.typeTag = new TSDocTagDefinition({
+            tagName: '@type', syntaxKind: TSDocTagSyntaxKind.BlockTag
+        });
+
+        const conf = new TSDocConfiguration();
+        conf.validation.reportUnsupportedTags = false;
+
+        conf.addTagDefinition(this.attributeTag);
+
+        tags.forEach((_, tag) => {
+            const tagDefinition = new TSDocTagDefinition({
+                tagName: `@${tag}`, syntaxKind: TSDocTagSyntaxKind.BlockTag, allowMultiple: false
+            });
+            conf.addTagDefinition(tagDefinition);
+        });
+
+        this.parser = new TSDocParser(conf);
+    }
+
+    /**
+     * Parses a JSdoc comment and extracts any associated metadata if the `@attribute`
+     * modifier is used. Returns null if this is not an attribute comment.
+     *
+     * @param {TextRange} comment - The comment to parse
+     * @param {ts.Node} node - The node to parse
+     * @param {ParsingError[]} errors - An array to store any parsing errors
+     * @param {boolean} requiresAttributeTag - Whether the comment must have an attribute tag
+     * @returns {*} - The extracted metadata or null if no metadata was found
+     */
+    parseAttributeComment(comment, node, errors = [], requiresAttributeTag = true) {
+
+        const parserContext = this.parser.parseRange(comment);
+        const docComment = parserContext.docComment;
+        const file = node.getSourceFile();
+
+        // Check if the comment has an attribute tag
+        if (!requiresAttributeTag || hasTag('attribute', node)) {
+
+            // Fetch the primary attribute metadata
+            const attribute = this.getNodeAsAttribute(node, errors);
+
+            if (!attribute) return;
+
+            // Extract the description from the summary section
+            const description = extractTextFromDocNode(docComment.summarySection);
+            if (description) {
+                attribute.description = description.split('.')[0];
+            }
+
+            // Extract metadata from custom tags
+            const jsDocTags = node.jsDoc?.[0]?.tags ?? [];
+            jsDocTags.forEach((tag) => {
+                let commentText = tag.comment;
+
+                // Check if the tag is a supported tag
+                if (this.tagTypeAnnotations.has(tag.tagName.text)) {
+                    try {
+                        const value = parseTag(commentText);
+                        const tagTypeAnnotation = this.tagTypeAnnotations.get(tag.tagName.text);
+
+                        // Tags like @resource with string values do not need quotations, so we need to manually add them
+                        if (typeof value === 'string') commentText = `"${commentText}"`;
+
+                        validateTag(commentText, tagTypeAnnotation, this.env);
+                        attribute[tag.tagName.text] = value;
+
+                    } catch (error) {
+                        const { line, character } = file.getLineAndCharacterOfPosition(tag.getStart());
+                        const parseError = new ParsingError(tag, `Error (${line}, ${character}): Parsing Tag '@${tag.tagName.text} ${commentText}' - ${error.message}`);
+                        errors.push(parseError);
+                    }
+                }
+            });
+
+            return attribute;
+        }
+    }
+
+    /**
+     * Returns the leading block comments for an typescript AST node.
+     *
+     * @param {import('typescript').Node} node - The node to parse
+     * @param {string[]} errors - An array to store any parsing errors
+     * @returns {import('@microsoft/tsdoc').DocComment} - The extracted doc comment;
+     */
+    getCommentsForNode(node, errors = []) {
+        const fullText = node.getSourceFile().getFullText();
+        const comments = getLeadingBlockCommentRanges(node, fullText);
+
+        if (comments.length === 0) return null;
+
+        const lastComment = comments[0];
+        const textRange = TextRange.fromStringRange(fullText, lastComment.pos, lastComment.end);
+
+        const parserContext = this.parser.parseRange(textRange);
+        const docComment = parserContext.docComment;
+
+        // Check for comment parsing errors and flag them
+        this.populateErrors(node, parserContext, errors);
+
+        return docComment;
+    }
+
+    /**
+     * Populates the errors array with any parsing errors found in the parser context.
+     *
+     * @param {import('typescript').Node} node - The node to parse
+     * @param {import('@microsoft/tsdoc').ParserContext} parserContext - The parser context to check for errors
+     * @param {string[]} errors - An array to store any parsing errors
+     */
+    populateErrors(node, parserContext, errors) {
+        if (parserContext.log.messages.length > 0) {
+            const sourceFile = node.getSourceFile();
+
+            const formattedErrors = parserContext.log.messages.map((message) => {
+                const location = sourceFile.getLineAndCharacterOfPosition(message.textRange.pos);
+                return `${sourceFile.fileName}(${location.line + 1},${location.character + 1}): [TSDoc] ${message}`;
+            });
+
+            errors.push(...formattedErrors);
+        }
+    }
+
+    /**
+     * Extract the public members and their values from a TypeScript node.
+     *
+     * @param {ts.Node} node - The node to parse
+     * @param {ParsingError[]} errors - An array to store any parsing errors
+     * @returns {*} - The extracted members
+     */
+    getNodeAsAttribute(node, errors = []) {
+
+        const name = node.name && ts.isIdentifier(node.name) && node.name.text;
+        const { type, name: typeName, array } = getType(node, this.typeChecker);
+        const enums = this.getEnumMembers(node, errors);
+        let value = null;
+
+        // we don't need to serialize the value for arrays
+        const serializer = !array && this.typeSerializerMap.get(type);
+        if (serializer) {
+            try {
+                value = serializer(node.initializer ?? node);
+            } catch (error) {
+                errors.push(error);
+                return;
+            }
+        }
+
+        return { type, name, typeName, array, value, enum: enums };
+    }
+
+    /**
+     * Extracts the members of an enum from a TypeScript node.
+     *
+     * @param {import('typescript').Node} node - The TypeScript node to analyze
+     * @param {ParsingError[]} errors - An array to store any parsing errors
+     * @returns {Object<string, *>[]} - An array of enum members
+     */
+    getEnumMembers(node, errors) {
+        let typeNode = null;
+        let members = [];
+
+        // Check if there's a type annotation directly on the variable declaration
+        if (ts.isVariableDeclaration(node) && node.type) {
+            typeNode = node.type;
+        } else {
+            // Check for JSDoc annotations
+            const jsDocs = ts.getJSDocTags(node);
+            const typeTag = jsDocs.find(doc => doc.tagName.text === "type");
+            if (typeTag && ts.isJSDocTypeTag(typeTag)) {
+                typeNode = typeTag.typeExpression;
+            }
+        }
+
+        if (typeNode && ts.isTypeReferenceNode(typeNode.type)) {
+
+            // resolve the symbol of the type
+            let symbol = this.typeChecker.getSymbolAtLocation(typeNode.type.typeName);
+
+            // Resolve aliases, which are common with imports
+            if (symbol && symbol.flags & ts.SymbolFlags.Alias) {
+                symbol = this.typeChecker.getAliasedSymbol(symbol);
+            }
+
+            if (symbol) {
+                const declarations = symbol.getDeclarations();
+
+                if (declarations && declarations.length > 0) {
+
+                    declarations.forEach((declaration) => {
+
+                        // Check if the declaration is a TypeScript enum
+                        if (ts.isEnumDeclaration(declaration)) {
+                            members = declaration.members.map(member => member.name.getText());
+                        }
+
+                        // Additionally check for JSDoc enum tag
+                        const jsDocs = ts.getJSDocTags(declaration);
+                        const enumTag = jsDocs.find(doc => doc.tagName.text === "enum");
+                        if (enumTag && ts.isJSDocEnumTag(enumTag)) {
+                            // This assumes enum members are correctly annotated in JSDoc comments
+                            // Example: @enum {type} EnumName { Member1, Member2 }
+                            members = members.concat(this.getMembers(declaration.initializer, errors)); // declaration.initializer.properties.map(member => member.name.getText());
+                        }
+                    });
+                }
+            }
+        }
+
+        return members;
+    }
+
+    /**
+     * Extract the public members and their values from a TypeScript node.
+     *
+     * @param {ts.Node} node - The node to parse
+     * @param {ParsingError[]} errors - An array to store any parsing errors
+     * @returns {any[]} - The extracted members
+     */
+    getMembers(node, errors = []) {
+
+        const members = [];
+
+        if (ts.isObjectLiteralExpression(node)) {
+            node.properties.forEach((property) => {
+
+                if (ts.isPropertyAssignment(property)) {
+                    const name = property.name && ts.isIdentifier(property.name) && property.name.text;
+                    let value;
+
+                    const node = property.initializer;
+
+                    // Enums can only contain primitives (string|number|boolean)
+                    if (ts.isNumericLiteral(node)) {
+                        value = parseFloat(node.getText());
+                    } else if (node.kind === ts.SyntaxKind.TrueKeyword || node.kind === ts.SyntaxKind.FalseKeyword) {
+                        value = node.kind === ts.SyntaxKind.TrueKeyword;
+                    } else {
+                        value = node.getText();
+                    }
+
+                    members.push({ [name]: value });
+                }
+            });
+        }
+
+        return members;
+    }
+}

package/src/parsers/parsing-error.js

@@ -0,0 +1,14 @@
+export class ParsingError {
+    constructor(node, message) {
+        this.node = node;        // AST node which caused the error
+        this.message = message;  // Description of the error
+    }
+
+    toString() {
+        // Ensure node.location exists and has the necessary properties
+        const file = this.node.getSourceFile();
+        const fileName = file.fileName;
+        const start = this.node.getSourceFile().getLineAndCharacterOfPosition(this.node.getStart());
+        return `ParsingError: ${this.message} at ${fileName}:${start.line + 1}:${start.character}`;
+    }
+}