@shaderfrog/core 0.0.2 → 0.1.1

package/README.md

@@ -1,3 +1,186 @@
 # Shaderfrog Core
 
-Experimental. Do not use this.
+🚨 This library is experimental! 🚨
+🚨 The API can change at any time! 🚨
+
+The core graph API that powers Shaderfrog. This API, built on top of the
+[@Shaderfrog/glsl-parser](https://github.com/ShaderFrog/glsl-parser), compiles
+Shaderfrog graphs into an intermediate result, which you then pass off to an
+_engine_ (aka a plugin), to create a running GLSL shader.
+
+### Graph
+
+```typescript
+interface Graph {
+  nodes: GraphNode[];
+  edges: Edge[];
+}
+```
+
+The Shaderfrog _graph_ is a list of nodes and edges. It represents all of the
+GLSL code and configurations in your material. Conceptually, a graph is similar
+to a dependency graph for source code, where edges represent relationships
+(including dependencies) between nodes.
+
+Each _node_ in the graph is some type of GLSL (raw source code) and configuration.
+Some graph node GLSL is hard coded, as in written by you, like in a
+`SourceNode`. Some source code is generated at runtime by an engine, and
+injected into a node right before the graph is compiled.
+
+Each _edge_ in the graph represents a dependency between two nodes. Edges have
+different types and meanings, based on which inputs and outputs they're
+connected to.
+
+The main API function for working with graphs are `compileGraph` and
+`computeGraphContext`:
+
+```typescript
+type compileGraph = (
+  engineContext: EngineContext,
+  engine: Engine,
+  graph: Graph
+): CompileGraphResult
+
+type computeGraphContext = async (
+  engineContext: EngineContext,
+  engine: Engine,
+  graph: Graph
+): void
+```
+
+A graph's _context_, more specifically a node's context, is the runtime /
+in-memory computed data associated with a graph node. It includes the parsed AST
+representation of the node, as well as the inputs found in that AST.
+
+### Parsers
+
+A graph is a vanilla Javscript object. To convert it to context, there's one
+"parser" per node type in the graph, defined in the engine configuration. A
+parser is an object with this interface:
+
+```typescript
+type NodeParser = {
+  // cacheKey?: (graph: Graph, node: GraphNode, sibling?: GraphNode) => string;
+  // Callback hook to manipulate the node right before it's compiled by the
+  // graph. Engines use this to dynamically generate node source code.
+  onBeforeCompile?: OnBeforeCompile;
+  // Callback hook to manipulate the parsed AST. Example use is to convert
+  // standalone GLSL programs into code that can be used in the graph, like
+  // turning `void main() { out = color; }` into `vec4 main() { return color; }`
+  manipulateAst?: ManipulateAst;
+  // Find the inputs for this node type. Done dynamically because it's based on
+  // the source code of the node.
+  findInputs?: FindInputs;
+  // Create the filler AST offered up to nodes that import this node.
+  produceFiller?: ProduceNodeFiller;
+};
+```
+
+### Engine
+
+Shaderfrog is a GLSL editor. It's not a Three.js editor, nor a Babylon.js
+editor, etc. The output of Shaderfrog is raw GLSL and metadata.
+
+To use shaders in your _engine_, like Three.js, or even your own home grown
+engine, you implement your engine as a _plugin_ to Shaderfrog. An engine
+definition is verbose and likely to change:
+
+```typescript
+export interface Engine {
+  // The name of your engine, like "three"
+  name: string;
+  // Which GLSL variables are defined in your engine's materials
+  preserve: Set<string>;
+  // Rules for how to merge source code from different nodes together
+  mergeOptions: MergeOptions;
+  // Parsers for your engine node types. These are combined with the
+  // core engine parsers
+  parsers: Record<string, NodeParser>;
+  // Functions to import graphs/code from other engines into your own
+  importers: EngineImporters;
+  // How to evaluate a node, like turning a node of { type: 'vec3' } into a
+  // THREE.Vector3
+  evaluateNode: (node: DataNode) => any;
+  // How to create specific nodes in your engine
+  constructors: {
+    [EngineNodeType]: NodeConstructor;
+  };
+}
+```
+
+### Inputs, Holes and Fillers
+
+Shaderfrog works by searching each node's AST for certain patterns, like
+`uniform` variables, and creating an interface where you can replace each
+`uniform` variable with the output of another node. 
+
+Each fillable part of the AST is called a __hole__. Holes are found by executing
+user defined _strategies_ against an AST. With a program such as:
+
+```glsl
+uniform vec2 uv;
+void main() {
+  vec2 someVar = uv * 2.0;
+}
+```
+
+If you apply the `uniform` strategy to this code, it will mark the AST nodes
+relevant to the uniform as _holes_:
+
+```glsl
+uniform vec2 [uv];
+void main() {
+  vec2 someVar = [uv] * 2.0;
+}
+```
+
+And it adds a new _input_ to your node, named `uv` in this case.
+
+When you plug in the output of another node into this input, it _"fills in"_ the
+hole with the _filler_ output of another node. A _filler_ is an AST node. For
+example, if you have another node like:
+
+```glsl
+vec2 myFn() {
+  return vec2(1.0, 1.0);
+}
+```
+
+And you plug in the `myFn` output into the `uv` input, the hole is _filled_,
+resulting in:
+
+```glsl
+vec2 myFn() {
+  return vec2(1.0, 1.0);
+}
+
+void main() {
+  vec2 someVar = myFn() * 2.0;
+}
+```
+
+Note that this is not a simple find and replace. Not only was the `uv` variable
+replaced, but the declaration line `uniform vec2 uv;` was removed, and `myFn`
+was inlined into the final program.
+
+Hole filling always produces a new AST, or more accurately, a new
+`ShaderSections`, which is the intermediary representation of the compilation
+process.
+
+### Static Monkeypatching
+
+This whole process allows Shaderfrog to monkeypatch engine shaders. When
+modifying an engine shader, the process is:
+
+- Shaderfrog creates a `BABYLON.PBRMaterial` or `Three.MeshPhysicalMaterial` (or
+  whatever built in material type you want)
+- Shaderfrog reads the engine material's generated GLSL, and then modifies it to
+  add new effects by injecting new GLSL
+- Shaderfrog dumps the new compiled GLSL back into the `BABYLON.PBRMaterial` or
+  `Three.MeshPhysicalMaterial`, and updates the material to add a new uniforms.
+
+Injecting new GLSL into an engine shader is essentially _monkeypatching_ it:
+your code is modifying an external library's code. I call this _static
+monkeypatching_ because compiles new source code. This is opposed to traditional
+monkeypatching in languages like Ruby, where you modify external modules by
+changing them at runtime.

package/dist/ast/manipulate.js

@@ -0,0 +1,328 @@
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+/**
+ * Utility functions to work with ASTs
+ */
+import { parser, generate } from '@shaderfrog/glsl-parser';
+import { visit, } from '@shaderfrog/glsl-parser/ast';
+var log = function () {
+    var _a;
+    var args = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        args[_i] = arguments[_i];
+    }
+    return (_a = console.log).call.apply(_a, __spreadArray([console, '\x1b[31m(core.manipulate)\x1b[0m'], __read(args), false));
+};
+export var findVec4Constructor = function (ast) {
+    var parent;
+    var visitors = {
+        function_call: {
+            enter: function (path) {
+                var _a, _b;
+                if (((_a = path.node.identifier) === null || _a === void 0 ? void 0 : _a.specifier).token === 'vec4') {
+                    parent = (_b = path.findParent(function (p) { return 'right' in p.node; })) === null || _b === void 0 ? void 0 : _b.node;
+                    path.skip();
+                }
+            },
+        },
+    };
+    visit(ast, visitors);
+    return parent;
+};
+export var findAssignmentTo = function (ast, assignTo) {
+    var assign;
+    var visitors = {
+        expression_statement: {
+            enter: function (path) {
+                var _a, _b;
+                if (((_b = (_a = path.node.expression) === null || _a === void 0 ? void 0 : _a.left) === null || _b === void 0 ? void 0 : _b.identifier) === assignTo) {
+                    assign = path.node;
+                }
+                path.skip();
+            },
+        },
+    };
+    visit(ast, visitors);
+    return assign;
+};
+export var findDeclarationOf = function (ast, declarationOf) {
+    var declaration;
+    var visitors = {
+        declaration_statement: {
+            enter: function (path) {
+                var _a, _b;
+                var foundDecl = (_b = (_a = path.node.declaration) === null || _a === void 0 ? void 0 : _a.declarations) === null || _b === void 0 ? void 0 : _b.find(function (decl) { var _a; return ((_a = decl === null || decl === void 0 ? void 0 : decl.identifier) === null || _a === void 0 ? void 0 : _a.identifier) === declarationOf; });
+                if (foundDecl) {
+                    declaration = foundDecl;
+                }
+                path.skip();
+            },
+        },
+    };
+    visit(ast, visitors);
+    return declaration;
+};
+export var from2To3 = function (ast, stage) {
+    var glOut = 'fragmentColor';
+    // TODO: add this back in when there's only one after the merge
+    // ast.program.unshift({
+    //   type: 'preprocessor',
+    //   line: '#version 300 es',
+    //   _: '\n',
+    // });
+    if (stage === 'fragment') {
+        ast.program.unshift(makeStatement("out vec4 ".concat(glOut)));
+    }
+    visit(ast, {
+        function_call: {
+            enter: function (path) {
+                var identifier = path.node.identifier;
+                if (identifier.type === 'identifier' &&
+                    identifier.identifier === 'texture2D') {
+                    identifier.identifier = 'texture';
+                }
+            },
+        },
+        identifier: {
+            enter: function (path) {
+                if (path.node.identifier === 'gl_FragColor') {
+                    path.node.identifier = glOut;
+                }
+            },
+        },
+        keyword: {
+            enter: function (path) {
+                if ((path.node.token === 'attribute' || path.node.token === 'varying') &&
+                    path.findParent(function (path) { return path.node.type === 'declaration_statement'; })) {
+                    path.node.token =
+                        stage === 'vertex' && path.node.token === 'varying' ? 'out' : 'in';
+                }
+            },
+        },
+    });
+};
+export var outDeclaration = function (name) { return ({
+    type: 'declaration_statement',
+    declaration: {
+        type: 'declarator_list',
+        specified_type: {
+            type: 'fully_specified_type',
+            qualifiers: [{ type: 'keyword', token: 'out', whitespace: ' ' }],
+            specifier: {
+                type: 'type_specifier',
+                specifier: { type: 'keyword', token: 'vec4', whitespace: ' ' },
+                quantifier: null,
+            },
+        },
+        declarations: [
+            {
+                type: 'declaration',
+                identifier: {
+                    type: 'identifier',
+                    identifier: name,
+                    whitespace: undefined,
+                },
+                quantifier: null,
+                operator: undefined,
+                initializer: undefined,
+            },
+        ],
+        commas: [],
+    },
+    semi: { type: 'literal', literal: ';', whitespace: '\n    ' },
+}); };
+export var makeStatement = function (stmt) {
+    // log(`Parsing "${stmt}"`);
+    var ast;
+    try {
+        ast = parser.parse("".concat(stmt, ";\n"), { quiet: true });
+    }
+    catch (error) {
+        console.error({ stmt: stmt, error: error });
+        throw new Error("Error parsing stmt \"".concat(stmt, "\": ").concat(error === null || error === void 0 ? void 0 : error.message));
+    }
+    // log(util.inspect(ast, false, null, true));
+    return ast.program[0];
+};
+export var makeFnStatement = function (fnStmt) {
+    var ast;
+    try {
+        ast = parser.parse("\n  void main() {\n      ".concat(fnStmt, ";\n    }"), { quiet: true });
+    }
+    catch (error) {
+        console.error({ fnStmt: fnStmt, error: error });
+        throw new Error("Error parsing fnStmt \"".concat(fnStmt, "\": ").concat(error === null || error === void 0 ? void 0 : error.message));
+    }
+    // log(util.inspect(ast, false, null, true));
+    return ast.program[0].body.statements[0];
+};
+export var makeExpression = function (expr) {
+    var ast;
+    try {
+        ast = parser.parse("void main() {\n          a = ".concat(expr, ";\n        }"), { quiet: true });
+    }
+    catch (error) {
+        console.error({ expr: expr, error: error });
+        throw new Error("Error parsing expr \"".concat(expr, "\": ").concat(error === null || error === void 0 ? void 0 : error.message));
+    }
+    return ast.program[0].body
+        .statements[0].expression.right;
+};
+export var makeExpressionWithScopes = function (expr) {
+    var ast;
+    try {
+        ast = parser.parse("void main() {\n          ".concat(expr, ";\n        }"), { quiet: true });
+    }
+    catch (error) {
+        console.error({ expr: expr, error: error });
+        throw new Error("Error parsing expr \"".concat(expr, "\": ").concat(error === null || error === void 0 ? void 0 : error.message));
+    }
+    // log(util.inspect(ast, false, null, true));
+    return {
+        scope: ast.scopes[1],
+        expression: ast.program[0].body
+            .statements[0].expression,
+    };
+};
+export var makeFnBodyStatementWithScopes = function (body) {
+    var ast;
+    try {
+        ast = parser.parse("void main() {\n".concat(body, "\n        }"), { quiet: true });
+    }
+    catch (error) {
+        console.error({ body: body, error: error });
+        throw new Error("Error parsing body \"".concat(body, "\": ").concat(error === null || error === void 0 ? void 0 : error.message));
+    }
+    // log(util.inspect(ast, false, null, true));
+    return {
+        scope: ast.scopes[1],
+        statements: ast.program[0].body.statements,
+    };
+};
+export var findFn = function (ast, name) {
+    return ast.program.find(function (stmt) {
+        return stmt.type === 'function' && stmt.prototype.header.name.identifier === name;
+    });
+};
+export var returnGlPosition = function (fnName, ast) {
+    return convertVertexMain(fnName, ast, 'vec4', function (assign) { return assign.expression.right; });
+};
+export var returnGlPositionHardCoded = function (fnName, ast, returnType, hardCodedReturn) {
+    return convertVertexMain(fnName, ast, returnType, function () {
+        return makeExpression(hardCodedReturn);
+    });
+};
+export var returnGlPositionVec3Right = function (fnName, ast) {
+    return convertVertexMain(fnName, ast, 'vec3', function (assign) {
+        var found;
+        visit(assign, {
+            function_call: {
+                enter: function (path) {
+                    var _a, _b, _c, _d, _e;
+                    log('returnGlPositionVec3Right', path.node);
+                    var node = path.node;
+                    if (((_b = (_a = node === null || node === void 0 ? void 0 : node.identifier) === null || _a === void 0 ? void 0 : _a.specifier) === null || _b === void 0 ? void 0 : _b.token) === 'vec4' &&
+                        ((_e = (_d = (_c = node === null || node === void 0 ? void 0 : node.args) === null || _c === void 0 ? void 0 : _c[2]) === null || _d === void 0 ? void 0 : _d.token) === null || _e === void 0 ? void 0 : _e.includes('1.'))) {
+                        found = node.args[0];
+                    }
+                },
+            },
+        });
+        if (!found) {
+            console.error(generate(ast));
+            throw new Error('Could not find position assignment to convert to return!');
+        }
+        return found;
+    });
+};
+var convertVertexMain = function (fnName, ast, returnType, generateRight) {
+    var mainReturnVar = "frogOut";
+    var main = findFn(ast, fnName);
+    if (!main) {
+        throw new Error("No ".concat(fnName, " fn found!"));
+    }
+    // Convert the main function to one that returns
+    main.prototype.header.returnType.specifier.specifier.token =
+        returnType;
+    // Find the gl_position assignment line
+    var assign = main.body.statements.find(function (stmt) {
+        var _a;
+        return stmt.type === 'expression_statement' &&
+            ((_a = stmt.expression.left) === null || _a === void 0 ? void 0 : _a.identifier) === 'gl_Position';
+    });
+    if (!assign) {
+        throw new Error("No gl position assign found in main fn!");
+    }
+    var rtnStmt = makeFnStatement("".concat(returnType, " ").concat(mainReturnVar, " = 1.0"));
+    rtnStmt.declaration.declarations[0].initializer =
+        generateRight(assign);
+    main.body.statements.splice(main.body.statements.indexOf(assign), 1, rtnStmt);
+    main.body.statements.push(makeFnStatement("return ".concat(mainReturnVar)));
+};
+export var convert300MainToReturn = function (suffix, ast) {
+    var mainReturnVar = "frogOut_".concat(suffix);
+    // Find the output variable, as in "pc_fragColor" from  "out highp vec4 pc_fragColor;"
+    var outName;
+    ast.program.find(function (line, index) {
+        var _a, _b;
+        var declaration = line === null || line === void 0 ? void 0 : line.declaration;
+        if (
+        // line.type === 'declaration_statement' &&
+        ((_b = (_a = declaration === null || declaration === void 0 ? void 0 : declaration.specified_type) === null || _a === void 0 ? void 0 : _a.qualifiers) === null || _b === void 0 ? void 0 : _b.find(function (n) { return n.token === 'out'; })) &&
+            declaration.specified_type.specifier.specifier.token ===
+                'vec4') {
+            // Remove the out declaration
+            ast.program.splice(index, 1);
+            outName = declaration.declarations[0].identifier.identifier;
+            return true;
+        }
+    });
+    if (!outName) {
+        console.error(generate(ast));
+        throw new Error('No "out vec4" line found in the fragment shader');
+    }
+    ast.program.unshift(makeStatement("vec4 ".concat(mainReturnVar)));
+    visit(ast, {
+        identifier: {
+            enter: function (path) {
+                if (path.node.identifier === outName) {
+                    path.node.identifier = mainReturnVar;
+                    // @ts-ignore
+                    path.node.doNotDescope = true; // hack because this var is in the scope which gets renamed later
+                }
+            },
+        },
+        function: {
+            enter: function (path) {
+                if (path.node.prototype.header.name.identifier === 'main') {
+                    path.node.prototype.header.returnType.specifier
+                        .specifier.token = 'vec4';
+                    path.node.body.statements.push(makeFnStatement("return ".concat(mainReturnVar)));
+                }
+            },
+        },
+    });
+};