@shaderfrog/core 0.0.1

package/src/ast/shader-sections.ts

@@ -0,0 +1,323 @@
+/**
+ * Categorizing / deduping parts of shaders to help merge them together
+ */
+import {
+  AstNode,
+  DeclarationStatementNode,
+  PreprocessorNode,
+} from '@shaderfrog/glsl-parser/ast';
+import { generate } from '@shaderfrog/glsl-parser';
+import { makeStatement } from './manipulate';
+import { Program } from '@shaderfrog/glsl-parser/ast';
+export interface ShaderSections {
+  precision: DeclarationStatementNode[];
+  version: AstNode[];
+  preprocessor: PreprocessorNode[];
+  structs: AstNode[];
+  inStatements: DeclarationStatementNode[];
+  outStatements: DeclarationStatementNode[];
+  uniforms: DeclarationStatementNode[];
+  program: AstNode[];
+}
+
+export const emptyShaderSections = (): ShaderSections => ({
+  precision: [],
+  preprocessor: [],
+  version: [],
+  structs: [],
+  program: [],
+  inStatements: [],
+  outStatements: [],
+  uniforms: [],
+});
+
+enum Precision {
+  highp = 2,
+  mediump = 1,
+  lowp = 0,
+}
+
+export const higherPrecision = (p1: Precision, p2: Precision): Precision =>
+  Precision[p1] > Precision[p2] ? p1 : p2;
+
+export const dedupeVersions = (nodes: AstNode[]): AstNode => nodes[0];
+export const highestPrecisions = (
+  nodes: DeclarationStatementNode[]
+): DeclarationStatementNode[] =>
+  Object.entries(
+    nodes.reduce(
+      (precisions, stmt) => ({
+        ...precisions,
+        // Like "float"
+        [stmt.declaration.specifier.specifier.token]: higherPrecision(
+          precisions[stmt.declaration.specifier.specifier.token],
+          stmt.declaration.qualifier.token
+        ),
+      }),
+      {} as { [type: string]: Precision }
+    )
+  ).map(
+    ([typeName, precision]) =>
+      makeStatement(
+        `precision ${precision} ${typeName}`
+      ) as DeclarationStatementNode
+  );
+
+export const dedupeQualifiedStatements = (
+  statements: DeclarationStatementNode[],
+  qualifier: string
+): any =>
+  Object.entries(
+    statements.reduce(
+      (stmts, stmt) => ({
+        ...stmts,
+        // Like "vec2"
+        [stmt.declaration.specified_type.specifier.specifier.token]: {
+          ...(stmts[
+            stmt.declaration.specified_type.specifier.specifier.token
+          ] || {}),
+          ...stmt.declaration.declarations.reduce(
+            (types: { [typeName: string]: string }, decl: any) => ({
+              ...types,
+              [decl.identifier.identifier]: true,
+            }),
+            {} as { [typeName: string]: string }
+          ),
+        },
+      }),
+      {} as { [key: string]: AstNode }
+    )
+  ).map(([type, varNames]) =>
+    makeStatement(`${qualifier} ${type} ${Object.keys(varNames).join(', ')}`)
+  );
+
+type UniformName = Record<string, { generated: string; hasInterface: boolean }>;
+type UniformGroup = Record<string, UniformName>;
+
+/**
+ * Merge uniforms together into lists of identifiers under the same type.
+ * There's special case handling for mixing of uniforms with "interface blocks"
+ * and those without when merging to make sure the interface block definition is
+ * preserved. Check out the tests for more.
+ *
+ * This function consumes uniforms as found by findShaderSections, so the
+ * definitions must line up
+ */
+export const dedupeUniforms = (statements: DeclarationStatementNode[]): any => {
+  const groupedByTypeName = Object.entries(
+    statements.reduce<UniformGroup>((stmts, stmt) => {
+      const { specified_type } = stmt.declaration;
+      const { identifier, interface_type } = stmt.declaration;
+
+      // This is the standard case, a uniform like "uniform vec2 x"
+      if (specified_type) {
+        const { specifier } = specified_type.specifier;
+        // Token is for "vec2", "identifier" is for custom names like struct
+        const type = (specifier.token || specifier.identifier) as string;
+
+        // Groups uniforms into their return type, and for each type, collapses
+        // uniform names into an object where the keys determine uniqueness
+        // "vec2": { x: x[1] }
+        const grouped = (
+          stmt.declaration.declarations as any[]
+        ).reduce<UniformName>(
+          (types, decl) => ({
+            ...types,
+            // There's probably a bug here where one shader declares x[1],
+            // another declares x[2], they both get collapsed under "x",
+            // and one is wrong
+            [decl.identifier.identifier as string]: stmts[type]?.[
+              decl.identifier.identifier as string
+            ]?.hasInterface
+              ? stmts[type]?.[decl.identifier.identifier as string]
+              : {
+                  hasInterface: false,
+                  generated:
+                    decl.identifier.identifier +
+                    (decl.quantifier
+                      ? `[${decl.quantifier.specifiers[0].expression.token}]`
+                      : ''),
+                },
+          }),
+          {}
+        );
+
+        return {
+          ...stmts,
+          [type]: {
+            ...(stmts[type] || {}),
+            ...grouped,
+          },
+        };
+        // This is the less common case, a uniform like "uniform Light { vec3 position; } name"
+      } else if (interface_type) {
+        // If this is an interface block only, like uniform Scene { mat4 view; };
+        // then group the interface block declaration under ''
+        const interfaceDeclaredUniform =
+          (identifier?.identifier?.identifier as string) || '';
+        return {
+          ...stmts,
+          [interface_type.identifier as string]: {
+            [interfaceDeclaredUniform]: {
+              generated: `${generate({
+                type: 'interface_declarator',
+                lp: stmt.declaration.lp,
+                declarations: stmt.declaration.declarations,
+                qualifiers: null,
+                interface_type: null,
+                rp: stmt.declaration.rp,
+              })}${interfaceDeclaredUniform}`,
+              hasInterface: true,
+            },
+          },
+        };
+      } else {
+        console.error('Unknown uniform AST', { stmt, code: generate(stmt) });
+        throw new Error(
+          'Unknown uniform AST encountered when merging uniforms'
+        );
+      }
+    }, {})
+  );
+
+  return groupedByTypeName.map(([type, variables]) => {
+    return makeStatement(
+      `uniform ${type} ${Object.values(variables)
+        .map((v) => v.generated)
+        .join(', ')}`
+    );
+  });
+};
+
+export const mergeShaderSections = (
+  s1: ShaderSections,
+  s2: ShaderSections
+): ShaderSections => {
+  return {
+    version: [...s1.version, ...s2.version],
+    precision: [...s1.precision, ...s2.precision],
+    preprocessor: [...s1.preprocessor, ...s2.preprocessor],
+    inStatements: [...s1.inStatements, ...s2.inStatements],
+    outStatements: [...s1.outStatements, ...s2.outStatements],
+    structs: [...s1.structs, ...s2.structs],
+    uniforms: [...s1.uniforms, ...s2.uniforms],
+    program: [...s1.program, ...s2.program],
+  };
+};
+
+export type MergeOptions = {
+  includePrecisions: boolean;
+  includeVersion: boolean;
+};
+
+export const shaderSectionsToProgram = (
+  sections: ShaderSections,
+  mergeOptions: MergeOptions
+): Program => ({
+  type: 'program',
+  scopes: [],
+  program: [
+    ...(mergeOptions.includeVersion ? [dedupeVersions(sections.version)] : []),
+    ...(mergeOptions.includePrecisions
+      ? highestPrecisions(sections.precision)
+      : []),
+    ...sections.preprocessor,
+    // Structs before ins and uniforms as they can reference structs
+    ...sections.structs,
+    ...dedupeQualifiedStatements(sections.inStatements, 'in'),
+    ...dedupeQualifiedStatements(sections.outStatements, 'out'),
+    ...dedupeUniforms(sections.uniforms),
+    ...sections.program,
+  ],
+});
+
+/**
+ * Group an AST into logical sections. The output of this funciton is consumed
+ * by the dedupe methods, namely dedupeUniforms, so the data shapes are coupled
+ */
+export const findShaderSections = (ast: Program): ShaderSections => {
+  const initialValue: ShaderSections = {
+    precision: [],
+    preprocessor: [],
+    version: [],
+    structs: [],
+    inStatements: [],
+    outStatements: [],
+    uniforms: [],
+    program: [],
+  };
+
+  return ast.program.reduce((sections, node) => {
+    if (node.type === 'preprocessor' && node.line.startsWith('#version')) {
+      return {
+        ...sections,
+        version: sections.version.concat(node),
+      };
+    } else if (
+      node.type === 'declaration_statement' &&
+      node.declaration.type === 'precision'
+    ) {
+      return {
+        ...sections,
+        precision: sections.precision.concat(node),
+      };
+    } else if (node.type === 'preprocessor') {
+      return {
+        ...sections,
+        preprocessor: sections.preprocessor.concat(node),
+      };
+    } else if (
+      node.type === 'declaration_statement' &&
+      node.declaration?.specified_type?.specifier?.specifier?.type === 'struct'
+    ) {
+      return {
+        ...sections,
+        structs: sections.structs.concat(node),
+      };
+      // This definition of a uniform lines up with the processing we do in
+      // dedupeUniforms
+    } else if (
+      node.type === 'declaration_statement' &&
+      // Ignore lines like "layout(std140,column_major) uniform;"
+      !node.declaration?.qualifiers?.find(
+        (q: any) => q.layout?.token === 'layout'
+      ) &&
+      // One of these checks is for a uniform with an interface block, and the
+      // other is for vanilla uniforms. I don't remember which is which
+      (node.declaration?.specified_type?.qualifiers?.find(
+        (n: any) => n.token === 'uniform'
+      ) ||
+        node.declaration?.qualifiers?.find((n: any) => n.token === 'uniform'))
+    ) {
+      return {
+        ...sections,
+        uniforms: sections.uniforms.concat(node),
+      };
+    } else if (
+      node.type === 'declaration_statement' &&
+      node.declaration?.specified_type?.qualifiers?.find(
+        (n: any) => n.token === 'in'
+      )
+    ) {
+      return {
+        ...sections,
+        inStatements: sections.inStatements.concat(node),
+      };
+    } else if (
+      node.type === 'declaration_statement' &&
+      node.declaration?.specified_type?.qualifiers?.find(
+        (n: any) => n.token === 'out'
+      )
+    ) {
+      return {
+        ...sections,
+        outStatements: sections.outStatements.concat(node),
+      };
+    } else {
+      return {
+        ...sections,
+        program: sections.program.concat(node),
+      };
+    }
+  }, initialValue);
+};

package/src/core/engine.ts

@@ -0,0 +1,214 @@
+import { Program } from '@shaderfrog/glsl-parser/ast';
+import { AstNode } from '@shaderfrog/glsl-parser/ast';
+import { MergeOptions } from '../ast/shader-sections';
+import { Graph, NodeParser } from './graph';
+import preprocess from '@shaderfrog/glsl-parser/preprocessor';
+import { generate, parser } from '@shaderfrog/glsl-parser';
+import { ShaderStage, GraphNode, NodeType } from './graph';
+import { CoreNode, NodeInput, NodePosition } from './nodes/core-node';
+import { DataNode, UniformDataType } from './nodes/data-nodes';
+import { CodeNode, SourceNode } from './nodes/code-nodes';
+import { Edge } from './nodes/edge';
+import groupBy from 'lodash.groupby';
+
+const log = (...args: any[]) =>
+  console.log.call(console, '\x1b[32m(core)\x1b[0m', ...args);
+
+export enum EngineNodeType {
+  toon = 'toon',
+  phong = 'phong',
+  physical = 'physical',
+  shader = 'shader',
+  binary = 'binary',
+}
+
+export type PhysicalNodeConstructor = (
+  id: string,
+  name: string,
+  groupId: string | null | undefined,
+  position: NodePosition,
+  uniforms: UniformDataType[],
+  stage: ShaderStage | undefined,
+  nextStageNodeId?: string
+) => CodeNode;
+
+export type ToonNodeConstructor = (
+  id: string,
+  name: string,
+  groupId: string | null | undefined,
+  position: NodePosition,
+  uniforms: UniformDataType[],
+  stage: ShaderStage | undefined,
+  nextStageNodeId?: string
+) => CodeNode;
+
+export interface Engine {
+  name: string;
+  preserve: Set<string>;
+  mergeOptions: MergeOptions;
+  // Component: FunctionComponent<{ engine: Engine; parsers: NodeParsers }>;
+  // nodes: NodeParsers;
+  parsers: Record<string, NodeParser>;
+  importers: EngineImporters;
+  evaluateNode: (node: DataNode) => any;
+  constructors: {
+    [EngineNodeType.physical]: PhysicalNodeConstructor;
+    [EngineNodeType.toon]: ToonNodeConstructor;
+  };
+}
+
+export type NodeContext = {
+  ast: AstNode | Program;
+  source?: string;
+  // Inputs are determined at parse time and should probably be in the graph,
+  // not here on the runtime context for the node
+  inputs?: NodeInput[];
+  id?: string;
+  name?: string;
+};
+
+// The context an engine builds as it evaluates. It can manage its own state
+// as the generic "RuntimeContext" which is passed to implemented engine methods
+export type EngineContext = {
+  engine: string;
+  nodes: Record<string, NodeContext>;
+  runtime: any;
+  debuggingNonsense: {
+    vertexSource?: string;
+    vertexPreprocessed?: string;
+    fragmentPreprocessed?: string;
+    fragmentSource?: string;
+  };
+};
+
+export type EngineImporter = {
+  convertAst(ast: Program, type?: ShaderStage): void;
+  nodeInputMap: Partial<Record<EngineNodeType, Record<string, string | null>>>;
+  edgeMap: { [oldInput: string]: string };
+};
+export type EngineImporters = {
+  [engine: string]: EngineImporter;
+};
+
+type EdgeUpdates = { [edgeId: string]: { oldInput: string; newInput: string } };
+
+export const convertNode = (
+  node: SourceNode,
+  converter: EngineImporter
+): SourceNode => {
+  log(`Converting ${node.name} (${node.id})`);
+  const preprocessed = preprocess(node.source, {
+    preserveComments: true,
+    preserve: {
+      version: () => true,
+      define: () => true,
+    },
+  });
+  const ast = parser.parse(preprocessed);
+  converter.convertAst(ast, node.stage);
+  const source = generate(ast);
+
+  return {
+    ...node,
+    source,
+  };
+};
+
+export const convertToEngine = (
+  oldEngine: Engine,
+  newEngine: Engine,
+  graph: Graph
+): Graph => {
+  const converter = newEngine.importers[oldEngine.name];
+  if (!converter) {
+    throw new Error(
+      `The engine ${newEngine.name} has no importer for ${oldEngine.name}`
+    );
+  }
+
+  log(`Attempting to convert from ${newEngine.name} to ${oldEngine.name}`);
+
+  // const edgeUpdates: EdgeUpdates = {};
+
+  const edgesByNodeId = groupBy(graph.edges, 'to');
+  const edgeUpdates: Record<string, Edge | null> = {};
+  const nodeUpdates: Record<string, GraphNode | null> = {};
+
+  graph.nodes.forEach((node) => {
+    // Convert engine nodes
+    if (node.type in EngineNodeType) {
+      if (node.type in newEngine.constructors) {
+        const source = node as SourceNode;
+        nodeUpdates[source.id] = // @ts-ignore
+        (newEngine.constructors[source.type] as PhysicalNodeConstructor)(
+          source.id,
+          source.name,
+          source.groupId,
+          source.position,
+          source.config.uniforms,
+          source.stage,
+          source.nextStageNodeId
+        );
+        // Bail if no conversion
+      } else {
+        throw new Error(
+          `Can't convert ${oldEngine.name} to ${newEngine.name} because ${newEngine.name} does not have a "${node.type}" constructor`
+        );
+      }
+    } else if (NodeType.SOURCE === node.type) {
+      nodeUpdates[node.id] = convertNode(node, converter);
+    }
+
+    // Then update input edges. We only care about engine nodes
+    if (node.type in converter.nodeInputMap) {
+      const map = converter.nodeInputMap[node.type as EngineNodeType]!;
+
+      (edgesByNodeId[node.id] || []).forEach((edge) => {
+        if (edge.input in map) {
+          const mapped = map[edge.input]!;
+          log('Converting edge', edge.input, 'to', map[edge.input]);
+          edgeUpdates[edge.id] = {
+            ...edge,
+            input: mapped,
+          };
+        } else {
+          log(
+            'Discarding',
+            edge.input,
+            'as there is no edge mapping in the',
+            newEngine.name,
+            'importer'
+          );
+          edgeUpdates[edge.id] = null;
+        }
+      });
+    }
+  });
+
+  graph.edges = graph.edges.reduce<Edge[]>((edges, edge) => {
+    if (edge.id in edgeUpdates) {
+      const res = edgeUpdates[edge.id];
+      if (res === null) {
+        return edges;
+      } else {
+        return [...edges, res];
+      }
+    }
+    return [...edges, edge];
+  }, []);
+
+  graph.nodes = graph.nodes.reduce<GraphNode[]>((nodes, node) => {
+    if (node.id in nodeUpdates) {
+      const res = nodeUpdates[node.id];
+      if (res === null) {
+        return nodes;
+      } else {
+        return [...nodes, res];
+      }
+    }
+    return [...nodes, node];
+  }, []);
+
+  log('Created converted graph', graph);
+  return graph;
+};

package/src/core/file.js

@@ -0,0 +1,53 @@
+const items = [
+  {
+    pairId: 1,
+    name: 'abc',
+    count: 10,
+  },
+  {
+    pairId: 1,
+    name: 'xyz',
+    count: 20,
+  },
+  {
+    pairId: 2,
+    name: 'abc',
+    count: 15,
+  },
+  {
+    pairId: 2,
+    name: 'xyz',
+    count: 25,
+  },
+  {
+    pairId: 3,
+    name: 'xyz',
+    count: 25,
+  },
+];
+
+// Group the items by pairId
+const groups = items.reduce(
+  (groups, item) => ({
+    ...groups,
+    [item.pairId]: [...(groups[item.pairId] || []), item],
+  }),
+  {}
+);
+
+// For each group, if has more than one element, append a new one with the sume of the previous elements.
+// Object.values() takes advantage of the facts that object keys are iterated over in insertion order, so
+// the resulting array is in the same order as the original one
+const result = Object.values(groups).flatMap((group) => [
+  ...group,
+  ...(group.length > 1
+    ? [
+        {
+          ...group[0],
+          count: group.reduce((sum, item) => sum + item.count, 0),
+        },
+      ]
+    : []),
+]);
+
+console.log(result);