@galacean/engine-shader-compiler 0.0.0-experimental-2.0-spine.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2020 - present Ant Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,43 @@
+## Installation
+
+```sh
+npm install @galacean/engine-shader-compiler
+```
+
+## Usage
+
+```typescript
+import { ShaderCompiler } from "@galacean/engine-shader-compiler";
+
+// Create shader compiler
+const shaderCompiler = new ShaderCompiler();
+
+// Create engine with shader compiler
+const engine = await WebGLEngine.create({ canvas: "canvas", shaderCompiler });
+
+......
+
+// Create shader by galacean shader code directly
+const shader = Shader.create(galaceanShaderCode);
+
+.......
+
+// Run engine
+engine.run()
+```
+
+There are two versions of the shader compiler: `Release` and `Verbose`. The `Verbose` version offers more user-friendly diagnostic information for debugging shader compilation errors, while the Release version provides superior performance.
+
+you can use `Verbose` version by import:
+
+```ts
+import { ShaderCompiler } from "@galacean/engine-shader-compiler/verbose";
+```
+
+## CFG Grammar conflict detection
+
+The Galacean shader syntax is defined using Context-Free Grammar (CFG) and is documented within the `\*.y` file. When modifications to the shader syntax are required, it is recommended to make changes to the existing CFG syntax file, and employ [Bison](https://www.gnu.org/software/bison/manual/bison.html) to detect any potential grammar conflicts.
+
+```sh
+bison ./Parser.y -r all
+```

package/bundler/cli.js

@@ -0,0 +1,341 @@
+#!/usr/bin/env node
+import { parseArgs } from 'node:util';
+import fs from 'node:fs';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+/** Normalize to forward slashes (Windows uses `\`; downstream regex/keys assume `/`). */ function normalizePath(p) {
+    return p.split(path.sep).join("/");
+}
+/** Recursively find all files with a given extension. */ function findFiles(dir, ext) {
+    const results = [];
+    if (!fs.existsSync(dir)) return results;
+    for (const entry of fs.readdirSync(dir, {
+        withFileTypes: true
+    })){
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            results.push(...findFiles(fullPath, ext));
+        } else if (entry.name.endsWith(ext)) {
+            results.push(normalizePath(fullPath));
+        }
+    }
+    return results;
+}
+/** `PBR.shaderc` → `PBRSource`,  `2D/Sprite.shaderc` → `SpriteSource`. */ function shadercPathToVarName(relPath) {
+    const normalized = normalizePath(relPath).replace(/\.shaderc$/, "");
+    const base = normalized.split("/").pop() ?? normalized;
+    const cleaned = base.replace(/[^A-Za-z0-9]/g, "");
+    if (cleaned.length === 0) return "Source";
+    return `${cleaned[0].toUpperCase()}${cleaned.slice(1)}Source`;
+}
+/** `Common/Common.glsl` → `Common_Common` — TS identifier for source-index imports. */ function pathToIdentifier(rel, ext) {
+    let id = rel.replace(new RegExp(`\\${ext}$`), "").replace(/[^A-Za-z0-9_]/g, "_");
+    if (/^[0-9]/.test(id)) id = "_" + id;
+    return id;
+}
+/** Remove empty parent directories upward from `startDir` until `stopDir` is hit. */ function pruneEmptyDirs(startDir, stopDir) {
+    let dir = startDir;
+    while(dir !== stopDir && fs.existsSync(dir) && fs.readdirSync(dir).length === 0){
+        fs.rmdirSync(dir);
+        dir = path.dirname(dir);
+    }
+}
+// One-shot mode exits non-zero on failure so CI breaks; watch mode logs and keeps running.
+async function precompile(options) {
+    const { failed } = await runFull(options);
+    if (options.watch) {
+        await startWatcher(options);
+        return;
+    }
+    console.log("[shader-compiler-bundler] Done.");
+    if (failed > 0) {
+        process.exit(1);
+    }
+}
+async function runFull(options) {
+    const inputDir = path.resolve(options.input);
+    const outputDir = path.resolve(options.output);
+    const platformTarget = options.platformTarget ?? 0;
+    if (!fs.existsSync(inputDir)) {
+        throw new Error(`Input directory does not exist: ${inputDir}`);
+    }
+    fs.mkdirSync(outputDir, {
+        recursive: true
+    });
+    // Emit source indexes before compile so collectIncludeMap can resolve the freshly-built sources.
+    if (options.emitSources) {
+        emitSources(inputDir);
+    }
+    const shaderCompiler = await loadShaderCompiler();
+    shaderCompiler._setIncludeMap(await collectIncludeMap(inputDir));
+    let failed = 0;
+    if (options.only) {
+        const target = path.resolve(options.only);
+        if (!compileSingle(shaderCompiler, target, inputDir, outputDir, platformTarget)) failed++;
+    } else {
+        const shaderFiles = findFiles(inputDir, ".shader");
+        console.log(`[shader-compiler-bundler] Precompiling ${shaderFiles.length} shader(s)...`);
+        for (const file of shaderFiles){
+            if (!compileSingle(shaderCompiler, file, inputDir, outputDir, platformTarget)) failed++;
+        }
+        if (options.clean) {
+            const removed = cleanOrphanedBundles(shaderFiles, inputDir, outputDir);
+            if (removed > 0) console.log(`[shader-compiler-bundler] Cleaned ${removed} orphaned .shaderc file(s).`);
+        }
+        if (failed > 0) console.warn(`[shader-compiler-bundler] ${failed} shader(s) failed to precompile.`);
+    }
+    if (options.emitIndex) {
+        emitIndex(outputDir);
+    }
+    return {
+        failed
+    };
+}
+async function startWatcher(options) {
+    const inputDir = path.resolve(options.input);
+    const outputDir = path.resolve(options.output);
+    const platformTarget = options.platformTarget ?? 0;
+    const shaderCompiler = await loadShaderCompiler();
+    shaderCompiler._setIncludeMap(await collectIncludeMap(inputDir));
+    // .glsl change → full recompile (can't cheaply track include graphs).
+    // .shader change → single-file recompile.
+    const handleInputChange = async (filename)=>{
+        if (!filename) return;
+        const norm = normalizePath(filename);
+        const fullPath = path.join(inputDir, filename);
+        if (norm.endsWith(".glsl")) {
+            console.log(`[shader-compiler-bundler] Include changed (${norm}), full recompile...`);
+            shaderCompiler._setIncludeMap(await collectIncludeMap(inputDir));
+            runFull(options).catch((e)=>console.error(e));
+            return;
+        }
+        if (!norm.endsWith(".shader")) return;
+        if (!fs.existsSync(fullPath)) {
+            removeBundleFor(fullPath, inputDir, outputDir);
+        } else {
+            compileSingle(shaderCompiler, fullPath, inputDir, outputDir, platformTarget);
+        }
+        if (options.emitSources) emitSources(inputDir);
+        if (options.emitIndex) emitIndex(outputDir);
+    };
+    console.log(`[shader-compiler-bundler] Watching ${inputDir} ...`);
+    fs.watch(inputDir, {
+        recursive: true
+    }, (_event, filename)=>{
+        handleInputChange(filename).catch((e)=>console.error(e));
+    });
+}
+async function loadShaderCompiler() {
+    // @ts-ignore — `../dist/main.js` is the compiled runtime entry; no .ts source.
+    // Bundler ships at `<pkg>/bundler/` and the runtime at `<pkg>/dist/`, so this
+    // resolves to the sibling `dist/` from `bundler/cli.js` at runtime.
+    const mod = await import('../dist/main.js');
+    const instance = new mod.ShaderCompiler();
+    if (typeof instance._precompile !== "function") {
+        throw new Error("ShaderCompiler._precompile is not available; rebuild @galacean/engine-shader-compiler first.");
+    }
+    return instance;
+}
+// Local .glsl come from inputDir; standard library comes from
+// `@galacean/engine-shader/sources` with a sibling-dir fallback for engine self-build cold-start.
+async function collectIncludeMap(inputDir) {
+    const map = {};
+    for (const file of findFiles(inputDir, ".glsl")){
+        const relPath = normalizePath(path.relative(inputDir, file));
+        map[relPath] = fs.readFileSync(file, "utf-8");
+    }
+    let usedRelease = false;
+    // Resolve from inputDir's node_modules so consumers without transitive linkage to shader-compiler can still find it.
+    try {
+        const requireFromInput = createRequire(path.join(inputDir, "package.json"));
+        const sourcesPath = requireFromInput.resolve("@galacean/engine-shader/sources");
+        const { shaderLibrary } = await import(pathToFileURL(sourcesPath).href);
+        for (const { source, path: chunkPath } of shaderLibrary){
+            map[chunkPath] = source;
+        }
+        usedRelease = true;
+    } catch  {
+    // engine self-build cold-start: dist/sources.* not yet built; use sibling source.
+    }
+    if (!usedRelease) {
+        const siblingLibrary = path.join(path.dirname(inputDir), "ShaderLibrary");
+        if (fs.existsSync(siblingLibrary)) {
+            for (const file of findFiles(siblingLibrary, ".glsl")){
+                const rel = normalizePath(path.relative(siblingLibrary, file));
+                map[`ShaderLibrary/${rel}`] = fs.readFileSync(file, "utf-8");
+            }
+        }
+    }
+    return map;
+}
+// Mirrors `ShaderPass._shaderRootPath` in engine; inlined to keep bundler engine-free.
+const SHADER_ROOT_PATH = "shaders://root/";
+function compileSingle(shaderCompiler, shaderPath, inputDir, outputDir, platformTarget) {
+    const source = fs.readFileSync(shaderPath, "utf-8");
+    const relativePath = normalizePath(path.relative(inputDir, shaderPath));
+    const bundleRelative = relativePath.replace(/\.shader$/, ".shaderc");
+    const bundlePath = path.join(outputDir, bundleRelative);
+    const basePathForIncludeKey = new URL(relativePath, SHADER_ROOT_PATH).href;
+    fs.mkdirSync(path.dirname(bundlePath), {
+        recursive: true
+    });
+    try {
+        const precompiled = shaderCompiler._precompile(source, platformTarget, basePathForIncludeKey);
+        fs.writeFileSync(bundlePath, JSON.stringify(precompiled));
+        console.log(`  ${relativePath} -> ${bundleRelative}`);
+        return true;
+    } catch (e) {
+        console.error(`  FAILED: ${relativePath}`);
+        console.error(e);
+        return false;
+    }
+}
+function cleanOrphanedBundles(shaderFiles, inputDir, outputDir) {
+    const aliveSet = new Set(shaderFiles.map((f)=>normalizePath(path.relative(inputDir, f)).replace(/\.shader$/, "")));
+    const bundleFiles = findFiles(outputDir, ".shaderc");
+    let removed = 0;
+    for (const bundleFile of bundleFiles){
+        const rel = normalizePath(path.relative(outputDir, bundleFile)).replace(/\.shaderc$/, "");
+        if (!aliveSet.has(rel)) {
+            fs.unlinkSync(bundleFile);
+            console.log(`  Removed orphaned: ${rel}.shaderc`);
+            removed++;
+            pruneEmptyDirs(path.dirname(bundleFile), outputDir);
+        }
+    }
+    return removed;
+}
+function emitIndex(outputDir) {
+    const bundleFiles = findFiles(outputDir, ".shaderc");
+    const entries = bundleFiles.map((file)=>{
+        const rel = normalizePath(path.relative(outputDir, file));
+        return {
+            varName: shadercPathToVarName(rel),
+            importPath: `./${rel}`
+        };
+    }).sort((a, b)=>a.varName.localeCompare(b.varName));
+    const imports = entries.map((e)=>`import ${e.varName} from "${e.importPath}";`).join("\n");
+    const exportBlock = `export {\n${entries.map((e)=>`  ${e.varName}`).join(",\n")}\n};`;
+    // prettier-ignore the export block: consumer's prettier config is unknown.
+    const content = `// Auto-generated by shader-compiler-precompile --emit-index — do not edit.\n${imports}\n\n// prettier-ignore\n${exportBlock}\n`;
+    const indexPath = path.join(outputDir, "index.ts");
+    const existing = fs.existsSync(indexPath) ? fs.readFileSync(indexPath, "utf-8") : "";
+    if (content !== existing) {
+        fs.writeFileSync(indexPath, content);
+        console.log("  Updated index.ts");
+    }
+}
+// Generate raw-source indexes:
+//   <inputDir>/index.ts           — `.shader` array (defines IShaderSource interface)
+//   <sibling>/ShaderLibrary/index.ts — `.glsl` array (only if the directory exists)
+function emitSources(inputDir) {
+    emitShaderSourceIndex(inputDir);
+    const libraryDir = path.join(path.dirname(inputDir), "ShaderLibrary");
+    if (fs.existsSync(libraryDir)) {
+        emitLibrarySourceIndex(libraryDir);
+    }
+}
+function emitShaderSourceIndex(rootDir) {
+    const srcDir = path.dirname(rootDir);
+    const files = findFiles(rootDir, ".shader").sort();
+    const importPaths = files.map((f)=>normalizePath(path.relative(rootDir, f)));
+    const paths = files.map((f)=>normalizePath(path.relative(srcDir, f)));
+    const ids = importPaths.map((p)=>pathToIdentifier(p, ".shader"));
+    const lines = [];
+    lines.push("// Auto-generated by shader-compiler-precompile --emit-sources — do not edit.");
+    lines.push("");
+    importPaths.forEach((rel, i)=>lines.push(`import ${ids[i]} from "./${rel}";`));
+    lines.push("");
+    lines.push("export interface IShaderSource {");
+    lines.push("  path: string;");
+    lines.push("  source: string;");
+    lines.push("}");
+    lines.push("");
+    lines.push("// prettier-ignore");
+    lines.push("export const shaders: IShaderSource[] = [");
+    paths.forEach((p, i)=>lines.push(`  { source: ${ids[i]}, path: ${JSON.stringify(p)} },`));
+    lines.push("];");
+    lines.push("");
+    const out = path.join(rootDir, "index.ts");
+    fs.writeFileSync(out, lines.join("\n"), "utf-8");
+    console.log(`  Emitted shaders index ${path.relative(process.cwd(), out)} (${files.length})`);
+}
+function emitLibrarySourceIndex(rootDir) {
+    const srcDir = path.dirname(rootDir);
+    const files = findFiles(rootDir, ".glsl").sort();
+    const importPaths = files.map((f)=>normalizePath(path.relative(rootDir, f)));
+    const paths = files.map((f)=>normalizePath(path.relative(srcDir, f)));
+    const ids = importPaths.map((p)=>pathToIdentifier(p, ".glsl"));
+    const lines = [];
+    lines.push("// Auto-generated by shader-compiler-precompile --emit-sources — do not edit.");
+    lines.push("");
+    lines.push(`import type { IShaderSource } from "../Shaders";`);
+    importPaths.forEach((rel, i)=>lines.push(`import ${ids[i]} from "./${rel}";`));
+    lines.push("");
+    lines.push("// prettier-ignore");
+    lines.push("export const shaderLibrary: IShaderSource[] = [");
+    paths.forEach((p, i)=>lines.push(`  { source: ${ids[i]}, path: ${JSON.stringify(p)} },`));
+    lines.push("];");
+    lines.push("");
+    const out = path.join(rootDir, "index.ts");
+    fs.writeFileSync(out, lines.join("\n"), "utf-8");
+    console.log(`  Emitted library index ${path.relative(process.cwd(), out)} (${files.length})`);
+}
+function removeBundleFor(shaderPath, inputDir, outputDir) {
+    const rel = normalizePath(path.relative(inputDir, shaderPath)).replace(/\.shader$/, ".shaderc");
+    const bundlePath = path.join(outputDir, rel);
+    if (fs.existsSync(bundlePath)) {
+        fs.unlinkSync(bundlePath);
+        console.log(`  Removed: ${rel}`);
+        pruneEmptyDirs(path.dirname(bundlePath), outputDir);
+    }
+}
+
+/**
+ * Entry point for the `shader-compiler-precompile` bin.
+ *
+ * Usage:
+ *   shader-compiler-precompile <input-dir> <output-dir>
+ *     [--clean] [--watch] [--only <file>] [--emit-index] [--emit-sources]
+ */ async function main() {
+    const { positionals, values } = parseArgs({
+        allowPositionals: true,
+        options: {
+            clean: {
+                type: "boolean"
+            },
+            watch: {
+                type: "boolean"
+            },
+            only: {
+                type: "string"
+            },
+            "emit-index": {
+                type: "boolean"
+            },
+            "emit-sources": {
+                type: "boolean"
+            }
+        }
+    });
+    const [input, output] = positionals;
+    if (!input || !output) {
+        console.error("Usage: shader-compiler-precompile <input-dir> <output-dir> [--clean] [--watch] [--only <file>] [--emit-index] [--emit-sources]");
+        process.exit(1);
+    }
+    await precompile({
+        input,
+        output,
+        clean: values.clean,
+        watch: values.watch,
+        only: values.only,
+        emitIndex: values["emit-index"],
+        emitSources: values["emit-sources"]
+    });
+}
+main().catch((err)=>{
+    console.error(err);
+    process.exit(1);
+});