firefly-compiler 0.5.79 → 0.5.81

package/compiler/Builder.ff

@@ -16,54 +16,40 @@ build(
     mainModules: List[ModuleKey]
     resolvedDependencies: ResolvedDependencies
     compilerModulePath: Option[Path]
-    tempPath: Path
     jsOutputPath: Path
     printMeasurements: Bool
     moduleCache: ModuleCache
 ): Unit {
-
-    if(tempPath.exists()) {tempPath.delete()}
-    tempPath.createDirectory()
-
-    let jsPathFile = tempPath.slash("js")
-    jsPathFile.createDirectory(createParentDirectories = True)
-
-    let success = do {
-        let compiler = Compiler.new(
-            emitTarget
-            system.mainTask()
-            compilerModulePath
-            jsPathFile
-            resolvedDependencies
-            Map.new()
-            moduleCache
-            lspHook = LspHook.disabled()
-        )
-        mainModules.each {moduleKey => compiler.emit(moduleKey, isMainModule = True)}
-        if(printMeasurements) {compiler.printMeasurements()}
-        resolvedDependencies.packagePaths.each {packagePair, packagePath =>
-            resolvedDependencies.packages.get(packagePair).each {packageInfo =>
-                processNodeModules(system, jsPathFile, packagePath, packageInfo)
-                if(emitTarget != EmitBrowser) {
-                    processIncludes(jsPathFile, packagePath, packageInfo)
-                }
+    jsOutputPath.createDirectory(createParentDirectories = True)
+    let compiler = Compiler.new(
+        emitTarget
+        system.mainTask()
+        compilerModulePath
+        jsOutputPath
+        resolvedDependencies
+        Map.new()
+        moduleCache
+        lspHook = LspHook.disabled()
+    )
+    mainModules.each {moduleKey => compiler.emit(moduleKey, isMainModule = True)}
+    if(printMeasurements) {compiler.printMeasurementsPerPhase()}
+    resolvedDependencies.packagePaths.each {packagePair, packagePath =>
+        resolvedDependencies.packages.get(packagePair).each {packageInfo =>
+            processNodeModules(system, jsOutputPath, packagePath, packageInfo)
+            if(emitTarget != EmitBrowser) {
+                processIncludes(jsOutputPath, packagePath, packageInfo)
             }
         }
-        True
     }
-
-    if(success) {
-        if(jsOutputPath.exists()) {jsOutputPath.delete()}
-        jsPathFile.renameTo(jsOutputPath)
-    }
-
 }
 
 processIncludes(jsPathFile: Path, packagePath: Path, info: PackageInfo) {
     info.includes.each {include =>
         let fromPath = packagePath.slash(".firefly").slash("include").slash(include.path)
         let toPath = jsPathFile.slash(info.package.packagePair.groupName("/")).slash(include.path)
-        toPath.createSymlinkTo(fromPath, junction = True)
+        if(!toPath.exists()) {
+            toPath.createSymlinkTo(fromPath, junction = True)
+        }
     }
 }
 
@@ -86,7 +72,14 @@ processNodeModules(system: NodeSystem, jsPathFile: Path, packagePath: Path, info
     }
 }
 
-buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFiles: List[Path], target: String) {
+buildViaBuildSystem(
+    system: NodeSystem
+    fireflyPath: Path
+    mainFiles: List[Path]
+    target: String
+    moduleCache: ModuleCache = ModuleCache.new(0)
+    printMeasurements: Bool = False
+) {
     let resolvedDependencies = Dependencies.process(
         system.httpClient()
         DependencyLock.new(system.mainTask())
@@ -115,10 +108,9 @@ buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFiles: List[Path]
         mainModules = mainModuleKeys
         resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
         compilerModulePath = None
-        tempPath = system.path(".firefly/temporary")
         jsOutputPath = system.path(".firefly/output").slash(target)
-        printMeasurements = False
-        moduleCache = ModuleCache.new(0)
+        printMeasurements = printMeasurements
+        moduleCache = moduleCache
     )
 }
 

package/compiler/Compiler.ff

@@ -132,6 +132,16 @@ extend self: Compiler {
         }
     }
 
+    printMeasurementsPerPhase(): Unit {
+        self.phaseDurations.toList().map {
+            | Pair(name, d) {name.split(' ') | [phase, module]} => Pair(phase, d)
+            | Pair(name, d) => Pair("Unknown", d)
+        }.group().each {phase, durations => 
+            let d = Duration(durations.map {_.seconds}.foldLeft(0.0, {_ + _}))
+            Log.debug(phase + " (" + durations.size() + "):\t" + d.show())
+        }
+    }
+    
     parse(moduleKey: ModuleKey, importedAt: Option[Location]): Module {
         self.cache.cacheParsedModule(self.packagePaths, moduleKey): path => 
         self.measure("Parse", moduleKey):
@@ -214,21 +224,21 @@ extend self: Compiler {
             self.infer(i.moduleKey)
         }
 
+        let packagePath = self.jsOutputPath.slash(moduleKey.packagePair.group).slash(moduleKey.packagePair.name)
+        let jsPath = moduleKey.folders.foldLeft(packagePath) {p, f => p.slash(f)}
         let allModules = [module, ...otherModules]
         let emitter = JsEmitter.new(
             otherModules = allModules
             emitTarget = self.emitTarget
             isMainModule = isMainModule
-            compilerModuleFileUrl = self.compilerModulePath.map {_.url()}
+            compilerModuleFileUrl = self.compilerModulePath.map {_.relativeUrlTo(jsPath)}
             moduleKey = moduleKey
         )
         emitter.emitModule(module)
-        let packagePath = self.jsOutputPath.slash(moduleKey.packagePair.group).slash(moduleKey.packagePair.name)
-        let jsPath = moduleKey.folders.foldLeft(packagePath) {p, f => p.slash(f)}
         let jsFile = jsPath.slash(moduleKey.name + ".mjs")
         let sourceMapFile = jsPath.slash(moduleKey.name + ".mjs.map")
         let source = Some(path.readText())
-        let jsAndSourceMap = emitter.makeOutputAndSourceMap(path.relativeListTo(jsPath).join("/"), source)
+        let jsAndSourceMap = emitter.makeOutputAndSourceMap(path.relativeUrlTo(jsPath), source)
         jsPath.createDirectory(createParentDirectories = True)
         jsFile.writeText(jsAndSourceMap.first + "\n\n//# sourceMappingURL=" + sourceMapFile.base())
         sourceMapFile.writeText(jsAndSourceMap.second.write(Some("    ")))

package/compiler/DevelopMode.ff

@@ -0,0 +1,413 @@
+import Main
+import ModuleCache
+import Dependencies
+import DependencyLock
+import Syntax
+import JsEmitter
+import Builder from ff:compiler
+
+capability Runner(
+    lock: Lock
+    lockCondition: LockCondition
+    mutable iteration: Int
+    mutable state: RunnerState
+    mutable changedSinceCompilationStarted: Set[String]
+    mutable recompile: Bool
+    mutable appRunning: Bool
+)
+
+data RunnerState {
+    CompilingState
+    CompileErrorState(at: Option[Location], output: String)
+    ApplicationRunningState
+    ApplicationCrashedState(output: String)
+}
+
+class EsbuildContext(mutable jsValue: Option[JsValue])
+
+run(
+    system: NodeSystem
+    fireflyPath: Path
+    proxyPort: Int
+    targetPort: Int
+    mainFile: String
+    arguments: List[String]
+) {
+    let esbuildContext = EsbuildContext(None)
+    let lock = system.mainTask().lock()
+    let lockCondition = lock.condition()
+    let runner = Runner(lock, lockCondition, 1, CompilingState, Set.new(), False, False)
+    startProxy(system, runner, proxyPort, targetPort)
+    startChangeListener(system, runner, system.path("."))
+    let moduleCache = ModuleCache.new(0)
+    while {True} {
+        let moduleKey = build(system, runner, mainFile, moduleCache)
+        let task = moduleKey.{
+            | None => 
+                system.mainTask().spawn {_ => }
+            | Some(key) => 
+                runner.lock.do {
+                    runner.state = ApplicationRunningState
+                }
+                startApp(system, runner, fireflyPath, esbuildContext, moduleCache, key, mainFile, arguments)
+        }
+        runner.lock.do {
+            if(runner.appRunning) {
+                while {runner.appRunning} {
+                    runner.lockCondition.sleep()
+                }
+            }
+            while {!runner.recompile} {
+                runner.lockCondition.sleep()
+            }
+            task.abort()
+            runner.changedSinceCompilationStarted.each {key =>
+                moduleCache.invalidate(key)
+            }
+            runner.state = CompilingState
+            runner.recompile = False
+            runner.changedSinceCompilationStarted = Set.new()
+            runner.iteration += 1
+        }
+    }
+}
+
+build(system: NodeSystem, runner: Runner, mainFile: String, moduleCache: ModuleCache): Option[ModuleKey] {
+    try {
+        Main.prepareFireflyDirectory(system.path("."))
+        let resolvedDependencies = Dependencies.process(
+            system.httpClient()
+            DependencyLock.new(system.mainTask())
+            system.path(mainFile)
+        )
+        let mainPath = system.path(mainFile)
+        Some(Main.buildScript(
+            system
+            mainPath
+            resolvedDependencies.mainPackagePair
+            EmitNode
+            resolvedDependencies
+            moduleCache
+            printMeasurements = False
+        ))
+    } tryCatch {| CompileError(at, message), error =>
+        Log.debug(message)
+        Log.debug(" at " + at.file.replace("./", "") + ":" + at.line + ":" + at.column)
+        runner.lock.do {
+            runner.state = CompileErrorState(Some(at), message)
+        }
+        None
+    } tryCatch {| CompileErrors(compileErrors), error =>
+        compileErrors.each {| CompileError(at, message) =>
+            Log.debug(message)
+            Log.debug(" at " + at.file.replace("./", "") + ":" + at.line + ":" + at.column)
+            runner.lock.do {
+                runner.state = CompileErrorState(Some(at), message)
+            }
+        }
+        None
+    } catchAny {error =>
+        Log.debug(error.message())
+        runner.lock.do {
+            runner.state = CompileErrorState(None, error.message())
+        }
+        None
+    }
+}
+
+startApp(
+    system: NodeSystem
+    runner: Runner
+    fireflyPath: Path
+    esbuildContext: EsbuildContext
+    moduleCache: ModuleCache
+    moduleKey: ModuleKey
+    mainFile: String
+    arguments: List[String]
+): Task {
+    let taskIteration = runner.iteration
+    system.mainTask().spawn {task =>
+        try {
+            let runFile = Main.locateRunFile(system, "node", moduleKey)
+            let runFilePath = if(runFile.contains("://")) {system.pathFromUrl(runFile)} else {system.path(runFile)}
+            let startPath = runFilePath.parent().grab().slash(runFilePath.base() + ".start.mjs")
+            startPath.writeText(
+                "import * as run from " + Json.string("./" + runFilePath.base()).write() + "\n" + 
+                "globalThis.ffDevelopMode = true\n" + 
+                // The following should be awaited, but esbuild doesn't support top level await - still seems to work though
+                "run.$run$(" + Json.string(fireflyPath.absolute()).write() + ", " + Json.toJson(arguments).write() + ")"
+            )
+            let esBuildPath = runFilePath.parent().grab().slash(runFilePath.base() + ".minified.js")
+            let context = esbuildContext.jsValue.else {
+                let jsValue = BuildSystem.internalNodeCallEsBuildContext(system, startPath.absolute(), esBuildPath.absolute(), minify = True)
+                esbuildContext.jsValue = Some(jsValue)
+                jsValue
+            }
+            Js.await(context->rebuild())
+            let relativeStartFile = esBuildPath.relativeTo(system.path("."))
+            let result = system.execute(relativeStartFile, arguments, node = Some({message, forkedProcess =>
+                if(message->ffDevelopMode === "internalCompile") {
+                    let mainFiles: List[String] = message->mainFiles?
+                    let mainPaths = mainFiles.map {system.path(_)}
+                    let target: String = message->target?
+                    runner.lock.do {
+                        if(taskIteration == runner.iteration):
+                        try {
+                            Builder.buildViaBuildSystem(system, fireflyPath, mainPaths, target, moduleCache, printMeasurements = False)
+                            forkedProcess->send(Js->(ffDevelopMode = "internalCompile"))
+                            Unit
+                        } tryCatch {| CompileError(at, message), error =>
+                            runner.state = CompileErrorState(Some(at), message)
+                        } tryCatch {| CompileErrors(compileErrors), error =>
+                            compileErrors.each {| CompileError(at, message) =>
+                                runner.state = CompileErrorState(Some(at), message)
+                            }
+                        } catchAny {error =>
+                            runner.state = CompileErrorState(None, error.message())
+                        }
+                    }
+                }
+            }))
+            let standardOut = result.standardOut.toString()
+            let standardError = result.standardError.toString()
+            runner.lock.do {
+                runner.appRunning = False
+                runner.state.{
+                    | ApplicationRunningState {taskIteration == runner.iteration} => 
+                        runner.state = ApplicationCrashedState(
+                            "Exited with code: " + result.exitCode + "\n\n" + standardOut + "\n\n" + standardError
+                        )
+                    | _ => 
+                }
+                runner.lockCondition.wakeAll()
+            }
+        } catchAny {error =>
+            runner.lock.do {
+                runner.appRunning = False
+                runner.lockCondition.wakeAll()
+            }
+        }
+    }
+}
+
+startChangeListener(
+    system: NodeSystem
+    runner: Runner
+    path: Path
+) {
+    let fs = Js.import("node:fs")
+    fs->watch(path.absolute(), Js->(recursive = True), Js->{eventType, fileName =>
+        if(!fileName.isNullOrUndefined()):
+        let file: String = fileName?
+        if(file.endsWith(".ff") || file.endsWith(".firefly-workspace")) {
+            let key = system.path(file).absolute()
+            runner.lock.do {
+                // Probably we should also listen for other files, e.g. resources
+                runner.changedSinceCompilationStarted = runner.changedSinceCompilationStarted.add(key)
+            }
+        }
+    })
+}
+
+startProxy(
+    system: NodeSystem
+    runner: Runner
+    proxyPort: Int
+    targetPort: Int
+) {
+    let net = Js.import("node:net")
+    let targetServer = "localhost" // 127.0.0.1
+    let proxyServer = net->createServer(Js->(pauseOnConnect = True), Js->{clientSocket =>
+        mutable targetSocket = Js.undefined()
+        mutable connected = False
+        clientSocket->on("error", Js->{err =>
+            if(!targetSocket.isUndefined()) {targetSocket->end()}
+            // Mute "Error: write EPIPE" and other errors that happen while writing a custom response to the client
+            //Log.debugDynamic(err)
+        })
+        function serveWaiterHtml() {
+            if(runner.changedSinceCompilationStarted.size() != 0) {
+                runner.recompile = True
+                system.mainTask().spawn {task =>
+                    runner.lock.do {
+                        runner.lockCondition.wakeAll()
+                    }
+                }
+            }
+            function escapeHtml(html: String): String {
+                html.replace("&", "&")
+                    .replace("'", "'")
+                    .replace("\"", """)
+                    .replace("<", "&lt;")
+                    .replace(">", "&gt;")
+            }
+            let status = runner.state.{
+                | _ {runner.recompile} => "Restarting..."
+                | ApplicationCrashedState(output) => "Application crashed!<br>" + escapeHtml(output)
+                | ApplicationRunningState => "Starting application..."
+                | CompileErrorState(Some(at), output) => 
+                    let location = escapeHtml(at.file + ":" + at.line + ":" + at.column)
+                    let relativeFile = system.path(at.file).relativeTo(system.path("."))
+                    let relativeLocation = escapeHtml(relativeFile + ":" + at.line + ":" + at.column)
+                    let link = "<a href='vscode://file/" + escapeHtml(location) + "'>" + relativeLocation + "</a>"
+                    escapeHtml(output) + "<br><br>at " + link
+                | CompileErrorState(None, output) => "Compiler crashed!<br>" + escapeHtml(output)
+                | CompilingState => "Compiling..."
+            }
+            
+            let waiterBuffer = waiterHtml.replace("[STATUS]", status).toBuffer()
+            clientSocket->write("HTTP/1.1 200 OK\r\n")
+            clientSocket->write("Content-Type: text/html\r\n")
+            clientSocket->write("Content-Length: " + waiterBuffer.size() + "\r\n")
+            clientSocket->write("x-firefly-develop-mode: true\r\n")
+            clientSocket->write("Connection: close\r\n")
+            clientSocket->write("\r\n")
+            clientSocket->write(
+                Js->Buffer->from(waiterBuffer!->buffer, waiterBuffer!->byteOffset, waiterBuffer!->byteLength)
+            )
+            clientSocket->end()
+        }
+        targetSocket = net->createConnection(targetPort, targetServer, Js->{
+            connected = True
+            let direct = runner.state.{
+                | ApplicationRunningState => !runner.recompile && runner.changedSinceCompilationStarted.size() == 0
+                | _ => False
+            }
+            if(direct) {
+                clientSocket->pipe(targetSocket)->pipe(clientSocket)
+                clientSocket->resume()?
+            } else {
+                serveWaiterHtml()
+            }
+        })
+        targetSocket->on("error", Js->{err =>
+            if(connected) {
+                clientSocket->end()?
+            } else {
+                serveWaiterHtml()
+            }
+        })?
+    })
+    
+    proxyServer->listen(proxyPort, Js->{
+        //print(system, "Proxy server running on port " + proxyPort)
+    })
+}
+
+waiterHtml = """<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Firefly develop mode</title>
+    <style>
+        body {
+            background-color: #121212;
+            color: #e0e0e0;
+            font-family: 'Courier New', monospace;
+            display: flex;
+            flex-direction: column;
+            justify-content: center;
+            align-items: center;
+            height: 100vh;
+            margin: 0;
+            background-image: radial-gradient(circle at center, #1a1a2e, #121212);
+        }
+
+        h1 {
+            color: #00f2ff;
+            text-shadow: 0 0 10px #00f2ff;
+            font-size: 3rem;
+            margin-bottom: 20px;
+            animation: glow 2s ease-in-out infinite alternate;
+        }
+
+        p {
+            background-color: #1a1a2e;
+            color: #00f2ff;
+            padding: 15px 30px;
+            border-radius: 25px;
+            font-size: 1.2rem;
+            border: 2px solid #00f2ff;
+            box-shadow: 0 0 15px #00f2ff;
+            animation: pulse 1.5s infinite;
+        }
+            
+        a {
+            color: #00f2ff;
+        }
+
+        @media only screen and (max-width: 600px) {
+            h1 {
+                font-size: 1.7rem;
+            }
+            p {
+                padding: 10px 20px;
+                border-radius: 50px;
+                font-size: 1.1rem;
+                max-width: 100%;
+                box-sizing: border-box;
+            }
+        }
+                    
+        @keyframes glow {
+            from {
+                text-shadow: 0 0 10px #00f2ff;
+            }
+            to {
+                text-shadow: 0 0 20px #00f2ff, 0 0 30px #00f2ff;
+            }
+        }
+
+        @keyframes pulse {
+            0% {
+                box-shadow: 0 0 0 0 rgba(0, 242, 255, 0.7);
+            }
+            70% {
+                box-shadow: 0 0 0 10px rgba(0, 242, 255, 0);
+            }
+            100% {
+                box-shadow: 0 0 0 0 rgba(0, 242, 255, 0);
+            }
+        }
+    </style>
+    <script>
+        let start = Date.now()
+        let appStarted = null
+        let delay = 10
+        let poll = async () => {
+            //delay *= 1.1
+            try {
+                let response = await fetch(".", {cache: 'no-store'})
+                if(!response.headers.has('x-firefly-develop-mode')) {
+                    let now = Date.now()
+                    let compiling = appStarted - start
+                    let appStarting = now - appStarted
+                    //window.alert("Reloading after: " + (now - start) + " ms. Compiling: " + compiling + " ms. Starting application: " + appStarting + " ms.")
+                    window.location.reload(true)
+                    return
+                } else {
+                    let html = await response.text()
+                    if(appStarted == null && html.includes("Starting application...")) appStarted = Date.now()
+                    let parser = new DOMParser()
+                    let d = parser.parseFromString(html, 'text/html')
+                    let bodyHtml = d.body.innerHTML
+                    if(document.body.innerHTML !== bodyHtml) {
+                        document.body.innerHTML = bodyHtml
+                    }
+                    setTimeout(poll, delay)
+                }
+            } catch (error) {
+                console.error("Polling error:", error)
+                setTimeout(poll, delay)
+            }
+        }
+        setTimeout(poll, delay)
+    </script>
+</head>
+<body>
+    <h1>Firefly develop mode</h1>
+    <p>[STATUS]</p>
+</body>
+</html>
+"""

package/compiler/JsEmitter.ff

@@ -206,6 +206,7 @@ extend self: JsEmitter {
                 "import {" + escapeKeyword(buildMain.signature.name) + "$} from './" + moduleName + ".mjs'"
             }.toList()
             "import {" + escapeKeyword(mainName) + "$} from './" + moduleName + ".mjs'"
+            self.emitImport(ModuleKey(PackagePair("ff", "core"), [], "Error"))
             "export async function $run$(fireflyPath_, arguments_) {"
             "Error.stackTraceLimit = 50"
             "const $task = {controller_: new AbortController(), subtasks_: new Set(), promise_: new Promise(() => {}), started_: performance.now() * 0.001}"
@@ -230,6 +231,11 @@ extend self: JsEmitter {
             ...if(self.emitTarget == EmitBuild) {[
                 "await $firefly_compiler.internalCreateExecutable_$(system, '.firefly/output/executable/Main.bundle.js', '.firefly/output', ['host'], system.assets_, $task)"
             ]} else {[]}
+            ...if(!willRunOnNode) {[]} else {[
+                "} catch(error) {"
+                "console.error(ff_core_Error.Error_stack(error))"
+                "process.exit(1)"
+            ]}
             "} finally {"
             ...if(self.emitTarget != EmitBrowser) {[
                 "$task.controller_.abort()"