firefly-compiler 0.5.79 → 0.5.80

package/compiler/Main.ff

@@ -11,10 +11,12 @@ import Inference
 import ModuleCache
 import LspHook
 import DependencyLock
+import DevelopMode
 
 data MainCommand {
     BootstrapCommand
     RunCommand(mainPath: String, argument: List[String])
+    DevelopCommand(mainPath: String, argument: List[String])
     BrowserCommand(mainPath: String)
     BuildCommand(mainPath: String)
     CheckCommand(filePath: String)
@@ -27,43 +29,6 @@ main(system: NodeSystem): Unit {
 
     let fireflyPath = detectFireflyPath(system)
 
-    function buildScript(
-        mainPath: Path
-        mainPackagePair: PackagePair
-        emitTarget: EmitTarget
-        resolvedDependencies: ResolvedDependencies
-    ): ModuleKey {
-        let fixedPackagePaths = if(resolvedDependencies.packagePaths.contains(PackagePair("ff", "core"))) {
-            resolvedDependencies.packagePaths
-        } else {
-            resolvedDependencies.packagePaths.add(PackagePair("ff", "core"), fireflyPath.slash("core"))
-        }
-        let compilerModulePath = if(emitTarget != EmitBrowser && emitTarget != EmitExecutable) {
-            fireflyPath.slash("output").slash("js").slash("ff").slash("compiler/Builder.mjs")
-        }
-        let targetName = emitTarget.{
-            | EmitBuild => "build"
-            | EmitNode => "node"
-            | EmitBrowser => "browser"
-            | EmitExecutable => "executable"
-        }
-        let folders = mainPath.parent().grab().relativeListTo(fixedPackagePaths.grab(mainPackagePair))
-        let name = mainPath.base().removeLast(".ff").grab()
-        let moduleKey = ModuleKey(mainPackagePair, folders, name)
-        Builder.build(
-            system = system
-            emitTarget = emitTarget
-            mainModules = [moduleKey]
-            resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
-            compilerModulePath = compilerModulePath
-            tempPath = system.path(".firefly").slash("temporary")
-            jsOutputPath = system.path(".firefly").path("output").path(targetName)
-            printMeasurements = False
-            moduleCache = ModuleCache.new(0)
-        )
-        moduleKey
-    }
-
     function runCommand(command: MainCommand) {
         | RunCommand(mainFile, arguments) =>
             let resolvedDependencies = Dependencies.process(
@@ -73,12 +38,15 @@ main(system: NodeSystem): Unit {
             )
             prepareFireflyDirectory(system.path("."))
             let mainPath = system.path(mainFile)
-            let moduleKey = buildScript(mainPath, resolvedDependencies.mainPackagePair, EmitNode, resolvedDependencies)
+            let moduleKey = buildScript(system, mainPath, resolvedDependencies.mainPackagePair, EmitNode, resolvedDependencies)
             if(!importAndRun(system, fireflyPath, "node", moduleKey, arguments)) {
                 let at = Location(system.path(mainFile).absolute(), 1, 1)
                 throw(CompileError(at, "This module does not contain a 'nodeMain' function"))
             }
 
+        | DevelopCommand(mainFile, arguments) =>
+            DevelopMode.run(system, fireflyPath, mainFile, arguments)
+            
         | BrowserCommand(mainFile) =>
             let resolvedDependencies = Dependencies.process(
                 system.httpClient()
@@ -88,7 +56,7 @@ main(system: NodeSystem): Unit {
             prepareFireflyDirectory(system.path("."))
             let mainPath = system.path(mainFile)
             let moduleKey = 
-                buildScript(mainPath, resolvedDependencies.mainPackagePair, EmitBrowser, resolvedDependencies)
+                buildScript(system, mainPath, resolvedDependencies.mainPackagePair, EmitBrowser, resolvedDependencies)
             bundleForBrowser(system, resolvedDependencies.mainPackagePair, moduleKey)
 
         | BuildCommand(mainFile) =>
@@ -99,8 +67,8 @@ main(system: NodeSystem): Unit {
             )
             prepareFireflyDirectory(system.path("."))
             let mainPath = system.path(mainFile)
-            let moduleKey = buildScript(mainPath, resolvedDependencies.mainPackagePair, EmitBuild, resolvedDependencies)
-            buildScript(mainPath, resolvedDependencies.mainPackagePair, EmitExecutable, resolvedDependencies)
+            let moduleKey = buildScript(system, mainPath, resolvedDependencies.mainPackagePair, EmitBuild, resolvedDependencies)
+            buildScript(system, mainPath, resolvedDependencies.mainPackagePair, EmitExecutable, resolvedDependencies)
             bundleForPkg(system, resolvedDependencies.mainPackagePair, mainPath.base())
             importAndRun(system, fireflyPath, "build", moduleKey, [])
 
@@ -141,7 +109,6 @@ main(system: NodeSystem): Unit {
                     singleFilePackages = [].toSet()
                 )
                 compilerModulePath = None
-                tempPath = workingDirectory.slash("output").slash("temporary")
                 jsOutputPath = workingDirectory.slash("output").slash("js")
                 printMeasurements = True
                 moduleCache = ModuleCache.new(0)
@@ -184,14 +151,55 @@ usageString = """
 usage: firefly <main-file> [<main-arguments>] | <command> [<command-arguments>]
 
 These are the commands:
-   run <main-file> [<main-arguments>]    Run the main file with the provided arguments
-   browser <main-file>                   Compile the main file for the browser
-   build <main-file>                     Build the main file
-   check <firefly-file>                  Check the firefly source file for errors
-   symbols <out-file> <firefly-file>     Print a .tsv with the symbols of a firefly source file
-   bootstrap                             Bootstrap the compiler
+   run <main-file> [<main-arguments>]       Run the main file with the provided arguments
+   develop <main-file> [<main-arguments>]   Develop the main file with the provided arguments
+   browser <main-file>                      Compile the main file for the browser
+   build <main-file>                        Build the main file
+   check <firefly-file>                     Check the firefly source file for errors
+   symbols <out-file> <firefly-file>        Print a .tsv with the symbols of a firefly source file
+   bootstrap                                Bootstrap the compiler
 """
 
+buildScript(
+    system: NodeSystem
+    mainPath: Path
+    mainPackagePair: PackagePair
+    emitTarget: EmitTarget
+    resolvedDependencies: ResolvedDependencies
+    moduleCache: ModuleCache = ModuleCache.new(0)
+    printMeasurements: Bool = False
+): ModuleKey {
+    let fireflyPath = detectFireflyPath(system)
+    let fixedPackagePaths = if(resolvedDependencies.packagePaths.contains(PackagePair("ff", "core"))) {
+        resolvedDependencies.packagePaths
+    } else {
+        resolvedDependencies.packagePaths.add(PackagePair("ff", "core"), fireflyPath.slash("core"))
+    }
+    let compilerModulePath = if(emitTarget != EmitBrowser && emitTarget != EmitExecutable) {
+        fireflyPath.slash("output").slash("js").slash("ff").slash("compiler/Builder.mjs")
+    }
+    let targetName = emitTarget.{
+        | EmitBuild => "build"
+        | EmitNode => "node"
+        | EmitBrowser => "browser"
+        | EmitExecutable => "executable"
+    }
+    let folders = mainPath.parent().grab().relativeListTo(fixedPackagePaths.grab(mainPackagePair))
+    let name = mainPath.base().removeLast(".ff").grab()
+    let moduleKey = ModuleKey(mainPackagePair, folders, name)
+    Builder.build(
+        system = system
+        emitTarget = emitTarget
+        mainModules = [moduleKey]
+        resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
+        compilerModulePath = compilerModulePath
+        jsOutputPath = system.path(".firefly").path("output").path(targetName)
+        printMeasurements = printMeasurements
+        moduleCache = moduleCache
+    )
+    moduleKey
+}
+
 parseCommandLine(arguments: List[String]): MainCommand {
     | [mainFile, ...mainArguments] {mainFile.removeLast(".ff") | Some(mainName)} =>
         RunCommand(mainFile, mainArguments)
@@ -203,6 +211,14 @@ parseCommandLine(arguments: List[String]): MainCommand {
                 "You must specify a Firefly file (.ff) as first argument to run." + usageString
             ))
         }
+    | ["develop", ...runArguments] =>
+        runArguments.{
+            | [mainFile, ...mainArguments] {mainFile.removeLast(".ff") | Some(mainName)} =>
+                DevelopCommand(mainFile, mainArguments)
+            | _ => throw(CommandLineError(
+                "You must specify a Firefly file (.ff) as first argument to develop." + usageString
+            ))
+        }
     | ["browser", ...browserArguments] =>
         browserArguments.{
             | [mainFile] {mainFile.removeLast(".ff") | Some(mainName)} =>
@@ -278,11 +294,7 @@ importAndRun(
     moduleKey: ModuleKey
     arguments: List[String]
 ): Bool {
-    let process = Js.await(Js.dynamicImport("process"))
-    let cwd = process->cwd()
-    let workingDirectory = if(cwd->indexOf(":") === 1) {"file:///" + cwd?} else {cwd?}
-    let packagePath = moduleKey.packagePair.groupName("/")
-    let runFile = workingDirectory + "/.firefly/output/" + target + "/" + packagePath + "/" + moduleKey.importName() + ".run.mjs"
+    let runFile = locateRunFile(system, target, moduleKey)
     let runFilePath = if(runFile.contains("://")) {system.pathFromUrl(runFile)} else {system.path(runFile)}
     if(runFilePath.exists()) {
         let main = Js.await(Js.dynamicImport(runFile))
@@ -298,6 +310,14 @@ importAndRun(
     }
 }
 
+locateRunFile(system: NodeSystem, target: String, moduleKey: ModuleKey): String {
+    let process = Js.await(Js.dynamicImport("process"))
+    let cwd = process->cwd()
+    let workingDirectory = if(cwd->indexOf(":") === 1) {"file:///" + cwd?} else {cwd?}
+    let packagePath = moduleKey.packagePair.groupName("/")
+    workingDirectory + "/.firefly/output/" + target + "/" + packagePath + "/" + moduleKey.importName() + ".run.mjs"
+}
+
 prepareFireflyDirectory(path: Path) {
     if(!path.slash(".firefly").slash("output").exists()) {
         if(!path.slash(".firefly").exists()) {

package/compiler/ModuleCache.ff

@@ -22,9 +22,11 @@ new(version: Int): ModuleCache {
 
 extend self: ModuleCache {
     
-    remove(keys: List[String]) {
+    remove(keys: List[String], removeParsed: Bool = True) {
         if(!keys.isEmpty()):
-        self.parsedModules = self.parsedModules.removeList(keys)
+        if(removeParsed) {
+            self.parsedModules = self.parsedModules.removeList(keys)
+        }
         self.resolvedModules = self.resolvedModules.removeList(keys)
         self.derivedModules = self.derivedModules.removeList(keys)
         self.inferredModules = self.inferredModules.removeList(keys)
@@ -32,13 +34,11 @@ extend self: ModuleCache {
     }
     
     invalidate(key: String) {
-        //Log.trace("Invalidate: " + uri)
         self.parsedModules.get(key).each: | Pair(module, _) =>
         self.remove([key])
         self.parsedModules.each {| k, Pair(m, _) =>
             if(m.imports.any {i => i.moduleKey == module.moduleKey}) {
-                //Log.trace("Invalidating due to import of invalidated module: " + m.packagePair.groupName() + "/" + m.file)
-                self.remove([k])
+                self.remove([k], removeParsed = False)
             }
         }
     }

package/core/.firefly/include/package.json

@@ -1,5 +1,5 @@
 {
     "dependencies": {
-        "esbuild": "^0.21.5"
+        "esbuild": "^0.25.9"
     }
 }

package/core/BuildSystem.ff

@@ -14,14 +14,48 @@ extend self: BuildSystem {
         internalCompile(self, mainFiles.map {internalPath(self, _)}, "browser")
         let browserOutputPath = internalPath(self, ".firefly/output/browser")
         let runPaths = internalListPath(browserOutputPath).filter {_.base().endsWith(".run.mjs")}.toList()
-        let outputPath = runPaths.grabFirst().parent().grab()
-        internalBrowserCallEsBuild(self, runPaths.map {_.absolute()}, outputPath.absolute(), minify, sourceMaps)
-        let bundlePaths = internalListPath(browserOutputPath).filter {p => 
-            p.base().endsWith(".bundle.js") || p.base().endsWith(".bundle.js.map")
-        }.toList()
-        AssetSystem(bundlePaths.map {p => 
-            Pair("/" + p.relativeTo(browserOutputPath).replace("\\", "/"), {p.readStream()})
-        }.toMap())
+        if(Js.globalThis()->ffDevelopMode.typeof() == "undefined") {
+            let outputPath = runPaths.grabFirst().parent().grab()
+            let start = self.mainTask().elapsed()
+            internalBrowserCallEsBuild(self, runPaths.map {_.absolute()}, outputPath.absolute(), minify, sourceMaps)
+            let bundlePaths = runPaths.flatMap {p =>
+                let outputPath = p.parent().grab()
+                let bundlePath = outputPath.slash(p.base().removeLast(".mjs").grab() + ".bundle.js")
+                let mapPath = outputPath.slash(p.base().removeLast(".mjs").grab() + ".bundle.js.map")
+                [bundlePath, mapPath]
+            }
+            AssetSystem(bundlePaths.map {p => 
+                Pair("/" + p.relativeListTo(browserOutputPath).join("/"), {p.readStream()})
+            }.toMap())
+        } else {
+            mutable built = Set.new()
+            let lock = self.mainTask().lock()
+            AssetSystem(runPaths.flatMap {p =>
+                let outputPath = p.parent().grab()
+                let bundlePath = outputPath.slash(p.base().removeLast(".mjs").grab() + ".bundle.js")
+                let mapPath = outputPath.slash(p.base().removeLast(".mjs").grab() + ".bundle.js.map")
+                let bundleAssetPath = "/" + bundlePath.relativeListTo(browserOutputPath).join("/")
+                let mapAssetPath = "/" + mapPath.relativeListTo(browserOutputPath).join("/")
+                function bundle() {
+                    lock.do {
+                        if(!built.contains(p.absolute())) {
+                            internalBrowserCallEsBuild(self, [p.absolute()], outputPath.absolute(), minify, sourceMaps)
+                            built = built.add(p.absolute())
+                        }
+                        bundlePath.readStream()
+                    }
+                }
+                let bundleEntry = Pair(bundleAssetPath, {
+                    bundle()
+                    bundlePath.readStream()
+                })
+                let mapEntry = Pair(mapAssetPath, {
+                    bundle()
+                    mapPath.readStream()
+                })
+                [bundleEntry, mapEntry]
+            }.toMap())
+        }
     }
 
     buildMode(): Bool {
@@ -82,16 +116,37 @@ internalNodeCallEsBuild(
     minify: Bool
 ): Unit {
     let esbuild = Js.import("esbuild")
-    esbuild->build(Js->(
+    Js.await(esbuild->build(Js->(
         entryPoints = [mainJsFile]
         bundle = True
         minify = minify
         sourcemap = True
         platform = "node"
         target = "es2017"
-        external = ["../../../node_modules/*"] // TODO
+        external = ["esbuild", "uws.js"]
+        loader = Js.object().with(".node", "copy")
         outfile = outputPath
-    ))
+    )))    
+}
+
+internalNodeCallEsBuildContext(
+    self: NodeSystem
+    mainJsFile: String
+    outputPath: String
+    minify: Bool
+): JsValue {
+    let esbuild = Js.import("esbuild")
+    Js.await(esbuild->context(Js->(
+        entryPoints = [mainJsFile]
+        bundle = True
+        minify = minify
+        sourcemap = True
+        platform = "node"
+        target = "es2017"
+        external = ["esbuild", "uws.js"]
+        loader = Js.object().with(".node", "copy")
+        outfile = outputPath
+    )))    
 }
 
 internalListDirectory(path: Path): List[Pair[String, () => Stream[Buffer]]] {
@@ -125,12 +180,28 @@ internalPath(buildSystem: BuildSystem, absoluteOrRelative: String): Path {
 }
 
 internalCompile(buildSystem: BuildSystem, mainFiles: List[Path], target: String): Unit {
+    if(Js.globalThis()->ffDevelopMode.typeof() != "undefined") {
+        Js->process->send(Js->(
+            ffDevelopMode = "internalCompile"
+            mainFiles = mainFiles.map {_.absolute()}
+            target = target
+        ))
+        Js.awaitCancellablePromise {resolve, reject, cleanup =>
+            Js->process->on("message", Js->{message => 
+                if(message->ffDevelopMode === "internalCompile") {
+                    resolve(Unit) // Handle errors?
+                }
+            })
+        }
+    } else:
     // Ported from the old FFI. It's quite fragile w.r.t. changes in code generation.
     Js.await(Js.rawIdentifier("$firefly_compiler")->"buildViaBuildSystem_$"(
         buildSystem! 
         internalPath(buildSystem, buildSystem!->"fireflyPath_"?)!
         mainFiles!
         target
+        Js.undefined()
+        Js.undefined()
         Js.currentTask()!
     ))
 }

package/core/NodeSystem.ff

@@ -116,7 +116,9 @@ extend self: NodeSystem {
         maxBuffer: Int = 16777216
         killSignal: Int = 9
         shell: Bool = False
+        node: Option[(JsValue, JsValue) => Unit] = None
     ): ProcessResult {
+        let inheritStdio = !node.isEmpty()
         let childProcess = Js.import("node:child_process")
         let env = environment.map {e =>
             let o = Js.object()
@@ -124,43 +126,58 @@ extend self: NodeSystem {
             o
         }.else {Js->process->env}
         Js.withSignal {signal => Js.awaitCancellablePromise {resolve, reject, onSettle =>
-            let newProcess = childProcess->spawn(command, arguments, Js->(
-                cwd = directory.map {_.absolutePath!}.else {Js.undefined()}
-                windowsHide = True
-                signal = signal
-                killSignal = killSignal
-                env = env
-                shell = shell
-            ))
+            let newProcess = if(!node.isEmpty()) {
+                let p = childProcess->fork(command, arguments, Js->(
+                    cwd = directory.map {_.absolutePath!}.else {Js.undefined()}
+                    signal = signal
+                    killSignal = killSignal
+                    env = env
+                    silent = True
+                    stdio = if(inheritStdio) {"inherit"} else {"pipe"}
+                ))
+                p->on("message", Js->{message => node.grab()(message, p)})
+                p
+            } else {
+                childProcess->spawn(command, arguments, Js->(
+                    cwd = directory.map {_.absolutePath!}.else {Js.undefined()}
+                    windowsHide = True
+                    signal = signal
+                    killSignal = killSignal
+                    env = env
+                    shell = shell
+                ))
+            }
             mutable size = 0
             let out = Array.new()
             let err = Array.new()
-            newProcess->stdout->on("data", Js->{data =>
-                if(size <= maxBuffer) {
-                    size += data->byteLength?
-                    if(size > maxBuffer) {
-                        newProcess->kill(killSignal)
-                    } else {
-                        out.push(data)
+            if(!inheritStdio) {
+                newProcess->stdout->on("data", Js->{data =>
+                    if(size <= maxBuffer) {
+                        size += data->byteLength?
+                        if(size > maxBuffer) {
+                            newProcess->kill(killSignal)
+                        } else {
+                            out.push(data)
+                        }
                     }
-                }
-                Unit
-            })
-            newProcess->stderr->on("data", Js->{data =>
-                if(size <= maxBuffer) {
-                    size += data->byteLength?
-                    if(size > maxBuffer) {
-                        newProcess->kill(killSignal)
-                    } else {
-                        err.push(data)
+                    Unit
+                })
+                newProcess->stderr->on("data", Js->{data =>
+                    if(size <= maxBuffer) {
+                        size += data->byteLength?
+                        if(size > maxBuffer) {
+                            newProcess->kill(killSignal)
+                        } else {
+                            err.push(data)
+                        }
                     }
+                    Unit
+                })
+                if(standardIn!->byteLength !== 0) {
+                    newProcess->stdin->write(standardIn!)
                 }
-                Unit
-            })
-            if(standardIn!->byteLength !== 0) {
-                newProcess->stdin->write(standardIn!)
+                newProcess->stdin->end()
             }
-            newProcess->stdin->end()
             newProcess->on("error", Js->{error =>
                 if(size > maxBuffer) {
                     reject(internalProcessError("maxBuffer exceeded"))

package/core/Path.ff

@@ -104,6 +104,11 @@ extend self: Path {
         let nodePath = Js.import("path")
         nodePath->relative(path.absolutePath, self.absolutePath)?
     }
+
+    relativeUrlTo(path: Path): String {
+        let relative = self.relativeListTo(path)
+        if(relative.first().any {_ == ".."}) {relative.join("/")} else {"./" + relative.join("/")}
+    }
     
     relativeListTo(path: Path): List[String] {
         let nodePath = Js.import("path")
@@ -299,7 +304,7 @@ extend self: PathEntry {
 
 }
 
-internalReadStream(createReadStream: () => JsValue): Stream[Buffer] {
+internalReadStream(createReadStream: () => JsValue, close: Bool = True): Stream[Buffer] {
     mutable readable = None
     mutable seenError = Js.null()
     let emptyResolve = {}
@@ -357,7 +362,7 @@ internalReadStream(createReadStream: () => JsValue): Stream[Buffer] {
         }
         go()
     } {
-        readable.each {_->destroy()}
+        if(close) {readable.each {_->destroy()}}
     }
 }
 

package/experimental/proxy/Main.ff

@@ -0,0 +1,60 @@
+nodeMain(system: NodeSystem) {
+    let net = Js.import("node:net")
+    let targetServer = "localhost" // 127.0.0.1
+    let targetPort = 8080
+    
+    function parseHeaders(headerData: JsValue): JsValue {
+        let headers = Js->()
+        let lines = headerData->split("\r\n")
+        if(lines.get(0)->endsWith(" HTTP/1.1")? || lines.get(0)->endsWith(" HTTP/1.0")?) {
+            lines->forEach(Js->{line =>
+                let index = line->indexOf(":")
+                if(index !== -1) {
+                    let key = line->substring(0, index)->trim()->toLowerCase()
+                    let value = line->substring(index + 1!)->trim()
+                    headers.set(key, value)
+                }
+            })
+        }
+        headers
+    }
+    
+    let proxyServer = net->createServer(Js->{clientSocket =>
+        mutable buffer = Js->Buffer->alloc(0)
+        mutable isHttpNavigateRequest = False
+        clientSocket->on("data", Js->{data =>
+            buffer = Js->Buffer->concat([buffer, data])
+            let headerEnd = buffer->indexOf("\r\n\r\n")
+            if(headerEnd !== -1 || buffer->length? >= 64 * 1024) {
+                let headerData = buffer->subarray(0, headerEnd)->toString()
+                let headers = parseHeaders(headerData)
+                
+                if(headers.get("sec-fetch-user") === "?1") {
+                    isHttpNavigateRequest = True
+                    Js->console->log("Detected HTTP request with Sec-Fetch-User")
+                }
+                
+                mutable targetSocket = Js.undefined()
+                targetSocket = net->createConnection(targetPort, targetServer, Js->{
+                    Js->console->log("Connected to target server")
+                    targetSocket->write(buffer)
+                    clientSocket->pipe(targetSocket)->pipe(clientSocket)
+                })
+
+                clientSocket->on("error", Js->{err =>
+                    Js->console->error("Client socket error:", err)
+                    targetSocket->end()
+                })
+
+                targetSocket->on("error", Js->{err =>
+                    Js->console->error("Target socket error:", err)
+                    clientSocket->end()
+                })
+            }
+        })
+    })
+    
+    proxyServer->listen(8081, Js->{
+        Js->console->log("Proxy server running on port 8081")
+    })
+}

package/experimental/proxy/Runner.ff

@@ -0,0 +1,11 @@
+data Runner(
+    state: RunnerState
+    changedSinceCompilationStarted: Set[String]
+)
+
+data RunnerState {
+    Compiling
+    CompileError(output: String)
+    AppRunning(output: String)
+    AppCrashed(output: String)
+}