firefly-compiler 0.5.49 → 0.5.51

package/bin/Release.ff

@@ -22,9 +22,9 @@ release(
     }
     requireNpmLoggedIn(system, system.path(".."))
     requireVsceLoggedIn(system, system.path("../vscode"))
-    runSuccessful(system, "node", ["output/js/ff/compiler/Main.mjs", "bootstrap"], system.path(".."))
-    runSuccessful(system, "node", ["output/js/ff/compiler/Main.mjs", "bootstrap"], system.path(".."))
-    runSuccessful(system, "node", ["output/js/ff/compiler/Main.mjs", "bootstrap"], system.path(".."))
+    runSuccessful(system, "node", ["output/js/ff/compiler/Main.run.mjs", "bootstrap"], system.path(".."))
+    runSuccessful(system, "node", ["output/js/ff/compiler/Main.run.mjs", "bootstrap"], system.path(".."))
+    runSuccessful(system, "node", ["output/js/ff/compiler/Main.run.mjs", "bootstrap"], system.path(".."))
     let version = bumpMinorVersion(system, system.path("../package.json"))
     bumpMinorVersion(system, system.path("../vscode/package.json"))
     runSuccessful(system, "npm", ["publish"], system.path(".."))

package/bin/firefly.mjs

@@ -1,2 +1,2 @@
 #!/usr/bin/env node --harmony-temporal
-import * as firefly from '../output/js/ff/compiler/Main.mjs';
+import * as firefly from '../output/js/ff/compiler/Main.run.mjs';

package/compiler/Builder.ff

@@ -14,7 +14,7 @@ build(
     system: NodeSystem
     emitTarget: EmitTarget
     mainPackage: PackagePair
-    mainModule: String
+    mainModules: List[String]
     resolvedDependencies: ResolvedDependencies
     compilerModulePath: Option[Path]
     tempPath: Path
@@ -40,7 +40,9 @@ build(
             moduleCache
             lspHook = LspHook.disabled()
         )
-        compiler.emit(mainPackage, mainModule, isMainModule = True)
+        mainModules.each {mainModule =>
+            compiler.emit(mainPackage, mainModule, isMainModule = True)
+        }
         if(printMeasurements) {compiler.printMeasurements()}
         resolvedDependencies.packagePaths.each {packagePair, packagePath =>
             resolvedDependencies.packages.get(packagePair).each {packageInfo =>
@@ -85,11 +87,15 @@ processNodeModules(system: NodeSystem, jsPathFile: Path, packagePath: Path, info
     }
 }
 
-buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFile: String, target: String) {
+buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFiles: List[String], target: String) {
+    let mainPaths = mainFiles.map {system.path(_).parent().grab().absolute()}
+    mainPaths.find {_ != mainPaths.grabFirst()}.each {path =>
+        panic("Can't build multiple main files in different directories: " + path + " vs. " + mainPaths.grabFirst())
+    }
     let resolvedDependencies = Dependencies.process(
         system.httpClient()
         DependencyLock.new(system.mainTask())
-        system.path(mainFile)
+        system.path(mainFiles.grabFirst())
     )
     let fixedPackagePaths = if(resolvedDependencies.packagePaths.contains(PackagePair("ff", "core"))) {
         resolvedDependencies.packagePaths
@@ -103,7 +109,7 @@ buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFile: String, tar
         system = system
         emitTarget = EmitBrowser
         mainPackage = resolvedDependencies.mainPackagePair
-        mainModule = mainFile.dropLast(".ff".size())
+        mainModules = mainFiles.map {_.dropLast(".ff".size())}
         resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
         compilerModulePath = None
         tempPath = system.path(".firefly/temporary")

package/compiler/Compiler.ff

@@ -209,7 +209,7 @@ extend self: Compiler {
     }
 
     emit(packagePair: PackagePair, moduleName: String, isMainModule: Bool): Unit {
-        self.cache.cacheEmittedModule(self.packagePaths, packagePair, moduleName): path => 
+        self.cache.cacheEmittedModule(self.packagePaths, packagePair, moduleName, isMainModule): path => 
         self.measure("Emit", packagePair, moduleName):
 
         let module = self.infer(packagePair, moduleName)
@@ -220,18 +220,29 @@ extend self: Compiler {
         }
 
         let allModules = [module, ...otherModules]
-        let js = JsEmitter.new(
+        let emitter = JsEmitter.new(
             otherModules = allModules
             emitTarget = self.emitTarget
             isMainModule = isMainModule
             compilerModuleFileUrl = self.compilerModulePath.map {_.url()}
             packagePair = packagePair
             moduleName = moduleName
-        ).emitModule(packagePair, module)
+        )
+        let js = emitter.emitModule(packagePair, module)
         let jsPath = self.jsOutputPath.slash(packagePair.group).slash(packagePair.name)
         let jsFile = jsPath.slash(moduleName + ".mjs")
         jsPath.createDirectory(createParentDirectories = True)
         jsFile.writeText(js)
+        if(isMainModule) {
+            let runJs = emitter.emitRun(
+                moduleName
+                module.functions
+                packagePair
+                packagePair.group == "ff" && packagePair.name == "compiler"
+            )
+            let jsRunFile = jsPath.slash(moduleName + ".run.mjs")
+            jsRunFile.writeText(runJs.map {_ + "\n"}.join())
+        }
     }
 
 }

package/compiler/JsEmitter.ff

@@ -67,9 +67,6 @@ extend self: JsEmitter {
             self.withEmittingAsync {module.functions.map {"export " + self.emitFunctionDefinition(_, True)}}
             module.extends.map {self.emitExtendsDefinition(_)}
             module.instances.map {self.emitInstanceDefinition(_)}
-            if(self.isMainModule) {
-                self.emitRun(module.functions, packagePair, packagePair.group == "ff" && packagePair.name == "compiler")
-            } else {[]}
         ]
         let ignoreJsImports = if(
             self.emitTarget == EmitExecutable &&
@@ -89,7 +86,7 @@ extend self: JsEmitter {
         }
     }
 
-    emitRun(functions: List[DFunction], mainPackagePair: PackagePair, bootstrapping: Bool): List[String] {
+    emitRun(moduleName: String, functions: List[DFunction], mainPackagePair: PackagePair, bootstrapping: Bool): List[String] {
         let buildMainFunction = functions.find {_.signature.name == "buildMain"}.filter {_ =>
             self.emitTarget != EmitBrowser && self.emitTarget != EmitExecutable
         }
@@ -98,6 +95,10 @@ extend self: JsEmitter {
         let mainFunction =
             functions.find {_.signature.name == targetMain}.orElse {functions.find {_.signature.name == "main"}}
         mainFunction.map {_.signature.name}.map {mainName => [[
+            ...buildMainFunction.map {buildMain =>
+                "import {" + escapeKeyword(buildMain.signature.name) + "$} from './" + moduleName + ".mjs'"
+            }.toList()
+            "import {" + escapeKeyword(mainName) + "$} from './" + moduleName + ".mjs'"
             "export async function $run$(fireflyPath_, arguments_) {"
             "Error.stackTraceLimit = 50"
             "const $task = {controller_: new AbortController(), subtasks_: new Set(), promise_: new Promise(() => {}), started_: performance.now() * 0.001}"
@@ -124,7 +125,7 @@ extend self: JsEmitter {
             ]} else {[]}
             "} finally {"
             ...if(self.emitTarget != EmitBrowser) {[
-                "ff_core_Task.Task_abort$($task)"
+                "$task.controller_.abort()"
                 "clearInterval(interval)"
             ]} else {[]}
             "}"
@@ -365,12 +366,6 @@ extend self: JsEmitter {
             let call = d + "." + n + "(" + [...emittedArguments, ...ds, ...controller].join(", ") + ")"
             if(await) {"(await " + call + ")"} else {call}
         | ECall(at, StaticCall(name, _, _), effect, typeArguments, arguments, dictionaries) =>
-            if(name.contains("bundleForBrowser")) { // TODO: Delete this test (for branch arraysonly)
-                if(!arguments.grab(0).name.contains("system")) {
-                    Log.debug("Wrong arguments for bundleForBrowser: " + Show.show(arguments.map {_.name}))
-                    throw(GrabException())
-                }
-            }
             detectIfElse(term).{
                 | [] =>
                     let await = async && effectTypeIsAsync(effect)

package/compiler/Main.ff

@@ -51,7 +51,7 @@ main(system: NodeSystem): Unit {
             system = system
             emitTarget = emitTarget
             mainPackage = mainPackagePair
-            mainModule = mainFile
+            mainModules = [mainFile]
             resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
             compilerModulePath = compilerModulePath
             tempPath = system.path(".firefly").slash("temporary")
@@ -71,7 +71,7 @@ main(system: NodeSystem): Unit {
             prepareFireflyDirectory(system.path("."))
             let localMainFile = system.path(mainFile).base()
             buildScript(localMainFile, resolvedDependencies.mainPackagePair, EmitNode, resolvedDependencies)
-            if(!importAndRun(fireflyPath, "node", resolvedDependencies.mainPackagePair, localMainFile, arguments)) {
+            if(!importAndRun(system, fireflyPath, "node", resolvedDependencies.mainPackagePair, localMainFile, arguments)) {
                 let at = Location(system.path(mainFile + ".ff").absolute(), 1, 1)
                 throw(CompileError(at, "This module does not contain a 'nodeMain' function"))
             }
@@ -98,7 +98,7 @@ main(system: NodeSystem): Unit {
             buildScript(localMainFile, resolvedDependencies.mainPackagePair, EmitBuild, resolvedDependencies)
             buildScript(localMainFile, resolvedDependencies.mainPackagePair, EmitExecutable, resolvedDependencies)
             bundleForPkg(system, resolvedDependencies.mainPackagePair, localMainFile)
-            importAndRun(fireflyPath, "build", resolvedDependencies.mainPackagePair, localMainFile, [])
+            importAndRun(system, fireflyPath, "build", resolvedDependencies.mainPackagePair, localMainFile, [])
 
         | CheckCommand(filePath) => 
             let errors = Builder.check(
@@ -124,7 +124,7 @@ main(system: NodeSystem): Unit {
                 system = system
                 emitTarget = EmitNode
                 mainPackage = PackagePair("ff", "compiler")
-                mainModule = "Main"
+                mainModules = ["Main"]
                 resolvedDependencies = ResolvedDependencies(
                     mainPackagePair = PackagePair("ff", "compiler")
                     packages = [Pair(
@@ -259,16 +259,11 @@ bundleForBrowser(system: NodeSystem, packagePair: PackagePair, mainFile: String)
     let prefix = ".firefly/output/browser/"
     let mainJsFile = prefix + packagePair.groupName("/") + "/" + mainFile + ".mjs"
     let file = prefix + "Main.bundle.js"
-    let browserCode = BrowserCode(
-        packageGroup = packagePair.group
-        packageName = packagePair.name
-        mainFile = system.path(mainFile)
-        assetSystem = AssetSystem([Pair("/", {system.path(".").readStream()})].toMap())
-    ) // TODO
-    BuildSystem.internalCallEsBuild(browserCode, mainJsFile, outputPath = file, minify = True, sourceMap = True)
+    BuildSystem.internalCallEsBuild(system, mainJsFile, outputPath = file, minify = True, sourceMap = True)
 }
 
 importAndRun(
+    system: NodeSystem
     fireflyPath: Path
     target: String
     packagePair: PackagePair
@@ -279,10 +274,10 @@ importAndRun(
     let cwd = process->cwd()
     let workingDirectory = if(cwd->indexOf(":") === 1) {"file:///" + cwd?} else {cwd?}
     let packagePath = packagePair.group + "/" + packagePair.name
-    let main = Js.await(Js.dynamicImport(
-        workingDirectory + "/.firefly/output/" + target + "/" + packagePath + "/" + mainFile + ".mjs"
-    ))
-    if(main->"$run$".typeof() != "undefined") {
+    let runFile = workingDirectory + "/.firefly/output/" + target + "/" + packagePath + "/" + mainFile + ".run.mjs"
+    let runFilePath = if(runFile.contains("://")) {system.pathFromUrl(runFile)} else {system.path(runFile)}
+    if(runFilePath.exists()) {
+        let main = Js.await(Js.dynamicImport(runFile))
         main->"$run$"(fireflyPath.absolutePath, arguments)
         True
     } else {

package/compiler/ModuleCache.ff

@@ -140,10 +140,11 @@ extend self: ModuleCache {
         packagePaths: Map[PackagePair, Path]
         packagePair: PackagePair
         moduleName: String
+        isMainModule: Bool
         body: Path => Unit
     ): Unit {
         let path = modulePath(packagePaths, packagePair, moduleName)
-        if(!self.emittedModules.contains(path.absolute())):
+        if(isMainModule || !self.emittedModules.contains(path.absolute())):
         self.emittedModules = self.emittedModules.add(path.absolute(), self.version)
         try {
             body(path)

package/core/Buffer.ff

@@ -144,7 +144,7 @@ extend self: Buffer {
         Js->TextDecoder->()->decode(self!)?
     }
 
-    toByteArray(): List[Int] {
+    toByteList(): List[Int] {
         Js->Uint8Array->(self!->buffer, self!->byteOffset, self!->byteLength).spreadToArray()!?
     }
     
@@ -158,14 +158,26 @@ extend self: Buffer {
     
     toBase64(): String {
         let view = Js->Uint8Array->(self!->buffer, self!->byteOffset, self!->byteLength)
-        let binaryString = Js->TextDecoder->("latin1")->decode(view)
+        //let binaryString = Js->TextDecoder->("latin1")->decode(view)
+        let binaryString = Js->Array->from(view, Js->{byte => Js->String->fromCodePoint(byte)})->join("");
         Js->btoa(binaryString)?
     }
+    
+    toLetter16(): String {
+        let result = Buffer.new(self.size().bitLeft(1))
+        0.until(self.size()).each {i =>
+            let b = self.grabUint8(i)
+            result.setUint16(i.bitLeft(1), (b.bitRightUnsigned(4).bitAnd(0xf) + 0x68).bitOr(
+                (b.bitAnd(0xf) + 0x68).bitLeft(8)
+            ), littleEndian = True)
+        }
+        result.toString()
+    }
 
 }
 
 fromByteList(array: List[Int]): Buffer {
-    Js->DataView->(Js->Uint8Array(array)->buffer)?
+    Js->DataView->(Js->Uint8Array->(array)->buffer)?
 }
 
 fromBufferList(array: List[Buffer]): Buffer {
@@ -193,10 +205,20 @@ fromHex(hex: String): Buffer {
 
 fromBase64(base64: String): Buffer {
     let binaryString = Js->atob(base64)
-    let bytes = Js->Uint8Array->from(binaryString, Js->{char => char->charCodeAt(0)})
+    let bytes = Js->Uint8Array->from(binaryString, Js->{char => char->codePointAt(0)})
     Js->DataView->(bytes->buffer)?
 }
 
+fromLetter16(letters: String): Buffer {
+    let buffer = Buffer.new(letters.size().bitRightUnsigned(1))
+    0.until(buffer.size()).each {i =>
+        buffer.setUint8(i, (letters.grab(i.bitLeft(1)).codeUnit.bitAnd(0x1f) - 0x8).bitLeft(4).bitOr(
+            letters.grab(i.bitLeft(1).bitOr(1)).codeUnit.bitAnd(0x1f) - 0x8
+        ))
+    }
+    buffer
+}
+
 instance Buffer: Show {
     show(value: Buffer): String {
         """Buffer.fromHex("""" + value.toHex() + """")"""

package/core/BuildSystem.ff

@@ -1,20 +1,25 @@
 capability BuildSystem {}
-capability BrowserCode(packageGroup: String, packageName: String, mainFile: Path, assetSystem: AssetSystem)
-capability BrowserBundle(assetSystem: AssetSystem)
 
 extend self: BuildSystem {
 
-    compileForBrowser(mainFile: String): BrowserCode {
+    compileForBrowser(mainFiles: List[String]): AssetSystem {
         // TODO: Check that the mainFile is in the current package directory
-        internalCompile(self, internalPath(self, mainFile), "browser")
+        internalCompile(self, mainFiles.map {internalPath(self, _)}, "browser")
         let streams = internalListDirectory(internalPath(self, ".firefly/output/browser"))
         let mainPackagePair = internalMainPackagePair(self)
-        BrowserCode(
-            packageGroup = mainPackagePair.first
-            packageName = mainPackagePair.second
-            mainFile = internalPath(self, mainFile)
-            assetSystem = AssetSystem(streams.toMap())
-        )
+        AssetSystem(streams.toMap())
+    }
+
+    bundleForBrowser(mainFiles: List[String], minify: Bool = True, sourceMaps: Bool = False): AssetSystem {
+        internalCompile(self, mainFiles.map {internalPath(self, _)}, "browser")
+        let browserOutputPath = internalPath(self, ".firefly/output/browser")
+        let runPaths = internalListPath(browserOutputPath).filter {_.base().endsWith(".run.mjs")}.toList()
+        let outputPath = runPaths.grabFirst().parent().grab()
+        internalBrowserCallEsBuild(self, runPaths.map {_.absolute()}, outputPath.absolute(), minify, sourceMaps)
+        let bundlePaths = internalListPath(browserOutputPath).filter {_.base().endsWith(".bundle.js")}.toList()
+        AssetSystem(bundlePaths.map {p => 
+            Pair("/" + p.relativeTo(browserOutputPath).replace("\\", "/"), {p.readStream()})
+        }.toMap())
     }
 
     buildMode(): Bool {
@@ -48,41 +53,9 @@ extend self: BuildSystem {
         
 }
 
-extend self: BrowserCode {
-
-    assets(): AssetSystem {
-        self.assetSystem
-    }
-
-    bundle(minify: Bool = True, sourceMap: Bool = False): BrowserBundle {
-        let prefix = ".firefly/output/browser"
-        let mainJsBaseFile = self.mainFile.base().removeLast(".ff").grab() + ".mjs"
-        let mainJsFile = prefix + "/" + self.packageGroup + "/" + self.packageName + "/" + mainJsBaseFile
-        let mainDirectory = self.mainFile.parent().grab()
-        let file = prefix + "/Main.bundle.js"
-        internalCallEsBuild(self, mainJsFile = mainJsFile, outputPath = file, minify = minify, sourceMap = sourceMap)
-        let assets = AssetSystem([
-            Pair(file.dropFirst(prefix.size()), {mainDirectory.path(file).readStream()})
-            ...if(sourceMap) {[
-                Pair(file.dropFirst(prefix.size()) + ".map", {mainDirectory.path(file + ".map").readStream()})
-            ]} else {[]}
-        ].toMap())
-        BrowserBundle(assets)
-    }
-
-}
-
-extend self: BrowserBundle {
-
-    assets(): AssetSystem {
-        self.assetSystem
-    }
-
-}
-
-
+// TODO: Remove
 internalCallEsBuild(
-    self: BrowserCode
+    self: NodeSystem
     mainJsFile: String
     outputPath: String
     minify: Bool
@@ -101,6 +74,26 @@ internalCallEsBuild(
     ))
 }
 
+internalBrowserCallEsBuild(
+    self: BuildSystem
+    mainJsFiles: List[String]
+    outputPath: String
+    minify: Bool
+    sourceMap: Bool
+): Unit {
+    let esbuild = Js.import("esbuild")
+    Js.await(esbuild->build(Js->(
+        entryPoints = mainJsFiles
+        bundle = True
+        minify = minify
+        sourcemap = sourceMap
+        platform = "browser"
+        target = "es2017"
+        outdir = outputPath
+        outExtension = Js.object().with(".js", ".bundle.js")
+    )))
+}
+
 internalNodeCallEsBuild(
     self: NodeSystem
     mainJsFile: String
@@ -135,17 +128,27 @@ internalListDirectory(path: Path): List[Pair[String, () => Stream[Buffer]]] {
     }.toList()
 }
 
+internalListPath(path: Path): Stream[Path] {
+    path.entries().flatMap {file =>
+        if(file.isDirectory()) {
+            internalListPath(file.path())
+        } else {
+            [file.path()].toStream()
+        }
+    }
+}
+
 internalPath(buildSystem: BuildSystem, absoluteOrRelative: String): Path {
     let nodePath = Js.import("path")
     Path(nodePath->resolve(absoluteOrRelative)?)
 }
 
-internalCompile(buildSystem: BuildSystem, mainFile: Path, target: String): Unit {
+internalCompile(buildSystem: BuildSystem, mainFiles: List[Path], target: String): Unit {
     // Ported from the old FFI. It's quite fragile w.r.t. changes in code generation.
     Js.await(Js.rawIdentifier("$firefly_compiler")->"buildViaBuildSystem_$"(
         buildSystem! 
         internalPath(buildSystem, buildSystem!->"fireflyPath_"?)!
-        mainFile.base()
+        mainFiles.map {_.base()}
         target
         Js.currentTask()!
     ))

package/core/JsValue.ff

@@ -333,6 +333,11 @@ extend self: JsValue {
         self->(a1, a2, a3, a4, a5, a6, a7, a8, a9)
     }
 
+    new10[A1: IsJsValue, A2: IsJsValue, A3: IsJsValue, A4: IsJsValue, A5: IsJsValue, A6: IsJsValue, A7: IsJsValue, A8: IsJsValue, A9: IsJsValue, A10: IsJsValue](a1: A1, a2: A2, a3: A3, a4: A4, a5: A5, a6: A6, a7: A7, a8: A8, a9: A9, a10: A10): JsValue {
+        // Special cased by the compiler
+        self->(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10)
+    }
+
 
     grabPairs(): List[Pair[String, JsValue]] {
         if(!self.isObject()) {throw(GrabException)}