firefly-compiler 0.4.4

package/.firefly-workspace

@@ -0,0 +1 @@
+ff:* .

package/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "editor.suggest.showWords": false,
+    "firefly.trace.server": "messages",
+    "editor.trimAutoWhitespace": false
+}

package/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2023 Joakim Ahnfelt-Rønne, Michael Werk Ravnsmed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,96 @@
+# Firefly programming language
+
+[Get the VSCode extension](https://marketplace.visualstudio.com/items?itemName=firefly-team.firefly-lang)
+
+## About the compiler
+This is a bootstrap transpiler for converting Firefly code to JavaScript code.
+
+Status: Firefly has been bootstrapped and now transpiles itself into JavaScript.
+
+You can create a whole webapp (frontend+backend) in a single file and it can be build as an executable for linux, windows and osx.
+
+[See an example webapp](https://github.com/Ahnfelt/firefly-boot/blob/master/experimental/random/rhymeapp/Main.ff)
+
+**Async/await inference:** https://www.ahnfelt.net/async-await-inference-in-firefly/
+
+
+## Running a main file
+
+```
+firefly.sh MyApp.ff
+```
+
+*You must stand in the project directory when running this command.*
+
+This will run `main` or `nodeMain` in MyApp.ff.
+
+If there is a `buildMain(system: BuildSystem)` method, it will be run before `main`,
+and if it calls `system.setAssets(assets)`, those assets will be available to the main file via `system.assets()`.
+
+You can also have a `browserMain(system: BrowserSystem)` in the file, which can be compiled to a browser JS-file and included as an asset.
+
+Missing dependencies are fetched from the central or configured package repository before building and running.
+
+## Building an executable
+
+```
+firefly.sh build MyApp.ff
+```
+
+*You must stand in the project directory when running this command.*
+
+This will generate a stand-alone executable file `MyApp` with the necessary assets and dependencies, compatible with most linux distributions, windows and osx.
+
+If there is a `buildMain(system: BuildSystem)` method, it will be run before producing the executables,
+and if it calls `system.setAssets(assets)`, those assets will be included in the executables and thus be available to the main file via `system.assets()`.
+
+## Building for the browser
+
+```
+firefly.sh browser MyApp.ff
+```
+
+*You must stand in the project directory when running this command.*
+
+This generates a single, minified `MyApp.min.js`, compatible with all modern browsers (ES6+).
+
+## Bootstrapping the compiler
+
+```
+firefly.sh bootstrap
+```
+
+*You must stand in the project directory when running this command.*
+
+If you botch the compiler, just roll back the output directory and try again.
+
+## Dependencies and imports
+
+You may list your dependencies either in the top of the main file or in `.firefly/package.ff`.
+
+In order to establish a package directory, and thus allow importing other files from the current package, you must place a `.firefly/package.ff` file. 
+
+Example (all lines optional):
+```
+package john:stuff:0.1.7
+dependency anne:goodies:2.1.0
+dependency chris:images:0.7.0
+```
+
+You don't have to manually install dependencies - they will be automatically downloaded when necessary.
+
+## Workspaces
+
+You may place a `.firefly-workspace` file somewhere in a parent directory.
+
+The workspace file configures where source code is fetched from. 
+
+It can be a local directory or a remote repository.
+
+Example:
+
+```
+foo:stuff projects/foo/stuff
+bar:* projects/foo
+* https://example.com/repository
+```

package/bin/firefly.mjs

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import * as firefly from '../output/js/ff/compiler/Main.mjs';

package/compiler/.firefly/package.ff

@@ -0,0 +1 @@
+package ff:compiler:0.0.0

package/compiler/Builder.ff

@@ -0,0 +1,218 @@
+import Tokenizer
+import Parser
+import Syntax
+import Resolver
+import Compiler
+import Unification
+import Dependencies
+import JsEmitter
+import LspHook
+
+build(
+    system: NodeSystem
+    emitTarget: EmitTarget
+    mainPackage: PackagePair
+    mainModule: String
+    resolvedDependencies: ResolvedDependencies
+    compilerModulePath: Option[Path]
+    tempPath: Path
+    jsOutputPath: Path
+    printMeasurements: Bool
+): Unit {
+
+    if(tempPath.exists()) {tempPath.delete()}
+    tempPath.createDirectory()
+
+    let jsPathFile = tempPath.slash("js")
+    jsPathFile.createDirectory(createParentDirectories = True)
+
+    let success = do {
+        let compiler = Compiler.make(
+            emitTarget
+            system.mainTask()
+            compilerModulePath
+            jsPathFile
+            resolvedDependencies
+            Map.empty()
+            lspHook = LspHook.disabled()
+        )
+        compiler.emit(mainPackage, mainModule, isMainModule = True)
+        if(printMeasurements) {compiler.printMeasurements()}
+        resolvedDependencies.packagePaths.each {packagePair, packagePath =>
+            resolvedDependencies.packages.get(packagePair).each {packageInfo =>
+                processIncludes(jsPathFile, packagePath, packageInfo)
+            }
+        }
+        True
+    }
+
+    if(success) {
+        if(jsOutputPath.exists()) {jsOutputPath.delete()}
+        jsPathFile.renameTo(jsOutputPath)
+    }
+
+}
+
+processIncludes(jsPathFile: Path, packagePath: Path, info: PackageInfo): Unit {
+    info.includes.each {include =>
+        let fromPath = packagePath.slash(".firefly").slash("include").slash(include.path)
+        let toPath = jsPathFile.slash(info.package.packagePair.groupName("/")).slash(include.path)
+        fromPath.copyTo(toPath)
+    }
+}
+
+buildViaBuildSystem(system: NodeSystem, fireflyPath: Path, mainFile: String, target: String) {
+    let resolvedDependencies = Dependencies.process(system.httpClient(), system.path(mainFile))
+    let fixedPackagePaths = if(resolvedDependencies.packagePaths.contains(PackagePair("ff", "core"))) {
+        resolvedDependencies.packagePaths
+    } else {
+        resolvedDependencies.packagePaths.add(PackagePair("ff", "core"), fireflyPath.slash("core"))
+    }
+    if(target != "browser") {
+        panic("buildViaBuildSystem is currently limited to browser target only - the restriction can be lifted")
+    }
+    build(
+        system = system
+        emitTarget = EmitBrowser
+        mainPackage = resolvedDependencies.mainPackagePair
+        mainModule = mainFile.dropLast(".ff".size())
+        resolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
+        compilerModulePath = None
+        tempPath = system.path(".firefly/temporary")
+        jsOutputPath = system.path(".firefly/output").slash(target)
+        printMeasurements = False
+    )
+}
+
+check(
+    system: NodeSystem
+    fireflyPath: Path
+    path: Path
+    virtualFiles: Map[String, String]
+    lspHook: LspHook
+    infer: Bool
+) {
+    let packages = path.isDirectory().{
+        | True => findPackageFiles(path)
+        | False {path.endsWith([".firefly", "package.ff"])} => [PackageFiles(path.parent().grab(), Some(path), [])]
+        | False => [PackageFiles(path.parent().grab(), None, [path])]
+    }
+
+    packages.filter {!_.files.isEmpty()}.each {package => 
+        let firstFile = package.files.grabFirst()
+        let resolvedDependencies = Dependencies.process(system.httpClient(), firstFile)
+        let fixedPackagePaths = if(resolvedDependencies.packagePaths.contains(PackagePair("ff", "core"))) {
+            resolvedDependencies.packagePaths
+        } else {
+            resolvedDependencies.packagePaths.add(PackagePair("ff", "core"), fireflyPath.slash("core"))
+        }
+        let fixedResolvedDependencies = resolvedDependencies.ResolvedDependencies(packagePaths = fixedPackagePaths)
+        let compiler = Compiler.make(
+            EmitBuild
+            system.mainTask() 
+            None
+            package.root.slash(".firefly").slash("temporary")
+            fixedResolvedDependencies
+            virtualFiles
+            lspHook
+        )
+        package.files.each {file =>
+            let localFile = file.base()
+            if(infer) {
+                compiler.infer(resolvedDependencies.mainPackagePair, localFile.dropLast(".ff".size()))
+            } else {
+                compiler.resolve(resolvedDependencies.mainPackagePair, localFile.dropLast(".ff".size()))
+            }
+        }
+    }
+
+}
+
+capability PackageFiles(
+    root: Path
+    packageFile: Option[Path]
+    files: List[Path]
+)
+
+
+findPackageFiles(path: Path): List[PackageFiles] {
+    let files = findFireflyFiles(path)
+    let split = files.partition {_.endsWith([".firefly", "package.ff"])}
+    let packageFiles = split.first
+    mutable singleFiles = split.second
+    let multiFileProjects = packageFiles.map {packageFile => 
+        let projectRoot = packageFile.parent().grab().parent().grab()
+        let files = singleFiles.partition {_.isInsideOf(projectRoot)}
+        singleFiles = files.second
+        PackageFiles(projectRoot, Some(packageFile), files.first)
+    }
+    let singleFileProjects = singleFiles.map {file => 
+        let projectRoot = file.parent().grab()
+        PackageFiles(projectRoot, None, [file])
+    }
+    [...multiFileProjects, ...singleFileProjects]
+}
+
+findFireflyFiles(path: Path): List[Path] {
+    let split = path.entries().toList().partition {_.isDirectory()}
+    let directories = split.first.map {_.path()}.filter {_.base().all {c => 
+        c == '.' || c.isAsciiLower() || c.isAsciiDigit()
+    }}
+    let fireflyFiles = split.second.map {_.path()}.filter {
+        _.extension() == ".ff"
+    }
+    [...fireflyFiles, ...directories.flatMap {findFireflyFiles(_)}]
+}
+
+internalCreateExecutable(
+    self: BuildSystem
+    mainJsFile: Path = ".firefly/output/executable/Main.bundle.js"
+    outputPath: Path = ".firefly/output"
+    targets: List[String] = ["host"]
+    assets: AssetSystem = AssetSystem.create()
+): Unit
+    target node {
+        let assetOutputPath = outputPath.slash("assets")
+        assets.files.pairs().each {| Pair(path, makeStream) =>
+            let p = assetOutputPath.slash(path)
+            p.parent().grab().createDirectory(createParentDirectories = True)
+            p.writeStream(makeStream())
+        }
+        let json = """{
+            "name": "main",
+            "bin": {
+                "firefly-main": "Main.bundle.js"
+            },
+            "devDependencies": {
+                "pkg": "^5.8.0"
+            },
+            "pkg": {
+                "scripts": "Main.bundle.js",
+                "outputPath": "bin",
+                "assets": ["../assets/**/*"],
+                "targets": [
+                    "node18-linux-x64",
+                    "node18-macos-x64",
+                    "node18-win-x64"
+                ]
+            }
+        }"""
+        let packageFile = outputPath.slash("executable/package.json")
+        packageFile.writeText(json)
+        internalCallPkg(self, packageFile, outputPath, targets)
+    }
+
+internalCallPkg(
+    self: BuildSystem
+    packageFile: Path
+    outputPath: Path
+    targets: List[String]
+): Unit
+    target node async """
+        import * as pkg from 'pkg'
+        return await pkg.exec([
+            packageFile_,
+            '--out-path', outputPath_,
+            '--target', ff_core_List.List_toArray(targets_).join(',')
+        ])
+    """

package/compiler/Compiler.ff

@@ -0,0 +1,241 @@
+import Syntax
+import Tokenizer
+import Parser
+import Resolver
+import Deriver
+import Inference
+import JsEmitter
+import Dictionaries
+import Dependencies
+import LspHook
+
+capability Compiler(
+    emitTarget: EmitTarget
+    task: Task
+    compilerModulePath: Option[Path]
+    jsOutputPath: Path
+    packagePaths: Map[PackagePair, Path]
+    singleFilePackages: Set[PackagePair]
+    virtualFiles: Map[String, String]
+    lspHook: LspHook
+    mutable parsedModules: Map[String, Module]
+    mutable resolvedModules: Map[String, Module]
+    mutable derivedModules: Map[String, Module]
+    mutable inferredModules: Map[String, Module]
+    mutable emittedModules: Set[String]
+    mutable phaseDurations: List[Pair[String, Duration]]
+    mutable phaseDurationDelta: Duration
+)
+
+make(
+    emitTarget: EmitTarget
+    task: Task
+    compilerModulePath: Option[Path]
+    jsOutputPath: Path
+    resolvedDependencies: ResolvedDependencies
+    virtualFiles: Map[String, String]
+    lspHook: LspHook
+): Compiler {
+    Compiler(
+        emitTarget = emitTarget
+        task = task
+        compilerModulePath = compilerModulePath
+        jsOutputPath = jsOutputPath
+        packagePaths = resolvedDependencies.packagePaths
+        singleFilePackages = resolvedDependencies.singleFilePackages
+        virtualFiles = virtualFiles
+        lspHook = lspHook
+        parsedModules = Map.empty()
+        resolvedModules = Map.empty()
+        derivedModules = Map.empty()
+        inferredModules = Map.empty()
+        emittedModules = Set.empty()
+        phaseDurations = []
+        phaseDurationDelta = Duration(0.0)
+    )
+}
+
+fail[T](at: Location, message: String): T {
+    panic(message + " " + at.show())
+}
+
+coreImports: List[DImport] =
+    [
+        "Any"
+        "Array"
+        "AssetSystem"
+        "Atomic"
+        "Bool"
+        "BrowserSystem"
+        "Buffer"
+        "BuildSystem"
+        "Channel"
+        "Char"
+        "Core"
+        "Duration"
+        "Equal"
+        "Error"
+        "FileHandle"
+        "Float"
+        "HttpClient"
+        "Instant"
+        "Int"
+        "IntMap"
+        "JsValue"
+        "JsSystem"
+        "List"
+        "Lock"
+        "Log"
+        "Map"
+        "NodeSystem"
+        "Nothing"
+        "Option"
+        "Ordering"
+        "Pair"
+        "Path"
+        "Serializable"
+        "Set"
+        "Show"
+        "Stack"
+        "Stream"
+        "String"
+        "StringMap"
+        "Task"
+        "Try"
+        "Unit"
+    ].map {moduleName =>
+        DImport(
+            at = Location("<prelude>", 1, 1)
+            alias = moduleName
+            package = PackagePair("ff", "core")
+            directory = []
+            file = moduleName
+        )
+    }
+
+extend self: Compiler {
+
+    measure[T](phase: String, packagePair: PackagePair, moduleName: String, body: () => T): T {
+        let start = Duration(self.task.elapsed().seconds - self.phaseDurationDelta.seconds)
+        let result = body()
+        let stop = Duration(self.task.elapsed().seconds - self.phaseDurationDelta.seconds)
+        let duration = Duration(stop.seconds - start.seconds)
+        self.phaseDurationDelta = Duration(self.phaseDurationDelta.seconds + duration.seconds)
+        let text = phase + " " + packagePair.groupName() + "/" + moduleName
+        self.phaseDurations = [Pair(text, duration), ...self.phaseDurations]
+        result
+    }
+
+    printMeasurements(): Unit {
+        let worst = self.phaseDurations.sortBy {(_.second.seconds + 1000000.0) + ""}.takeLast(5).reverse()
+        worst.each {| Pair(text, duration) =>
+            Log.debug(text + ":\t" + duration.show())
+        }
+    }
+
+    parse(packagePair: PackagePair, moduleName: String, importedAt: Option[Location]): Module {
+        let packageName = packagePair.groupName()
+        self.parsedModules.get(packageName + ":" + moduleName).else:
+        self.measure("Parse", packagePair, moduleName):
+        let packagePath = self.packagePaths.get(packagePair).else {
+            panic("Internal error - package path missing: " + packagePair.groupName())
+        }
+        let file = moduleName + ".ff"
+        let path = packagePath.slash(file)
+        let code = self.virtualFiles.get(path.absolute()).else {
+            importedAt.each {at => if(!path.exists()) {
+                throw(CompileError(at, "Imported module not found: " + packageName + "/" + moduleName))
+            }}
+            path.readText()
+        }
+        let completionAt = if(self.lspHook.isEnabled() && self.lspHook.insertIdentifier) {self.lspHook.at}
+        let tokens = Tokenizer.tokenize(path.absolute(), code, completionAt, self.lspHook.isEnabled())
+        let parser = Parser.make(packagePair, file, tokens, self.emitTarget != EmitBrowser, self.lspHook)
+        let module = if(self.singleFilePackages.contains(packagePair)) {
+            parser.parseModuleWithPackageInfo().module
+        } else {
+            parser.parseModuleWithoutPackageInfo()
+        }
+        let result = module.Module(
+            imports = [...coreImports, ...module.imports]
+        )
+        self.parsedModules = self.parsedModules.add(packageName + ":" + moduleName, result)
+        result
+    }
+
+    imports(module: Module): List[Module] {
+        module.imports.map {import =>
+            let newPackagePair = import.package
+            let newModuleName = import.directory.map {_ + "/"}.join("") + import.file
+            if(!self.packagePaths.contains(newPackagePair)) {
+                throw(CompileError(import.at, "Missing dependency declaration for: " + newPackagePair.groupName()))
+            }
+            self.parse(newPackagePair, newModuleName, Some(import.at))
+        }
+    }
+
+    resolve(packagePair: PackagePair, moduleName: String): Module {
+        let packageName = packagePair.groupName()
+        self.resolvedModules.get(packageName + ":" + moduleName).else:
+        self.measure("Resolve", packagePair, moduleName):
+
+        let module = self.parse(packagePair, moduleName, None)
+        let otherModules = self.imports(module)
+        let resolver = Resolver.make(self.lspHook)
+        let result = resolver.resolveModule(module, otherModules)
+
+        self.resolvedModules = self.resolvedModules.add(packageName + ":" + moduleName, result)
+        result
+    }
+
+    derive(packagePair: PackagePair, moduleName: String): Module {
+        let packageName = packagePair.groupName()
+        self.derivedModules.get(packageName + ":" + moduleName).else:
+        self.measure("Derive", packagePair, moduleName):
+
+        let module = self.resolve(packagePair, moduleName)
+        let result = Deriver.make().deriveModule(module)
+
+        self.derivedModules = self.derivedModules.add(packageName + ":" + moduleName, result)
+        result
+    }
+
+    infer(packagePair: PackagePair, moduleName: String): Module {
+        let packageName = packagePair.groupName()
+        self.inferredModules.get(packageName + ":" + moduleName).else:
+        self.measure("Infer", packagePair, moduleName):
+
+        let module = self.derive(packagePair, moduleName)
+        let otherModules = self.imports(module).map {i =>
+            self.derive(i.packagePair, i.file.dropLast(".ff".size()))
+        }
+        let inference = Inference.make([module, ...otherModules], self.lspHook)
+        let inferredModule = inference.inferModule(module, otherModules)
+        let result = Dictionaries.make([module, ...otherModules]).processModule(inferredModule, otherModules)
+
+        self.inferredModules = self.inferredModules.add(packageName + ":" + moduleName, result)
+        result
+    }
+
+    emit(packagePair: PackagePair, moduleName: String, isMainModule: Bool): Unit {
+        let packageName = packagePair.groupName()
+        if(self.emittedModules.contains(packageName + ":" + moduleName)) {} else:
+        self.measure("Emit", packagePair, moduleName):
+        self.emittedModules = self.emittedModules.add(packageName + ":" + moduleName)
+
+        let module = self.infer(packagePair, moduleName)
+        let otherModules = self.imports(module).map {i =>
+            let newModuleName = i.file.dropLast(".ff".size())
+            self.emit(i.packagePair, newModuleName, isMainModule = False)
+            self.infer(i.packagePair, newModuleName)
+        }
+
+        let js = JsEmitter.make([module, ...otherModules], self.emitTarget, isMainModule, self.compilerModulePath)
+            .emitModule(packagePair, module)
+        let jsPath = self.jsOutputPath.slash(packagePair.group).slash(packagePair.name)
+        let jsFile = jsPath.slash(moduleName + ".mjs")
+        jsPath.createDirectory(createParentDirectories = True)
+        jsFile.writeText(js)
+    }
+
+}