firefly-compiler 0.4.4

package/experimental/random/Patterns.ff

@@ -0,0 +1,226 @@
+data Body(term: Int)
+
+data Constructor(
+    name: String
+    arity: Int
+    span: Int
+)
+
+data Pattern {
+    PVariable(name: String)
+    PConstructor(
+        constructor: Constructor
+        patterns: List[Pattern]
+    )
+}
+
+data Match(
+    rules: List[Pair[Pattern, Body]]
+)
+
+data TermDescription {
+    Positive(
+        constructor: Constructor
+        descriptions: List[TermDescription]
+    )
+    Negative(constructors: Set[String])
+}
+
+data Context(
+    list: List[Pair[Constructor, List[TermDescription]]]
+)
+
+data MatchResult {
+    Yes
+    No
+    Maybe
+}
+
+data Access{
+    Object
+    Select(
+        index: Int
+        access: Access
+    )
+}
+
+data Decision {
+    Failure
+    Success(body: Body)
+    IfEqual(
+        access: Access
+        constructor: Constructor
+        left: Decision
+        right: Decision
+    )
+}
+
+data Work(
+    patterns: List[Pattern]
+    objects: List[Access]
+    descriptions: List[TermDescription]
+)
+
+staticMatch(
+    constructor: Constructor
+    description: TermDescription
+) : MatchResult {
+    | c1, Positive(c2, _) { c1 == c2 } => Yes
+    | c1, Positive(_, _) => No
+    | c1, Negative(noneSet) { noneSet.contains(c1.name) } => No
+    | c1, Negative(noneSet) { c1.span == 1 + noneSet.size() } => Yes
+    | _, Negative(_) => Maybe
+}
+
+augment(context: Context, description: TermDescription): Context {
+    | Context([]), _ => Context([])
+    | Context([Pair(c, ds), ...rest]), d => Context([Pair(c, [d, ...ds]), ...rest])
+}
+
+norm(context: Context): Context {
+    | Context([Pair(c, ds), ...rest]) => augment(Context(rest), Positive(c, ds.reverse()))
+    | _ => panic("Norm is unexhaustive(!)")
+}
+
+buildDescription(
+    context: Context
+    description: TermDescription
+    works: List[Work]
+): TermDescription {
+    | Context([]), d, [] => d
+    | Context([Pair(c, ds1), ...rest]), d, [Work(_, _, ds2), ...works] =>
+        buildDescription(Context(rest), Positive(c, ds1.reverse() ++ [d, ...ds2]), works)
+    | _, _, _ =>  panic("buildDescription is unexhaustive: buildDescription(\n" +
+         "    context = " + magicShow(context) + "\n" +
+         "    description = " + magicShow(description) + "\n" +
+         "    works = " + magicShow(works) + "\n)"
+     )
+}
+
+addNegative(description : TermDescription, constructor : Constructor) : TermDescription {
+    | Negative(noneSet), c {
+        !noneSet.contains(c.name) && noneSet.add(c.name).size() < c.span
+    } => Negative(noneSet.add(constructor.name))
+    | _, _ => panic("Negative TermDescription expected, got " + magicShow(description))
+}
+
+fail(
+    description: TermDescription
+    rules: List[Pair[Pattern, Body]]
+): Decision {
+    | _, [] => Failure
+    | d, [Pair(p, body), ...rules] =>
+        match(
+            pattern = p
+            object = Object
+            description = d
+            context = Context([])
+            works = []
+            body = body
+            rules = rules
+        )
+   | _, rs => panic("fail is unexhaustive:: rs=" + magicShow(rs))
+
+}
+
+succeed(
+    context: Context
+    works: List[Work]
+    body: Body
+    rules: List[Pair[Pattern, Body]]
+): Decision {
+    works.{
+        | [] => Success(body)
+        | [Work([], [], []), ...ws] => succeed(norm(context), ws, body, rules)
+        | [Work([p, ...ps], [o, ...os], [d, ...ds]), ...ws] =>
+            match(
+               pattern = p
+               object = o
+               description = d
+               context = context
+               works = [Work(ps, os, ds), ...ws]
+               body = body
+               rules = rules
+           )
+       | ws => panic("succeed is unexhaustive: ws=" + magicShow(ws))
+    }
+}
+
+match(
+    pattern: Pattern
+    object: Access
+    description: TermDescription
+    context: Context
+    works : List[Work]
+    body: Body
+    rules: List[Pair[Pattern, Body]]
+): Decision {
+    pattern.{
+        | PVariable(_) =>
+            succeed(augment(context, description), works, body, rules)
+        | PConstructor(c, ps) =>
+            function args[R](f: Int => R): List[R] {
+                List.range(c.arity).map(f)
+            }
+
+            function getDescriptionArguments(description: TermDescription) : List[TermDescription] {
+                | Negative(_) => args {_ => Negative(Set.empty())}
+                | Positive(c, ps) => ps
+            }
+
+            function getoargs(): List[Access] {
+                args {i => Select(i + 1, object)}
+            }
+
+            function doSucceed() : Decision {
+                let work = Work(ps, getoargs(), getDescriptionArguments(description))
+                succeed(
+                    Context([Pair(c, []), ...context.list]),
+                    [work, ...works],
+                    body,
+                    rules
+                )
+            }
+
+            function doFail(newDescription : TermDescription): Decision {
+                Log.debug("doFail: " + magicShow(newDescription))
+                fail(buildDescription(context, newDescription, works), rules)
+            }
+
+            staticMatch(c, description).{
+                | Yes => doSucceed()
+                | No => doFail(description)
+                | Maybe => IfEqual(
+                    access = object,
+                    constructor = c,
+                    left = doSucceed(),
+                    right = doFail(addNegative(description, c))
+                )
+            }
+    }
+}
+
+compile(input: Match): Decision {
+    fail(Negative(Set.empty()), input.rules)
+}
+
+main(system: NodeSystem): Unit {
+    //type Color {Red, Green, Blue}
+    let red = PConstructor(Constructor("Red", 0, 3), [])
+    let green = PConstructor(Constructor("Green", 0, 3), [])
+    let blue = PConstructor(Constructor("Blue", 0, 3), [])
+    function pair(first : Pattern, second : Pattern) : Pattern {
+        PConstructor(Constructor("()", 2, 1), [first, second])
+    }
+    let match = Match(rules = [
+        //Pair(red, Body(1))
+        //Pair(pair(red, blue), Body(1)),
+        //Pair(pair(green, blue), Body(2))
+        Pair(pair(blue, green), Body(3)),
+        Pair(pair(PVariable("x"), PVariable("y")), Body(4))
+        //Pair(red, Body(1)),
+        //Pair(PVariable("x"), Body(2))
+    ])
+
+    Log.debug(compile(match))
+}

package/experimental/random/ReadBytesTest.ff

@@ -0,0 +1,10 @@
+main(system: NodeSystem) {
+    read(read(system.readStream()))
+}
+
+read(input: Stream[Buffer]): Stream[Buffer] {
+    let pair = input.readBytes(10)
+    let arrayBuffer = pair.first
+    Log.show(arrayBuffer)
+    pair.second
+}

package/experimental/random/RunLength.ff

@@ -0,0 +1,65 @@
+main(system: NodeSystem) {
+    let inBuffers = 1.to(100).map {_ => Buffer.fromByteArray([5, 5, 3, 1, 2].toArray())}
+    let inStream = inBuffers.toStream()
+    let outStream = toRunLength(inStream, 3)
+    printRunLength(outStream)
+}
+
+printRunLength(outStream: Stream[Buffer]) {
+    let buffer = Buffer.fromBufferArray(outStream.toArray())
+    Log.debug(buffer.toHex())
+    let stack = Stack.make()
+    mutable i = 0
+    while {i < buffer.size()} {
+        1.to(buffer.grabUint8(i)).each {_ => stack.push(buffer.grabUint8(i + 1))}
+        i += 2
+    }
+    Log.debug(Buffer.fromByteArray(stack.drain()).toHex())
+}
+
+toRunLength(stream: Stream[Buffer], bufferSize: Int = 65536): Stream[Buffer] {
+    mutable outBuffer = Buffer.make(bufferSize)
+    mutable outOffset = 0
+    let outBuffers = Stack.make()
+    function writeByte(byte: Int): Unit {
+        if(outOffset >= outBuffer.size()) {
+            outBuffers.push(outBuffer)
+            outBuffer = Buffer.make(bufferSize)
+            outOffset = 0
+        }
+        outBuffer.setUint8(outOffset, byte)
+        outOffset += 1
+    }
+    mutable extraCount = 0
+    mutable value = 0
+    let result = stream.flatMap {inBuffer =>
+        mutable i = 0
+        while {i < inBuffer.size()} {
+            value = if(extraCount > 0) {value} else {inBuffer.grabUint8(i)}
+            mutable j = if(extraCount > 0) {-1} else {0}
+            doWhile {
+                j += 1
+                j + extraCount < 256 && i + j < inBuffer.size() && inBuffer.grabUint8(i + j) == value
+            }
+            i += j
+            if(i < inBuffer.size() || j + extraCount == 256) {
+                writeByte(j + extraCount)
+                writeByte(value)
+                extraCount = 0
+            } else {
+                extraCount += j
+            }
+        }
+        outBuffers.drain().toStream()
+    }
+    result.addAll(Stream.init {
+        if(extraCount > 0) {
+            writeByte(extraCount)
+            writeByte(value)
+        }
+        if(outOffset > 0) {
+            outBuffers.push(outBuffer.view(0, outOffset))
+        }
+        outBuffers.drain().toStream()
+    })
+}

package/experimental/random/Scrape.ff

@@ -0,0 +1,51 @@
+
+nodeMain(system: NodeSystem) {
+    
+    let urlFile = system.arguments().grab(0)
+    let urls = system.path(urlFile).readText().lines()
+    
+    let urlChannel = system.mainTask().channel(urls.size())
+    let htmlChannel = system.mainTask().channel(urls.size())
+    let resultChannel = system.mainTask().channel(urls.size())
+    
+    1.to(1000).each {_ => 
+        system.mainTask().spawn {task =>
+            let url = urlChannel.read()
+            let html = system.httpClient().fetch(url).readText()
+            htmlChannel.write(html)
+        }
+    }
+
+    1.to(10).each {_ => 
+        system.mainTask().spawn {task =>
+            let html = htmlChannel.read()
+            let result = scrape(html)
+            resultChannel.write(result)
+        }
+    }
+
+    urls.each {url => urlChannel.write(url)}
+    let results = urls.map {_ => resultChannel.read()}
+    system.writeText(results.join("\n"))
+    
+}
+
+scrape(html: String): String {
+    html // Uhm...
+}
+
+foo() {
+    
+    let result = urls.toStream()
+        .parallelMap(threads = 1000) {url => download(url)}
+        .parallelMap(threads = 10) {result => result.map(scrape)}
+
+    let result = urls.toStream()
+        .parallelMap(threads = 1000) {task, url => task.timeout(Duration(5)) {download(url)}}
+        .parallelMap(threads = 10) {_, result => result.map(scrape)}
+
+    let result = urls.toStream()
+        .parallelMap(1000) {task, url => task.deadline(task.now().add(Duration(5))) {download(url)}}
+        .parallelMap(10) {_, result => result.map(scrape)}
+    
+}

package/experimental/random/Serialization.ff

@@ -0,0 +1,217 @@
+data Person(id: Int, male: Bool, name: String)
+
+instance Person: Serializable {
+    serializeUsing(serialization: Serialization, value: Person): Unit {
+        serialization.checksum = (31 * serialization.checksum + 7020).bitOr(0)
+        serialization.buffer.setUint8(serialization.offset, 1)
+        serialization.offset += 1
+        serializeUsing(serialization, value.id)
+        serializeUsing(serialization, value.male)
+        serializeUsing(serialization, value.name)
+    }
+    deserializeUsing(serialization: Serialization): Person {
+        let constructorNumber = serialization.buffer.grabUint8(serialization.offset)
+        serialization.offset += 1
+        constructorNumber.{
+            | 0 =>
+                serialization.checksum = (31 * serialization.checksum + 7020).bitOr(0)
+                Person(
+                    deserializeUsing(serialization)
+                    deserializeUsing(serialization)
+                    deserializeUsing(serialization)
+                )
+        }
+    }
+}
+
+data DeserializationChecksumException()
+
+main(system: BrowserSystem) {
+    let p = Person(5, True, "Hans")
+    Log.show(p)
+    Log.show(serialize(p))
+    Log.show(deserialize[Person](serialize(p)))
+
+    let numbers = 1.to(1000).toArray()
+
+    let serialized = serialize(numbers)
+    let deserialized = deserialize(serialized)
+    Log.show(numbers == deserialized)
+
+    function logDuration[T](label: String, body: () => T): T {
+        let started = system.time().elapsed()
+        let result = body()
+        Log.debug(label + ": " + Show.show(Duration(system.time().elapsed().seconds - started.seconds)))
+        result
+    }
+
+    function test() {
+        let persons = logDuration("create") {
+            Array.fillBy(1000000) {n =>
+                Person(n, n % 2 == 0, "Person called " + n)
+            }
+        }
+
+        let started = system.time().elapsed()
+        let personsSerialized = logDuration("serialize") {
+            serialize(persons)
+        }
+        let personsDeserialized = logDuration("deserialize") {
+            deserialize[Array[Person]](personsSerialized)
+        }
+        Log.show(persons == personsDeserialized)
+        //Log.show(personsDeserialized1 == personsDeserialized2)
+        //Log.show(persons.toList().takeFirst(2))
+        //Log.show(personsDeserialized.toList().takeFirst(2))
+    }
+    test()
+    //system.tasks().sleep(Duration(10.0))
+    //test()
+    //system.tasks().sleep(Duration(10.0))
+    //test()
+}
+
+serialize[T: Serializable](value: T, initialBufferSize: Int = 1024): Buffer {
+    let serialization = Serialization(Buffer.make(initialBufferSize), 0, 0)
+    serializeUsing(serialization, value)
+    serialization.autoResize(4)
+    serialization.buffer.setInt32(serialization.offset, serialization.checksum)
+    serialization.offset += 4
+    serialization.buffer.view(0, serialization.offset)
+}
+
+deserialize[T: Serializable](buffer: Buffer): T {
+    let serialization = Serialization(buffer, 0, 0)
+    let result = deserializeUsing(serialization)
+    let checksum = serialization.buffer.grabInt32(serialization.offset)
+    if(checksum != serialization.checksum) {
+        throw(DeserializationChecksumException)
+    }
+    result
+}
+
+trait T: Serializable {
+    serializeUsing(serialization: Serialization, value: T): Unit
+    deserializeUsing(serialization: Serialization): T
+}
+
+data Serialization(
+    mutable buffer: Buffer
+    mutable offset: Int
+    mutable checksum: Int
+)
+
+extend self: Serialization {
+    autoResize(minSpareCapacity: Int) {
+        if(self.offset + minSpareCapacity > self.buffer.size()) {
+            let minSize = self.buffer.size() + minSpareCapacity
+            let newBuffer = Buffer.make((self.buffer.size() * 2).max(minSize))
+            newBuffer.setAll(0, self.buffer)
+            self.buffer = newBuffer
+        }
+    }
+}
+
+instance Int: Serializable {
+    serializeUsing(serialization: Serialization, value: Int): Unit {
+        serialization.autoResize(8)
+        serialization.buffer.setInt64(serialization.offset, value)
+        serialization.offset += 8
+    }
+    deserializeUsing(serialization: Serialization): Int {
+        let result = serialization.buffer.grabInt64(serialization.offset)
+        serialization.offset += 8
+        result
+    }
+}
+
+instance Bool: Serializable {
+    serializeUsing(serialization: Serialization, value: Bool): Unit {
+        serialization.autoResize(1)
+        serialization.buffer.setUint8(serialization.offset, if(value) {1} else {0})
+        serialization.offset += 1
+    }
+    deserializeUsing(serialization: Serialization): Bool {
+        let result = serialization.buffer.grabUint8(serialization.offset)
+        serialization.offset += 1
+        result == 1
+    }
+}
+
+instance Array[T: Serializable]: Serializable {
+    serializeUsing(serialization: Serialization, value: Array[T]): Unit {
+        if(value.size() < 255) {
+            serialization.autoResize(1)
+            serialization.buffer.setUint8(serialization.offset, value.size())
+            serialization.offset += 1
+        } elseIf {value.size() < 1073741824} {
+            serialization.autoResize(1 + 4)
+            serialization.buffer.setUint8(serialization.offset, 255)
+            serialization.buffer.setUint32(1 + serialization.offset, value.size())
+            serialization.offset += 1 + 4
+        } else {
+            panic("Can't serialize arrays where size() >= 1073741824")
+        }
+        value.each {serializeUsing(serialization, _)}
+    }
+    deserializeUsing(serialization: Serialization): Array[T] {
+        let smallSize = serialization.buffer.grabUint8(serialization.offset)
+        if(smallSize != 255) {
+            serialization.offset += 1
+            Array.fillBy(smallSize) {_ => deserializeUsing(serialization)}
+        } else {
+            let size = serialization.buffer.grabUint32(serialization.offset + 1)
+            serialization.offset += 1 + 4
+            Array.fillBy(size) {_ => deserializeUsing(serialization)}
+        }
+    }
+}
+
+instance String: Serializable {
+    serializeUsing(serialization: Serialization, value: String): Unit {
+        serialization.autoResize(1 + value.size())
+        serialization.buffer.setUint8(serialization.offset, value.size())
+        if(value.size() < 255 && internalSetLatin1(serialization.buffer, serialization.offset + 1, value)) {
+            serialization.offset += 1 + value.size()
+        } elseIf {value.size() < 1073741824} {
+            let stringBuffer = value.toBuffer()
+            serialization.autoResize(5 + stringBuffer.size())
+            serialization.buffer.setUint8(serialization.offset, 255)
+            serialization.buffer.setUint32(serialization.offset + 1, stringBuffer.size())
+            serialization.buffer.setAll(serialization.offset + 5, stringBuffer)
+            serialization.offset += 5 + stringBuffer.size()
+        } else {
+            panic("Can't serialize strings where size() >= 1073741824")
+        }
+    }
+    deserializeUsing(serialization: Serialization): String {
+        let smallSize = serialization.buffer.grabUint8(serialization.offset)
+        if(smallSize != 255) {
+            let result = internalGrabLatin1(serialization.buffer, serialization.offset + 1, smallSize)
+            serialization.offset += 1 + smallSize
+            result
+        } else {
+            let size = serialization.buffer.grabUint32(serialization.offset + 1)
+            let stringBuffer = serialization.buffer.view(serialization.offset + 5, serialization.offset + 5 + size)
+            serialization.offset += 5 + size
+            stringBuffer.toString()
+        }
+    }
+}
+
+
+internalSetLatin1(self: Buffer, byteOffset: Int, value: String): Bool
+    target js sync """
+        for(let i = 0; i < value_.length; i++) {
+            let charCode = value_.charCodeAt(i)
+            if(charCode >= 256) return false
+            self_.setUint8(byteOffset_ + i, charCode)
+        }
+        return true
+    """
+
+internalGrabLatin1(self: Buffer, byteOffset: Int, size: Int): String
+    target js sync """
+        const codeUnits = new Uint8Array(self_.buffer, self_.byteOffset + byteOffset_, size_)
+        return String.fromCharCode.apply(null, codeUnits)
+    """

package/experimental/random/SerializationTest.ff

@@ -0,0 +1,46 @@
+data Person(id: Int, male: Bool, name: String)
+
+main(system: BrowserSystem) {
+    let p = Person(5, True, "Hans")
+    Log.show(p)
+    Log.show(Serializable.serialize(p))
+    Log.show(Serializable.deserialize[Person](Serializable.serialize(p)))
+
+    let numbers = 1.to(1000).toArray()
+
+    let serialized = Serializable.serialize(numbers)
+    let deserialized = Serializable.deserialize(serialized)
+    Log.show(numbers == deserialized)
+
+    function logDuration[T](label: String, body: () => T): T {
+        let started = system.time().elapsed()
+        let result = body()
+        Log.debug(label + ": " + Show.show(Duration(system.time().elapsed().seconds - started.seconds)))
+        result
+    }
+
+    function test() {
+        let persons = logDuration("create") {
+            Array.fillBy(1000000) {n =>
+                Person(n, n % 2 == 0, "Person called " + n)
+            }
+        }
+
+        let started = system.time().elapsed()
+        let personsSerialized = logDuration("serialize") {
+            Serializable.serialize(persons)
+        }
+        let personsDeserialized = logDuration("deserialize") {
+            Serializable.deserialize[Array[Person]](personsSerialized)
+        }
+        Log.show(persons == personsDeserialized)
+        //Log.show(personsDeserialized1 == personsDeserialized2)
+        //Log.show(persons.toList().takeFirst(2))
+        //Log.show(personsDeserialized.toList().takeFirst(2))
+    }
+    test()
+    //system.tasks().sleep(Duration(10.0))
+    //test()
+    //system.tasks().sleep(Duration(10.0))
+    //test()
+}

package/experimental/random/Serializer.ff

@@ -0,0 +1,36 @@
+// Binary format each starts with a single byte.
+// tttvvvvv - the ttt bits describe the type and the vvvvv bits describe the value.
+// If vvvvv is 111cc, then 2^cc bytes follow with the actual value. Otherwise the value is vvvvv-3.
+// If it's a string (UTF-8) or a buffer, the value describes the length, and the raw bytes then follow.
+
+capability Serializer[T](
+    writeBool: Bool => Unit
+    writeChar: Char => Unit
+    writeInt: Int => Unit
+    writeFloat: Float => Unit
+    writeString: String => Unit
+    writeBuffer: Buffer => Unit
+    grab: () => T
+)
+
+trait T: Serializable {
+    makeSerializer(): Serializer[T]
+    serialize[R](serializer: Serializer[R], value: T): Unit
+}
+
+
+
+
+
+data Drawing {
+    Circle(radius: Float) // 0
+    Rectangle(width: Float, height: Float) // 1
+    Rotate(drawing: Drawing, angle: Float) // 2
+}
+
+// Then Rotate(Rectangle(5.0, 7.0), 0.3) could be serialized as:
+serializer.writeInt(2) // Tag for the third constructor
+serializer.writeInt(1) // Tag for the second constructor
+serializer.writeFloat(5.0)
+serializer.writeFloat(7.0)
+serializer.writeFloat(0.3)

package/experimental/random/StdInOutErr.ff

@@ -0,0 +1,4 @@
+main(system: NodeSystem) {
+    //let stream = system.readStream()
+    //system.writeStream(stream)
+}