jscall 1.0.1 → 1.2.0

data/lib/jscall/main.mjs

@@ -1,12 +1,18 @@
 // Copyright (C) 2022- Shigeru Chiba.  All rights reserved.
 
-const cmd_eval = 1
-const cmd_call = 2
-const cmd_reply = 3
+export const cmd_eval = 1
+export const cmd_call = 2
+export const cmd_reply = 3
+export const cmd_async_call = 4
+export const cmd_async_eval = 5
+export const cmd_retry = 6
+export const cmd_reject = 7
+
 const param_array = 0
 const param_object = 1
 const param_local_object = 2
 const param_error = 3
+const param_hash = 4
 
 const table_size = 100
 
@@ -64,8 +70,9 @@ const exported = new class {
     }
 }
 
-const RemoteRef = class {
+class RemoteRef extends Function {
     constructor(id) {
+        super()
         this.id = id
     }
 }
@@ -74,12 +81,20 @@ const remoteRefHandler = new class {
     get(obj, name) {
         if (name === '__self__')
             return obj
+        else if (name === 'then')
+            // to prevent the Promise from handling RemoteRefs as thenable
+            // e.g., `Jscall.exec("{ call: (x) => Promise.resolve(x) }").call(obj)' should return that obj itself
+            return undefined
         else
             return (...args) => {
                 // this returns Promise
                 return funcall_to_ruby(obj.id, name, args)
             }
     }
+    apply(obj, self, args) {
+        // this returns Promise
+        return funcall_to_ruby(obj.id, 'call', args)
+    }
 }
 
 const imported = new class {
@@ -94,7 +109,8 @@ const imported = new class {
         if (obj !== null && obj !== undefined)
             return obj
         else {
-            const rref = new Proxy(new RemoteRef(index), remoteRefHandler)
+            const ref = new RemoteRef(index)
+            const rref = new Proxy(ref, remoteRefHandler)
             this.objects[index] = new WeakRef(rref)
             return rref
         }
@@ -104,14 +120,15 @@ const imported = new class {
 
     // collect reclaimed RemoteRef objects.
     dead_references() {
-        if (this.canary === null || this.canary.deref() === undefined)
+        // In Safari, deref() may return null
+        if (this.canary === null || this.canary.deref() == undefined)
             this.canary = new WeakRef(new RemoteRef(-1))
         else
             return []   // the canary is alive, so no GC has happened yet.
 
         const deads = []
         this.objects.forEach((wref, index) => {
-            if (wref !== null && wref !== undefined && wref.deref() === undefined) {
+            if (wref !== null && wref !== undefined && wref.deref() == undefined) {
                 this.objects[index] = null
                 deads.push(index)
             }
@@ -127,15 +144,21 @@ const encode_obj = obj => {
         return null
     else if (obj.constructor === Array)
         return [param_array, obj.map(e => encode_obj(e))]
+    else if (obj instanceof Map) {
+        const hash = {}
+        for (const [key, value] of obj.entries())
+            hash[key] = value
+        return [param_hash, hash]
+    }
     else if (obj instanceof RemoteRef)
         return [param_local_object, exported.export_remoteref(obj)]
     else
         return [param_object, exported.export(obj)]
 }
 
-const encode_error = msg => [param_error, msg]
+const encode_error = msg => [param_error, msg.toString()]
 
-const RubyError = class {
+class RubyError {
     constructor(msg) { this.message = msg }
     get() { return 'RubyError: ' + this.message }
 }
@@ -146,6 +169,12 @@ const decode_obj = obj => {
     else if (obj.constructor === Array && obj.length == 2)
         if (obj[0] == param_array)
             return obj[1].map(e => decode_obj(e))
+        else if (obj[0] == param_hash) {
+            const hash = {}
+            for (const [key, value] of Object.entries(obj[1]))
+                hash[key] = decode_obj(value)
+            return hash
+        }
         else if (obj[0] == param_object)
             return imported.import(obj[1])
         else if (obj[0] == param_local_object)
@@ -156,12 +185,20 @@ const decode_obj = obj => {
     throw `decode_obj: unsupported value, ${obj}`
 }
 
+export const decode_obj_or_error = obj => {
+    const result = decode_obj(obj)
+    if (result instanceof RubyError)
+        return result.get()
+    else
+        return result
+}
+
 const js_eval = eval
 
-const funcall_from_ruby = cmd => {
-    const receiver = decode_obj(cmd[1])
-    const name = cmd[2]
-    const args = cmd[3].map(e => decode_obj(e))
+export const funcall_from_ruby = cmd => {
+    const receiver = decode_obj(cmd[2])
+    const name = cmd[3]
+    const args = cmd[4].map(e => decode_obj(e))
     if (name.endsWith('=')) {
         const name2 = name.substring(0, name.length - 1)
         if (receiver === null)
@@ -189,34 +226,41 @@ const funcall_from_ruby = cmd => {
                 return f    // obtain a propety
     }
 
-    throw `unknown function/method was called, ${name}`
+    throw `unknown JS function/method was called: ${name} on <${receiver}>`
 }
 
-let stdout_puts = console.log
+export let stdout_puts = console.log
+let num_generated_ids = 0
 
-const reply = value => {
-    if (value instanceof Promise)
-        value.then(result => { reply(result) })
-             .catch(err => reply_error(err))
-    else {
-        if (last_callback_stack_depth < callback_stack.length)
-            throw 'Ruby code was called without await.  Call Jscall.close for recovery'
+const fresh_id = () => {
+    num_generated_ids += 1
+    return num_generated_ids
+}
 
+export const reply = (message_id, value, sync_mode) => {
+    if (sync_mode && value instanceof Promise)
+        value.then(result => { reply(message_id, result, true) })
+             .catch(err => reply_error(message_id, err))
+    else {
         try {
-            const cmd = reply_with_piggyback([cmd_reply, encode_obj(value)])
+            const cmd = reply_with_piggyback([cmd_reply, message_id, encode_obj(value)])
             const data = JSON.stringify(cmd)
             stdout_puts(data)
         } catch (e) {
-            reply_error(e)
+            reply_error(message_id, e)
         }
     }
 }
 
-const reply_error = e => {
-    const cmd = reply_with_piggyback([cmd_reply, encode_error(e)])
+export const reply_error = (message_id, e) => {
+    const cmd = reply_with_piggyback([cmd_reply, message_id, encode_error(e)])
     stdout_puts(JSON.stringify(cmd))
 }
 
+const puts_retry_cmd = msg_id => {
+    stdout_puts(JSON.stringify([cmd_retry, msg_id, encode_obj(false)]))
+}
+
 export const scavenge_references = () => {
     reply_counter = 200
     imported.kill_canary()
@@ -230,7 +274,7 @@ const reply_with_piggyback = cmd => {
         const dead_refs = imported.dead_references()
         if (dead_refs.length > 0) {
             const cmd2 = cmd.concat()
-            cmd2[4] = dead_refs
+            cmd2[5] = dead_refs
             return cmd2
         }
     }
@@ -239,58 +283,159 @@ const reply_with_piggyback = cmd => {
 }
 
 const callback_stack = []
-let last_callback_stack_depth = 0
 let reply_counter = 0
 
 export const exec = src => {
     return new Promise((resolve, reject) => {
-        const cmd = reply_with_piggyback([cmd_eval, src])
-        callback_stack.push([resolve, reject])
+        const cmd = make_cmd_eval(src)
+        const message_id = cmd[1]
+        callback_stack.push([message_id, resolve, reject])
         stdout_puts(JSON.stringify(cmd))
     })
 }
 
-const funcall_to_ruby = (receiver_id, name, args) => {
+export const make_cmd_eval = src => {
+    const message_id = fresh_id()
+    return reply_with_piggyback([cmd_eval, message_id, src])
+}
+
+let funcall_to_ruby = (receiver_id, name, args) => {
     return new Promise((resolve, reject) => {
-        const receiver = [param_local_object, receiver_id]
-        const encoded_args = args.map(e => encode_obj(e))
-        const cmd = reply_with_piggyback([cmd_call, receiver, name, encoded_args])
-        callback_stack.push([resolve, reject])
+        const cmd = make_cmd_call(receiver_id, name, args)
+        const message_id = cmd[1]
+        callback_stack.push([message_id, resolve, reject])
         stdout_puts(JSON.stringify(cmd))
     })
 }
 
+export const set_funcall_to_ruby = f => { funcall_to_ruby = f }
+
+export const make_cmd_call = (receiver_id, name, args) => {
+    const message_id = fresh_id()
+    const receiver = [param_local_object, receiver_id]
+    const encoded_args = args.map(e => encode_obj(e))
+    return reply_with_piggyback([cmd_call, message_id, receiver, name, encoded_args])
+}
+
 const returned_from_callback = cmd => {
-    const result = decode_obj(cmd[1])
-    const callback = callback_stack.pop()
-    if (result instanceof RubyError)
-        callback[1](result.get())
-    else
-        callback[0](result)
+    const message_id = cmd[1]
+    const result = decode_obj(cmd[2])
+    for (let i = callback_stack.length - 1; i >= 0; i--) {
+        // check the most recent element first since callbacks are
+        // assumed to be synchronously executed
+        if (callback_stack[i][0] === message_id) {
+            const [[_, resolve, reject]] = callback_stack.splice(i, 1)
+            if (result instanceof RubyError)
+                reject(result.get())
+            else
+                resolve(result)
+        }
+    }
 }
 
-class JsonReader {
+export class MessageReader {
+    static HeaderSize = 6
+
     constructor(stream) {
-        this.stream     = stream
-        this.acc        = ""
+        this.stream = stream
+        this.state = "header"
+        this.acc = ""
+        this.bodySize = 0
+    }
+
+    parseHeader(pos) {
+        // skip leading whitespace characters as a countermeasure against leftover "\n"
+        while (pos < this.acc.length && /\s/.test(this.acc[pos]))
+            pos++
+
+        if (this.acc.length >= MessageReader.HeaderSize) {
+            const start = pos
+            pos += MessageReader.HeaderSize
+            return [parseInt(this.acc.slice(start, pos), 16), pos]
+        }
+        else
+            return undefined
+    }
+
+    parseBody(pos) {
+        if (this.acc.length >= this.bodySize) {
+            const start = pos
+            pos += this.bodySize
+            return [this.acc.slice(start, pos), pos]
+        }
+        else
+            return undefined
+    }
+
+    consume(pos) {
+        if (pos > 0)
+            this.acc = this.acc.slice(pos)
     }
 
     async *[Symbol.asyncIterator]() {
-        for await (let data of this.stream) {
+        for await (const data of this.stream) {
             this.acc += data
-            try {
-                yield JSON.parse(this.acc)
-                this.acc = ""
-            } catch {
-                // keep data in this.acc
+            let pos = 0
+            while (true) {
+                const result = this.iteratorBody(pos)
+                if (result[0] === false)
+                    break
+                else if (result[0] !== true)
+                    yield result[0]     // result[0] is a string
+
+                pos = result[1]
+                if (this.checkEmptiness(pos))
+                    break
+            }
+        }
+        this.checkEOS()
+    }
+
+    iteratorBody(pos) {
+        if (this.state === "header") {
+            const header = this.parseHeader(pos)
+            if (header === undefined) {
+                this.consume(pos)
+                return [false, pos]
+            } else {
+                this.bodySize = header[0]
+                pos = header[1]
+                this.state = "body"
             }
         }
-        if (this.acc != "") {
-            yield JSON.parse(this.acc)
+        if (this.state === "body") {
+            const body = this.parseBody(pos)
+            if (body === undefined) {
+                this.consume(pos)
+                return [false, pos]
+            } else {
+                this.state = "header"
+                return body
+            }
+        }
+        return [true, pos]
+    }
+
+    checkEmptiness(pos) {
+        if (pos == this.acc.length || (pos == this.acc.length - 1
+                                       && this.acc[this.acc.length - 1] === "\n")) {
+            this.acc = ""
+            return true
         }
+        else
+            return false
+    }
+
+    checkEOS() {
+        if (this.acc.length > 0)
+            throw new Error("The pipe closed after receiving an incomplete message")
     }
 }
 
+let make_message_reader = (stdin) => new MessageReader(stdin)
+
+export const set_make_message_reader = (f) => { make_message_reader = f }
+
 export const start = async (stdin, use_stdout) => {
     if (use_stdout)
         console.log = console.error         // on node.js
@@ -298,30 +443,41 @@ export const start = async (stdin, use_stdout) => {
         stdout_puts = (m) => stdin.puts(m)  // on browser
 
     stdin.setEncoding('utf8')
-    for await (const cmd of new JsonReader(stdin)) {
+    for await (const json_data of make_message_reader(stdin)) {
+        let cmd
         try {
-            last_callback_stack_depth = callback_stack.length
+            cmd = JSON.parse(json_data)
 
             // scavenge remote references
-            if (cmd.length > 4)
-                cmd[4].forEach(i => exported.remove(i))
+            if (cmd.length > 5)
+                cmd[5].forEach(i => exported.remove(i))
 
             if (cmd[0] == cmd_eval) {
-                const result = js_eval(cmd[1])
-                reply(result)
+                const result = js_eval(cmd[2])
+                reply(cmd[1], result, true)
             }
             else if (cmd[0] == cmd_call) {
                 const result = funcall_from_ruby(cmd)
-                reply(result)
+                reply(cmd[1], result, true)
             }
             else if (cmd[0] == cmd_reply)
                 returned_from_callback(cmd)
-            else
-                reply_error('invalid command')
+            else if (cmd[0] == cmd_async_call) {
+                const result = funcall_from_ruby(cmd)
+                reply(cmd[1], result, false)
+            }
+            else if (cmd[0] == cmd_async_eval) {
+                const result = js_eval(cmd[2])
+                reply(cmd[1], result, false)
+            }
+            else if (cmd[0] == cmd_reject)
+                puts_retry_cmd(cmd[1])
+            else // cmd_retry and other unknown commands
+                reply_error(cmd[1], `invalid command ${cmd[0]}`)
         } catch (error) {
             const msg = typeof error === 'string' ? error : error.toString() +
                                                             '\n  ---\n' + error.stack
-            reply_error(msg)
+            reply_error(cmd[1], msg)
         }
     }
 }

data/lib/jscall/synch.mjs

@@ -0,0 +1,185 @@
+// Copyright (C) 2022- Shigeru Chiba.  All rights reserved.
+// This works only with node.js on Linux
+
+import * as main from './main.mjs'
+import { readSync, openSync } from 'fs'
+
+class SynchronousStdin {
+    constructor() {
+        this.buf_size = 4096
+        this.buffer = Buffer.alloc(this.buf_size);
+        this.stdin = openSync('/dev/stdin', 'rs') 
+    }
+
+    *[Symbol.iterator]() {
+        let str
+        while ((str = this.readOne()) !== null)
+            yield str
+    }
+
+    readOne() {
+        while (true) {
+            try {
+                const nbytes = readSync(this.stdin, this.buffer, 0, this.buf_size)
+                if (nbytes > 0)
+                    return this.buffer.toString('utf-8', 0, nbytes)
+                else
+                    return null     // maybe EOF on macOS
+            }
+            catch (e) {
+                if (e.code === 'EOF')
+                    return null
+                else if (e.code !== 'EAGAIN')
+                    throw e
+            }
+        }
+    }
+}
+
+class SyncMessageReader extends main.MessageReader {
+    constructor(stream) {
+        super(stream)
+        this.stdin = new SynchronousStdin()
+        this.generator = null
+    }
+
+    gets_function() {
+        const iterator = this[Symbol.iterator]()
+        return () => {
+            const v = iterator.next()
+            if (v.done)
+                return null
+            else
+                return v.value
+        }
+    }
+
+    async *[Symbol.asyncIterator]() {
+        if (this.generator !== null) {
+            while (true) {
+                const v = this.generator.next()
+                if (v.done)
+                    break
+                else
+                    yield v.value
+            }
+        }
+        for await (const data of this.stream) {
+            this.acc += data
+            this.generator = this.generatorBody()
+            while (true) {
+                const v = this.generator.next()
+                if (v.done)
+                    break
+                else
+                    yield v.value
+            }
+        }
+        this.checkEOS()
+    }
+
+    *[Symbol.iterator]() {
+        if (this.generator !== null) {
+            while (true) {
+                const v = this.generator.next()
+                if (v.done)
+                    break
+                else
+                    yield v.value
+            }
+        }
+        for (const data of this.stdin) {
+            this.acc += data
+            this.generator = this.generatorBody()
+            while (true) {
+                const v = this.generator.next()
+                if (v.done)
+                    break
+                else
+                    yield v.value
+            }
+        }
+        this.checkEOS()
+    }
+
+    *generatorBody() {
+        let pos = 0
+        while (true) {
+            const result = this.iteratorBody(pos)
+            if (result[0] === false)
+                break
+            else if (result[0] !== true)
+                yield result[0]     // result[0] is a string
+
+            pos = result[1]
+            if (this.checkEmptiness(pos))
+                break
+        }
+    }
+}
+
+export const start = main.start
+
+let stdin_gets = null
+
+main.set_make_message_reader((stdin) => {
+    const reader = new SyncMessageReader(stdin)
+    stdin_gets = reader.gets_function()
+    return reader
+})
+
+const js_eval = eval
+const exported = main.get_exported_imported()[0]
+
+const event_loop = () => {
+    let json_data
+    while ((json_data = stdin_gets()) !== null) {
+        let cmd
+        try {
+            cmd = JSON.parse(json_data)
+
+            // scavenge remote references
+            if (cmd.length > 5)
+                cmd[5].forEach(i => exported.remove(i))
+
+            if (cmd[0] == main.cmd_eval || cmd[0] == main.cmd_async_eval) {
+                const result = js_eval(cmd[2])
+                main.reply(cmd[1], result, false)
+            }
+            else if (cmd[0] == main.cmd_call || cmd[0] == main.cmd_async_call) {
+                const result = main.funcall_from_ruby(cmd)
+                main.reply(cmd[1], result, false)
+            }
+            else if (cmd[0] == main.cmd_reply)
+                return main.decode_obj_or_error(cmd[2])
+            else { // cmd_retry, cmd_reject, and other unknown commands
+                console.error(`*** node.js; bad message received: ${json_data}`)
+                break
+            }
+        } catch (error) {
+            const msg = typeof error === 'string' ? error : error.toString() +
+                                                            '\n  ---\n' + error.stack
+            main.reply_error(cmd[1], msg)
+        }
+    }
+    return undefined
+}
+
+export const exec = src => {
+    const cmd = main.make_cmd_eval(src)
+    main.stdout_puts(JSON.stringify(cmd))
+    return event_loop()
+}
+
+main.set_funcall_to_ruby((receiver_id, name, args) => {
+    const cmd = main.make_cmd_call(receiver_id, name, args)
+    main.stdout_puts(JSON.stringify(cmd))
+    return event_loop()
+})
+
+export const scavenge_references = main.scavenge_references
+
+// for testing and debugging
+export const get_exported_imported = main.get_exported_imported
+
+export const dyn_import = main.dyn_import

data/lib/jscall/version.rb

@@ -3,5 +3,5 @@
 # Copyright (C) 2022- Shigeru Chiba.  All rights reserved.
 
 module Jscall
-  VERSION = "1.0.1"
+  VERSION = "1.2.0"
 end