@naanlang/naan 1.2.1 → 1.3.0

package/frameworks/node/apiserver.nlg

@@ -83,7 +83,16 @@ closure APIServer(options, local server, chroot) {
         if server.guid
             server.oururl = server.oururl.concat("?guid=", server.guid)
         App.api = server                                                    // used remotely by client/relaycon.nlg
-        server.ws = WebSocketServer(server)
+        if options.gateway {                                                // processes at /wsgw endpoint
+            server.ws = WebSocketGatewayServer()
+            `(error) = server.ws.open(server, options.gateway)
+            if error {
+                error = Error("APIServer: cannot establish websocket gateway", options.gateway, error)
+                ErrorDebuglog(error)
+                return (list(error))
+            }
+        } else                                                              // default is native websocket protocols
+            server.ws = WebSocketServer(server)
         WesRelay(server)                                                    // registers as subsystem
         server.psm = psmServerMod.MakeServerPSM(server, options.name)
         `(error, hostfs) = server.psm.conns.NodeFS.connect("Host", "NideFS", list(""))
@@ -267,7 +276,7 @@ closure APIServer(options, local server, chroot) {
 
     server.app.all("/psm", closure(req, res, local guid) {
         guid = req.get("x-naanlang-api-guid")
-        if server.guid && guid != server.guid && req.method !== "OPTIONS"
+        if server.guid && guid != server.guid
             res.status(403).send("unauthorized")
         else if !server.hostfs
             res.status(503).send("host filesystem not available")
@@ -275,12 +284,187 @@ closure APIServer(options, local server, chroot) {
             server.hostfs.api(req, res)
     })
 
+    //  "/wsgw"
+
+    server.app.all("/wsgw", closure(req, res) {
+        if !server.ws.requested
+            res.status(503).send("websocket gateway not available")
+        else
+            server.ws.requested(req, res)
+        res.set('Content-Type', 'application/json')
+        res.status(200).send("")
+    })
+
     // finis
 
     server
 };
 
 
+/* 
+ * WebSocketGatewayServer
+ *
+ * Process websocket operations forwarded by an API Gateway. This connects to the specified gateway
+ * component to process the details.
+ *
+ * Gateway handler methods:
+ *      handler = Gateway(server)               -- constructor, returns gateway object
+ *      `(error) = open(wsgate)                 -- initialize for any API Gateway
+ *      `(error, data) = requested(req, res)    -- process incoming requests
+ *      `(error) = send(connID, message)        -- send a message
+ *      close()                                 -- close the gateway for all connections
+ *
+ */
+
+closure WebSocketGatewayServer(local wsgate) {
+    wsgate = new(object, this)
+    wsgate.conns = {}                                                       // our connections indexed by connID
+    wsgate.subsystems = { }                                                 // registered subsystems
+
+    // open
+    //
+    // Open the server with the specified gateway.
+    //
+    wsgate.open = closure open(server, gatewayPath) {
+        try {
+            wsgate.handler = require(gatewayPath).Gateway(server)
+            result = wsgate.handler.open()
+            if result.0
+                return (list(result.0))
+        } catch {
+            return (list(exception))
+        }
+        wsgate.guid = server.guid
+        list(false, { ok: true })
+    }
+
+    // requested
+    //
+    // We have an incoming request translated from WS by the API Gateway. This finds out from our
+    // gateway handler the client this pertains to and connects/disconnects/messages as appropriate.
+    //
+    wsgate.requested = closure requested(req, res, local error, data, conn, message) {
+        // reqOpen
+        // Get our connection or make one.
+        function reqOpen(connID) {
+            conn = wsgate.conns[connID]
+            if !conn {
+                conn = WebSocketConn(wsgate, connID)
+                wsgate.conns[connID] = conn
+                for serverID in wsgate.subsystems
+                     wsgate.subsystems[serverID].wsOpen(conn)
+            }
+            conn
+        }
+
+        `(error, data) = wsgate.handler.requested(req, res)
+        if data.op == "connect"
+            conn = reqOpen(data.connID)
+        else if data.op == "message"
+            reqOpen(data.connID).msgin(data.message)
+        else if data.op == "disconnect" {
+            conn = wsgate.conns[data.connID]
+            wsgate.conns[data.connID] = undefined
+            if conn {
+                conn.close(data.code, data.reason)
+                for serverID in wsgate.subsystems
+                    wsgate.subsystems[serverID].wsClose(conn)
+            }
+        }
+    }
+    
+    // close
+    //
+    // Respond to server closing.
+    //
+    wsgate.close = function close(conn, local serverID) {
+        wsgate.handler.close()
+        wsgate.handler = false
+        for serverID in wsgate.subsystems
+            wsgate.subsystems[serverID].wsClose()
+    }
+
+    // finis
+    wsgate
+};
+
+
+/*
+ * WebSocketConn
+ *
+ *     WebSocketConn objects represent a connection with a particular client. These are created when
+ * a new client connects. The protocol is JSON with the top level being a dictionary having the
+ * following static keys specified by the client:
+ *     serverID:    defined by the server, this string defines where requests should be routed.
+ *     clientID:    defined by the client, this string defines where replies should be sent.
+ * These are class (code) designators. Further keys can be defined to identify a specific instance on
+ * the client or server.
+ *
+ */
+
+closure WebSocketConn(wsgate, connID, local wsconn) {
+    global()
+    wsconn = new(object, this)
+    wsconn.info = {
+        destID: UUID()
+    }
+
+    // msgin
+    //
+    // This finds the appropriate target for an incoming message, which can be a relay message that
+    // we send on to the next node on the network.
+    //
+    wsconn.msgin = closure msgin(message, local error) {
+        if wsgate.guid && message.guid != wsgate.guid
+            error = Error("unauthorized")
+        else if message.init {
+            if message.init.destID {
+                wsconn.relay = true
+                wsgate.subsystems.Relay.wsChangeDestID(conn, wsconn.info.destID, message.init.destID)
+            }
+            wsconn.info = merge(wsconn.info, message.init)
+        }
+        else if message.respOp && wsgate.subsystems.Relay
+            wsgate.subsystems.Relay.wsReceive(message, wsconn)          // with respOp, must be a relay
+        else if message.clientID && wsgate.subsystems[message.serverID]
+            wsgate.subsystems[message.serverID].wsReceive(message, wsconn)
+        else
+            error = Error("can't route message")
+        if error
+            ErrorDebuglog("WebSocketConn.msgin error:", error)
+    }
+
+    // on close
+    //
+    // Connection has closed.
+    //
+    wsconn.close = closure close(code, reason) {
+        if code != 1000 && code != 1001 && code != 1005 && code != 1006
+            debuglog("WebSocketConn: connection has closed:", code, reason, JSONstringify(wsconn.info))
+        wsconn.closed = {
+            code: code
+            reason: reason
+        }
+    }
+
+    // send
+    //
+    // Send a dictionary, which must have a clientID defined.
+    //
+    wsconn.send = function send(message) {
+        if !message.clientID || !message.serverID
+            throw("WebSocketConn.send: client and server IDs required")
+        if wsgate.closed
+            list(Error("cannot send - websocket closed"))
+        wsgate.handler.send(connID, JSONstringify(message))
+        list(false, { ok: true })
+    }
+
+    // finis
+    wsconn
+};
+
+
 /* 
  * WebSocketServer
  *
@@ -322,8 +506,9 @@ closure WebSocketServer(server, local websocks) {
     })
 
     // closeConn
+    //
     // respond to connection closing
-
+    //
     websocks.closeConn = function closeConn(conn, local serverID) {
         websocks.conns = websocks.conns.filter(function(item) {
             item !== conn
@@ -641,7 +826,8 @@ function apisLoadLibs() {
 };
 
 function ApisInit(local manifest) {
-    manifest = `(APIServer, WebSocketServer, WesCon, WesRelay, apisLoadLibs, ApisInit)
+    manifest = `(APIServer, WebSocketGatewayServer, WebSocketConn, WebSocketServer, WesCon, WesRelay,
+        apisLoadLibs, ApisInit)
 
     Naan.module.build(module.id, "apiserver", function(modobj, compobj) {
         require("./node.nlg")

package/frameworks/node/worker.nlg

@@ -18,7 +18,7 @@
  *
  */
 
-closure workCurrent(workerID, startup, local current) {
+closure workCurrent(workco, workerID, startup, local current) {
     global()
     current = new(object, this)
     current.workerID = workerID
@@ -31,9 +31,22 @@ closure workCurrent(workerID, startup, local current) {
         debugout:   ONdebugOut.proc
         messageout: ONmessageOut.proc
     })
+    current.guid = 1
+    current.host = js.t                                                     // our magic Naan controller reference
     
-    current.host =  js.t                                                    // our magic Naan controller reference
-    
+    // replyHook
+    //
+    //     Intercept the ReplyMessage function to see if we should handle it.
+    //
+    function replyHook(data) {
+        if data._guid
+            workco.uworkers[data._guid].msgport.postMessage({
+                id: "targetreceive"
+                data: data
+            })
+    }
+    current.host.OnReplyHook(replyHook.proc)
+
     // ONtextOut
     // Process console text output by Naan.
     
@@ -79,6 +92,8 @@ closure workCurrent(workerID, startup, local current) {
     // Process message output.
     
     function ONmessageOut(data) {
+        if data._guid
+            return                                                          // already replied
         sendAll({
             id: "targetout",
             data: data
@@ -98,6 +113,7 @@ closure workCurrent(workerID, startup, local current) {
     // Notify all links that we have terminated.
     
     function exitNotify(exitCode, local link) {
+        workco.uworkers[current.guid] = undefined
         for link in current.links
             link.exitWorker(exitCode)
     }
@@ -157,7 +173,7 @@ closure workCurrent(workerID, startup, local current) {
             if !(linkref.link eq thislink)
                 linkref.responder(msg)
     }
-    
+
     //
     // vitalUpdate
     //
@@ -172,6 +188,81 @@ closure workCurrent(workerID, startup, local current) {
 };
 
 
+/*
+ * workMainProxy
+ *
+ *     Make a nodejs proxy representing the main thread. This runs inside the worker and allows it
+ * to do remote execution with Main.
+ *
+ */
+
+closure workMainProxy(track, name, options, local target) {
+    global(js)
+    target = require("../running/executors.nlg").ExecutorBase(track, "mainProxy", name)
+    target.name = name
+
+    //
+    // target.PostMessage
+    //
+    //     Post a message to the main thread.
+    //
+    target.PostMessage = function PostMessage(data) {
+        js.t.DispatchMessage(data)
+        list(false, { posted: true })
+    }
+
+    //
+    // onreceive
+    //
+    //     Process a received message from the main thread, typically in response.
+    //
+    function onreceive(data) {
+        target.execReceived(data)
+    }
+    
+    // destroy
+    //
+    //     Destroy the execution instance.
+    //
+    target.destroy = function destroy() {
+        target.execClosed(Error("instance destroyed"))
+        track.delete(target)
+    }
+
+    js.t.OnReceive(onreceive.proc)
+    track.add(target)
+
+    // finis
+
+    list(false, target)
+};
+
+
+/*
+ * WorkerToMain
+ *
+ *     Connect our worker to the main thread, returning a result tuple with an executor representing
+ * Main. Use this to create a context object for Main, allowing remote evaluation.
+ *
+ */
+
+closure WorkerToMain(local nideRunning, track) {
+    // mainProxy -- link tracker to main thread proxy
+    function mainProxy(track, name, options) {
+        workMainProxy(track, name, options)
+    }
+
+    if !js.g {
+        error = Error("WorkerToMain only available in node workers")
+        return (list(error))
+    }
+    nideRunning = require("naanlib:frameworks/running/executors.nlg")
+    track = nideRunning.ExecutorTracker()
+    track.register(mainProxy, "proxy")
+    track.spawn("proxy")                                                    // returns result tuple
+};
+
+
 /*
  * workThread
  *
@@ -179,12 +270,13 @@ closure workCurrent(workerID, startup, local current) {
  *
  */
 
-closure workThread(workerID, startup, local worker, options, subChannel, deathWatcher) {
+closure workThread(workco, workerID, startup, local worker, options, subChannel, deathWatcher) {
     global(threads, JSpath)
     worker = new(object, this)
     worker.name = startup.name
     worker.workerID = workerID
     worker.links = []
+    worker.guid = UUID()
 
     // create our worker thread
 
@@ -207,6 +299,7 @@ closure workThread(workerID, startup, local worker, options, subChannel, deathWa
     // Notify all links that we have terminated.
     
     function exitNotify(exitCode, local link) {
+        workco.uworkers[worker.guid] = undefined
         for link in worker.links
             link.exitWorker(exitCode)
         worker.links = []
@@ -262,6 +355,12 @@ closure workThread(workerID, startup, local worker, options, subChannel, deathWa
             worker.links[0].responder({                                     // only send this to first link
                 id: "started"                                               // don't want multiple initializers
             }) }
+        else if msg.id == "targetsend" {
+            msg.data._guid = worker.guid
+            js.t.DispatchMessage({
+                id: "targetin"
+                data: msg.data
+            }) }
         else {
             if msg.id == "status"
                 worker.lastStatus = milliseconds()
@@ -403,6 +502,7 @@ closure workController(api, local workco) {
     workco = new(object, this)
     workco.serverID = "Workers"                                             // our serverID, used for communications
     workco.workers = { }                                                    // dictionary of workers by workerID
+    workco.uworkers = { }                                                   // dictionary of workers by GUID
     workco.links = new(weakmap)
     api.Register(workco.serverID, workco)
     
@@ -487,9 +587,10 @@ closure workController(api, local workco) {
             else if message.workerOp == "spawn" {
                 if !worker {                                                // create worker if needed
                     if message.workerID == "NideServer"
-                        worker = workco.current = workCurrent(message.workerID, message.payload)
+                        worker = workco.current = workCurrent(workco, message.workerID, message.payload)
                     else
-                        worker = workThread(message.workerID, message.payload)
+                        worker = workThread(workco, message.workerID, message.payload)
+                    workco.uworkers[worker.guid] = worker
                     workco.workers[message.workerID] = worker }
                 if !link
                     makeLink(message, conn)                                 // make sure we have a link for responses
@@ -521,7 +622,7 @@ closure workController(api, local workco) {
                     return } }
             error = Error("invalid parameters", message.workerOp, message.workerID)
         }
-        call(makeSender(conn, message), "error", [error])
+        ErrorDebuglog("workController.wsReceive error:", error)
     }
     
     //
@@ -562,12 +663,13 @@ closure workController(api, local workco) {
  */
 
 function workInit(local manifest) {
-
-    manifest = `(workCurrent, workThread, workLink, workController, workInit)
+    manifest = `(workCurrent, workMainProxy, WorkerToMain, workThread, workLink, workController,
+        workInit)
 
     Naan.module.build(module.id, "worker", function(modobj, compobj) {
         require("./node.nlg")
         compobj.manifest = manifest
         modobj.exports.WorkerController = workController
+        modobj.exports.WorkerToMain = WorkerToMain
     })
 }();

package/frameworks/node/ws_client.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2024 by Richard C. Zulch
  *
  */
 
@@ -25,6 +25,8 @@
  *      onmessage:  <proc>(e, message)          // message received
  *      onerror:    <proc>(e)                   // error occurred
  *      onclose:    <proc>(e)                   // connection has closed
+ *      query:      <string>                    // query string starting with ? for insecure connections
+ *      querys:     <string>                    // query string starting with ? for secure connections
  *  }
  *
  */
@@ -45,9 +47,9 @@ closure WebSocket(host, options, local wsock) {
         if error
             return (list(error))
         if options.wss
-            url = "wss://${host}"
+            url = "wss://${host}${options.querys||''}"
         else
-            url = "ws://${host}"
+            url = "ws://${host}${options.query||''}"
         wsock.ws = xnew(ws.default, url)
         pending = new(nonce)
 

package/frameworks/project/build.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2023 by Richard C. Zulch
+ * Copyright (c) 2019-2024 by Richard C. Zulch
  *
  */
 
@@ -728,6 +728,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                     tasks: group.tasks
                     readpath: infile,
                     writepath: outfile
+                    follow: group.follow || undefined
                 }
                 if group.input.startsWith("@")
                     entry.insource = JSpath.join(group.input.substring(1), group.sources[sdex])
@@ -982,6 +983,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                         overwrite: true                                     // overwrite files
                         force: true                                         // ignore modify date
                         idmode: true                                        // preserve ids and mode
+                        follow: entry.follow                                // optional follow links
                     })
                     if error                                                // deepcopy returns a list of errors
                         error = error.0                                     // but we're only doing one item

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.2.1+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.3.0+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info

package/frameworks/running/executors.nlg

@@ -56,9 +56,11 @@ closure ExecutorBase(track, type, name, local exec) {
     
     exec.execReceived = closure execReceived(data, local pending) {
         pending = exec.pending[data.xid]
-        if !pending {
+        if !pending {                                                       // not waiting on message
             if data.xmsg == "xready"
                 exec.pending.ready = true
+            if data.xmsg == "xattention"
+                { }
             return
         }
         exec.pending[data.xid] = undefined

package/frameworks/running/taskexec.nlg

@@ -28,12 +28,16 @@ closure TaskDispatcher(replier, local contexts, gohome, util) {
         reply = {
             xmsg: "error"
             xid: data.xid
+            _guid: data._guid || undefined
             error: "unknown operation"
         }
         try {
             restorens = makeActivator()
             if data.xmsg == "xstart"
-                reply = { xmsg: "xready" }                                  // ready to accept messages
+                reply = {                                                   // ready to accept messages
+                    xmsg: "xready"
+                     _guid: data._guid || undefined
+                }
             else if data.xmsg == "register" {
                 //
                 // register a new context to default namespace

package/lib/browser/env_web.js

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2017-2022 by Richard C. Zulch
+ * Copyright (c) 2017-2024 by Richard C. Zulch
  *
  */
 
@@ -223,7 +223,17 @@ exports.NaanControllerWeb = function() {
         }, 1);
     };
 
+    var replyHook;
+    this.OnReplyHook = function OnReplyHook(proc) {
+        replyHook = proc;
+        return (proc);
+    };
+
     this.ReplyMessage = function ReplyMessage(data) {                       // "remote" -> "local"
+        if (replyHook)
+            setTimeout(function() {
+                replyHook(data);
+            }, 1);
         termlist.forEach(function(term) {
             if (term.OnMessageOut)
                 term.OnMessageOut(data);
@@ -526,7 +536,7 @@ exports.NaanControllerWeb = function() {
         statedoc.curversion = kStateCurrentVersion;
         statedoc.firstver = kStateFirstVersion;
         statedoc.licensee = "MIT-License";
-        statedoc.verstring = "1.2.1+1";
+        statedoc.verstring = "1.3.0+1";
         statedoc.date = new Date().toISOString();
         statedoc.prefs = prefs;
         statedoc.naan = naanlib.saveState(false);                           // true to optimize, which is a bit slower
@@ -544,7 +554,7 @@ exports.NaanControllerWeb = function() {
             || statedoc.firstver > kStateCurrentVersion
             || statedoc.curversion < kStateFirstVersion
             || statedoc.licensee != "MIT-License"
-            || statedoc.verstring != "1.2.1+1")
+            || statedoc.verstring != "1.3.0+1")
         {
             localStorage.removeItem("NaanState_Hosted");
             return (false);
@@ -603,7 +613,7 @@ exports.NaanControllerWeb = function() {
                     op: "VsiteOpen",
                     name: vsiteName,
                     naancont: contSelf,
-                    title: vsiteName + " 1.2.1+1"
+                    title: vsiteName + " 1.3.0+1"
                 });
             }
         } catch (e) {
@@ -617,7 +627,7 @@ exports.NaanControllerWeb = function() {
                     op: "VsiteClose",
                     name: vsiteName,
                     naancont: contSelf,
-                    title: vsiteName + " 1.2.1+1"
+                    title: vsiteName + " 1.3.0+1"
                 });
                 termTextOut("\x1b[90m\x1b[3m".concat("\nwindow closed", "\x1b[0m\n"));
             }
@@ -632,8 +642,8 @@ exports.NaanControllerWeb = function() {
         naanlib.banner();
         var hostpath = naanlib.js.r("path").dirname(window.location.href);
         naanlib.start({
-            cmd: 'App.version = "1.2.1+1";;\r\n'
-                + 'App.cache = "f22e60a842b7a65ac309da378d469eed";;\r\n'
+            cmd: 'App.version = "1.3.0+1";;\r\n'
+                + 'App.cache = "49a9b89fb804111a36026df163e15009";;\r\n'
                 + 'Naan.module.requireQuery({ naanver: App.cache });;\r\n'
                 + 'Naan.module.webparse("naan_init.nlg", "' + hostpath + '", { naanver: App.cache });;\r\n'
         });

package/lib/browser/env_webworker.js

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2022 by Richard C. Zulch
+ * Copyright (c) 2019-2024 by Richard C. Zulch
  *
  */
 
@@ -83,13 +83,27 @@ exports.NaanControllerWebWorker = function NaanControllerWebWorker() {
         });
         return (data);
     };
-    
+
     var targetListener;
     targSelf.OnMessage = function OnMessage(proc) {
         targetListener = proc;
         return (proc);
     };
-    
+
+    this.DispatchMessage = function DispatchMessage(data) {
+        postMessage({                                                       // target is sending a message
+            id: "targetsend",
+            data: data
+        });
+        return (data);
+    };
+
+    var targetReceiver;
+    targSelf.OnReceive = function OnReceive(proc) {
+        targetReceiver = proc;
+        return (proc);
+    };
+
     targSelf.ReplyDebugger = function ReplyDebugger(data) {
         postMessage({                                                       // target is sending debug data
             id: "debugout",
@@ -149,6 +163,11 @@ exports.NaanControllerWebWorker = function NaanControllerWebWorker() {
             if (debugListener)
                 debugListener(msg.data);                                    // target received a message
         }
+        else if (msg.id == "targetreceive")
+        {
+            if (targetReceiver)
+                targetReceiver(msg.data);
+        }
     };
     
     //