@naanlang/naan 1.2.0 → 1.3.0

package/frameworks/node/apiserver.nlg

@@ -83,7 +83,16 @@ closure APIServer(options, local server, chroot) {
         if server.guid
             server.oururl = server.oururl.concat("?guid=", server.guid)
         App.api = server                                                    // used remotely by client/relaycon.nlg
-        server.ws = WebSocketServer(server)
+        if options.gateway {                                                // processes at /wsgw endpoint
+            server.ws = WebSocketGatewayServer()
+            `(error) = server.ws.open(server, options.gateway)
+            if error {
+                error = Error("APIServer: cannot establish websocket gateway", options.gateway, error)
+                ErrorDebuglog(error)
+                return (list(error))
+            }
+        } else                                                              // default is native websocket protocols
+            server.ws = WebSocketServer(server)
         WesRelay(server)                                                    // registers as subsystem
         server.psm = psmServerMod.MakeServerPSM(server, options.name)
         `(error, hostfs) = server.psm.conns.NodeFS.connect("Host", "NideFS", list(""))
@@ -267,7 +276,7 @@ closure APIServer(options, local server, chroot) {
 
     server.app.all("/psm", closure(req, res, local guid) {
         guid = req.get("x-naanlang-api-guid")
-        if server.guid && guid != server.guid && req.method !== "OPTIONS"
+        if server.guid && guid != server.guid
             res.status(403).send("unauthorized")
         else if !server.hostfs
             res.status(503).send("host filesystem not available")
@@ -275,12 +284,187 @@ closure APIServer(options, local server, chroot) {
             server.hostfs.api(req, res)
     })
 
+    //  "/wsgw"
+
+    server.app.all("/wsgw", closure(req, res) {
+        if !server.ws.requested
+            res.status(503).send("websocket gateway not available")
+        else
+            server.ws.requested(req, res)
+        res.set('Content-Type', 'application/json')
+        res.status(200).send("")
+    })
+
     // finis
 
     server
 };
 
 
+/* 
+ * WebSocketGatewayServer
+ *
+ * Process websocket operations forwarded by an API Gateway. This connects to the specified gateway
+ * component to process the details.
+ *
+ * Gateway handler methods:
+ *      handler = Gateway(server)               -- constructor, returns gateway object
+ *      `(error) = open(wsgate)                 -- initialize for any API Gateway
+ *      `(error, data) = requested(req, res)    -- process incoming requests
+ *      `(error) = send(connID, message)        -- send a message
+ *      close()                                 -- close the gateway for all connections
+ *
+ */
+
+closure WebSocketGatewayServer(local wsgate) {
+    wsgate = new(object, this)
+    wsgate.conns = {}                                                       // our connections indexed by connID
+    wsgate.subsystems = { }                                                 // registered subsystems
+
+    // open
+    //
+    // Open the server with the specified gateway.
+    //
+    wsgate.open = closure open(server, gatewayPath) {
+        try {
+            wsgate.handler = require(gatewayPath).Gateway(server)
+            result = wsgate.handler.open()
+            if result.0
+                return (list(result.0))
+        } catch {
+            return (list(exception))
+        }
+        wsgate.guid = server.guid
+        list(false, { ok: true })
+    }
+
+    // requested
+    //
+    // We have an incoming request translated from WS by the API Gateway. This finds out from our
+    // gateway handler the client this pertains to and connects/disconnects/messages as appropriate.
+    //
+    wsgate.requested = closure requested(req, res, local error, data, conn, message) {
+        // reqOpen
+        // Get our connection or make one.
+        function reqOpen(connID) {
+            conn = wsgate.conns[connID]
+            if !conn {
+                conn = WebSocketConn(wsgate, connID)
+                wsgate.conns[connID] = conn
+                for serverID in wsgate.subsystems
+                     wsgate.subsystems[serverID].wsOpen(conn)
+            }
+            conn
+        }
+
+        `(error, data) = wsgate.handler.requested(req, res)
+        if data.op == "connect"
+            conn = reqOpen(data.connID)
+        else if data.op == "message"
+            reqOpen(data.connID).msgin(data.message)
+        else if data.op == "disconnect" {
+            conn = wsgate.conns[data.connID]
+            wsgate.conns[data.connID] = undefined
+            if conn {
+                conn.close(data.code, data.reason)
+                for serverID in wsgate.subsystems
+                    wsgate.subsystems[serverID].wsClose(conn)
+            }
+        }
+    }
+    
+    // close
+    //
+    // Respond to server closing.
+    //
+    wsgate.close = function close(conn, local serverID) {
+        wsgate.handler.close()
+        wsgate.handler = false
+        for serverID in wsgate.subsystems
+            wsgate.subsystems[serverID].wsClose()
+    }
+
+    // finis
+    wsgate
+};
+
+
+/*
+ * WebSocketConn
+ *
+ *     WebSocketConn objects represent a connection with a particular client. These are created when
+ * a new client connects. The protocol is JSON with the top level being a dictionary having the
+ * following static keys specified by the client:
+ *     serverID:    defined by the server, this string defines where requests should be routed.
+ *     clientID:    defined by the client, this string defines where replies should be sent.
+ * These are class (code) designators. Further keys can be defined to identify a specific instance on
+ * the client or server.
+ *
+ */
+
+closure WebSocketConn(wsgate, connID, local wsconn) {
+    global()
+    wsconn = new(object, this)
+    wsconn.info = {
+        destID: UUID()
+    }
+
+    // msgin
+    //
+    // This finds the appropriate target for an incoming message, which can be a relay message that
+    // we send on to the next node on the network.
+    //
+    wsconn.msgin = closure msgin(message, local error) {
+        if wsgate.guid && message.guid != wsgate.guid
+            error = Error("unauthorized")
+        else if message.init {
+            if message.init.destID {
+                wsconn.relay = true
+                wsgate.subsystems.Relay.wsChangeDestID(conn, wsconn.info.destID, message.init.destID)
+            }
+            wsconn.info = merge(wsconn.info, message.init)
+        }
+        else if message.respOp && wsgate.subsystems.Relay
+            wsgate.subsystems.Relay.wsReceive(message, wsconn)          // with respOp, must be a relay
+        else if message.clientID && wsgate.subsystems[message.serverID]
+            wsgate.subsystems[message.serverID].wsReceive(message, wsconn)
+        else
+            error = Error("can't route message")
+        if error
+            ErrorDebuglog("WebSocketConn.msgin error:", error)
+    }
+
+    // on close
+    //
+    // Connection has closed.
+    //
+    wsconn.close = closure close(code, reason) {
+        if code != 1000 && code != 1001 && code != 1005 && code != 1006
+            debuglog("WebSocketConn: connection has closed:", code, reason, JSONstringify(wsconn.info))
+        wsconn.closed = {
+            code: code
+            reason: reason
+        }
+    }
+
+    // send
+    //
+    // Send a dictionary, which must have a clientID defined.
+    //
+    wsconn.send = function send(message) {
+        if !message.clientID || !message.serverID
+            throw("WebSocketConn.send: client and server IDs required")
+        if wsgate.closed
+            list(Error("cannot send - websocket closed"))
+        wsgate.handler.send(connID, JSONstringify(message))
+        list(false, { ok: true })
+    }
+
+    // finis
+    wsconn
+};
+
+
 /* 
  * WebSocketServer
  *
@@ -322,8 +506,9 @@ closure WebSocketServer(server, local websocks) {
     })
 
     // closeConn
+    //
     // respond to connection closing
-
+    //
     websocks.closeConn = function closeConn(conn, local serverID) {
         websocks.conns = websocks.conns.filter(function(item) {
             item !== conn
@@ -641,7 +826,8 @@ function apisLoadLibs() {
 };
 
 function ApisInit(local manifest) {
-    manifest = `(APIServer, WebSocketServer, WesCon, WesRelay, apisLoadLibs, ApisInit)
+    manifest = `(APIServer, WebSocketGatewayServer, WebSocketConn, WebSocketServer, WesCon, WesRelay,
+        apisLoadLibs, ApisInit)
 
     Naan.module.build(module.id, "apiserver", function(modobj, compobj) {
         require("./node.nlg")

package/frameworks/node/worker.nlg

@@ -18,7 +18,7 @@
  *
  */
 
-closure workCurrent(workerID, startup, local current) {
+closure workCurrent(workco, workerID, startup, local current) {
     global()
     current = new(object, this)
     current.workerID = workerID
@@ -31,9 +31,22 @@ closure workCurrent(workerID, startup, local current) {
         debugout:   ONdebugOut.proc
         messageout: ONmessageOut.proc
     })
+    current.guid = 1
+    current.host = js.t                                                     // our magic Naan controller reference
     
-    current.host =  js.t                                                    // our magic Naan controller reference
-    
+    // replyHook
+    //
+    //     Intercept the ReplyMessage function to see if we should handle it.
+    //
+    function replyHook(data) {
+        if data._guid
+            workco.uworkers[data._guid].msgport.postMessage({
+                id: "targetreceive"
+                data: data
+            })
+    }
+    current.host.OnReplyHook(replyHook.proc)
+
     // ONtextOut
     // Process console text output by Naan.
     
@@ -79,6 +92,8 @@ closure workCurrent(workerID, startup, local current) {
     // Process message output.
     
     function ONmessageOut(data) {
+        if data._guid
+            return                                                          // already replied
         sendAll({
             id: "targetout",
             data: data
@@ -98,6 +113,7 @@ closure workCurrent(workerID, startup, local current) {
     // Notify all links that we have terminated.
     
     function exitNotify(exitCode, local link) {
+        workco.uworkers[current.guid] = undefined
         for link in current.links
             link.exitWorker(exitCode)
     }
@@ -157,7 +173,7 @@ closure workCurrent(workerID, startup, local current) {
             if !(linkref.link eq thislink)
                 linkref.responder(msg)
     }
-    
+
     //
     // vitalUpdate
     //
@@ -172,6 +188,81 @@ closure workCurrent(workerID, startup, local current) {
 };
 
 
+/*
+ * workMainProxy
+ *
+ *     Make a nodejs proxy representing the main thread. This runs inside the worker and allows it
+ * to do remote execution with Main.
+ *
+ */
+
+closure workMainProxy(track, name, options, local target) {
+    global(js)
+    target = require("../running/executors.nlg").ExecutorBase(track, "mainProxy", name)
+    target.name = name
+
+    //
+    // target.PostMessage
+    //
+    //     Post a message to the main thread.
+    //
+    target.PostMessage = function PostMessage(data) {
+        js.t.DispatchMessage(data)
+        list(false, { posted: true })
+    }
+
+    //
+    // onreceive
+    //
+    //     Process a received message from the main thread, typically in response.
+    //
+    function onreceive(data) {
+        target.execReceived(data)
+    }
+    
+    // destroy
+    //
+    //     Destroy the execution instance.
+    //
+    target.destroy = function destroy() {
+        target.execClosed(Error("instance destroyed"))
+        track.delete(target)
+    }
+
+    js.t.OnReceive(onreceive.proc)
+    track.add(target)
+
+    // finis
+
+    list(false, target)
+};
+
+
+/*
+ * WorkerToMain
+ *
+ *     Connect our worker to the main thread, returning a result tuple with an executor representing
+ * Main. Use this to create a context object for Main, allowing remote evaluation.
+ *
+ */
+
+closure WorkerToMain(local nideRunning, track) {
+    // mainProxy -- link tracker to main thread proxy
+    function mainProxy(track, name, options) {
+        workMainProxy(track, name, options)
+    }
+
+    if !js.g {
+        error = Error("WorkerToMain only available in node workers")
+        return (list(error))
+    }
+    nideRunning = require("naanlib:frameworks/running/executors.nlg")
+    track = nideRunning.ExecutorTracker()
+    track.register(mainProxy, "proxy")
+    track.spawn("proxy")                                                    // returns result tuple
+};
+
+
 /*
  * workThread
  *
@@ -179,12 +270,13 @@ closure workCurrent(workerID, startup, local current) {
  *
  */
 
-closure workThread(workerID, startup, local worker, options, subChannel, deathWatcher) {
+closure workThread(workco, workerID, startup, local worker, options, subChannel, deathWatcher) {
     global(threads, JSpath)
     worker = new(object, this)
     worker.name = startup.name
     worker.workerID = workerID
     worker.links = []
+    worker.guid = UUID()
 
     // create our worker thread
 
@@ -207,6 +299,7 @@ closure workThread(workerID, startup, local worker, options, subChannel, deathWa
     // Notify all links that we have terminated.
     
     function exitNotify(exitCode, local link) {
+        workco.uworkers[worker.guid] = undefined
         for link in worker.links
             link.exitWorker(exitCode)
         worker.links = []
@@ -262,6 +355,12 @@ closure workThread(workerID, startup, local worker, options, subChannel, deathWa
             worker.links[0].responder({                                     // only send this to first link
                 id: "started"                                               // don't want multiple initializers
             }) }
+        else if msg.id == "targetsend" {
+            msg.data._guid = worker.guid
+            js.t.DispatchMessage({
+                id: "targetin"
+                data: msg.data
+            }) }
         else {
             if msg.id == "status"
                 worker.lastStatus = milliseconds()
@@ -403,6 +502,7 @@ closure workController(api, local workco) {
     workco = new(object, this)
     workco.serverID = "Workers"                                             // our serverID, used for communications
     workco.workers = { }                                                    // dictionary of workers by workerID
+    workco.uworkers = { }                                                   // dictionary of workers by GUID
     workco.links = new(weakmap)
     api.Register(workco.serverID, workco)
     
@@ -487,9 +587,10 @@ closure workController(api, local workco) {
             else if message.workerOp == "spawn" {
                 if !worker {                                                // create worker if needed
                     if message.workerID == "NideServer"
-                        worker = workco.current = workCurrent(message.workerID, message.payload)
+                        worker = workco.current = workCurrent(workco, message.workerID, message.payload)
                     else
-                        worker = workThread(message.workerID, message.payload)
+                        worker = workThread(workco, message.workerID, message.payload)
+                    workco.uworkers[worker.guid] = worker
                     workco.workers[message.workerID] = worker }
                 if !link
                     makeLink(message, conn)                                 // make sure we have a link for responses
@@ -521,7 +622,7 @@ closure workController(api, local workco) {
                     return } }
             error = Error("invalid parameters", message.workerOp, message.workerID)
         }
-        call(makeSender(conn, message), "error", [error])
+        ErrorDebuglog("workController.wsReceive error:", error)
     }
     
     //
@@ -562,12 +663,13 @@ closure workController(api, local workco) {
  */
 
 function workInit(local manifest) {
-
-    manifest = `(workCurrent, workThread, workLink, workController, workInit)
+    manifest = `(workCurrent, workMainProxy, WorkerToMain, workThread, workLink, workController,
+        workInit)
 
     Naan.module.build(module.id, "worker", function(modobj, compobj) {
         require("./node.nlg")
         compobj.manifest = manifest
         modobj.exports.WorkerController = workController
+        modobj.exports.WorkerToMain = WorkerToMain
     })
 }();

package/frameworks/node/ws_client.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2024 by Richard C. Zulch
  *
  */
 
@@ -20,10 +20,13 @@
  *
  * Options:
  *  {
+ *      wss:        <boolean>                   // true to use secure connection
  *      onopen:     <proc>(e)                   // connection has opened
  *      onmessage:  <proc>(e, message)          // message received
  *      onerror:    <proc>(e)                   // error occurred
  *      onclose:    <proc>(e)                   // connection has closed
+ *      query:      <string>                    // query string starting with ? for insecure connections
+ *      querys:     <string>                    // query string starting with ? for secure connections
  *  }
  *
  */
@@ -37,13 +40,17 @@ closure WebSocket(host, options, local wsock) {
     //
     // Connect with the host if not already connected, returning a result tuple.
     //
-    wsock.connect = closure connect(local pending, error, ws) {
+    wsock.connect = closure connect(local error, ws, url, pending) {
         if wsock.ws
             return (list(false, { open: true }))
         `(error, ws) = await(js.i("ws"))
         if error
             return (list(error))
-        wsock.ws = xnew(ws.default, "ws://${host}")
+        if options.wss
+            url = "wss://${host}${options.querys||''}"
+        else
+            url = "ws://${host}${options.query||''}"
+        wsock.ws = xnew(ws.default, url)
         pending = new(nonce)
 
         // onopen

package/frameworks/project/build.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2023 by Richard C. Zulch
+ * Copyright (c) 2019-2024 by Richard C. Zulch
  *
  */
 
@@ -118,7 +118,7 @@
  *          ]
  *      }
  *
- * build = Builder(rulesDict, operation, fs, srcpath, destpath)             // create a builder object
+ * build = Builder(rulesDict, operation, fs, srcpath, destpath, track)      // create a builder object
  * build.increment()                                                        // increment the build counter
  * build.make()                                                             // run the build
  * build.clean()                                                            // clear all output folders
@@ -140,7 +140,7 @@
  *
  */
 
-closure Builder(rules, operation, fs, srcpath, destpath, local build, group, newgroup) {
+closure Builder(rules, operation, fs, srcpath, destpath, track, local build, group, newgroup) {
     build = new(object, this)
     build.srcpath = srcpath
     build.destpath = destpath
@@ -393,7 +393,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
 
     closure zip(files, options, local result) {
         if !build.zipper
-            build.zipper = ExecutorParallel({                               // create worker for zipping
+            build.zipper = ExecutorParallel(track, {                        // create worker for zipping
                 maxcount: 1                                                 // stupid Zip lib can't do more than one
                 basename: "zip"
                 initeval: `(false, (zip1))
@@ -529,7 +529,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
 
     closure parseAndPack(text, fpath, options, local result) {
         if !build.parallel
-            build.parallel = ExecutorParallel({                             // create worker pool
+            build.parallel = ExecutorParallel(track, {                      // create worker pool
                 basename: "build"
                 initeval: `(false, (parseAndPack1))
             }) 
@@ -728,6 +728,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
                     tasks: group.tasks
                     readpath: infile,
                     writepath: outfile
+                    follow: group.follow || undefined
                 }
                 if group.input.startsWith("@")
                     entry.insource = JSpath.join(group.input.substring(1), group.sources[sdex])
@@ -982,6 +983,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
                         overwrite: true                                     // overwrite files
                         force: true                                         // ignore modify date
                         idmode: true                                        // preserve ids and mode
+                        follow: entry.follow                                // optional follow links
                     })
                     if error                                                // deepcopy returns a list of errors
                         error = error.0                                     // but we're only doing one item
@@ -1190,7 +1192,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     //
     // Run the current build product by spawning a new worker.
     //
-    build.runSpawn = function runSpawn(track local error, maintext, executor, vpath, site) {
+    build.runSpawn = function runSpawn(local error, maintext, executor, vpath, site) {
         `(error, maintext) = fs.readFile(JSpath.join(build.destpath, build.runway.main))
         if error
             error = Error("Builder: cannot read main file", error)
@@ -1228,7 +1230,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     //
     // Run the current build product by opening a new window.
     //
-    build.runWindow = function runWindow(track
+    build.runWindow = function runWindow(
         local title, main, url, features, vpath, error, window, site) {
         if !js.w
             return (list(Error('Run method "window" requires browser host')))
@@ -1260,13 +1262,13 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     // Run the current build product, which would be after the make call. Returns a standard result
     // tuple.
     //
-    build.run = function run(track) {
+    build.run = function run() {
         if build.runway.open
             fs.shellOpen(JSpath.join(build.destpath, build.runway.open), build.runway.args)
         else if build.runway.spawn
-            build.runSpawn(track)
+            build.runSpawn()
         else if build.runway.window
-            build.runWindow(track)
+            build.runWindow()
         else
             list(false, { ok: true })
     }
@@ -1276,7 +1278,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     //
     // Refresh after updating the v-site, returning true iff an existing vsite was found.
     //
-    build.refresh = function refresh(track local url_origin, found) {
+    build.refresh = function refresh(local url_origin, found) {
         if build.runway."v-site" {
             url_origin = js.w.location.origin.concat("/run", build.runway."v-site")
             for target in track.enumlist() {

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.2.0+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.3.0+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -614,7 +614,7 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
         rules.typeID = procon.where_typeID
         if !rules.operation
             rules.operation = "debug"
-        `(error, builder) = Builder(rules, rules.operation, fs, "", buildpath)
+        `(error, builder) = Builder(rules, rules.operation, fs, "", buildpath, projman.track)
         if error
             return (list(error))
         procon.builder = builder
@@ -628,10 +628,10 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
             buildnum: buildnum
         })
         if !error {
-            if builder.refresh(projman.track)
+            if builder.refresh()
                 buildonly = !buildonly                                      // reverse if refresh
             if !buildonly
-                `(error) = builder.run(projman.track)
+                `(error) = builder.run()
         }
         builder.destroy()
         procon.builder = false
@@ -662,7 +662,9 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
         publish = rules.publish
         where = procon.localDict.publish[stage].where
         if !rules || !where
-            return (list(Error("no project publish location configured")))
+            return (list(Error("no project publish location set")))
+        if !publish.input || !publish.sources || !publish.output
+            return (list(Error("project configuration has no publish instructions")))
         if publish.cacheControl
             outOptions = {
                 cacheControl: publish.cacheControl