@naanlang/naan 1.0.16 → 1.2.0

package/frameworks/node/node.nlg

@@ -6,11 +6,66 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2024 by Richard C. Zulch
  *
  */
 
 
+/*
+ * NaanlangDir
+ *
+ *     Ensure the naanlang directory exists, returning a result tuple.
+ *
+ */
+
+closure NaanlangDir(local path, errmkdir, errstat, stat) {
+    global(JSpath, os, App, fs)
+    path = JSpath.join(os.homedir(), ".naanlang/")
+    `(errmkdir) = await(fs.promises.mkdir(path))
+    if errmkdir.code == "EEXIST" {
+        `(errstat, stat) = await(fs.promises.stat(path))
+        if errstat || !stat.isDirectory()
+            return (list(Error("mkdir conflict:", errmkdir)))               // file in the way
+    }
+    else if errmkdir
+        return (list(Error("mkdir failed:", errmkdir)))
+    await(fs.promises.mkdir(JSpath.join(path, "nidecom")))                  // ignore errors here
+    App.config = merge({
+        prefsdir: path
+    })
+    list(false, path)
+};
+
+
+/*
+ * NaanlangRC
+ *
+ *     Return the naanlang RC file if found, or an empty dictionary otherwise.
+ *
+ */
+
+closure NaanlangRC(local path, error, config) {
+    global(JSpath, fs, App)
+    `(error, path) = NaanlangDir()
+    if path {
+        path = JSpath.join(path, "naanlangrc")
+        `(error, config) = await(fs.promises.readFile(path, {
+            encoding: "utf8"
+        }))
+        if !error
+            `(error, config) = JsonParse(config)
+    }
+    if config
+        App.config = merge({
+            rcpath: path
+            config: config
+        })
+    else
+        config = { }
+    config
+};
+
+
 /*
  * NodeStreamLiner
  *
@@ -63,7 +118,7 @@ closure NodeStreamLiner(endings, local liner) {
  */
 
 function nodrInit(local manifest) {
-    manifest = `(NodeStreamLiner, nodrInit)
+    manifest = `(NaanlangDir, NaanlangRC, NodeStreamLiner, nodrInit)
 
     if !js.g
         throw("module frameworks/node cannot be used from a browser")
@@ -77,7 +132,10 @@ function nodrInit(local manifest) {
             stream = js.r("stream")
             nodeHttps = js.r("https")
             nodeUrl= js.r("url")
+            threads = js.r("worker_threads")
         }()
         module.reload = nodrReload
+        module.exports.NaanlangDir = NaanlangDir
+        module.exports.NaanlangRC = NaanlangRC
     })
 } ();

package/frameworks/node/psm_server.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2022 by Richard C. Zulch
+ * Copyright (c) 2019-2023 by Richard C. Zulch
  *
  */
 
@@ -450,8 +450,8 @@ closure psmsFsConnector(psm, api, local connClassID, connector, watch) {
     connector.info = function info(resID, local info, error, creds) {
         info = {
             classID: connClassID
-            type: "remote filesystem"
-            services: ["NideFS", "NideHostAPI"]
+            type: "remote server"
+            services: ["NideFS", "NideHostAPI", "NideTerminal"]
         }
         if resID == "Host" {
             info.name = "Host"
@@ -517,13 +517,17 @@ closure psmsFsConnector(psm, api, local connClassID, connector, watch) {
  *
  */
 
-function MakeServerPSM(api, local importPSM) {
+function MakeServerPSM(api, name, local importPSM, error, path, psm) {
     importPSM = require("../storage/psm.nlg")
-    App.psm = importPSM.MakePSM(JSpath.resolve(js.d, "NideDB")).1
-    if App.psm {
+    `(error, path) = NaanlangDir()                                          // put database in ~/.naanlang/
+    if !error
+        `(error, psm) = importPSM.MakePSM(JSpath.join(path, "${name}DB_${api.port.tostring}")) 
+    if !error {
+        App.psm = psm
         psmsFsConnector(App.psm, api)
         require("frameworks/storage/dbt_pouch.nlg").DbConnector(App.psm)    // Pouch databases as filesystems
-    }
+    } else
+        ErrorDebuglog("Cannot create PSM", error)
     App.psm
 };
 

package/frameworks/node/worker.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2021 by Richard C. Zulch
+ * Copyright (c) 2019-2024 by Richard C. Zulch
  *
  */
 
@@ -18,9 +18,11 @@
  *
  */
 
-closure workCurrent(workerID, local current) {
+closure workCurrent(workerID, startup, local current) {
+    global()
     current = new(object, this)
     current.workerID = workerID
+    current.name = startup.name
     current.links = []
     current.termif = xnew({
         textout:    ONtextOut.proc
@@ -177,14 +179,17 @@ closure workCurrent(workerID, local current) {
  *
  */
 
-closure workThread(workerID, startup, local worker, subChannel, deathWatcher) {
+closure workThread(workerID, startup, local worker, options, subChannel, deathWatcher) {
+    global(threads, JSpath)
     worker = new(object, this)
+    worker.name = startup.name
     worker.workerID = workerID
     worker.links = []
 
     // create our worker thread
 
-    worker.thread = xnew(threads.Worker, JSpath.resolve(js.d, "node_worker.js"))
+    // options = { execArgv: ["--inspect-brk"] }                            // uncomment for NodeJS debugging
+    worker.thread = xnew(threads.Worker, JSpath.resolve(js.d, "node_worker.js"), options)
     subChannel = xnew(threads.MessageChannel)
     worker.thread.postMessage({ hereIsYourPort: subChannel.port1 }, [subChannel.port1])
     worker.msgport = subChannel.port2
@@ -224,7 +229,7 @@ closure workThread(workerID, startup, local worker, subChannel, deathWatcher) {
 
     worker.thread.on("error", function onerror(error) {
         debuglog("worker terminated with error", workerID, error)
-        exitNotify(exitCode)
+        exitNotify(-1)
     })
     
     //
@@ -341,6 +346,7 @@ closure workThread(workerID, startup, local worker, subChannel, deathWatcher) {
  */
 
 closure workLink(workco, conn, worker, sender, local link) {
+    global()
     link = new(object, this)
 
     //
@@ -393,6 +399,7 @@ closure workLink(workco, conn, worker, sender, local link) {
  */
 
 closure workController(api, local workco) {
+    global()
     workco = new(object, this)
     workco.serverID = "Workers"                                             // our serverID, used for communications
     workco.workers = { }                                                    // dictionary of workers by workerID
@@ -480,7 +487,7 @@ closure workController(api, local workco) {
             else if message.workerOp == "spawn" {
                 if !worker {                                                // create worker if needed
                     if message.workerID == "NideServer"
-                        worker = workco.current = workCurrent(message.workerID)
+                        worker = workco.current = workCurrent(message.workerID, message.payload)
                     else
                         worker = workThread(message.workerID, message.payload)
                     workco.workers[message.workerID] = worker }
@@ -489,7 +496,14 @@ closure workController(api, local workco) {
                 if message.workerID == "NideServer"
                     workco.current.vitalUpdate({
                         id: "workerlist",                                   // send the initial list of workers
-                        workerIDs: workco.workers.*
+                        workers: let(workers, workerID, worker) {
+                            workers = { }
+                            for `(workerID, worker) in workco.workers
+                                workers[workerID] = {
+                                    name: worker.name
+                                }
+                            workers
+                        }()
                     })
                 else {
                     worker.vitalWatch(termination)
@@ -516,7 +530,7 @@ closure workController(api, local workco) {
     //     Note that the connection with the client was closed.
     //
     
-    workco.wsClose = function wsClose(conn, local workerID) {
+    workco.wsClose = function wsClose(conn, local workerID, link) {
         for workerID in workco.workers {
             link = workco.links[conn][workerID]
             if link
@@ -553,7 +567,6 @@ function workInit(local manifest) {
 
     Naan.module.build(module.id, "worker", function(modobj, compobj) {
         require("./node.nlg")
-        threads = js.r("worker_threads")                                    // ### should be on reload
         compobj.manifest = manifest
         modobj.exports.WorkerController = workController
     })

package/frameworks/node/ws_client.nlg

@@ -0,0 +1,127 @@
+/*
+ * ws_client.nlg
+ *
+ *     WebSocket client operations for NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2023 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * WebSocket
+ *
+ * Create a WebSocket client connection object using the existing HTTPS connection.
+ *
+ * Required:
+ *      host:       <string>                    // host:port
+ *
+ * Options:
+ *  {
+ *      onopen:     <proc>(e)                   // connection has opened
+ *      onmessage:  <proc>(e, message)          // message received
+ *      onerror:    <proc>(e)                   // error occurred
+ *      onclose:    <proc>(e)                   // connection has closed
+ *  }
+ *
+ */
+
+closure WebSocket(host, options, local wsock) {
+    global(js)
+    wsock = new(object, this)
+    options = new(options) || { }
+
+    // connect
+    //
+    // Connect with the host if not already connected, returning a result tuple.
+    //
+    wsock.connect = closure connect(local pending, error, ws) {
+        if wsock.ws
+            return (list(false, { open: true }))
+        `(error, ws) = await(js.i("ws"))
+        if error
+            return (list(error))
+        wsock.ws = xnew(ws.default, "ws://${host}")
+        pending = new(nonce)
+
+        // onopen
+        // The WebSocket connection is now open.
+        wsock.ws.onopen = closure onopen(e) {
+            pending.signal(list(false, { ok: true }))
+            options.onopen(e, e.target, e.type)
+        }
+
+        // onmessage
+        // A message was received from the worker via WebSockets.
+        wsock.ws.onmessage = closure onmessage(e, local message) {
+            try {
+                message = new(JSONparse(e.data))
+            } catch {
+                debuglog("can't decode incoming ws message:", exception)
+                options.onerror(Error("can't decode incoming message:", exception))
+                return
+            }
+            options.onmessage(e, message)
+        }
+
+        // onerror
+        // An error occurred on the WebSocket connection.
+        wsock.ws.onerror = closure onerror(e, local error) {
+            error = Error("websocket failed", e)
+            options.onerror(e, error)
+            pending.signal(list(error))                                     // ensure connect completes
+        }
+
+        // onclose
+        // The WebSocket connection was closed.
+        wsock.ws.onclose = closure onclose(e) {
+            options.onclose(e, e.code, e.reason)
+            pending.signal(list(Error("connection closed")))                // ensure connect completes
+            wsock.ws = false
+        }
+
+        pending.wait()
+    }
+
+    // send
+    //
+    //     Send a message dictionary.
+    //
+    wsock.send = function send(message) {
+        wsock.ws.send(JSONstringify(message))
+    }
+
+    // close
+    //
+    //     Close the connection.
+    //
+    wsock.close = function close(reason) {
+        if !reason
+            reason = 1000                                                   // normal close
+        wsock.ws.close(reason)
+    }
+
+    // finis
+
+    wsock
+};
+
+
+/*
+ * ws_clientInit
+ *
+ *     Initialize WebSocket client operations for browsers.
+ *
+ */
+
+function ws_clientInit(local manifest) {
+    manifest = `(WebSocket, ws_clientInit)
+
+    Naan.module.build(module.id, "ws_client", function(modobj, compobj) {
+        require("node.nlg")
+        compobj.manifest = manifest
+        modobj.exports.WebSocket = WebSocket
+    })
+}();

package/frameworks/project/build.nlg

@@ -933,7 +933,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
                 xvrFileDefs = { }
             xvrFileDefs.BuildNumber = buildno
             `(error, verFileDefs) = readSubstitutionFile(build.verfilepath, xvrFileDefs)
-            build.version = verFileDefs["Version"]
+            build.version = verFileDefs["Version"].trim()
             verFileDefs["CacheBuster"] = HashMD5(build.version.concat(Math.random()))
         }
         if build.tasks.catenate > 0 || build.tasks.zip > 0 {
@@ -1195,7 +1195,8 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
         if error
             error = Error("Builder: cannot read main file", error)
         else {
-            `(error, executor) = track.spawn(build.runway.spawn.0, build.runway.spawn.1, "Build-".concat(build.buildnum), {
+            `(error, executor) = track.spawn(build.runway.spawn.0, "${rules.name} ${build.version}", {
+                workerID: "Run-${build.buildnum}"
                 startup: {
                     initcmds: maintext
                     dirpath: fs.path.resolve(fs.path.sep, fs.rootpath, build.destpath)

package/frameworks/project/projects.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2016-2022 by Richard C. Zulch
+ * Copyright (c) 2016-2024 by Richard C. Zulch
  *
  */
 
@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.0.16+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.2.0+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -602,6 +602,8 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
     //
     procon.run = closure run(stage, buildonly, 
         local error, rules, builder, buildpath, buildnum) {
+        if procon.builder
+            return (list(Error("build in progress")))
         `(error, rules) = buildrules(stage)
         if error
             return (list(error))
@@ -615,6 +617,7 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
         `(error, builder) = Builder(rules, rules.operation, fs, "", buildpath)
         if error
             return (list(error))
+        procon.builder = builder
         `(error, buildnum) = builder.increment()
         if error {
             ErrorDebuglog("build number increment failed", error)
@@ -631,6 +634,7 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
                 `(error) = builder.run(projman.track)
         }
         builder.destroy()
+        procon.builder = false
         if error
             list(error)
         else
@@ -650,6 +654,8 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
     //
     procon.publish = closure publish(stage, 
         local error, rules, where, publish, outOptions, outfs, inpath) {
+        if procon.builder
+            return (list(Error("cannot publish while building")))
         `(error, rules) = buildrules(stage)
         if error
             return (list(error))

package/frameworks/running/debugnub.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021 by Richard C. Zulch
+ * Copyright (c) 2021-2024 by Richard C. Zulch
  *
  */
 
@@ -480,7 +480,7 @@ closure DebugNub(local nub, debug, paused) {
             locals = localist(frame)
             ns = frame.3.1.namespace
             if ns !== nsactive().0 {
-                mod = module.owner.findModule(ns)
+                mod = module.owner.find(function(mod){ mod.namespace === ns })
                 if mod {
                     gohome = makeActivator(nsactive())
                     chns(mod.id)

package/frameworks/running/debugutil.nlg

@@ -458,7 +458,7 @@ closure DebugRemote(commander, local remoter, here, there, library) {
     // .undefined
     // Execute methods remotely, returning a standard `(error, data) tuple.
     
-    remoter[".undefined"] = closure remoter args {
+    remoter@[".undefined"] = closure remoter args {
         closure (selector, arglist, local error, context, result) {
             if !there[selector] {                                           // not over there yet
                 proc = here[selector]

package/frameworks/running/executors.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2024 by Richard C. Zulch
  *
  */
 
@@ -18,6 +18,7 @@
  */
 
 closure ExecutorBase(track, type, name, local exec) {
+    global()
     exec = new(object, this)
     exec.type = type
     exec.name = name
@@ -36,7 +37,7 @@ closure ExecutorBase(track, type, name, local exec) {
     // Send a message to the running instance and return a nonce to wait on. If data is not a
     // dictionary then it is ignored.
     
-    exec.execSend = closure execSend(msg, data, local pending) {
+    exec.execSend = closure execSend(msg, data, local pending, error) {
         if !dictionary(data)
             data = { }
         data.xmsg = msg
@@ -44,7 +45,9 @@ closure ExecutorBase(track, type, name, local exec) {
         pending = new(nonce)
         pending.msgout = data
         exec.pending[data.xid] = pending
-        exec.PostMessage(data)
+        `(error) = exec.PostMessage(data)
+        if error
+            pending.signal(Error("execSend failed", error))
         pending
     }
     
@@ -62,10 +65,19 @@ closure ExecutorBase(track, type, name, local exec) {
         pending.signal(list(false, data))
     }
     
+    // execFailed
+    // Communication has failed.
+    
+    exec.execFailed = closure execFailed(error, local xid, pending) {
+        exec.pending.error = error
+        for `(xid, pending) in exec.pending
+            pending.signal(list(error))
+    }
+    
     // execClosed
     // Note that the connection has closed, so signal all the waiters.
     
-    exec.execClosed = closure execClosed(error, local xid) {
+    exec.execClosed = closure execClosed(error, local xid, pending) {
         for `(xid, pending) in exec.pending {
             pending.signal(list(error))
             pending[xid] = undefined
@@ -76,18 +88,25 @@ closure ExecutorBase(track, type, name, local exec) {
     // Return a context object for the instance, or an error, in a result tuple. Currently this does
     // not ever reclaim old contexts on the instance end of things, but it should. ###
 
-    exec.context = closure context(local context, retries, ms) {
+    exec.context = closure context(local error, retries, ms, delta, excon) {
         retries = 0
         ms = milliseconds()
+        delta = 1
         while !exec.pending.ready {
-            exec.PostMessage({
+            `(error) = exec.PostMessage({
                 xmsg: "xstart"
             })
+            if error
+                return (Error("exec.context failed", error))
             if ++retries > 100
-                return (list(Error("timeout creating remote worker context", type, name)))
-            sleep(100) }
-        context = ExecutorContext(exec, exec.nextContextID++)
-        context.register()
+                return (list(Error("timeout creating remote context", type, name)))
+            if exec.pending.error
+                return (list(Error("cannot create remote context", exec.pending.error)))
+            sleep(delta)
+            if delta < 100
+                delta *= 2 }
+        excon = ExecutorContext(exec, exec.nextContextID++)
+        excon.register()
     }
 
     // attention
@@ -117,6 +136,7 @@ closure ExecutorBase(track, type, name, local exec) {
  */
 
 closure ExecutorContext(exec, contextID, local context, util) {
+    global(apply)
     context = new(object, this)
     util = modlist().Util.exports
 
@@ -172,13 +192,35 @@ closure ExecutorContext(exec, contextID, local context, util) {
     context.evalq = macro evalq(expr, put, get) {
         evalf(expr, eval(put), eval(get))
     }
+                
+    // apply
+    // Apply list of arguments to the specified procure evaluated remotely. If the procedure is a
+    // not a tuple then this uses the tuple definition if there is one.
+    
+    context.apply = closure applyf(proc, args, put, get) {
+        if !tuple(proc) && tuple(proc.proc)
+            proc = proc.proc
+        evalf(list(apply, proc, args), eval(put), eval(get))
+    }
+
+    // call
+    // Call the specified function remotely using local arguments:
+    //      context.call(proc, args...)
     
+    context.call = macro callf args {
+        closure (proc, arglist, local eargs) {
+            while arglist
+                push(eval(pop(arglist)), eargs)
+            applyf(eval(proc), eargs)
+        } (pop(args), reverse(args))
+    }
+
     // rpc
     // Execute a remote procedure call without package translation, to reduce overhead. This will not
     // properly transfer Naan-specific types like symbols, and the results will be native JavaScript.
     // The procedure must be specified as a symbol or string.
     
-    context.rpc = closure rpc(proc, args, local data, result) {
+    context.rpc = closure rpc(proc, args, local data, pending, result) {
         data = {
             contextID: contextID
             proc: proc
@@ -187,8 +229,12 @@ closure ExecutorContext(exec, contextID, local context, util) {
         result = pending.wait()
         if result.1.error
             list(Error(result.1.error))
-        else
-            list(false, totuple(result.1.result))
+        else {
+            result = result.1.result
+            if Array.isArray(result)
+                result = totuple(result)
+            list(false, result)
+        }
     }
 
     // finis
@@ -225,6 +271,7 @@ closure ExecutorContext(exec, contextID, local context, util) {
  */
 
 closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, crValue) {
+    global()
     maxcount = options.maxcount
     if !maxcount
         maxcount = 4                                                        // default is 4 workers
@@ -251,7 +298,7 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
             return (false)
         parallel.workers[workdex] = false                                   // reserve our spot
         name = basename.concat("-", workdex+1)
-        worker = App.nide.track.spawn("Local", name, "ExecutorParallel")
+        worker = App.nide.track.spawn("Local", name, "ExecutorParallel")    // ### App.nide.track.spawn may not exist
         if worker.0 {                                                       // ### wasting space on workers list
             debuglog("execParallel: cannot spawn worker:", ErrorString(worker.0))
             return (false)
@@ -401,6 +448,7 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
  */
 
 closure ExecutorTracker(local xtrak) {
+    global()
     xtrak = new(object, this)
     xtrak.instances = []
     xtrak.watchers = []
@@ -451,7 +499,7 @@ closure ExecutorTracker(local xtrak) {
     // enumlist
     // Enumerate the list of executors.
     
-    xtrak.enumlist = function enumlist(local results) {
+    xtrak.enumlist = function enumlist(local results, instance) {
         results = []
         for instance in xtrak.instances
             results.push(instance.executor)
@@ -509,6 +557,7 @@ closure ExecutorTracker(local xtrak) {
  */
 
 closure execLambda(track, url, name, workerID, local target, nlregex, clientID, serverID) {
+    global()
     target = ExecutorBase(track, "Lambda", name)
     clientID = "DebugWorkers"
     serverID = "Workers"
@@ -670,11 +719,15 @@ closure execLambda(track, url, name, workerID, local target, nlregex, clientID,
     //     Post a message to the target.
     //
 
-    target.PostMessage = function PostMessage(data) {
+    target.PostMessage = function PostMessage(data, local result) {
+        result = connect()
+        if result.0
+            return (result)
         target.ws.send({                                                    // send user-entered text to worker
             id: "msgin",
             data: data
         })
+        list(false, { posted: true })
     }
     
     //
@@ -753,7 +806,6 @@ closure execLambda(track, url, name, workerID, local target, nlregex, clientID,
  */
 
 function execInit(local manifest) {
-
     manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda, execInit)
 
     Naan.module.build(module.id, "executors", function(modobj, compobj) {