@naanlang/naan 1.1.0 → 1.2.1

package/frameworks/running/executors.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2023 by Richard C. Zulch
+ * Copyright (c) 2020-2024 by Richard C. Zulch
  *
  */
 
@@ -18,6 +18,7 @@
  */
 
 closure ExecutorBase(track, type, name, local exec) {
+    global()
     exec = new(object, this)
     exec.type = type
     exec.name = name
@@ -51,7 +52,7 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execReceived
-    // THe specified message was received from the instance, so signal it.
+    // The specified message was received from the instance, so signal it.
     
     exec.execReceived = closure execReceived(data, local pending) {
         pending = exec.pending[data.xid]
@@ -64,6 +65,13 @@ closure ExecutorBase(track, type, name, local exec) {
         pending.signal(list(false, data))
     }
     
+    // execSetStatus
+    // Received a status update from the instance.
+    
+    exec.execSetStatus = function execSetStatus(status) {
+        exec.xstatus = status
+    }
+    
     // execFailed
     // Communication has failed.
     
@@ -76,7 +84,7 @@ closure ExecutorBase(track, type, name, local exec) {
     // execClosed
     // Note that the connection has closed, so signal all the waiters.
     
-    exec.execClosed = closure execClosed(error, local xid) {
+    exec.execClosed = closure execClosed(error, local xid, pending) {
         for `(xid, pending) in exec.pending {
             pending.signal(list(error))
             pending[xid] = undefined
@@ -135,6 +143,7 @@ closure ExecutorBase(track, type, name, local exec) {
  */
 
 closure ExecutorContext(exec, contextID, local context, util) {
+    global(apply)
     context = new(object, this)
     util = modlist().Util.exports
 
@@ -212,12 +221,13 @@ closure ExecutorContext(exec, contextID, local context, util) {
             applyf(eval(proc), eargs)
         } (pop(args), reverse(args))
     }
+
     // rpc
     // Execute a remote procedure call without package translation, to reduce overhead. This will not
     // properly transfer Naan-specific types like symbols, and the results will be native JavaScript.
     // The procedure must be specified as a symbol or string.
     
-    context.rpc = closure rpc(proc, args, local data, result) {
+    context.rpc = closure rpc(proc, args, local data, pending, result) {
         data = {
             contextID: contextID
             proc: proc
@@ -260,20 +270,22 @@ closure ExecutorContext(exec, contextID, local context, util) {
  *
  * Allowable options are:
  *  {
- *      maxcount: <integer>                                                 // maximum workers, default 4
- *      initeval: `(<expr>, <put>, <get>)                                   // initializes each new worker
- *      basename: <string>                                                  // worker label for UI, default "batch"
+ *      spawnLoc: <string>                      // e.g. "Local" (default) or "Host"
+ *      maxcount: <integer>                     // maximum workers, default 4
+ *      initeval: `(<expr>, <put>, <get>)       // initializes each new worker
+ *      basename: <string>                      // worker label for UI, default "batch"
  *  }
  *
  */
 
-closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, crValue) {
+closure ExecutorParallel(track, options, local maxcount, parallel, exqueue, crValue) {
+    global()
+    options = merge({
+        spawnLoc: "Local"
+        maxcount: 4
+        basename: "batch"
+    }, options)
     maxcount = options.maxcount
-    if !maxcount
-        maxcount = 4                                                        // default is 4 workers
-    basename = options.basename
-    if !string(basename)
-        basename = "batch"
     exqueue = []                                                            // waiting for execution
     parallel = new(object, this)
     parallel.workers = []                                                   // array of worker executors
@@ -284,36 +296,41 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
     //
     // Return the next available worker. If no workers are available and we haven't hit the maximum 
     // then create an additional worker and return it. If we cannot allocate another worker then this 
-    // returns false.
+    // returns { wait: true }.
     
     closure acquireWorker(local workdex, name, worker, context, result) {
         if parallel.available.length > 0
-            return (parallel.available.pop())
+            return (list(false, parallel.available.pop()))
         workdex = parallel.workers.length
         if workdex >= maxcount
-            return (false)
+            return (list(false, { wait: true }))
         parallel.workers[workdex] = false                                   // reserve our spot
-        name = basename.concat("-", workdex+1)
-        worker = App.nide.track.spawn("Local", name, "ExecutorParallel")
-        if worker.0 {                                                       // ### wasting space on workers list
+        name = options.basename.concat("-", workdex+1)
+        worker = track.spawn(options.spawnLoc, name, {
+            workerID: "ExecutorParallel"
+        })
+        if worker.0 {
+            parallel.workers.splice(workdex, 1)
             debuglog("execParallel: cannot spawn worker:", ErrorString(worker.0))
-            return (false)
+            return (list(worker.0))
         }
         worker = worker.1
         context = worker.context()
         if context.0 {
-            debuglog("execParallel: cannot create worker context:", ErrorString(context.0))
+            parallel.workers.splice(workdex, 1)
             worker.destroy()
-            return (false)
+            debuglog("execParallel: cannot create worker context:", ErrorString(context.0))
+            return (list(context.0))
         }
         context = context.1
         parallel.workers[workdex] = worker
         parallel.contexts[worker] = context
         if tuple(options.initeval) {
             result = apply(context.eval, options.initeval)
-            if result.0
-                debuglog("execParallel: cannot initialize worker", name, ErrorString(result.0)) }
-        return (worker)
+            if result.0 {
+                debuglog("execParallel: cannot initialize worker", name, ErrorString(result.0))
+                return (list(result.0)) } }
+        return (list(false, worker))
     }
     
     // releaseWorker
@@ -333,23 +350,26 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
     // don't remove work from the queue until we already have a worker.
     //
 
-    closure execNext(local worker, result) {
+    closure execNext(local error, worker, result) {
         if exqueue.length > 0 {
-            worker = acquireWorker()                                        // false if none available
-            if !worker {
-                if maxcount > 0 && parallel.workers.length == 0             // maxcount zero when being destroyed
-                    throw("execParallel: cannot allocate any workers at all")
-                return                                                      // we're busy, so try again later
+            `(error, worker) = acquireWorker()
+            if error {
+                if maxcount > 0 && parallel.workers.length == 0 {
+                    exqueue = []
+                    throw("execParallel: cannot allocate any workers at all") }
+                return                                                      // one failed, so try again later
             }
+            if worker.wait
+                return                                                      // we're busy, so try again later
             future(function(local next, context) {
                 next = exqueue.splice(0, 1).0                               // remove oldest entry, from start
                 if !next {
                     releaseWorker(worker)                                   // someone else already processed this
                     return
-               }
+                }
                 context = parallel.contexts[worker]
                 if next.eval
-                    result = apply(context.eval, next.args)
+                    result = apply(context.eval, next.eval)
                 else if next.proc
                     result = context.rpc(next.proc, next.args)
                 else
@@ -368,7 +388,7 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
     //
 
     parallel.eval = closure evalf(expr, put, get, local pending) {
-        if !maxcount
+        if maxcount == 0
             return (crValue)                                                // already destroyed
         pending = new(nonce)
         exqueue.push({
@@ -388,13 +408,55 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
         evalf(expr, eval(put), eval(get))
     }
     
+    // exBatch
+    //
+    // Remote helper to execute a batch function.
+    //
+    
+    closure exBatch(expr, local pending) {
+        // evaluate the expression in batch mode
+        function execbat() {
+            try {
+                pending.signal(list(false, evalactive(expr)))
+            } catch {
+                pending.signal(list(Error(exception)))
+            }
+        }
+
+        pending = new(nonce)
+        exec(list(execbat))
+        pending.wait()
+    }
+    
+    // batch
+    //
+    // Execute eval above in a remote in batch mode, waiting until it is complete.
+    //
+
+    parallel.batch = closure batchf(expr, put, get, local result) {
+        result = evalf(list(exBatch, list(quote, expr)), `(exBatch).append(put), get)
+        if result.0                                                         // unpack our error wrapper
+            result
+        else
+            result.1
+    }
+        
+    // batchq
+    //
+    // Evaluate the specified QUOTED expression in a remote in batch mode.
+    //
+    
+    parallel.batchq = macro batchq(expr, put, get) {
+        batchf(expr, eval(put), eval(get))
+    }
+
     // rpc
     //
     // Execute a procedure call in a remote worker.
     //
     
     parallel.rpc = closure rpc(proc, args, local pending) {
-        if !maxcount
+        if maxcount == 0
             return (crValue)                                                // already destroyed
         pending = new(nonce)
         exqueue.push({
@@ -406,13 +468,36 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
         pending.wait()
     }
 
+    // status
+    //
+    // Report status of workers. Each worker has a dictionary:
+    //  {
+    //      state:  <string>                    // "active" or "idle"
+    //      status: <dict>                      // like runaverage(1)
+    //  }
+    //
+
+    parallel.status = function status(local output, worker) {
+        output = []
+        for worker in parallel.workers {
+            output.push({
+                name: worker.name
+                type: worker.type
+                state: if parallel.available.indexOf(worker) >= 0
+                    "idle" else "active"
+                status: worker.xstatus
+            })
+        }
+        output
+    }
+
     // destroy
     //
     // Destroy the object by destroying the workers and failing pending requests with cancelResult.
     //
     
     parallel.destroy = closure destroy(cancelResult, local worker, next) {
-        if !maxcount
+        if maxcount == 0
             return (false)                                                  // already destroyed
         crValue = cancelResult
         maxcount = 0
@@ -444,6 +529,7 @@ closure ExecutorParallel(options, local maxcount, basename, parallel, exqueue, c
  */
 
 closure ExecutorTracker(local xtrak) {
+    global()
     xtrak = new(object, this)
     xtrak.instances = []
     xtrak.watchers = []
@@ -494,7 +580,7 @@ closure ExecutorTracker(local xtrak) {
     // enumlist
     // Enumerate the list of executors.
     
-    xtrak.enumlist = function enumlist(local results) {
+    xtrak.enumlist = function enumlist(local results, instance) {
         results = []
         for instance in xtrak.instances
             results.push(instance.executor)
@@ -537,6 +623,13 @@ closure ExecutorTracker(local xtrak) {
         } (pop(args), args)
     }
     
+    // parallel
+    // Return a new parallel executor with specified options.
+    
+    xtrak.parallel = function parallel(options) {
+        ExecutorParallel(xtrak, options)
+    }
+    
     xtrak.register(execLambda, "Lambda")
     
     // finis
@@ -552,6 +645,7 @@ closure ExecutorTracker(local xtrak) {
  */
 
 closure execLambda(track, url, name, workerID, local target, nlregex, clientID, serverID) {
+    global()
     target = ExecutorBase(track, "Lambda", name)
     clientID = "DebugWorkers"
     serverID = "Workers"
@@ -800,7 +894,6 @@ closure execLambda(track, url, name, workerID, local target, nlregex, clientID,
  */
 
 function execInit(local manifest) {
-
     manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda, execInit)
 
     Naan.module.build(module.id, "executors", function(modobj, compobj) {

package/frameworks/running/taskexec.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2023 by Richard C. Zulch
+ * Copyright (c) 2021-2024 by Richard C. Zulch
  *
  */
 
@@ -20,10 +20,11 @@
  */
 
 closure TaskDispatcher(replier, local contexts, gohome, util) {
+    global()
     contexts = { }
     gohome = makeActivator()
     util = modlist().Util.exports
-    closure(data, local reply, restorens, expr, roots, proc, args, result) {
+    closure(data, local reply, restorens, context, expr, roots, proc, args, result) {
         reply = {
             xmsg: "error"
             xid: data.xid

package/frameworks/storage/psm.nlg

@@ -20,6 +20,7 @@
  */
 
 closure MakePSMutil(fs, local util) {
+    global()
     util = new(object, this)
     util.fs = fs
 
@@ -310,7 +311,7 @@ closure MakePSMutil(fs, local util) {
         //
         if options.erase {
             asyncArray(keys(folders).toarray, 10, closure eraser(destpath
-                local error, outfolder, folder, namesin, removes, delopt) {
+                local error, outfolder, folder, namesin, node, removes, delopt) {
                 `(error, outfolder) = outfs.dirList(destpath)               // existing files in our destination
                 if error
                     return

package/frameworks/storage/psm_dbtables.nlg

@@ -45,6 +45,14 @@ closure FSview(table, rootpath, local view) {
     view.rootpath = rootpath
     view.path = JSpath.posix                                                // for view.path.join(), etc.
 
+    // dirPath
+    //
+    // True iff the path is a dirpath. If the path is empty this checks the rootpath.
+    
+    function dirPath(path) {
+        path.slice(-1) == "/" || path == "" && rootpath.slice(-1) == "/"
+    }
+
     // dirName
     //
     // Return the directory portion of the specified path.
@@ -163,7 +171,7 @@ closure FSview(table, rootpath, local view) {
         else
             proc = view.table.get
         `(error, data) = proc(path)
-        if error && error.status == 404 && path.slice(-1) != "/"
+        if error && error.status == 404 && !dirPath(path)
             `(error, data) = proc(path.concat("/"))
         callback(error, data)
     }
@@ -200,7 +208,7 @@ closure FSview(table, rootpath, local view) {
                         birthtime: nowstr
                     }
                 }
-                if path.slice(-1) == "/"
+                if dirPath(path)
                     metadata.finfo.type = "directory"
                 else
                     metadata.finfo.type = "file"
@@ -268,7 +276,7 @@ closure FSview(table, rootpath, local view) {
             return (asyncResult(callback, false, { created: false }))       // root always already exists
         if checkPath(path, callback)
             return
-        if path.slice(-1) != "/"
+        if !dirPath(path)
             path = path.concat("/")
         view.table.get(path, function(error, data, local pathdirs, buildpath, item) {
             if !error
@@ -286,7 +294,7 @@ closure FSview(table, rootpath, local view) {
                                 name: "conflict"
                                 path: buildpath
                             })))
-                    if buildpath.slice(-1) != "/"
+                    if !dirPath(buildpath)
                         buildpath = buildpath.concat("/")
                     `(error, data) = view.table.head(buildpath)             // get desired directory
                     if error.status == 404
@@ -311,7 +319,7 @@ closure FSview(table, rootpath, local view) {
         if dest.indexOf("/") < 0
             newpath = dirName(srcpath).concat(dest)                         // renaming within same directory
         else {
-            if dest.slice(-1) == "/"                                        // moving to new directory location:
+            if dirPath(dest)                                                // moving to new directory location:
                 newpath = dest.concat(baseName(srcpath))                    //      new path is new pathname
             else {                                                          // moving and renaming:
                 newpath = dest                                              //      new path is new pathname
@@ -361,7 +369,7 @@ closure FSview(table, rootpath, local view) {
     //
     closure copyDirectory(srcpath, dest, options,
         local error, nodes, data, destpath, errors, docpath, newpath, srcroot, item) {
-        if srcpath.slice(-1) != "/"
+        if !dirPath(srcpath)
             srcpath = srcpath.concat("/")                                   // normalize source as directory
         `(error, nodes) = view.table.records({ prefix: srcpath })           // each item has key/_id/_rev, not doc
         if error
@@ -380,13 +388,13 @@ closure FSview(table, rootpath, local view) {
             destpath = dirName(srcpath).concat(dest)                        // new directory name in same parent directory
             dest = dirName(srcpath) }                                       // dest is new parent pathname
         else {
-            if dest.slice(-1) == "/"
+            if dirPath(dest)
                 destpath = dest.concat(baseName(srcpath))                   // same name; new location
             else {
                 destpath = dest                                             // new full pathname
                 dest = dirName(dest) }                                      // dest is new parent pathname
         }
-        if destpath != "" && destpath.slice(-1) != "/"
+        if destpath != "" && !dirPath(destpath)
             destpath = destpath.concat("/")                                 // get actual path of new directory
         //
         // Change srcpath to destpath for all records. Note that dest is the name of our new parent
@@ -489,13 +497,13 @@ closure FSview(table, rootpath, local view) {
                     finfo.length = data.length
                 else if data._length
                     finfo.length = data._length
-                if finfo.length
+                if dirPath(path)
+                    finfo.type = "directory"
+                else if finfo.length
                     finfo.type = "file" }
             else if path == "" {
                 error = false                                               // we were asked for info on superoot
                 finfo = {
-                    md5: HashMD5("")
-                    length: 0
                     type: "directory"
                 } }
             callback(error, finfo)
@@ -553,7 +561,7 @@ closure FSview(table, rootpath, local view) {
             return
         if !content
             return (asyncResult(callback, Error("content required")))
-        if path.slice(-1) == "/"
+        if dirPath(path)
             return (asyncResult(callback, Error("cannot write to a directory", { path: path })))
         if !validName(path, true)
             return (asyncResult(callback, Error("invalid filename", { path: path })))
@@ -631,7 +639,7 @@ closure FSview(table, rootpath, local view) {
                     name: "forbidden"
                     path: path
                 })))
-            if path.slice(-1) != "/"
+            if !dirPath(path)
                 path = path.concat("/")                                     // avoid deleting longer filenames with same parent and prefix
             `(error, data) = dbDirRecords(path, true)
             if error
@@ -739,6 +747,8 @@ closure FSview(table, rootpath, local view) {
             prefix: path
             deep: true
         }
+        if name == "" && rootpath != ""
+            name = rootpath.split("/").slice(0,-1).pop() || ""
         treedict = {
             name: name
             data: {

package/lib/browser/env_web.js

@@ -526,7 +526,7 @@ exports.NaanControllerWeb = function() {
         statedoc.curversion = kStateCurrentVersion;
         statedoc.firstver = kStateFirstVersion;
         statedoc.licensee = "MIT-License";
-        statedoc.verstring = "1.1.0+1";
+        statedoc.verstring = "1.2.1+1";
         statedoc.date = new Date().toISOString();
         statedoc.prefs = prefs;
         statedoc.naan = naanlib.saveState(false);                           // true to optimize, which is a bit slower
@@ -544,7 +544,7 @@ exports.NaanControllerWeb = function() {
             || statedoc.firstver > kStateCurrentVersion
             || statedoc.curversion < kStateFirstVersion
             || statedoc.licensee != "MIT-License"
-            || statedoc.verstring != "1.1.0+1")
+            || statedoc.verstring != "1.2.1+1")
         {
             localStorage.removeItem("NaanState_Hosted");
             return (false);
@@ -603,7 +603,7 @@ exports.NaanControllerWeb = function() {
                     op: "VsiteOpen",
                     name: vsiteName,
                     naancont: contSelf,
-                    title: vsiteName + " 1.1.0+1"
+                    title: vsiteName + " 1.2.1+1"
                 });
             }
         } catch (e) {
@@ -617,7 +617,7 @@ exports.NaanControllerWeb = function() {
                     op: "VsiteClose",
                     name: vsiteName,
                     naancont: contSelf,
-                    title: vsiteName + " 1.1.0+1"
+                    title: vsiteName + " 1.2.1+1"
                 });
                 termTextOut("\x1b[90m\x1b[3m".concat("\nwindow closed", "\x1b[0m\n"));
             }
@@ -632,8 +632,8 @@ exports.NaanControllerWeb = function() {
         naanlib.banner();
         var hostpath = naanlib.js.r("path").dirname(window.location.href);
         naanlib.start({
-            cmd: 'App.version = "1.1.0+1";;\r\n'
-                + 'App.cache = "241e1d15dfb0de008135e2de76886ccc";;\r\n'
+            cmd: 'App.version = "1.2.1+1";;\r\n'
+                + 'App.cache = "f22e60a842b7a65ac309da378d469eed";;\r\n'
                 + 'Naan.module.requireQuery({ naanver: App.cache });;\r\n'
                 + 'Naan.module.webparse("naan_init.nlg", "' + hostpath + '", { naanver: App.cache });;\r\n'
         });