@naanlang/naan 1.3.2 → 1.4.1

package/frameworks/running/executors.nlg

@@ -24,19 +24,20 @@ closure ExecutorBase(track, type, name, local exec) {
     exec.name = name
     
     // execReset
+    //
     // Initialize remote communications for a new or restarted instance.
-    
+    //
     exec.execReset = function execReset() {
         exec.pending = { }
         exec.nextSendID = toint(RandomBytes(3))                             // keep our sequence code unique for shared servers
-        exec.nextContextID = 1
     }
     execReset()
 
     // execSend
+    //
     // Send a message to the running instance and return a nonce to wait on. If data is not a
     // dictionary then it is ignored.
-    
+    //
     exec.execSend = closure execSend(msg, data, local pending, error) {
         if !dictionary(data)
             data = { }
@@ -52,8 +53,9 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execReceived
+    //
     // The specified message was received from the instance, so signal it.
-    
+    //
     exec.execReceived = closure execReceived(data, local pending) {
         pending = exec.pending[data.xid]
         if !pending {                                                       // not waiting on message
@@ -68,15 +70,17 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execSetStatus
+    //
     // Received a status update from the instance.
-    
+    //
     exec.execSetStatus = function execSetStatus(status) {
         exec.xstatus = status
     }
     
     // execFailed
+    //
     // Communication has failed.
-    
+    //
     exec.execFailed = closure execFailed(error, local xid, pending) {
         exec.pending.error = error
         for `(xid, pending) in exec.pending
@@ -84,8 +88,10 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execClosed
+    //
+    //
     // Note that the connection has closed, so signal all the waiters.
-    
+    //
     exec.execClosed = closure execClosed(error, local xid, pending) {
         for `(xid, pending) in exec.pending {
             pending.signal(list(error))
@@ -94,10 +100,13 @@ closure ExecutorBase(track, type, name, local exec) {
     }
         
     // context
-    // Return a context object for the instance, or an error, in a result tuple. Currently this does
-    // not ever reclaim old contexts on the instance end of things, but it should. ###
-
-    exec.context = closure context(local error, retries, ms, delta, excon) {
+    //
+    // Return a context object for the instance, or an error, in a result tuple. The optional state
+    // dictionary argument initializes the context with the state of a previous one to facilitate
+    // persistent reconnection. Currently this does not ever reclaim old contexts on the instance
+    // end of things, but it should. ###
+    //
+    exec.context = closure context(state, local error, retries, ms, delta, excon) {
         retries = 0
         ms = milliseconds()
         delta = 1
@@ -114,13 +123,14 @@ closure ExecutorBase(track, type, name, local exec) {
             sleep(delta)
             if delta < 100
                 delta *= 2 }
-        excon = ExecutorContext(exec, exec.nextContextID++)
+        excon = ExecutorContext(exec, state)
         excon.register()
     }
 
     // attention
+    //
     // Focus the UI on this instance because something needs attention.
-
+    //
     exec.attention = function attention() {
         track.attention(exec)
     }
@@ -141,20 +151,37 @@ closure ExecutorBase(track, type, name, local exec) {
  * ability to track namespace changes as the result of evaluating an expression. They do not override
  * normal namespace protections. The contexts exposed here preserve the active namespace after the
  * expression is evaluated.
+ *     The optional state argument preserves the context state so that it can be restored for
+ * resuming persistent connections.
  *
  */
 
-closure ExecutorContext(exec, contextID, local context, util) {
+closure ExecutorContext(exec, state, local context, coder, util) {
     global(apply)
     context = new(object, this)
+    if !state || !state.contextID
+        state = {
+            contextID: UUID()
+        }
+    context.state = state
+    coder = packageCoder(evalf)
     util = modlist().Util.exports
 
+    // getState
+    //
+    // Get context state for future initialization
+    //
+    context.getState = function getState() {
+        context.state
+    }
+
     // register
+    //
     // Register or re-register our context with the worker.
-
+    //
     context.register = closure register(local pending, result) {
         pending = exec.execSend("register", {
-            contextID: contextID
+            contextID: state.contextID
         })
         result = pending.wait()
         if result.1
@@ -164,10 +191,11 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
     
     // eval
+    //
     // Evaluate the specified expression in the current worker context. The optional put argument
     // if non-false is a tuple of symbols that should be transfered to the remote end, while get is
     // an optional tuple of symbols to retrieve after evaluation.
-
+    //
     context.eval = closure evalf(expr, put, get, local roots, pkg, data, pending, result) {
         if tuple(put)
             roots = cons(`expr, put)
@@ -175,9 +203,11 @@ closure ExecutorContext(exec, contextID, local context, util) {
             roots = `(expr)
         pkg = util.pkgSave(`expr, roots, {
             nsignore: true
-            comments: true })
+            comments: true
+            objectEncoder: coder.encodeObject
+        })
         data = {
-            contextID: contextID
+            contextID: state.contextID
             expr: pkg }
         if tuple(get)
             data.get = get
@@ -190,22 +220,25 @@ closure ExecutorContext(exec, contextID, local context, util) {
                 localroots: false
                 intern: false
                 execute: false
+                objectDecoder: coder.decodeObject
             })))
         else
             result
     }
-    
+
     // evalq
+    //
     // Evaluate the specified QUOTED expression in the current worker context.
-    
+    //
     context.evalq = macro evalq(expr, put, get) {
         evalf(expr, eval(put), eval(get))
     }
-                
+
     // apply
+    //
     // Apply list of arguments to the specified procure evaluated remotely. If the procedure is a
     // not a tuple then this uses the tuple definition if there is one.
-    
+    //
     context.apply = closure applyf(proc, args, put, get) {
         if !tuple(proc) && tuple(proc.proc)
             proc = proc.proc
@@ -213,9 +246,10 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
 
     // call
+    //
     // Call the specified function remotely using local arguments:
     //      context.call(proc, args...)
-    
+    //
     context.call = macro callf args {
         closure (proc, arglist, local eargs) {
             while arglist
@@ -225,13 +259,14 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
 
     // rpc
+    //
     // Execute a remote procedure call without package translation, to reduce overhead. This will not
     // properly transfer Naan-specific types like symbols, and the results will be native JavaScript.
     // The procedure must be specified as a symbol or string.
-    
+    //
     context.rpc = closure rpc(proc, args, local data, pending, result) {
         data = {
-            contextID: contextID
+            contextID: state.contextID
             proc: proc
             args: args }
         pending = exec.execSend("rpc", data)
@@ -647,7 +682,7 @@ closure ExecutorTracker(local xtrak) {
  */
 
 closure execLambda(track, url, name, workerID, local target, nlregex, clientID, serverID) {
-    global()
+    global(js)
     target = ExecutorBase(track, "Lambda", name)
     clientID = "DebugWorkers"
     serverID = "Workers"
@@ -896,7 +931,8 @@ closure execLambda(track, url, name, workerID, local target, nlregex, clientID,
  */
 
 function execInit(local manifest) {
-    manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda, execInit)
+    manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda,
+        execInit)
 
     Naan.module.build(module.id, "executors", function(modobj, compobj) {
         require("./running.nlg")

package/frameworks/running/running.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2023 by Richard C. Zulch
+ * Copyright (c) 2021-2024 by Richard C. Zulch
  *
  */
 
@@ -19,6 +19,7 @@
  */
 
 function rootproc(procdef, local parent) {
+    global()
     loop {                                                                  // can only find out proc
         parent = procdef.1@\.parent
         if !parent
@@ -38,6 +39,7 @@ function rootproc(procdef, local parent) {
  */
 
 function baseproc(procdef) {
+    global()
     if procdef.1@\.base.proc
         { }
     else
@@ -58,6 +60,7 @@ function baseproc(procdef) {
  */
 
 function pathmatch(parent, path, local segment, nth, child) {
+    global()
     if empty(path)
         return (parent)
     segment = pop(path)
@@ -83,6 +86,7 @@ function pathmatch(parent, path, local segment, nth, child) {
  */
 
 function procpath(procdef, local path, segment, parent, peers, nth, peer) {
+    global()
     loop {
         segment = procdef.1
         parent = segment@\.parent
@@ -114,6 +118,7 @@ function procpath(procdef, local path, segment, parent, peers, nth, peer) {
  */
 
 function procname(name, local parent) {
+    global()
     parent = name@\.parent
     name = tostring(name)
     if parent
@@ -136,6 +141,7 @@ function procname(name, local parent) {
  */
     
 function procstring(procdef, options, local name, output, parm) {
+    global()
     if options.fullname || !defined(options, `fullname)
         name = procname(procdef.1)                                          // default is full name
     else
@@ -156,6 +162,82 @@ function procstring(procdef, options, local name, output, parm) {
 }
 
 
+/*
+ * packageCoder
+ *
+ *     Encode and decode objects for the NaaN packager to establish proxies for remote access.
+ *
+ */
+
+closure packageCoder(evalf, local coder) {
+    global()
+    coder = new(object, this)
+    coder.wmapEncoding = new(weakmap)
+    coder.mapDecoding = { }
+
+    // encodeObject
+    //
+    // Encode an object into the ident dictionary so that the remote end can create a proxy.
+    //
+    coder.encodeObject = closure encodeObject(obj, local objectID) {
+        if coder.wmapEncoding[obj]
+            { }
+        else {
+            objectID = UUID()
+            coder.mapDecoding[objectID] = obj
+            coder.wmapEncoding[obj] = {
+                name: obj@\.closure || undefined
+                objectID: objectID
+            }
+        }
+    }
+
+    // decodeObject
+    //
+    // Decode the ident dictionary into a proxy object allowing method invocation.
+    //
+    coder.decodeObject = closure decodeObject(ident, local x, obj) {
+        if coder.mapDecoding[ident.objectID]
+            { }
+        else if ident.name {
+            putproc(ident.name, false)
+            obj = coder.mapDecoding[ident.objectID] = closure(nn, objectID) {
+                call(enclose(list(`closure, nn, list(`local, `x),
+                    quote(x=new(object, nn)),
+                    quote(x@\.tostring = function() { "{proxy ${nn}}" }),
+                    quote(x@\.undefined = closure undef args {
+                        let (result) {
+                            result = evalf(list(`\.execute\-method, objectID, args))
+                            if result.0
+                                throw("remoting failure: ${ErrorString(result.0)}")
+                            else
+                                result.1
+                        } ()
+                    }), `x)))
+            } (ident.name, ident.objectID)
+            coder.wmapEncoding[obj] = ident
+            obj
+        } else
+            coder.mapDecoding[ident.objectID] = new(object)
+    }
+    
+    // invokeMethod
+    //
+    // Invoke a local object method from the remote proxy, returning a result tuple.
+    //
+    coder.invokeMethod = closure invokeMethod(objectID, args, local obj) {
+        obj = coder.mapDecoding[objectID]
+        if obj
+            eval(list(deref, obj, pop(args), args))
+        else
+            throw("remoting error: object ${objectID} not found")
+    }
+
+    // finis
+    coder
+};
+
+
 /*
  * runnInit
  *
@@ -164,8 +246,8 @@ function procstring(procdef, options, local name, output, parm) {
  */
 
 function runnInit(local manifest) {
-    manifest = `(rootproc, baseproc, pathmatch, procpath, procname, procstring, runnInit)
-
+    manifest = `(rootproc, baseproc, pathmatch, procpath, procname, procstring, packageCoder,
+        runnInit)
     Naan.module.build(module.id, "running", closure(modobj, compobj) {
         require("../common").LiveImport()
         compobj.manifest = manifest

package/frameworks/running/sourcecode.nlg

@@ -87,12 +87,14 @@ closure CodeManager(options,  local coman, cache)
                 output.push(false)
                 continue
             }
+            if text.startsWith("!")
+                text = "(${text})"                                          // no shell escapes here
             try {
                 source = new(textstream, text.concat(" ;"))                 // avoid EOF
                 if locals
                     source.setsymbols(locals)
                 registry = modlist().Lib.exports.Registry                   // dialect registry
-                dialect = registry.findName(registry.dialects().0)          // ### default dialecct
+                dialect = registry.findName(registry.dialects().0)          // ### default dialect
                 output.push(dialect.parse(source))
             } catch {
                 if true {

package/frameworks/running/taskexec.nlg

@@ -19,12 +19,27 @@
  *
  */
 
-closure TaskDispatcher(replier, local contexts, gohome, util) {
+closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
     global()
     contexts = { }
     gohome = makeActivator()
+    coder = packageCoder(evalf)
     util = modlist().Util.exports
-    closure(data, local reply, restorens, context, expr, roots, proc, args, result) {
+
+    // sendEval
+    //
+    // Send an evaluation request to the original execution context.
+    //
+    closure evalf(expr) {
+        debuglog("TaskDispatcher doesn't yet implement reverse proxy objects")
+        list(Error("### reverse object proxy not implemented"))
+    }
+
+    // dispatcher
+    //
+    // Return a new task dispatcher.
+    //
+    closure dispatcher(data, local reply, restorens, context, expr, roots, proc, args, result) {
         reply = {
             xmsg: "error"
             xid: data.xid
@@ -70,10 +85,14 @@ closure TaskDispatcher(replier, local contexts, gohome, util) {
                         intern: true                                        // intern symbols
                         localroots: true                                    // make roots local if needed to intern
                         execute: true                                       // execute auto-running procedures
+                        objectDecoder: coder.decodeObject
                     })
                     expr = caar(expr)
+                    if expr.0 == `\.execute\-method
+                        expr = coder.invokeMethod(expr.1, expr.2)           // object method, args
+                    else
+                        expr = evalactive(expr)                             // evaluate expression
                     reply.error = undefined
-                    expr = evalactive(expr)
                     if data.get
                         roots = new(data.get).map(function(sym){            // new() to get symbol from compress
                             compress(sym)
@@ -84,7 +103,8 @@ closure TaskDispatcher(replier, local contexts, gohome, util) {
                         roots = `(expr)
                     reply.result = util.pkgSave(`expr, roots, {
                         nsignore: true                                      // don't report namespace errors
-                        comments: true })                                   // include comments
+                        comments: true
+                        objectEncoder: coder.encodeObject})                 // include comments
                     if context.namespace !== nsactive().0
                     {                                                       // evaluation changed namespace
                         context.namespace = nsactive().0

package/frameworks/storage/dbt_pouch.nlg

@@ -34,13 +34,17 @@
  *
  */
 
-closure dapoTable(database, path, local table, prefix) {
+closure dapoTable(database, rootpath, local table, prefix) {
     global()
+    if !database || !string(rootpath)
+        return (list(Error("invalid arguments")))
     table = new(object, this)
     table.type = database.type
     table.db = database
-    table.path = path
-    table.prefix = makePrefix(path)
+    if rootpath != "" && rootpath.slice(-1) != "/"
+        rootpath = rootpath.concat("/")                                     // "" or "<path>/"
+    table.rootpath = rootpath
+    table.prefix = makePrefix(rootpath)
         
     // makePrefix
     //
@@ -80,7 +84,7 @@ closure dapoTable(database, path, local table, prefix) {
             return (syncAdapter(add, key, content, metadata))
         id = makeID(key)
         if !string(id)
-            return (asyncResult(callback, id))                                 // error from makeID
+            return (asyncResult(callback, id))                              // error from makeID
         if metadata && !dictionary(metadata)
             return (asyncResult(callback, Error("invalid arguments")))
         if !database.pouch
@@ -291,7 +295,7 @@ closure dapoTable(database, path, local table, prefix) {
     // only immediate children are listed, with directories at the end--and no metadata for them.
     //
 
-    table.records = closure records(options, callback, local prefix, pouchOpts, output) {
+    table.records = closure records(options, callback, local prefix, prelen, pouchOpts, output) {
         if !callback
             return (syncAdapter(records, options))
         output = {
@@ -300,9 +304,11 @@ closure dapoTable(database, path, local table, prefix) {
             if !string(options.prefix)
                 return (asyncResult(callback, Error("prefix option must be string")))
             output.prefix = options.prefix
-            prefix = table.prefix.concat(makePrefix(options.prefix)) }
-        else
+            prefix = table.prefix.concat(makePrefix(options.prefix))
+            prelen = options.prefix.length }
+        else {
             prefix = table.prefix
+            prelen = 0 }
         if !database.pouch
             return (asyncResult(callback, Error("database closed")))
         pouchOpts = {
@@ -321,19 +327,24 @@ closure dapoTable(database, path, local table, prefix) {
                         key = obj.id.substring(table.prefix.length)
                     else
                         key = "*key error*"                                 // should not happen
-                    if !options.deep {
-                        offset = key.indexOf("/", prefix.length)
-                        if offset >= 0 {
-                                key = key.substring(0, offset+1)            // child directory name
-                                if !firstkids[key]
-                                    firstkids[key] = key                    // first time we've seen this child directory
-                                else
-                                    continue } }                            // ignore deeper child
                     item = {
                         key:        key
                         _id:        obj.id
                         _rev:       obj.value.rev
                     }
+                    if !options.deep {
+                        offset = key.indexOf("/", prelen)                   // start looking past options.prefix
+                        if offset >= 0 {                                    // directory child
+                            if length(key) > offset+1 {                     // transitive child (not directory itself)
+                                obj.doc = undefined                         // just record directory name
+                                item._id = undefined
+                                item._rev = undefined }
+                            key = key.substring(0, offset+1)                // child directory name
+                            if !firstkids[key]
+                                firstkids[key] = key                        // first time we've seen this child directory
+                            else
+                                continue                                    // ignore deeper child
+                            item.key = key } }
                     if obj.doc {                                            // docs were requested
                         if obj.doc.content {
                             item._md5 = HashMD5(obj.doc.content)
@@ -508,7 +519,7 @@ closure ListBrowserDBs(local pending, output, item) {
  */
 
 closure DbConnector(psm, local connClassID, connector, watch) {
-    global()
+    global(js)
     connClassID = "PouchDB"
     connector = new(object, this)
     connector.psm = psm
@@ -741,6 +752,8 @@ closure DbConnector(psm, local connClassID, connector, watch) {
     //
     // Connect to a database, returning a PSM view.
     connector.connect = function connect(resID, service, args, local error, creds, db, rootpath, table) {
+        if !resID || args && !tuple(args)
+            return (list(Error("invalid arguments")))
         `(error, creds) = connector.access(resID)
         if !error
             `(error, db) = PouchDB(creds)
@@ -751,7 +764,7 @@ closure DbConnector(psm, local connClassID, connector, watch) {
         else if service == "NideFS" {
             rootpath = args.0                                               // connect to a NideFS filesystem
             if !rootpath
-                return (Error("mandatory rootpath missing"))
+                return (list(Error("mandatory rootpath missing")))
             `(error, table) = db.table(rootpath)
             if error
                 list(error)

package/frameworks/storage/file_manager.nlg

@@ -766,7 +766,7 @@ function fimaInit(local manifest) {
 
     manifest = `(FileManager, fimaStore, fimaInit)
 
-    Naan.module.build(module.id, "fileManager", function(modobj, compobj) {
+    Naan.module.build(module.id, "file_manager", function(modobj, compobj) {
         require("./storage.nlg")
         commonWatching = require("../common/watching.nlg")
         compobj.manifest = manifest

package/frameworks/storage/psm.nlg

@@ -260,6 +260,16 @@ closure MakePSMutil(fs, local util) {
 
     util.deepcopy = closure deepcopy(inpath, sources, outfs, outpath, options, callback,
         local errors, copies, fsopt, folders) {
+        
+        // fixpath
+        // Fix the path to match the destination.
+        function fixpath(path) {
+            if fs.path.sep == outfs.path.sep
+                path
+            else
+                path.replaceAll(fs.path.sep, outfs.path.sep)
+        }
+
         if !callback
             return (syncAdapter(deepcopy, inpath, sources, outfs, outpath, options))
         if sources && !array(sources) || !string(inpath) || !string(outpath)
@@ -281,7 +291,7 @@ closure MakePSMutil(fs, local util) {
         asyncArray(sources, 10, closure mklist(entry,
             local srcpath, destpath, epath, error, stat, folder, node) {
             srcpath = fs.path.join(inpath, entry)
-            destpath = outfs.path.join(outpath, entry)
+            destpath = fixpath(outfs.path.join(outpath, entry))
             if destpath == "."
                 destpath = ""                                               // joined empty strings
             `(error, stat) = fs.info(srcpath, fsopt)