@naanlang/naan 1.3.1 → 1.4.0

package/frameworks/node/worker.nlg

@@ -10,6 +10,13 @@
  *
  */
 
+/*
+ * globals
+ *
+ */
+
+gWorkers;
+
 
 /*
  * workCurrent
@@ -19,8 +26,8 @@
  */
 
 closure workCurrent(workco, workerID, startup, local current) {
-    global()
-    current = new(object, this)
+    global(js)
+    current = require("../running/executors.nlg").ExecutorBase(workco, workerID, startup.name)
     current.workerID = workerID
     current.name = startup.name
     current.links = []
@@ -94,6 +101,7 @@ closure workCurrent(workco, workerID, startup, local current) {
     function ONmessageOut(data) {
         if data._guid
             return                                                          // already replied
+        current.execReceived(data)
         sendAll({
             id: "targetout",
             data: data
@@ -153,7 +161,17 @@ closure workCurrent(workco, workerID, startup, local current) {
         if current.links.length == 0
             current.host.Detach(current.termif)                             // attached while one link is active
     }
-    
+
+    //
+    // target.PostMessage
+    //
+    //     Post a message to the main thread.
+    //
+    current.PostMessage = function PostMessage(data) {
+        js.t.DispatchMessage(data)
+        list(false, { posted: true })
+    }
+
     //
     // msgin
     //
@@ -188,6 +206,18 @@ closure workCurrent(workco, workerID, startup, local current) {
 };
 
 
+/*
+ * WorkerConnect
+ *
+ *     Return the existing web worker for a workerGUID, or false if it does not exist.
+ *
+ */
+
+function WorkerConnect(workerGUID) {
+    gWorkers[workerGUID]
+}
+
+
 /*
  * workMainProxy
  *
@@ -270,13 +300,14 @@ closure WorkerToMain(local nideRunning, track) {
  *
  */
 
-closure workThread(workco, workerID, startup, local worker, options, subChannel, deathWatcher) {
-    global(threads, JSpath)
-    worker = new(object, this)
+closure workThread(workco, workerID, startup, guid
+    local worker, options, subChannel, deathWatcher) {
+    global(js, threads, JSpath)
+    worker = require("../running/executors.nlg").ExecutorBase(workco, workerID, startup.name)
     worker.name = startup.name
     worker.workerID = workerID
     worker.links = []
-    worker.guid = UUID()
+    worker.guid = guid || UUID()
 
     // create our worker thread
 
@@ -346,6 +377,8 @@ closure workThread(workco, workerID, startup, local worker, options, subChannel,
                 state: false                                                // resume state, if we have any
                 altcmd: false
                 dirpath: js.d
+                workerID: workerID
+                workerGUID: worker.guid
             }
             if startup.initcmds
                 startmsg.altcmd = startup.initcmds.concat("\n")
@@ -364,6 +397,8 @@ closure workThread(workco, workerID, startup, local worker, options, subChannel,
         else {
             if msg.id == "status"
                 worker.lastStatus = milliseconds()
+            else if msg.id == "targetout"
+                worker.execReceived(msg.data)
             sendAll(msg) }                                                  // default is just return message to the client
     })
     
@@ -401,6 +436,19 @@ closure workThread(workco, workerID, startup, local worker, options, subChannel,
         removeLink(link)
     }
 
+    //
+    // PostMessage
+    //
+    //     Post a message to the main thread.
+    //
+    worker.PostMessage = function PostMessage(data) {
+        worker.msgport.postMessage({                                        // send data to worker
+            id: "targetin",
+            data: data
+        })
+        list(false, { posted: true })
+    }
+
     //
     // msgin
     //
@@ -498,7 +546,7 @@ closure workLink(workco, conn, worker, sender, local link) {
  */
 
 closure workController(api, local workco) {
-    global()
+    global(gWorkers)
     workco = new(object, this)
     workco.serverID = "Workers"                                             // our serverID, used for communications
     workco.workers = { }                                                    // dictionary of workers by workerID
@@ -559,12 +607,47 @@ closure workController(api, local workco) {
         workco.workers[workerID] = undefined                                // it's like this worker has never been
     }
 
-    //
+    // spawn
+    //
+    //     Spawn a new worker with the specified configuration dictionary and optional GUID.
+    //
+    workco.spawn = closure spawn(workerID, config, guid, local worker) {
+        if !dictionary(gWorkers)
+            gWorkers = {}
+        worker = workco.workers[workerID]
+        if !worker {                                                        // create worker if needed
+            if workerID == "NideServer"
+                worker = workco.current = workCurrent(workco, workerID, config)
+            else
+                worker = workThread(workco, workerID, config, guid)
+            workco.uworkers[worker.guid] = worker
+            workco.workers[workerID] = worker
+            gWorkers[worker.guid] = worker }
+        if workerID == "NideServer"
+            workco.current.vitalUpdate({
+                id: "workerlist",                                           // send the initial list of workers
+                workers: let(workers, workerID, worker) {
+                    workers = { }
+                    for `(workerID, worker) in workco.workers
+                        workers[workerID] = {
+                            name: worker.name
+                        }
+                    workers
+                }()
+            })
+        else {
+            worker.vitalWatch(termination)
+            workco.current.vitalUpdate({
+                id: "spawned",                                              // note a new worker thread
+                name: config.name
+                workerID: workerID
+            }) }
+    }
+
     // wsReceive
     //
     //     Process a message from the websocket API.
     //
-
     workco.wsReceive = function wsReceive(message, conn, local worker, link, error) {
         if !message.workerID
             error = Error("workerID required", message.workerOp)
@@ -576,50 +659,25 @@ closure workController(api, local workco) {
                 if worker && message.payload {
                     if !link
                         makeLink(message, conn)                             // make sure we have a link for responses
-                    worker.msgin(message.payload)                           // send message to worker
-                    return } }
+                    worker.msgin(message.payload) }                         // send message to worker
+                return }
             else if message.workerOp == "echo" {
                 if worker && message.payload {
                     if !link
                         makeLink(message, conn)                             // make sure we have a link for responses
-                    worker.echo(message.payload, link)                      // echo message to other links
-                    return } }
+                    worker.echo(message.payload, link) }                    // echo message to other links
+                return }
             else if message.workerOp == "spawn" {
-                if !worker {                                                // create worker if needed
-                    if message.workerID == "NideServer"
-                        worker = workco.current = workCurrent(workco, message.workerID, message.payload)
-                    else
-                        worker = workThread(workco, message.workerID, message.payload)
-                    workco.uworkers[worker.guid] = worker
-                    workco.workers[message.workerID] = worker }
+                spawn(message.workerID, message.payload, message.workerGUID)
                 if !link
                     makeLink(message, conn)                                 // make sure we have a link for responses
-                if message.workerID == "NideServer"
-                    workco.current.vitalUpdate({
-                        id: "workerlist",                                   // send the initial list of workers
-                        workers: let(workers, workerID, worker) {
-                            workers = { }
-                            for `(workerID, worker) in workco.workers
-                                workers[workerID] = {
-                                    name: worker.name
-                                }
-                            workers
-                        }()
-                    })
-                else {
-                    worker.vitalWatch(termination)
-                    workco.current.vitalUpdate({
-                        id: "spawned",                                      // note a new worker thread
-                        name: message.payload.name
-                        workerID: message.workerID
-                    }) }
                 return }
             else if message.workerOp == "terminate" {
                 if worker {
                     if !link
                         makeLink(message, conn)                             // make sure we have a link for responses
-                    worker.terminate()                                      // echo message to other links
-                    return } }
+                    worker.terminate() }                                    // echo message to other links
+                return }
             error = Error("invalid parameters", message.workerOp, message.workerID)
         }
         ErrorDebuglog("workController.wsReceive error:", error)
@@ -663,13 +721,13 @@ closure workController(api, local workco) {
  */
 
 function workInit(local manifest) {
-    manifest = `(workCurrent, workMainProxy, WorkerToMain, workThread, workLink, workController,
-        workInit)
-
+    manifest = `(workCurrent, WorkerConnect, workMainProxy, WorkerToMain, workThread, workLink,
+        workController, workInit)
     Naan.module.build(module.id, "worker", function(modobj, compobj) {
         require("./node.nlg")
         compobj.manifest = manifest
         modobj.exports.WorkerController = workController
+        modobj.exports.WorkerConnect = WorkerConnect
         modobj.exports.WorkerToMain = WorkerToMain
     })
 }();

package/frameworks/project/build.nlg

@@ -142,8 +142,8 @@
 
 closure Builder(rules, operation, fs, srcpath, destpath, track, local build, group, newgroup) {
     build = new(object, this)
-    build.srcpath = srcpath
-    build.destpath = destpath
+    build.srcpath = JSpath.normalize(srcpath)
+    build.destpath = JSpath.normalize(destpath)
     build.intpath = JSpath.join(destpath, "tmp")
     build.tasks = {                                                         // all tasks counter
         copy: 0,
@@ -694,13 +694,13 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                                         if !file.info.hidden
                                             addgroup.sources.push(file.name)    // add non-hidden files found in specified directory
                                     if addgroup.input
-                                        addgroup.input = JSpath.join(addgroup.input, source, listing.data.pathsep)
+                                        addgroup.input = JSpath.join(addgroup.input, source, "/")
                                     else
-                                        addgroup.input = source.concat(listing.data.pathsep)
+                                        addgroup.input = JSpath.join("/")
                                     if group.destfile
-                                        addgroup.output = JSpath.join(addgroup.output, listing.data.pathsep)
+                                        addgroup.output = JSpath.join(addgroup.output, "/")
                                     else
-                                        addgroup.output = JSpath.join(addgroup.output, source, listing.data.pathsep)
+                                        addgroup.output = JSpath.join(addgroup.output, source, "/")
                                     addgroup.groupno = build.nextgno++      // needed for concat/zip
                                     groups.push(addgroup)
                                 }
@@ -1145,6 +1145,11 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     build.make = closure make(options, local sink, xchan, entry, outext, error) {
         sink = new(textstream, "", `string)                                 // log all output
         xchan = textstreams(list(sink, sink))
+        `(error) = fs.mkdir(build.destpath)                                 // make sure our directory exists
+        if error {
+            error = Error("cannot make build products folder", error)
+            return (list(error))
+        }
         try {
             resetlist()
             analyze(options.debugMake)
@@ -1201,7 +1206,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
                 workerID: "Run-${build.buildnum}"
                 startup: {
                     initcmds: maintext
-                    dirpath: fs.path.resolve(fs.path.sep, fs.rootpath, build.destpath)
+                    dirpath: fs.path.resolve("/", fs.rootpath, build.destpath)
                 }
             })
             if error

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.3.1+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.4.0+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -574,8 +574,12 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
         if error
             return (list(error))
         rules = procon.configDict."build-rules"[stage]
-        if rules.inherit
-            rules = js.o.assign(procon.configDict."build-rules"[rules.inherit], rules)
+        if rules.inherit {
+            data = procon.configDict."build-rules"[rules.inherit]
+            if !data
+                return (list(Error("build inherit rules '${rules.inherit}' do not exist")))
+            rules = merge(data, rules)
+        }
         else if !rules
             rules = procon.configDict.build                                 // fallback to old config structure
         rules = new(rules)

package/frameworks/running/executors.nlg

@@ -24,19 +24,20 @@ closure ExecutorBase(track, type, name, local exec) {
     exec.name = name
     
     // execReset
+    //
     // Initialize remote communications for a new or restarted instance.
-    
+    //
     exec.execReset = function execReset() {
         exec.pending = { }
         exec.nextSendID = toint(RandomBytes(3))                             // keep our sequence code unique for shared servers
-        exec.nextContextID = 1
     }
     execReset()
 
     // execSend
+    //
     // Send a message to the running instance and return a nonce to wait on. If data is not a
     // dictionary then it is ignored.
-    
+    //
     exec.execSend = closure execSend(msg, data, local pending, error) {
         if !dictionary(data)
             data = { }
@@ -52,8 +53,9 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execReceived
+    //
     // The specified message was received from the instance, so signal it.
-    
+    //
     exec.execReceived = closure execReceived(data, local pending) {
         pending = exec.pending[data.xid]
         if !pending {                                                       // not waiting on message
@@ -68,15 +70,17 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execSetStatus
+    //
     // Received a status update from the instance.
-    
+    //
     exec.execSetStatus = function execSetStatus(status) {
         exec.xstatus = status
     }
     
     // execFailed
+    //
     // Communication has failed.
-    
+    //
     exec.execFailed = closure execFailed(error, local xid, pending) {
         exec.pending.error = error
         for `(xid, pending) in exec.pending
@@ -84,8 +88,10 @@ closure ExecutorBase(track, type, name, local exec) {
     }
     
     // execClosed
+    //
+    //
     // Note that the connection has closed, so signal all the waiters.
-    
+    //
     exec.execClosed = closure execClosed(error, local xid, pending) {
         for `(xid, pending) in exec.pending {
             pending.signal(list(error))
@@ -94,10 +100,13 @@ closure ExecutorBase(track, type, name, local exec) {
     }
         
     // context
-    // Return a context object for the instance, or an error, in a result tuple. Currently this does
-    // not ever reclaim old contexts on the instance end of things, but it should. ###
-
-    exec.context = closure context(local error, retries, ms, delta, excon) {
+    //
+    // Return a context object for the instance, or an error, in a result tuple. The optional state
+    // dictionary argument initializes the context with the state of a previous one to facilitate
+    // persistent reconnection. Currently this does not ever reclaim old contexts on the instance
+    // end of things, but it should. ###
+    //
+    exec.context = closure context(state, local error, retries, ms, delta, excon) {
         retries = 0
         ms = milliseconds()
         delta = 1
@@ -114,13 +123,14 @@ closure ExecutorBase(track, type, name, local exec) {
             sleep(delta)
             if delta < 100
                 delta *= 2 }
-        excon = ExecutorContext(exec, exec.nextContextID++)
+        excon = ExecutorContext(exec, state)
         excon.register()
     }
 
     // attention
+    //
     // Focus the UI on this instance because something needs attention.
-
+    //
     exec.attention = function attention() {
         track.attention(exec)
     }
@@ -141,20 +151,37 @@ closure ExecutorBase(track, type, name, local exec) {
  * ability to track namespace changes as the result of evaluating an expression. They do not override
  * normal namespace protections. The contexts exposed here preserve the active namespace after the
  * expression is evaluated.
+ *     The optional state argument preserves the context state so that it can be restored for
+ * resuming persistent connections.
  *
  */
 
-closure ExecutorContext(exec, contextID, local context, util) {
+closure ExecutorContext(exec, state, local context, coder, util) {
     global(apply)
     context = new(object, this)
+    if !state || !state.contextID
+        state = {
+            contextID: UUID()
+        }
+    context.state = state
+    coder = packageCoder(evalf)
     util = modlist().Util.exports
 
+    // getState
+    //
+    // Get context state for future initialization
+    //
+    context.getState = function getState() {
+        context.state
+    }
+
     // register
+    //
     // Register or re-register our context with the worker.
-
+    //
     context.register = closure register(local pending, result) {
         pending = exec.execSend("register", {
-            contextID: contextID
+            contextID: state.contextID
         })
         result = pending.wait()
         if result.1
@@ -164,10 +191,11 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
     
     // eval
+    //
     // Evaluate the specified expression in the current worker context. The optional put argument
     // if non-false is a tuple of symbols that should be transfered to the remote end, while get is
     // an optional tuple of symbols to retrieve after evaluation.
-
+    //
     context.eval = closure evalf(expr, put, get, local roots, pkg, data, pending, result) {
         if tuple(put)
             roots = cons(`expr, put)
@@ -175,9 +203,11 @@ closure ExecutorContext(exec, contextID, local context, util) {
             roots = `(expr)
         pkg = util.pkgSave(`expr, roots, {
             nsignore: true
-            comments: true })
+            comments: true
+            objectEncoder: coder.encodeObject
+        })
         data = {
-            contextID: contextID
+            contextID: state.contextID
             expr: pkg }
         if tuple(get)
             data.get = get
@@ -190,22 +220,25 @@ closure ExecutorContext(exec, contextID, local context, util) {
                 localroots: false
                 intern: false
                 execute: false
+                objectDecoder: coder.decodeObject
             })))
         else
             result
     }
-    
+
     // evalq
+    //
     // Evaluate the specified QUOTED expression in the current worker context.
-    
+    //
     context.evalq = macro evalq(expr, put, get) {
         evalf(expr, eval(put), eval(get))
     }
-                
+
     // apply
+    //
     // Apply list of arguments to the specified procure evaluated remotely. If the procedure is a
     // not a tuple then this uses the tuple definition if there is one.
-    
+    //
     context.apply = closure applyf(proc, args, put, get) {
         if !tuple(proc) && tuple(proc.proc)
             proc = proc.proc
@@ -213,9 +246,10 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
 
     // call
+    //
     // Call the specified function remotely using local arguments:
     //      context.call(proc, args...)
-    
+    //
     context.call = macro callf args {
         closure (proc, arglist, local eargs) {
             while arglist
@@ -225,13 +259,14 @@ closure ExecutorContext(exec, contextID, local context, util) {
     }
 
     // rpc
+    //
     // Execute a remote procedure call without package translation, to reduce overhead. This will not
     // properly transfer Naan-specific types like symbols, and the results will be native JavaScript.
     // The procedure must be specified as a symbol or string.
-    
+    //
     context.rpc = closure rpc(proc, args, local data, pending, result) {
         data = {
-            contextID: contextID
+            contextID: state.contextID
             proc: proc
             args: args }
         pending = exec.execSend("rpc", data)
@@ -647,7 +682,7 @@ closure ExecutorTracker(local xtrak) {
  */
 
 closure execLambda(track, url, name, workerID, local target, nlregex, clientID, serverID) {
-    global()
+    global(js)
     target = ExecutorBase(track, "Lambda", name)
     clientID = "DebugWorkers"
     serverID = "Workers"
@@ -896,7 +931,8 @@ closure execLambda(track, url, name, workerID, local target, nlregex, clientID,
  */
 
 function execInit(local manifest) {
-    manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda, execInit)
+    manifest = `(ExecutorBase, ExecutorContext, ExecutorParallel, ExecutorTracker, execLambda,
+        execInit)
 
     Naan.module.build(module.id, "executors", function(modobj, compobj) {
         require("./running.nlg")