@naanlang/naan 1.4.1 → 1.4.3

package/frameworks/browser/sworker.js

@@ -36,7 +36,7 @@
  *
  */
 
-var CurrentCacheName = "Naanlang-1.4.1+1";
+var CurrentCacheName = "Naanlang-1.4.3+1";
 
 
 //
@@ -92,18 +92,18 @@ var terminated;                                 // flag that this service worker
         var pubVersion = event.data.hereIsMyVersion;
         var sourceId = event.source.id;                                     // client id of sender of message
         msgPorts[sourceId] = msgport;
-        console.log("[1.4.1+1] received new msgport for", sourceId, pubID, "-", pubVersion);
-        if (pubVersion != "1.4.1+1") {
+        console.log("[1.4.3+1] received new msgport for", sourceId, pubID, "-", pubVersion);
+        if (pubVersion != "1.4.3+1") {
             if (upgradeMsgPorts) {                                          // if not activated
-                console.log("[1.4.1+1] update pending for", sourceId);
+                console.log("[1.4.3+1] update pending for", sourceId);
                 upgradeMsgPorts[sourceId] = msgport;
             }
             else {
                 msgport.postMessage({                                       // notify new version available
                     id: "upgrade",
-                    version: "1.4.1+1"
+                    version: "1.4.3+1"
                 });
-                console.log("[1.4.1+1] update sent for:", sourceId);
+                console.log("[1.4.3+1] update sent for:", sourceId);
             }
         }
         Reaper();                                                           // clean up obsolete info
@@ -134,10 +134,10 @@ var terminated;                                 // flag that this service worker
         msgport.onmessage = function(msg) {
             msg = msg.data;
             if (msg.id == "skipWaiting") {                                  // try to activate this SW now
-                console.log("[1.4.1+1] attempting skipWaiting");
+                console.log("[1.4.3+1] attempting skipWaiting");
                 self.skipWaiting();
             } else if (msg.id == "terminate") {                             // termiante this SW now
-                console.log("[1.4.1+1] terminated");
+                console.log("[1.4.3+1] terminated");
                 terminate();
             } else if (msg.id == "abortURL") {                              // abort an I/O transaction
                 var href = new URL(msg.url).href;
@@ -150,7 +150,7 @@ var terminated;                                 // flag that this service worker
             } else if (msg.id == "response")                                // response from fetch request
                 processResponse(msg);
             else if (msg.id == "text")                                      // just log some text
-                console.log("[1.4.1+1] msg received:", msg.text);
+                console.log("[1.4.3+1] msg received:", msg.text);
         };
         
         // send text to IDE log
@@ -161,7 +161,7 @@ var terminated;                                 // flag that this service worker
             // don't clutter the log
             msgport.postMessage({
                 id: "text",
-                text: "port received by 1.4.1+1",
+                text: "port received by 1.4.3+1",
             });
             */
         if (--pending === 0)
@@ -178,7 +178,7 @@ var terminated;                                 // flag that this service worker
         includeUncontrolled: true
     }).then(function(clientList) {
         clientList.every(function(client) {
-            console.log("[1.4.1+1] requesting new msgport for", client.id);
+            console.log("[1.4.3+1] requesting new msgport for", client.id);
             ++pending;
             client.postMessage({                                            // tell client(s) we need this fetch source
                 msg: "Naan_need_fetch_port",
@@ -224,12 +224,12 @@ function Reaper() {
             clients[clientList[clidex].id] = clientList[clidex];
         for (var sourceId in msgPorts)
             if (!clients[sourceId]) {
-                console.log("[1.4.1+1] source gone:", sourceId);
+                console.log("[1.4.3+1] source gone:", sourceId);
                 delete msgPorts[sourceId];                                  // no longer a source
             }
         for (var clientId in fetchPorts)
             if (!clients[clientId]) {
-                console.log("[1.4.1+1] client gone:", clientId);
+                console.log("[1.4.3+1] client gone:", clientId);
                 delete fetchPorts[clientId];                                // no longer a client
             }
         for (var fqdex = 0; fqdex < fetchQueue.length; ++fqdex) {
@@ -263,16 +263,16 @@ function ClearCaches() {
             return (Promise.all(
                 cacheNames.map(function(cacheName) {
                     if (cacheName != CurrentCacheName) {
-                        console.log('[1.4.1+1] deleting old cache:', cacheName);
+                        console.log('[1.4.3+1] deleting old cache:', cacheName);
                         return (caches.delete(cacheName));
                     }
                 })
             ));
         }).then(function() {                                                // claim all clients
-            console.log('[1.4.1+1] claiming clients');
+            console.log('[1.4.3+1] claiming clients');
             return (self.clients.claim());
         }).then(function() {
-            console.log('[1.4.1+1] clients claimed');
+            console.log('[1.4.3+1] clients claimed');
             return (Promise.resolve(true));
         });
     return (promise);
@@ -315,7 +315,7 @@ function GetClientResponse(event, urlpath) {
             msgport.postMessage({
                 id: "fetch",
                 seq: seqno,
-                version: "1.4.1+1",
+                version: "1.4.3+1",
                 request: {
                     method: event.request.method,
                     url: event.request.url
@@ -363,7 +363,7 @@ function GetClientResponse(event, urlpath) {
  */
 
 self.addEventListener('install', function(event) {
-    console.log("[1.4.1+1] install");
+    console.log("[1.4.3+1] install");
     self.skipWaiting();
 });
 
@@ -376,20 +376,20 @@ self.addEventListener('install', function(event) {
  */
 
 self.addEventListener('activate', function(event) {
-    console.log("[1.4.1+1] activate");
+    console.log("[1.4.3+1] activate");
     self.clients.matchAll({                                                 // for debugging, list controlled clients           
         includeUncontrolled: true
     }).then(function(clientList) {
         var urls = clientList.map(function(client) {
             return (client.url);
         });
-        console.log('[1.4.1+1] matching clients:', urls.join(', '));
+        console.log('[1.4.3+1] matching clients:', urls.join(', '));
     });
     var promise = ClearCaches().then(function() {
         for (var sourceId in upgradeMsgPorts) {
             upgradeMsgPorts[sourceId].postMessage({                         // notify new version available
                 id: "upgrade",
-                version: "1.4.1+1"
+                version: "1.4.3+1"
             });
         }
         upgradeMsgPorts = false;
@@ -421,7 +421,7 @@ self.addEventListener('fetch', function(event) {
             var promise;
             var url = new URL(request.url);
             var nocache = request.method != "GET"
-                || url.search.length !== 0 && url.searchParams.get("naanver") !== "348dfbc247edf982d7b9f50b51d55de4"
+                || url.search.length !== 0 && url.searchParams.get("naanver") !== "3b6364f71856e32f6cb8edae0e67bbc2"
                 || request.headers.get('range');
             if (url.pathname.startsWith("/run/")) {
                 nocache = true;
@@ -455,7 +455,7 @@ self.addEventListener('fetch', function(event) {
                             statusText: "Request Cancelled"
                         }));
                     }
-                    console.log("[1.4.1+1] fetch failed", request.url, e);
+                    console.log("[1.4.3+1] fetch failed", request.url, e);
                     return (new Response(undefined, {
                         status: 404,
                         statusText: "Fetch Failed"

package/frameworks/browser/terminals.nlg

@@ -152,9 +152,9 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
 
         // onerror
         // An error occurred on the WebSocket connection.
+        // This can happen in normal operation when the connection times out.
         target.ws.onerror = function onerror(e) {
             pending.signal(list(Error("websocket failed", e)))
-            debuglog("ws error")
         }
         
         // onclose
@@ -655,6 +655,7 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
     //     Destroy the execution instance.
     
     target.destroy = function destroy() {
+        gWorkers[target.guid] = undefined
         target.execClosed(Error("instance destroyed"))
         track.delete(target)
         termDetach()
@@ -865,6 +866,10 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
             if inbound
                 inbound.active = active
         }
+        
+        connection.WriteLn = function WriteLn(text) {
+            term.WriteLn(text)
+        }
 
         connection.termDetach = function termDetach() {                     // disconnect the terminal
             db.detach(target)
@@ -921,7 +926,46 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
         }
         target.execReset()                                                  // reset our parent class
     }
-    
+
+    // targetClose
+    //
+    //     Mark the target window as closed.
+    //
+    function targetClose(local conn) {
+        for conn in connected {
+            conn.db.detach(target)
+            conn.WriteLn("\x1b[90m\x1b[3m".concat("\ntarget closed", "\x1b[0m"))
+        }
+        target.execReset()
+        target.naancont = naancont = false
+    }
+
+    // setCheckStatus
+    //
+    //     Begin or end checking status on the worker to see if stopped responding. The challenge
+    // here is that we open vsite windows in the main thread, but there is no reliable event to learn
+    // when they are closed. The solution is to start checking if the window is closed after it gets
+    // a visibilityChange event to hidden. This terminates when the window gets marked closed or when
+    // it becomes visible again--which is quite unlikely unless someone uses the explicit hide/show
+    // window methods.
+    //
+    target.setCheckStatus = closure setCheckStatus(hidden, window) {
+        if hidden {
+            if !target.statusPoll
+                target.statusPoll = future(function() {
+                    while target.statusPoll {
+                        if window.closed {
+                            targetClose()
+                            target.statusPoll = false
+                            break
+                        } else
+                            sleep(100)                                      // check 10x/second
+                    }
+                }, 10)                                                      // start after 10 msec
+        } else
+            target.statusPoll = false
+    }
+
     // vsiteRefresh
     //
     //     The vsite upon which we are based has changed.
@@ -969,6 +1013,11 @@ closure termInstallVirtualWatcher(track, naancont) {
             if target
                 target.updateController()
         }
+        else if msg.op == "VsiteVisibilityChange" {
+            target = findIDEtarget(track, msg.name, "NaanVsite", msg.naancont)
+            if target
+                target.setCheckStatus(msg.hidden, msg.window)
+        }
     }
     
     require("../running/executors.nlg").TaskOnMessageHook(watchVsites)

package/frameworks/common/common.nlg

@@ -182,53 +182,6 @@ function JsonStringify(data) {
     }
 };
 
- 
-/*
- * JsLoadScript
- *
- *     Load the specified script, or return its main entry point if already loaded. This blocks the
- * caller until the script load completes. The return value is false if an error occurred.
- *
- */
-
-jsScripts = { };                                                            // script values become false on reload
- 
-closure JsLoadScript(libpath, globalID, local savens, hostpath, pending, scriptTag) {
-    global(jsScripts, JSpath, js)
-    if jsScripts[libpath]
-        return                                                              // already loaded
-    savens = nsactive()
-    hostpath = js.w.location.origin.concat(JSpath.join(JSpath.dirname(js.w.location.pathname), libpath))
-    if js.s.importScripts && tostring(js.s.importScripts) == "[Function importScripts]" {
-        try {
-            js.s.importScripts(hostpath)                                    // load in worker context
-            jsScripts[libpath] = js.s[globalID]
-        } catch {
-            debuglog("JsLoadScript:", libpath, "failed", exception)
-            js.s.console.log("JsLoadScript:", libpath, "failed", exception) // ensure visibility in JS console
-        }
-    } else if js.w {                                                        // load in browser context
-        pending = new(nonce)
-        scriptTag = js.w.document.createElement("script")
-        scriptTag.src = hostpath.concat(requireQuery())
-        scriptTag.onload = function (event) {
-            jsScripts[libpath] = js.w[globalID]
-            pending.signal(js.w[globalID])
-            true
-        }
-        scriptTag.onerror = function (event) {
-            debuglog("JsLoadScript:", libpath, "not found")
-            pending.signal(false)
-            true
-        }
-        js.w.document.body.appendChild(scriptTag)
-        pending.wait()
-    } else if js.g
-        jsScripts[libpath] = js.r(libpath)                                  // load in NodeJS context
-    nsactive(savens)
-    jsScripts[libpath]
-};
-
 
 /*
  * LoadComponentOnDemand
@@ -365,7 +318,7 @@ Uint8ArrayFromString = false;
 
 function comrInit(local manifest) {
     manifest = `(EncodeQuery, DecodeQuery, ContentTypeFromFileExt, UUID, VersionCheck, JsonParse,
-        JsonStringify, JsLoadScript, LoadComponentOnDemand, LiveExport, comrInit)
+        JsonStringify, LoadComponentOnDemand, LiveExport, comrInit)
 
     Naan.module.build(module.id, "common", closure(modobj, compobj) {
         compobj.manifest = manifest
@@ -373,8 +326,9 @@ function comrInit(local manifest) {
         modobj.exports.DecodeQuery = DecodeQuery
         modobj.exports.ContentTypeFromFileExt = ContentTypeFromFileExt
         modobj.exports.UUID = UUID
-        modobj.exports.JsLoadScript = JsLoadScript
         modobj.exports.LoadComponentOnDemand = LoadComponentOnDemand
+        letimport(module.owner.list.Util.exports.JsLoadScript)              // use the Util version
+        modobj.exports.JsLoadScript = JsLoadScript
         modobj.exports.LiveExport = LiveExport                              // allow others to create export updaters
         updateExports = LiveExport()                                        // export LiveImport() so we can update
 
@@ -481,13 +435,12 @@ function comrInit(local manifest) {
                 }
                 //
                 // MD5 functions
-                if symbol(NideBuild) {                                      // execute if we are not building
-                    JsLoadScript("frameworks/browser/spark-md5/spark-md5.min.js", "SparkMD5")
-                }
                 //
                 // hasMD5 in hex
                 HashMD5 = function browserMD5(data, raw) {
                     data = binaryFrom(data)
+                    if !wsg.SparkMD5
+                        JsLoadScript("frameworks/browser/spark-md5/spark-md5.min.js", "SparkMD5")
                     if xobject(data)
                         wsg.SparkMD5.ArrayBuffer.hash(data, raw)
                     else

package/frameworks/project/build.nlg

@@ -230,17 +230,26 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //      <substitute text, one or more lines>
     //      [File EOF]
     //
+    // Special line starts:
+    //      # comment
+    //      \# line starts with #
+    //      \\ line starts with \
+    //
     // Note that the newlines before and after the substitute text are not included. This returns a
     // a dictionary of variables and replacement text. The defs argument is optional, and defines a
     // substitution dictionary applied to the replacement text defined in this file.
     //
-    function parseSubstitutionLines(lines, defs, local ignoregex, varegex, line, find, subs, curvar, sub) {
+    function parseSubstitutionLines(lines, defs,
+        local ignoregex, escapex, varegex, line, find, subs, curvar, sub) {
         subs = { }
         ignoregex = RegExp("^#")
+        escapex = RegExp("^\\\\(#|\\\\)")
         varegex = RegExp("^[$]([^$]+)[$]")
         for line in lines {
-            if ignoregex.exec(line)
-                continue
+            if ignoregex.test(line)
+                continue                                                        // comment
+            if escapex.test(line)
+                line = line.slice(1)                                            // not-a-comment escape
             if find = line.match(varegex) {
                 curvar = find[1]
                 subs[curvar] = ""
@@ -271,6 +280,11 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //      <substitute text, one or more lines>
     //      [File EOF]
     //
+    // Special line starts:
+    //      # comment
+    //      \# line starts with #
+    //      \\ line starts with \
+    //
     // Note that the newlines before and after the substitute text are not included. This returns a
     // standard `(error, subs) result tuple where subs is a dictionary of variables and replacement
     // text. The defs argument is optional, and defines a substitution dictionary applied to the
@@ -293,7 +307,7 @@ closure Builder(rules, operation, fs, srcpath, destpath, track, local build, gro
     //
     function applySubstitutions(text, subs, local subbed) {
         subbed = { }
-        list(text.replace(RegExp("[$][-A-Za-z0-9_./()\\\\]+[$]", "g"), function(match, local subst) {
+        list(text.replace(RegExp("[$][^$\\s]+[$]", "g"), function(match, local subst) {
             subst = subs[match.slice(1,-1)]
             if subst {
                 subbed[match] = true

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.4.1+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.4.3+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -563,6 +563,17 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
         }
     }
 
+    // persist
+    //
+    //    Return a dictionary for persistent local project state, like view configuration.
+    //
+    procon.persist = function persist() {
+        if procon.localDict.persist {
+        }
+        else
+            procon.localDict.persist = {}
+    }
+
     // buildrules
     //
     //     Get the build rules for the named stage, reloading the project configuration if changed.

package/frameworks/running/executors.nlg

@@ -71,10 +71,14 @@ closure ExecutorBase(track, type, name, local exec) {
     
     // execSetStatus
     //
-    // Received a status update from the instance.
+    // Received a status update from the instance. This makes a copy of the specified dictionary for
+    // garbage collection, because if we store the status from a separate vsite window then it can
+    // cause the main window to retain a vsite window after it's closed. The last status update sets
+    // an object in the context of the detached window, and then it never gets updated until the
+    // executor goes away. That doesn't always happen as soon as you think.
     //
     exec.execSetStatus = function execSetStatus(status) {
-        exec.xstatus = status
+        exec.xstatus = new(status)                                          // for GC (!)
     }
     
     // execFailed
@@ -218,7 +222,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
         else if result.1.result
             list(false, caar(util.pkgLoad(result.1.result, {
                 localroots: false
-                intern: false
+                intern: "symbols"
                 execute: false
                 objectDecoder: coder.decodeObject
             })))

package/frameworks/running/running.nlg

@@ -207,12 +207,26 @@ closure packageCoder(evalf, local coder) {
                     quote(x@\.tostring = function() { "{proxy ${nn}}" }),
                     quote(x@\.undefined = closure undef args {
                         let (result) {
-                            result = evalf(list(`\.execute\-method, objectID, args))
+                            result = evalf(list(`\.invoke\-method, objectID, args))
                             if result.0
                                 throw("remoting failure: ${ErrorString(result.0)}")
                             else
                                 result.1
                         } ()
+                    }),
+                    quote(x@\.\.undefined = function (key, local result) {
+                        result = evalf(list(`\.invoke\-method, objectID, list(`\.\.undefined, key)))
+                        if result.0
+                            throw("remoting failure: ${ErrorString(result.0)}")
+                        else
+                            result.1
+                    }),
+                    quote(x@\.\=undefined = function (key, value, local result) {
+                        result = evalf(list(`\.invoke\-method, objectID, list(`\.\=undefined, key, value)))
+                        if result.0
+                            throw("remoting failure: ${ErrorString(result.0)}")
+                        else
+                            result.1
                     }), `x)))
             } (ident.name, ident.objectID)
             coder.wmapEncoding[obj] = ident
@@ -225,11 +239,17 @@ closure packageCoder(evalf, local coder) {
     //
     // Invoke a local object method from the remote proxy, returning a result tuple.
     //
-    coder.invokeMethod = closure invokeMethod(objectID, args, local obj) {
+    coder.invokeMethod = closure invokeMethod(objectID, args, local obj, method) {
         obj = coder.mapDecoding[objectID]
-        if obj
-            eval(list(deref, obj, pop(args), args))
-        else
+        method = pop(args)
+        if obj {
+            if method == `\.\.undefined
+                obj[args.0]
+            else if method == `\.\=undefined
+                obj[args.0] = args.1
+            else
+                eval(list(deref, obj, method, args))
+        } else
             throw("remoting error: object ${objectID} not found")
     }
 

package/frameworks/running/taskexec.nlg

@@ -88,7 +88,7 @@ closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
                         objectDecoder: coder.decodeObject
                     })
                     expr = caar(expr)
-                    if expr.0 == `\.execute\-method
+                    if expr.0 == `\.invoke\-method
                         expr = coder.invokeMethod(expr.1, expr.2)           // object method, args
                     else
                         expr = evalactive(expr)                             // evaluate expression

package/frameworks/storage/dbt_pouch.nlg

@@ -529,7 +529,7 @@ closure DbConnector(psm, local connClassID, connector, watch) {
     
     // localDBcred
     //
-    // True iff the specified credential dictionary is for a local browser DB.
+    // True iff the specified credential dictionary is for a local DB.
     function localDBcred(creds) {
         !creds.authName && creds.urlName.indexOf("://") < 0
     }
@@ -722,8 +722,13 @@ closure DbConnector(psm, local connClassID, connector, watch) {
         if creds {
             if creds.label
                 info.name = creds.label
-            if localDBcred(creds)
-                info.type = "browser storage"
+            if localDBcred(creds) {
+                if js.w
+                    info.type = "PouchDB-browser"
+                else
+                    info.type = "PouchDB-filesystem"
+            } else
+                info.type = "PouchDB-network"
             if creds.locked
                 info.locked = true
             if creds.hidden

package/frameworks/storage/psm_dbtables.nlg

@@ -715,8 +715,8 @@ closure FSview(table, rootpath, local view) {
                             finfo.type = "directory"
                     } else if !finfo.type
                         finfo.type = "file"                                 // everything is a file in UNIX
-                    finfo.md5 = item.doc._md5 || undefined
-                    finfo.length = item.doc._length || item.doc.length || undefined
+                    finfo.md5 = item._md5 || item.doc._md5 || undefined
+                    finfo.length = item._length || item.doc._length || item.doc.length || undefined
                     if item.key.length > offset                             // not the directory itself
                         dirdict.children.push({
                             name: name