@naanlang/naan 1.3.1 → 1.4.0

package/frameworks/browser/sworker.js

@@ -36,7 +36,7 @@
  *
  */
 
-var CurrentCacheName = "Naanlang-1.3.1+1";
+var CurrentCacheName = "Naanlang-1.4.0+1";
 
 
 //
@@ -92,18 +92,18 @@ var terminated;                                 // flag that this service worker
         var pubVersion = event.data.hereIsMyVersion;
         var sourceId = event.source.id;                                     // client id of sender of message
         msgPorts[sourceId] = msgport;
-        console.log("[1.3.1+1] received new msgport for", sourceId, pubID, "-", pubVersion);
-        if (pubVersion != "1.3.1+1") {
+        console.log("[1.4.0+1] received new msgport for", sourceId, pubID, "-", pubVersion);
+        if (pubVersion != "1.4.0+1") {
             if (upgradeMsgPorts) {                                          // if not activated
-                console.log("[1.3.1+1] update pending for", sourceId);
+                console.log("[1.4.0+1] update pending for", sourceId);
                 upgradeMsgPorts[sourceId] = msgport;
             }
             else {
                 msgport.postMessage({                                       // notify new version available
                     id: "upgrade",
-                    version: "1.3.1+1"
+                    version: "1.4.0+1"
                 });
-                console.log("[1.3.1+1] update sent for:", sourceId);
+                console.log("[1.4.0+1] update sent for:", sourceId);
             }
         }
         Reaper();                                                           // clean up obsolete info
@@ -134,10 +134,10 @@ var terminated;                                 // flag that this service worker
         msgport.onmessage = function(msg) {
             msg = msg.data;
             if (msg.id == "skipWaiting") {                                  // try to activate this SW now
-                console.log("[1.3.1+1] attempting skipWaiting");
+                console.log("[1.4.0+1] attempting skipWaiting");
                 self.skipWaiting();
             } else if (msg.id == "terminate") {                             // termiante this SW now
-                console.log("[1.3.1+1] terminated");
+                console.log("[1.4.0+1] terminated");
                 terminate();
             } else if (msg.id == "abortURL") {                              // abort an I/O transaction
                 var href = new URL(msg.url).href;
@@ -150,7 +150,7 @@ var terminated;                                 // flag that this service worker
             } else if (msg.id == "response")                                // response from fetch request
                 processResponse(msg);
             else if (msg.id == "text")                                      // just log some text
-                console.log("[1.3.1+1] msg received:", msg.text);
+                console.log("[1.4.0+1] msg received:", msg.text);
         };
         
         // send text to IDE log
@@ -161,7 +161,7 @@ var terminated;                                 // flag that this service worker
             // don't clutter the log
             msgport.postMessage({
                 id: "text",
-                text: "port received by 1.3.1+1",
+                text: "port received by 1.4.0+1",
             });
             */
         if (--pending === 0)
@@ -178,7 +178,7 @@ var terminated;                                 // flag that this service worker
         includeUncontrolled: true
     }).then(function(clientList) {
         clientList.every(function(client) {
-            console.log("[1.3.1+1] requesting new msgport for", client.id);
+            console.log("[1.4.0+1] requesting new msgport for", client.id);
             ++pending;
             client.postMessage({                                            // tell client(s) we need this fetch source
                 msg: "Naan_need_fetch_port",
@@ -224,12 +224,12 @@ function Reaper() {
             clients[clientList[clidex].id] = clientList[clidex];
         for (var sourceId in msgPorts)
             if (!clients[sourceId]) {
-                console.log("[1.3.1+1] source gone:", sourceId);
+                console.log("[1.4.0+1] source gone:", sourceId);
                 delete msgPorts[sourceId];                                  // no longer a source
             }
         for (var clientId in fetchPorts)
             if (!clients[clientId]) {
-                console.log("[1.3.1+1] client gone:", clientId);
+                console.log("[1.4.0+1] client gone:", clientId);
                 delete fetchPorts[clientId];                                // no longer a client
             }
         for (var fqdex = 0; fqdex < fetchQueue.length; ++fqdex) {
@@ -263,16 +263,16 @@ function ClearCaches() {
             return (Promise.all(
                 cacheNames.map(function(cacheName) {
                     if (cacheName != CurrentCacheName) {
-                        console.log('[1.3.1+1] deleting old cache:', cacheName);
+                        console.log('[1.4.0+1] deleting old cache:', cacheName);
                         return (caches.delete(cacheName));
                     }
                 })
             ));
         }).then(function() {                                                // claim all clients
-            console.log('[1.3.1+1] claiming clients');
+            console.log('[1.4.0+1] claiming clients');
             return (self.clients.claim());
         }).then(function() {
-            console.log('[1.3.1+1] clients claimed');
+            console.log('[1.4.0+1] clients claimed');
             return (Promise.resolve(true));
         });
     return (promise);
@@ -315,7 +315,7 @@ function GetClientResponse(event, urlpath) {
             msgport.postMessage({
                 id: "fetch",
                 seq: seqno,
-                version: "1.3.1+1",
+                version: "1.4.0+1",
                 request: {
                     method: event.request.method,
                     url: event.request.url
@@ -363,7 +363,7 @@ function GetClientResponse(event, urlpath) {
  */
 
 self.addEventListener('install', function(event) {
-    console.log("[1.3.1+1] install");
+    console.log("[1.4.0+1] install");
     self.skipWaiting();
 });
 
@@ -376,20 +376,20 @@ self.addEventListener('install', function(event) {
  */
 
 self.addEventListener('activate', function(event) {
-    console.log("[1.3.1+1] activate");
+    console.log("[1.4.0+1] activate");
     self.clients.matchAll({                                                 // for debugging, list controlled clients           
         includeUncontrolled: true
     }).then(function(clientList) {
         var urls = clientList.map(function(client) {
             return (client.url);
         });
-        console.log('[1.3.1+1] matching clients:', urls.join(', '));
+        console.log('[1.4.0+1] matching clients:', urls.join(', '));
     });
     var promise = ClearCaches().then(function() {
         for (var sourceId in upgradeMsgPorts) {
             upgradeMsgPorts[sourceId].postMessage({                         // notify new version available
                 id: "upgrade",
-                version: "1.3.1+1"
+                version: "1.4.0+1"
             });
         }
         upgradeMsgPorts = false;
@@ -421,7 +421,7 @@ self.addEventListener('fetch', function(event) {
             var promise;
             var url = new URL(request.url);
             var nocache = request.method != "GET"
-                || url.search.length !== 0 && url.searchParams.get("naanver") !== "ecae8a62f4a0ae0408f1cbd16082d4ea"
+                || url.search.length !== 0 && url.searchParams.get("naanver") !== "fae76573fc7eeb3020b11e0d88fe69d4"
                 || request.headers.get('range');
             if (url.pathname.startsWith("/run/")) {
                 nocache = true;
@@ -455,7 +455,7 @@ self.addEventListener('fetch', function(event) {
                             statusText: "Request Cancelled"
                         }));
                     }
-                    console.log("[1.3.1+1] fetch failed", request.url, e);
+                    console.log("[1.4.0+1] fetch failed", request.url, e);
                     return (new Response(undefined, {
                         status: 404,
                         statusText: "Fetch Failed"

package/frameworks/browser/terminals.nlg

@@ -15,7 +15,7 @@
  *
  */
 
-workers;
+gWorkers;
 
 
 /*
@@ -154,7 +154,7 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
         // An error occurred on the WebSocket connection.
         target.ws.onerror = function onerror(e) {
             pending.signal(list(Error("websocket failed", e)))
-            debuglog("ws error:", e)
+            debuglog("ws error")
         }
         
         // onclose
@@ -431,9 +431,9 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
 
 closure TermLocal(track, name, options, local target, nlregex, listener)
 {
-    global(js, window, workers)
-    if !dictionary(workers)
-        workers = {}
+    global(js, window, gWorkers)
+    if !dictionary(gWorkers)
+        gWorkers = {}
     options = merge({
         type: "Local"
     }, options)
@@ -456,14 +456,15 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
             // ### tell target we are closing (if needed)
         })
         listener = window
+        target.guid = 1
     } else {
         if not window.Worker
             return (list(Error("Local environment does not support workers")))
         target.worker = xnew(window.Worker, "env_webworker.js")
         listener = target.worker
         target.guid = UUID()
-        workers[target.guid] = target
     }
+    gWorkers[target.guid] = target
 
     // onmessage
     // A message was received from the worker.
@@ -551,6 +552,8 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
                     id: "start",
                     state: false                                            // resume state, if we have any
                     altcmd: options.startup.initcmds
+                    workerID: target.workerID
+                    workerGUID: target.guid
                 })
             target.started = true
         } else if msg.id == "targetsend" {
@@ -667,10 +670,21 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
 };
 
 
+/*
+ * WorkerConnect
+ *
+ *     Return the existing web worker for a workerGUID, or false if it does not exist.
+ *
+ */
+
+function WorkerConnect(workerGUID) {
+    gWorkers[workerGUID]
+};
+
+
 /*
  * termMainProxy
  *
- * column positioning:                          //                          //                      !
  *     Make a browser proxy representing the main thread. This runs inside the worker and allows it
  * to do remote execution with Main.
  *
@@ -769,7 +783,7 @@ closure findIDEtarget(track, name, workerID, naancont, local target) {
  */
 
 closure termIDE(track, name, workerID, naancont, title, local target, connected) {
-    global(workers)
+    global(js, gWorkers)
     target = findIDEtarget(track, name, workerID, naancont)
     if target {                                                             // new naancont on existing target
         target.updateController(naancont)
@@ -790,7 +804,7 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     //
     function replyHook(data) {
         if data._guid
-            workers[data._guid].worker.postMessage({
+            gWorkers[data._guid].worker.postMessage({
                 id: "targetreceive"
                 data: data
             })
@@ -969,14 +983,15 @@ closure termInstallVirtualWatcher(track, naancont) {
  */
 
 function termInit(local manifest) {
-    manifest = `(TermHost, TermLocal, termMainProxy, WorkerToMain, findIDEtarget, termIDE,
-        termInstallVirtualWatcher, termInit)
+    manifest = `(TermHost, TermLocal, WorkerConnect, termMainProxy, WorkerToMain, findIDEtarget,
+        termIDE, termInstallVirtualWatcher, termInit)
 
     Naan.module.build(module.id, "terminals", function(modobj, compobj) {
         require("./browser.nlg")
         compobj.manifest = manifest
         modobj.exports.TermLocal = TermLocal
         modobj.exports.TermHost = TermHost
+        modobj.exports.WorkerConnect = WorkerConnect
         modobj.exports.WorkerToMain = WorkerToMain
     })
 } ();

package/frameworks/browser/workers.nlg

@@ -371,10 +371,13 @@ closure TrackedScope(track, scope, fs, rootpath, local trope) {
             "jpg":  "image/jpeg",
             "js":   "text/javascript",
             "json": "application/json",
+            "mov":  "video/mp4",
             "mp3":  "audio/mpeg",
             "mp4":  "video/mp4",
             "nlg":  "text/plain",
             "npk":  "application/json",
+            "ogg":  "video/ogg",
+            "png":  "image/png",
             "svg":  "image/svg+xml",
             "text": "text/plain",
             "txt":  "text/plain",

package/frameworks/browser/ws_client.nlg

@@ -51,7 +51,7 @@ closure WebSocket(options, local wsock) {
     
         // onopen
         // The WebSocket connection is now open.
-        wsock.ws.onopen = closure onopen(e, local spawner) {
+        wsock.ws.onopen = closure onopen(e) {
             pending.signal(list(false, { ok: true }))
             options.onopen(e)
         }

package/frameworks/client/psm_client.nlg

@@ -430,7 +430,7 @@ closure psmcFsView(api, rootpath, local view, pathmod) {
  */
 
 closure psmcFsConnector(psm, api, local connClassID, connector, watch) {
-    global()
+    global(App)
     connClassID = "NodeFS"
     connector = new(object, this)
     connector.psm = psm

package/frameworks/client/relaycon.nlg

@@ -43,7 +43,7 @@ wsMod;;                                         // websocket module
 
 closure RelayCon(host, secret, options, local relay, nlregex)
 {
-    global(execsMod, wsMod)
+    global(App, execsMod, wsMod)
     relay = new(object, this)
     options = merge({
         name: "NaanIDE Server"
@@ -301,6 +301,7 @@ function hocoInit(local manifest) {
     manifest = `(RelayCon, hocoInit)
 
     Naan.module.build(module.id, "relaycon", function(modobj, compobj) {
+        require("./client.nlg")
         execsMod = require("../running/executors.nlg")
         if js.w
             wsMod = require("../browser/ws_client.nlg")

package/frameworks/common/common.nlg

@@ -83,6 +83,7 @@ function ContentTypeFromFileExt(filepath, local mimeType, encoding) {
         "nlg":  "text/plain",
         "npk":  "application/json",
         "ogg":  "video/ogg",
+        "png":  "image/png",
         "svg":  "image/svg+xml",
         "text": "text/plain",
         "txt":  "text/plain",
@@ -192,20 +193,15 @@ function JsonStringify(data) {
 
 jsScripts = { };                                                            // script values become false on reload
  
-closure JsLoadScript(libpath, globalID, local savens, fpath, hostpath, pending, scriptTag) {
-    global(jsScripts, js, JSpath)
+closure JsLoadScript(libpath, globalID, local savens, hostpath, pending, scriptTag) {
+    global(jsScripts, JSpath, js)
     if jsScripts[libpath]
         return                                                              // already loaded
     savens = nsactive()
-    if libpath.startsWith("/")
-        fpath = libpath
-    else
-        fpath = "/".concat(libpath)
-    hostpath = js.w.location.origin.concat(JSpath.dirname(js.w.location.pathname))
+    hostpath = js.w.location.origin.concat(JSpath.join(JSpath.dirname(js.w.location.pathname), libpath))
     if js.s.importScripts && tostring(js.s.importScripts) == "[Function importScripts]" {
-        fpath = hostpath.concat(fpath)
         try {
-            js.s.importScripts(fpath)                                       // load in worker context
+            js.s.importScripts(hostpath)                                    // load in worker context
             jsScripts[libpath] = js.s[globalID]
         } catch {
             debuglog("JsLoadScript:", libpath, "failed", exception)
@@ -214,7 +210,7 @@ closure JsLoadScript(libpath, globalID, local savens, fpath, hostpath, pending,
     } else if js.w {                                                        // load in browser context
         pending = new(nonce)
         scriptTag = js.w.document.createElement("script")
-        scriptTag.src = hostpath.concat(fpath, requireQuery())
+        scriptTag.src = hostpath.concat(requireQuery())
         scriptTag.onload = function (event) {
             jsScripts[libpath] = js.w[globalID]
             pending.signal(js.w[globalID])

package/frameworks/common/preferences.nlg

@@ -88,7 +88,7 @@ closure Preferences(nicc, dbname, local prefs, error, data) {
         folder = "plugins/"
         path = folder.concat("plugins.cfg")
         pending = new(nonce)
-        nicc.api.readFile(path, function(error, req, event, local config) {
+        nicc.api.readFile(path, function(error, req, event) {
             if error
                 ErrorDebuglog("Can't read ".concat(path), error)
             else

package/frameworks/node/apiserver.nlg

@@ -30,7 +30,7 @@
  */
 
 closure APIServer(options, local server, chroot) {
-    global(JSpath, js, bodyParser, express, psmServerMod, workerMod)
+    global(App, JSpath, js, bodyParser, express, psmServerMod, workerMod)
     apisLoadLibs()
     server = new(object, this)
     server.app = express()
@@ -116,7 +116,7 @@ closure APIServer(options, local server, chroot) {
             return (list(error))
         }
         server.hostfs = hostfs
-        workerMod.WorkerController(server)
+        server.workco = workerMod.WorkerController(server)
         list(false, port)
     }
 
@@ -550,7 +550,7 @@ closure WebSocketServer(server, local websocks) {
  */
 
 closure WesCon(server, websocks, socket, request, local wescon) {
-    global()
+    global(js)
     wescon = new(object, this)
     wescon.socket = socket
     wescon.info = {

package/frameworks/node/https_request.nlg

@@ -31,7 +31,8 @@
  *          key:        <data>
  *          passphrase: <data>                  // optional
  *      }
- *      allowSelfSigned: <boolean>              // allow a self-signed certificate at the servera
+ *      allowSelfSigned: <boolean>              // allow a self-signed certificate at the server
+ *      streamHook:     <proc>                  // called for incoming chunks of text/event-stream
  *      debug:          <boolean>               // log errors and status results
  *  }
  *
@@ -100,11 +101,26 @@ closure httpsApiReqOnce(url, options,
     pending = new(nonce)
     chunks = []
     request = nodeHttps.request(url, reqOptions, closure (response) {
+        // test if response is a SSE stream
+        function stream() { response.headers["content-type"].startsWith("text/event-stream") }
+
         response.on("data", function (chunk) {
-            chunks.push(chunk)
+            if options.streamHook && stream()
+                call(options.streamHook, chunk)                             // got some data
+            else
+                chunks.push(chunk)
         })
-        response.on("end", function (local content, error, headers) {
-            content = js.g.Buffer.concat(chunks)
+        response.on("end", function (local state, content, error, headers) {
+            state = {
+                headers: response.headers
+                status: response.statusCode
+                contentType: response.headers["content-type"]
+                complete: response.complete
+            }
+            if options.streamHook && stream()
+                call(options.streamHook, false, state)                      // no more data
+            else
+                content = js.g.Buffer.concat(chunks)
             if response.statusCode < 200 || response.statusCode >= 299 {    // 299 is "cancelled"
                 if options.debug
                     debuglog("HttpsApiRequest status:", url, response.statusCode, "message:", content)
@@ -120,7 +136,7 @@ closure httpsApiReqOnce(url, options,
                         error = Error("HttpsApiRequest decode:", url, error)
                     else
                         content = new(content)
-                    if array(content) {                                     // deencapsulate to get API result
+                    if array(content) {                                     // naan: decapsulate to get API result
                         content = totuple(content)
                         error = content.0
                         content = content.1
@@ -129,11 +145,7 @@ closure httpsApiReqOnce(url, options,
             }
             else
                 error = Error("HttpsApiRequest terminated prematurely:", url, response.statusCode)
-            pending.signal(list(error, content, {
-                headers: response.headers
-                status: response.statusCode
-                contentType: response.headers["content-type"]
-            }))
+            pending.signal(list(error, content, state))
         })
     })
     request.on("error", function hre(error) {
@@ -149,6 +161,66 @@ closure httpsApiReqOnce(url, options,
 };
 
 
+/*
+ * sseParserFactory
+ *
+ * Returns an Server-Sent Events parser that calls the specified message procedure each time an event
+ * is recognized. Call the returned parser object's write method to add data for parsing.
+ *
+ */
+
+closure sseParserFactory(message, local parser) {
+    global()
+    parser = new(object, this)
+    parser.buffer = ""
+    parser.eomrx = RegExp("(?:\\r(?!\\n)|\\r\\n|\\n){2}")
+    parser.eolrx = RegExp("(?:\\r(?!\\n)|\\r\\n|\\n){1}")
+    parser.divrx = RegExp("([^:]*)[:] ?(.*)")
+
+    // write
+    //
+    // Add a string to the parser input, calling message for any completed messages.
+    //
+    parser.write = function write(text,
+        local messages, msg, output, line, found, field, value) {
+        parser.buffer = parser.buffer.concat(text)
+        messages = parser.buffer.split(parser.eomrx)
+        loop {
+            msg = messages.shift()
+            if messages.length == 0 {
+                parser.buffer = msg
+                break
+            }
+            output = { }
+            for line in msg.split(parser.eolrx) {
+                found = line.match(parser.divrx)
+                if found {
+                    field = found.1
+                    value = found.2
+                } else {
+                    field = line
+                    value = ""
+                }
+                if field == "data" {
+                    if output.data
+                        output.data = output.data.concat("\n", value)
+                    else
+                        output.data = value
+                }
+                else if field == "event"
+                    output.type = value
+                else if field == "id"
+                    output.lastEventID = value
+            }
+            call(message, output)
+        }
+    }
+
+    // finis
+    parser
+};
+
+
 /*
  * HttpsApiRequest
  *
@@ -157,11 +229,24 @@ closure httpsApiReqOnce(url, options,
  * Options:
  *  {
  *      retrier: function(error) { error.status == 502 }                    // retry 502 errors
+ *      sseHook: function(msg) { }                                          // process SSE messages
  *  }
  *
  */
 
-closure HttpsApiRequest(url, options, local result) {
+closure HttpsApiRequest(url, options, local parser, decoder, result) {
+    if options.sseHook {
+        parser = sseParserFactory(options.sseHook)
+        decoder = xnew(stringDecoder.StringDecoder, "utf8")
+        options = merge(options, {
+            streamHook: function(chunk) {
+                if chunk
+                    parser.write(decoder.write(chunk))                      // data arrived
+                else
+                    parser.write(decoder.end())                             // end of data
+            }
+        })
+    }
     loop {
         result = httpsApiReqOnce(url, options)
         if result.0 && result.0.status {                                    // check for retries
@@ -182,7 +267,7 @@ closure HttpsApiRequest(url, options, local result) {
  */
 
 function https_nodeInit(local manifest) {
-    manifest = `(httpsApiReqOnce, HttpsApiRequest, https_nodeInit)
+    manifest = `(httpsApiReqOnce, sseParserFactory, HttpsApiRequest, https_nodeInit)
 
     Naan.module.build(module.id, "https_request", function(modobj, compobj) {
         require("./node.nlg")

package/frameworks/node/node.nlg

@@ -133,6 +133,7 @@ function nodrInit(local manifest) {
             nodeHttps = js.r("https")
             nodeUrl= js.r("url")
             threads = js.r("worker_threads")
+            stringDecoder = js.r("string_decoder")
         }()
         module.reload = nodrReload
         module.exports.NaanlangDir = NaanlangDir