@naanlang/naan 1.2.1 → 1.3.1

package/frameworks/browser/https_request.nlg

@@ -24,6 +24,7 @@
  *      encoding:       <string>                // "binary" to receive binary data as arrayBuffer
  *      range:          <string>                // sets range header
  *      headers:        [`(key, value)]         // add header(s) to the request, overriding defaults
+ *      keepalive:      <boolean>               // true to send data at unload
  *      debug:          <boolean>               // log errors and status results
  *  }
  *
@@ -55,6 +56,7 @@ closure HttpsApiRequest(url, options,
         headers: { }
         redirect: 'follow'
         referrerPolicy: 'no-referrer'
+        keepalive: options.keepalive || undefined
     }
     if options.method
         fetchOptions.method = options.method

package/frameworks/browser/sworker.js

@@ -36,7 +36,7 @@
  *
  */
 
-var CurrentCacheName = "Naanlang-1.2.1+1";
+var CurrentCacheName = "Naanlang-1.3.1+1";
 
 
 //
@@ -92,18 +92,18 @@ var terminated;                                 // flag that this service worker
         var pubVersion = event.data.hereIsMyVersion;
         var sourceId = event.source.id;                                     // client id of sender of message
         msgPorts[sourceId] = msgport;
-        console.log("[1.2.1+1] received new msgport for", sourceId, pubID, "-", pubVersion);
-        if (pubVersion != "1.2.1+1") {
+        console.log("[1.3.1+1] received new msgport for", sourceId, pubID, "-", pubVersion);
+        if (pubVersion != "1.3.1+1") {
             if (upgradeMsgPorts) {                                          // if not activated
-                console.log("[1.2.1+1] update pending for", sourceId);
+                console.log("[1.3.1+1] update pending for", sourceId);
                 upgradeMsgPorts[sourceId] = msgport;
             }
             else {
                 msgport.postMessage({                                       // notify new version available
                     id: "upgrade",
-                    version: "1.2.1+1"
+                    version: "1.3.1+1"
                 });
-                console.log("[1.2.1+1] update sent for:", sourceId);
+                console.log("[1.3.1+1] update sent for:", sourceId);
             }
         }
         Reaper();                                                           // clean up obsolete info
@@ -134,10 +134,10 @@ var terminated;                                 // flag that this service worker
         msgport.onmessage = function(msg) {
             msg = msg.data;
             if (msg.id == "skipWaiting") {                                  // try to activate this SW now
-                console.log("[1.2.1+1] attempting skipWaiting");
+                console.log("[1.3.1+1] attempting skipWaiting");
                 self.skipWaiting();
             } else if (msg.id == "terminate") {                             // termiante this SW now
-                console.log("[1.2.1+1] terminated");
+                console.log("[1.3.1+1] terminated");
                 terminate();
             } else if (msg.id == "abortURL") {                              // abort an I/O transaction
                 var href = new URL(msg.url).href;
@@ -150,7 +150,7 @@ var terminated;                                 // flag that this service worker
             } else if (msg.id == "response")                                // response from fetch request
                 processResponse(msg);
             else if (msg.id == "text")                                      // just log some text
-                console.log("[1.2.1+1] msg received:", msg.text);
+                console.log("[1.3.1+1] msg received:", msg.text);
         };
         
         // send text to IDE log
@@ -161,7 +161,7 @@ var terminated;                                 // flag that this service worker
             // don't clutter the log
             msgport.postMessage({
                 id: "text",
-                text: "port received by 1.2.1+1",
+                text: "port received by 1.3.1+1",
             });
             */
         if (--pending === 0)
@@ -178,7 +178,7 @@ var terminated;                                 // flag that this service worker
         includeUncontrolled: true
     }).then(function(clientList) {
         clientList.every(function(client) {
-            console.log("[1.2.1+1] requesting new msgport for", client.id);
+            console.log("[1.3.1+1] requesting new msgport for", client.id);
             ++pending;
             client.postMessage({                                            // tell client(s) we need this fetch source
                 msg: "Naan_need_fetch_port",
@@ -224,12 +224,12 @@ function Reaper() {
             clients[clientList[clidex].id] = clientList[clidex];
         for (var sourceId in msgPorts)
             if (!clients[sourceId]) {
-                console.log("[1.2.1+1] source gone:", sourceId);
+                console.log("[1.3.1+1] source gone:", sourceId);
                 delete msgPorts[sourceId];                                  // no longer a source
             }
         for (var clientId in fetchPorts)
             if (!clients[clientId]) {
-                console.log("[1.2.1+1] client gone:", clientId);
+                console.log("[1.3.1+1] client gone:", clientId);
                 delete fetchPorts[clientId];                                // no longer a client
             }
         for (var fqdex = 0; fqdex < fetchQueue.length; ++fqdex) {
@@ -263,16 +263,16 @@ function ClearCaches() {
             return (Promise.all(
                 cacheNames.map(function(cacheName) {
                     if (cacheName != CurrentCacheName) {
-                        console.log('[1.2.1+1] deleting old cache:', cacheName);
+                        console.log('[1.3.1+1] deleting old cache:', cacheName);
                         return (caches.delete(cacheName));
                     }
                 })
             ));
         }).then(function() {                                                // claim all clients
-            console.log('[1.2.1+1] claiming clients');
+            console.log('[1.3.1+1] claiming clients');
             return (self.clients.claim());
         }).then(function() {
-            console.log('[1.2.1+1] clients claimed');
+            console.log('[1.3.1+1] clients claimed');
             return (Promise.resolve(true));
         });
     return (promise);
@@ -315,7 +315,7 @@ function GetClientResponse(event, urlpath) {
             msgport.postMessage({
                 id: "fetch",
                 seq: seqno,
-                version: "1.2.1+1",
+                version: "1.3.1+1",
                 request: {
                     method: event.request.method,
                     url: event.request.url
@@ -363,7 +363,7 @@ function GetClientResponse(event, urlpath) {
  */
 
 self.addEventListener('install', function(event) {
-    console.log("[1.2.1+1] install");
+    console.log("[1.3.1+1] install");
     self.skipWaiting();
 });
 
@@ -376,20 +376,20 @@ self.addEventListener('install', function(event) {
  */
 
 self.addEventListener('activate', function(event) {
-    console.log("[1.2.1+1] activate");
+    console.log("[1.3.1+1] activate");
     self.clients.matchAll({                                                 // for debugging, list controlled clients           
         includeUncontrolled: true
     }).then(function(clientList) {
         var urls = clientList.map(function(client) {
             return (client.url);
         });
-        console.log('[1.2.1+1] matching clients:', urls.join(', '));
+        console.log('[1.3.1+1] matching clients:', urls.join(', '));
     });
     var promise = ClearCaches().then(function() {
         for (var sourceId in upgradeMsgPorts) {
             upgradeMsgPorts[sourceId].postMessage({                         // notify new version available
                 id: "upgrade",
-                version: "1.2.1+1"
+                version: "1.3.1+1"
             });
         }
         upgradeMsgPorts = false;
@@ -421,7 +421,7 @@ self.addEventListener('fetch', function(event) {
             var promise;
             var url = new URL(request.url);
             var nocache = request.method != "GET"
-                || url.search.length !== 0 && url.searchParams.get("naanver") !== "ecffb6bced5a08a6d2fdd188410b1b50"
+                || url.search.length !== 0 && url.searchParams.get("naanver") !== "ecae8a62f4a0ae0408f1cbd16082d4ea"
                 || request.headers.get('range');
             if (url.pathname.startsWith("/run/")) {
                 nocache = true;
@@ -455,7 +455,7 @@ self.addEventListener('fetch', function(event) {
                             statusText: "Request Cancelled"
                         }));
                     }
-                    console.log("[1.2.1+1] fetch failed", request.url, e);
+                    console.log("[1.3.1+1] fetch failed", request.url, e);
                     return (new Response(undefined, {
                         status: 404,
                         statusText: "Fetch Failed"

package/frameworks/browser/terminals.nlg

@@ -10,6 +10,13 @@
  *
  */
 
+/*
+ * globals
+ *
+ */
+
+workers;
+
 
 /*
  * TermHost
@@ -55,17 +62,19 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
 
     // connect
     // Connect with the host if not already connected.
-    closure connect(local host, url, pending) {
+    closure connect(local url, pending) {
         if target.ws
             return(list(false, { connected: true }))
-        host = xnew(window.URL, api.url).host
-        if !host || host == ""
-            return(list(Error("invalid URL ${api.url}")))
-        if window.location.protocol == "https:"
-            url = "wss://${host}"
+        try {
+            url = xnew(window.URL, api.url)
+        } catch {
+            return(list(Error("invalid URL ${api.url}", exception)))
+        }
+        if url.protocol == "https:"
+            url.protocol = "wss:"
         else
-            url = "ws://${host}"
-        target.ws = xnew(window.WebSocket, url)
+            url.protocol = "ws:"
+        target.ws = xnew(window.WebSocket, url.href)
         pending = new(nonce)
 
         // onopen
@@ -422,7 +431,9 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
 
 closure TermLocal(track, name, options, local target, nlregex, listener)
 {
-    global(js, window)
+    global(js, window, workers)
+    if !dictionary(workers)
+        workers = {}
     options = merge({
         type: "Local"
     }, options)
@@ -450,6 +461,8 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
             return (list(Error("Local environment does not support workers")))
         target.worker = xnew(window.Worker, "env_webworker.js")
         listener = target.worker
+        target.guid = UUID()
+        workers[target.guid] = target
     }
 
     // onmessage
@@ -540,8 +553,11 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
                     altcmd: options.startup.initcmds
                 })
             target.started = true
+        } else if msg.id == "targetsend" {
+            msg.data._guid = target.guid
+            js.t.DispatchMessage(msg.data)
         }
-    }
+     }
     
     //
     // target.DebugCmd
@@ -651,6 +667,82 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
 };
 
 
+/*
+ * termMainProxy
+ *
+ * column positioning:                          //                          //                      !
+ *     Make a browser proxy representing the main thread. This runs inside the worker and allows it
+ * to do remote execution with Main.
+ *
+ */
+
+closure termMainProxy(track, name, options, local target) {
+    global(js)
+    target = require("../running/executors.nlg").ExecutorBase(track, "mainProxy", name)
+    target.name = name
+
+    //
+    // target.PostMessage
+    //
+    //     Post a message to the main thread.
+    //
+    target.PostMessage = function PostMessage(data) {
+        js.t.DispatchMessage(data)
+        list(false, { posted: true })
+    }
+
+    //
+    // onreceive
+    //
+    //     Process a received message from the main thread, typically in response.
+    //
+    function onreceive(data) {
+        target.execReceived(data)
+    }
+    
+    // destroy
+    //
+    //     Destroy the execution instance.
+    //
+    target.destroy = function destroy() {
+        target.execClosed(Error("instance destroyed"))
+        track.delete(target)
+    }
+
+    js.t.OnReceive(onreceive.proc)
+    track.add(target)
+
+    // finis
+
+    list(false, target)
+};
+
+
+/*
+ * WorkerToMain
+ *
+ *     Connect our worker to the main thread, returning a result tuple with an executor representing
+ * Main. Use this to create a context object for Main, allowing remote evaluation.
+ *
+ */
+
+closure WorkerToMain(local nideRunning, track) {
+    // mainProxy -- link tracker to main thread proxy
+    function mainProxy(track, name, options) {
+        termMainProxy(track, name, options)
+    }
+
+    if !js.s {
+        error = Error("WorkerToMain only available in browser workers")
+        return (list(error))
+    }
+    nideRunning = require("frameworks/running/executors.nlg")
+    track = nideRunning.ExecutorTracker()
+    track.register(mainProxy, "proxy")
+    track.spawn("proxy")                                                    // returns result tuple
+};
+
+
 /*
  * findIDEtarget
  *
@@ -677,7 +769,7 @@ closure findIDEtarget(track, name, workerID, naancont, local target) {
  */
 
 closure termIDE(track, name, workerID, naancont, title, local target, connected) {
-    global()
+    global(workers)
     target = findIDEtarget(track, name, workerID, naancont)
     if target {                                                             // new naancont on existing target
         target.updateController(naancont)
@@ -691,23 +783,31 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     target.workerID = workerID
     target.naancont = naancont
     connected = []
-
+    
+    // replyHook
+    //
+    //     Intercept the ReplyMessage function to see if we should handle it.
     //
+    function replyHook(data) {
+        if data._guid
+            workers[data._guid].worker.postMessage({
+                id: "targetreceive"
+                data: data
+            })
+    }
+
     // target.DebugCmd
     //
     //     Post a debug command to the target, ourselves.
     //
-
     target.DebugCmd = function DebugCmd(data) {
         naancont.DispatchDebugger(data)
     }
 
-    //
     // target.PostMessage
     //
     //     Post a message to the target, ourselves.
     //
-
     target.PostMessage = function PostMessage(data) {
         naancont.DispatchMessage(data)
         list(false, { posted: true })
@@ -716,7 +816,7 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     // termAttach
     //
     // Create a closure for an attaching terminal.
-
+    //
     target.termAttach = closure termAttach(term, db, statusCB, local connection, inbound) {
         connection = new(object, this)
 
@@ -793,7 +893,7 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     // updateController
     //
     //     Update the naan controller on this target, which means updating for all the terminals.
-    
+    //
     target.updateController = function updateController(newnc, local conn) {
         naancont = newnc
         target.naancont = newnc
@@ -811,7 +911,7 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     // vsiteRefresh
     //
     //     The vsite upon which we are based has changed.
-    
+    //
     target.vsiteRefresh = function vsiteRefresh(url_origin) {
         naancont.VsiteRefresh(url_origin)
     }
@@ -819,12 +919,13 @@ closure termIDE(track, name, workerID, naancont, title, local target, connected)
     // destroy
     //
     //     Destroy the execution instance.
-    
+    //
     target.destroy = function destroy() {
         target.execClosed(Error("instance destroyed"))
         track.delete(target)
     }
 
+    js.t.OnReplyHook(replyHook.proc)
     track.add(target)
 
     // finis
@@ -868,13 +969,14 @@ closure termInstallVirtualWatcher(track, naancont) {
  */
 
 function termInit(local manifest) {
-
-    manifest = `(TermHost, TermLocal, findIDEtarget, termIDE, termInstallVirtualWatcher, termInit)
+    manifest = `(TermHost, TermLocal, termMainProxy, WorkerToMain, findIDEtarget, termIDE,
+        termInstallVirtualWatcher, termInit)
 
     Naan.module.build(module.id, "terminals", function(modobj, compobj) {
         require("./browser.nlg")
         compobj.manifest = manifest
         modobj.exports.TermLocal = TermLocal
         modobj.exports.TermHost = TermHost
+        modobj.exports.WorkerToMain = WorkerToMain
     })
 } ();

package/frameworks/browser/workers.nlg

@@ -234,7 +234,10 @@ closure ScopedServiceWorker(pubID, pubVersion, callback, local scoper, result) {
     scoper = new(object, ScopedServiceWorker)
     scoper.scopes = {}
     scoper.serv = ServiceWorker(pubID, pubVersion)
-    result = scoper.serv.register("/", closure(error, data, local response, filepath, scope) {
+    scoper.root = js.r("path").dirname(js.w.location.pathname)              // normally "/", but could be subfolder
+    if scoper.root.slice(-1) != "/"
+        scoper.root = scoper.root.concat("/")                               // must end in "/"
+    result = scoper.serv.register(scoper.root, closure(error, data, local response, filepath, scope) {
         if data.message == "fetch" {
             response = {
                 id: "response"

package/frameworks/browser/ws_client.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2024 by Richard C. Zulch
  *
  */
 
@@ -22,6 +22,8 @@
  *      onmessage:  <proc>(e, message)          // message received
  *      onerror:    <proc>(e)                   // error occurred
  *      onclose:    <proc>(e)                   // connection has closed
+ *      query:      <string>                    // query string starting with ? for insecure connections
+ *      querys:     <string>                    // query string starting with ? for secure connections
  *  }
  *
  */
@@ -41,9 +43,9 @@ closure WebSocket(options, local wsock) {
         if wsock.ws
             return (list(false, { open: true }))
         if window.location.protocol == "https:"
-            url = "wss://${options.host}"
+            url = "wss://${options.host}${options.querys||''}"
         else
-            url = "ws://${options.host}"
+            url = "ws://${options.host}${options.query||''}"
         wsock.ws = xnew(window.WebSocket, url)
         pending = new(nonce)
     

package/frameworks/client/apiclient.nlg

@@ -22,7 +22,9 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
     global()
     client = new(object, this)
     if !url.match(RegExp("^https?:\/\/", "i"))
-        url = "http://${url}"
+        url = "http://${url}"                                               // a protocol is required
+    if url.slice(-1) != "/"
+        url = url.concat("/")                                               // must end in "/"
     client.url = url
     client.guid = guid
     client.notifyAfter = 0
@@ -42,7 +44,7 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
     //
 
     client.docs = closure docs(local error, content, extra) {
-        `(error, content, extra) = clientRequest(url.concat("/"))
+        `(error, content, extra) = clientRequest(url)
         printline(content)
     }
 
@@ -51,7 +53,7 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
     //
 
     client.status = closure status(cbStatus, timeoutms, local query, error, content, extra, queued) {
-        query = EncodeQuery("/status/?", {
+        query = EncodeQuery("status/?", {
             timeout: timeoutms
             after: client.notifyAfter
         })
@@ -75,7 +77,7 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
     //
 
     client.get = closure get(path, options, local error, content, extra) {
-        `(error, content, extra) = clientRequest(url.concat("/", path))
+        `(error, content, extra) = clientRequest(url.concat(path))
         if !error && extra.status != 200
             error = Error("readFile failed: ".concat(extra.status), {
                     status: extra.status
@@ -91,7 +93,7 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
     //
 
     client.readFile = closure readFile(filepath, options, local query, error, content, extra) {
-        query = EncodeQuery("/readfile/?", {
+        query = EncodeQuery("readfile/?", {
             path: filepath
         })
         `(error, content, extra) = clientRequest(url.concat(query), options)
@@ -112,7 +114,7 @@ closure NideAPIclient(url, guid, apiRequester, local client) {
 
     client.psmRemote = closure psmRemote(options, data, callback, 
         local query, reqOptions, error, content, extra) {
-        query = EncodeQuery("/psm/?", options)
+        query = EncodeQuery("psm/?", options)
         reqOptions = {
             putdata: data
         }

package/frameworks/client/relaycon.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2024 by Richard C. Zulch
  *
  */
 
@@ -80,6 +80,8 @@ closure RelayCon(host, secret, options, local relay, nlregex)
             onmessage: onmessage
             onerror: onerror
             onclose: onclose
+            query: undefined                                                // don't put secret on unsecure connections
+            querys: secret && EncodeQuery("?", { guid: secret })
         })
         result = relay.wscon.connect()
         if !result.0

package/frameworks/common/common.nlg

@@ -19,6 +19,7 @@
  */
 
 function EncodeQuery(prefix, items, local result, item) {
+    global()
     result = []
     for item in items
         result.push(EncodeURIComponent(item).concat("=", EncodeURIComponent(items[item])))
@@ -38,6 +39,7 @@ function EncodeQuery(prefix, items, local result, item) {
  */
 
 function DecodeQuery(text, local output, rpreg, qarr, item, pair) {
+    global()
     output = {}
     rpreg = RegExp("\\+", "g")
     qarr = text.split("&")
@@ -62,6 +64,7 @@ function DecodeQuery(text, local output, rpreg, qarr, item, pair) {
  */
 
 function ContentTypeFromFileExt(filepath, local mimeType, encoding) {
+    global(JSpath)
     mimeType = {
         "css":  "text/css",
         "csv":  "text/csv",
@@ -103,6 +106,7 @@ function ContentTypeFromFileExt(filepath, local mimeType, encoding) {
  */
  
 function UUID(local rando) {
+    global(RandomBytes)
     if !RandomBytes
         return (false)
     rando = RandomBytes(36)                                                 // length of UUID
@@ -128,6 +132,7 @@ function UUID(local rando) {
  */
 
 function VersionCheck(testv, refv, local rxver, test, ref) {
+    global()
     rxver = RegExp("([0-9]+)[.]([0-9]+)[.]([0-9]+)-?[A-Za-z_]*[+]?([0-9]+)")
     function numerify(x){ Number.parseInt(x) }
     test = testv.match(rxver).map(numerify.proc)
@@ -147,6 +152,7 @@ function VersionCheck(testv, refv, local rxver, test, ref) {
  */
 
 function JsonParse(data) {
+    global(exception)
     try {
         data = JSONparse(data)
         list(false, data)
@@ -165,6 +171,7 @@ function JsonParse(data) {
  */
 
 function JsonStringify(data) {
+    global(exception)
     try {
         data = JSONstringify(data)
         list(false, data)
@@ -185,15 +192,18 @@ function JsonStringify(data) {
 
 jsScripts = { };                                                            // script values become false on reload
  
-closure JsLoadScript(libpath, globalID, local scriptTag, fpath) {
+closure JsLoadScript(libpath, globalID, local savens, fpath, hostpath, pending, scriptTag) {
+    global(jsScripts, js, JSpath)
     if jsScripts[libpath]
         return                                                              // already loaded
+    savens = nsactive()
     if libpath.startsWith("/")
         fpath = libpath
     else
         fpath = "/".concat(libpath)
+    hostpath = js.w.location.origin.concat(JSpath.dirname(js.w.location.pathname))
     if js.s.importScripts && tostring(js.s.importScripts) == "[Function importScripts]" {
-        fpath = JSpath.dirname(js.w.location.href).concat(fpath)
+        fpath = hostpath.concat(fpath)
         try {
             js.s.importScripts(fpath)                                       // load in worker context
             jsScripts[libpath] = js.s[globalID]
@@ -204,7 +214,7 @@ closure JsLoadScript(libpath, globalID, local scriptTag, fpath) {
     } else if js.w {                                                        // load in browser context
         pending = new(nonce)
         scriptTag = js.w.document.createElement("script")
-        scriptTag.src = JSpath.dirname(js.w.location.href).concat(fpath, requireQuery())
+        scriptTag.src = hostpath.concat(fpath, requireQuery())
         scriptTag.onload = function (event) {
             jsScripts[libpath] = js.w[globalID]
             pending.signal(js.w[globalID])
@@ -219,6 +229,7 @@ closure JsLoadScript(libpath, globalID, local scriptTag, fpath) {
         pending.wait()
     } else if js.g
         jsScripts[libpath] = js.r(libpath)                                  // load in NodeJS context
+    nsactive(savens)
     jsScripts[libpath]
 };
 
@@ -234,6 +245,7 @@ closure JsLoadScript(libpath, globalID, local scriptTag, fpath) {
  */
 
 macro LoadComponentOnDemand(functionKeys, componentPath, local modobj, key) {
+    global()
     modobj = car(nsrights().lookup("module"))
     functionKeys = eval(functionKeys)
     componentPath = eval(componentPath)
@@ -281,6 +293,7 @@ macro LoadComponentOnDemand(functionKeys, componentPath, local modobj, key) {
  */
 
 macro LiveExport() {
+    global()
     closure (exporterMod, local importList) {
         importList = []
         //