@naanlang/naan 1.0.2 → 1.0.5

package/frameworks/browser/https_request.nlg

@@ -0,0 +1,120 @@
+/*
+ * https_browser.nlg
+ *
+ *     Https operations for the browser.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2021-2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * HttpsApiRequest
+ *
+ * Perform an API request from the browser. This operates the same as HttpsApiRequest for NodeJS.
+ *
+ * Options:
+ *  {
+ *      method:         <string>                // HTTP method (defaults to GET/PUT)
+ *      query:          <dictionary>            // query variables added to the URL
+ *      putdata:        <data>                  // data to put
+ *      contentType:    <string>                // sets Content-Type header, otherwise auto-determined
+ *      encoding:       <string>                // "binary" to receive binary data as arrayBuffer
+ *      range:          <string>                // sets range header
+ *      headers:        [`(key, value)]         // add header(s) to the request, overriding defaults
+ *  }
+ *
+ */
+
+closure HttpsApiRequest(url, options,
+    local urlq, fetchOptions, putdata, error, response, contentType, content) {
+    if !options
+        options = { }
+    if options.query
+        urlq = url.concat(EncodeQuery("?", options.query))
+    else
+        urlq = url
+    putdata = options.putdata
+    fetchOptions = {
+        mode: 'cors'
+        cache: 'no-cache'
+        credentials: 'same-origin'
+        headers: { }
+        redirect: 'follow'
+        referrerPolicy: 'no-referrer'
+    }
+    if options.method
+        fetchOptions.method = options.method
+    else if putdata
+        fetchOptions.method = "POST"
+    else
+        fetchOptions.method = "GET"
+    if putdata {
+        if options.contentType
+            fetchOptions.headers['Content-Type'] = options.contentType
+        else if jsTypedArray(options.putdata)
+            fetchOptions.headers['Content-Type'] = "application/octet-stream"
+        else if member(typeof(options.putdata), `(dictionary, array, xobject)) {
+            fetchOptions.headers['Content-Type'] = "application/json"
+            `(error, putdata) = JsonStringify(options.putdata)
+            if error
+                return (list(Error("HttpsApiRequest encode:", url)))
+        }
+        fetchOptions.body = putdata
+    }
+    if options.range
+        fetchOptions.headers.Range = options.range
+    if array(options.headers)
+        for item in options.headers
+            fetchOptions.headers[item.0] = item.1
+    //
+    // perform the fetch and return when complete
+    //
+    `(error, response) = await(js.w.fetch(urlq, fetchOptions))
+    if error
+        return (list(Error("HttpsApiRequest fetch failed:", error, url)))
+    contentType = response.headers.get("content-type")
+    if response.status < 200 || response.status >= 300
+        return (list(Error("HttpsApiRequest status:", url, response.status, {
+            status: response.status
+        })))
+    if contentType.startsWith("application/json") {
+        `(error, content) = await(response.text())
+        if !error
+            `(error, content) = JsonParse(content)
+            if error
+                error = Error("HttpsApiRequest decode:", url, error)
+            else
+                content = new(content)
+            if array(content) {                                             // deencapsulate to get API result
+                content = totuple(content)
+                error = content.0
+                content = content.1
+            }
+    }
+    else if options.encoding == "binary"
+        `(error, content) = await(response.arrayBuffer())
+    else
+        `(error, content) = await(response.text())
+    list(error, content)
+};
+
+
+/*
+ * https_browserInit
+ *
+ * Initialize HTTPS request operations for browsers.
+ *
+ */
+
+function https_browserInit(local manifest) {
+    manifest = `(HttpsApiRequest, https_browserInit)
+
+    Naan.module.build(module.id, "https_request", function(modobj, compobj) {
+        require("./browser.nlg")
+        compobj.manifest = manifest
+        modobj.exports.HttpsApiRequest = HttpsApiRequest
+    })
+}();

package/frameworks/browser/sworker.js

@@ -4,66 +4,309 @@
  *
  *     Nide service worker script.
  *
+ * The goal of this service worker is to 1) cache fetch requests for static content, and 2) relay
+ * fetch requests in the /run/ URL path namespace to instances of the IDE that can provide it. The
+ * identifier for the various browser windows is by "clientId", which is a GUID reported by the
+ * service worker and message channel APIs. Here we use the term clientId to refer to windows that
+ * consume data in the /run/ path, i.e. the IDE targets, and the term sourceId to refer to windows
+ * that provide data, i.e. the IDE instances.
+ *     When the service worker is first executed it sends messages to all known windows requesting
+ * that they establish a message channel and forward the service worker's port to it. This secondary
+ * channel is used to communicate fetch requests from the service worker to the source IDE and then
+ * get the response. If an IDE is executed subsequently then it registers itself by sending the port
+ * unilatterally, instead of waiting to be requested.
+ *     The service worker gets a fetch request when a non-cached resource is requested by a client
+ * window. If the URL path is not within /run/ then this executes a normal network request. IF the
+ * URL path is within /run/ then this asks the IDE source(s) to fetch it. Initially every IDE source
+ * is sent the request, but when one provides a 200 response then this stores that source port under
+ * the clientId in fetchPorts so that subsequent requests can go only to the correct IDE. This works
+ * great until an IDE goes away or becomes unresponsive.
+ *     Unfortunately there is no event that tells us when a window is closed or goes away. The first
+ * indication of trouble is when an IDE fails to respond within a reasonable amount of time, which
+ * could be anything. We can enumerates the valid clients, which gives us a chance to validate our
+ * knowledge of which ones are responding. If a request takes longer than a second to respond then
+ * this executes a validate operation, which eliminates any clients that have gone away and rejects
+ * their pending operations.
+ *     One potential problem is stale caches. When an older version is discovered then we notify the
+ * client and invalidate the old cache.
+ *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020 by Richard C. Zulch
+ * Copyright (c) 2020-2022 by Richard C. Zulch
  *
  */
 
-var CurrentCacheName = "Naan IDE 1.0.2-1"
+var CurrentCacheName = "Naanlang-1.0.5-2";
 
 
 //
 // locals
 //
 
-var fetchQueue = [];
-var fetchNextSeq = 1;
-var msgport;
+var fetchQueue = [];                            // queue for outstanding requests to sources
+var fetchNextSeq = 1;                           // sequence number for source requests
+var msgPorts = {};                              // message ports keyed by sourceId
+var fetchPorts = {};                            // message ports keyed by clientId
+var waitingForInit = [];                        // initialization functions, or false after done
 
 
 /*
- * message event
+ * initialize
  *
- *     Received when someone sends us a.
+ *     Initiate listening for incoming messages from our clients. This waits for any client responses
+ * and then quits. Any clients that want to participate after that must unilaterally send a port for
+ * communication.
  *
  */
 
-var promiseport = new Promise(function (resolve, reject) {
+(function initialize() {
+    if (!waitingForInit)
+        return;                                                             // already initialized
+    var pending = 0;
+    
+    // doneInit
+    //
+    // Call this to resolve any waiting promises and release them for attempting to fetch from the
+    // IDEs (sources.)
+    //
+    function doneInit() {
+        if (!waitingForInit)
+            return;                                                         // already done
+        var waiters = waitingForInit;
+        waitingForInit = false;                                             // no more waiters allowed
+        for (var wdex in waiters)
+            waiters[wdex]();                                                // call each waiter
+    }
+
+    // self.onmessage events
+    //
+    // This receives channel port messages from the IDEs as they either 1) respond to a request to
+    // register, or 2) spontaneously register themselves as they notice that we're running.
+    //
     self.addEventListener('message', function(event) {
-        console.log("[1.0.2-1] port received");
-        msgport = event.data.hereIsYourPort;
+        var msgport = event.data.hereIsYourPort;                            // message port of a client
+        var pubID = event.data.hereIsMyID;
+        var pubVersion = event.data.hereIsMyVersion;
+        var sourceId = event.source.id;                                     // client id of sender of message
+        msgPorts[sourceId] = msgport;
+        console.log("[1.0.5-2] received new msgport for", sourceId, pubID, "-", pubVersion);
+        if (pubVersion != "1.0.5-2")
+            msgport.postMessage({                                           // notify new version available
+                id: "upgrade",
+                version: "1.0.5-2"
+            });
+        Reaper();                                                           // clean up obsolete info
+        
+        // msgport.onmessage events
+        //
+        // This receives channel port messages that are responding to our fetch requests. The IDE
+        // can also send a text message for logging, but that is used only for debugging.
+        //
         msgport.onmessage = function(msg) {
             msg = msg.data;
             if (msg.id == "response")
-                postResponse(msg);
+                processResponse(msg);
             else if (msg.id == "text")
-                console.log("[1.0.2-1] msg received:", msg.text);
+                console.log("[1.0.5-2] msg received:", msg.text);
         };
-/*
-        // don't clutter the log
-        msgport.postMessage({
-            id: "text",
-            text: "port received by 1.0.2-1",
-        });
-*/
-        resolve(msgport);
-
+        
+        // send text to IDE log
         //
-        // postResponse
+        // This is only for debugging.
         //
-        
-        function postResponse(msg) {
-            var fqdex, response, item;
-            for (fqdex = 0; fqdex < fetchQueue.length; ++fqdex)
-                if (fetchQueue[fqdex].seq === msg.seq) {
-                    item = fetchQueue.splice(fqdex, 1);
-                    item[0].resolve(new Response(msg.body, msg.init));
-                    break;
-                }
+            /*
+            // don't clutter the log
+            msgport.postMessage({
+                id: "text",
+                text: "port received by 1.0.5-2",
+            });
+            */
+        if (--pending === 0)
+             doneInit();                                                    // we finished all known clients
+    });
+
+    // initialize known IDE instances.
+    //
+    // This sends a port request to each known IDE instance. Fetches that occur prior to completion
+    // of initialization will wait for these instances to respond. A timeout prevents one errant IDE
+    // from hanging all requests, by releasing the fetches even before all source IDEs respond.
+    //
+    self.clients.matchAll({
+        includeUncontrolled: true
+    }).then(function(clientList) {
+        clientList.every(function(client) {
+            console.log("[1.0.5-2] requesting new msgport for", client.id);
+            ++pending;
+            client.postMessage({                                            // tell client(s) we need this fetch source
+                msg: "Naan_need_fetch_port",
+            });
+        });
+        setTimeout(doneInit, 10000);                                        // release fetches
+    });
+
+    // processResponse
+    //
+    // An IDE source has responded to our fetch request. If this was successful then we record the
+    // IDE (sourceId) so that subsequent requests from that target (clientId) will go to the same
+    // source.
+    //
+    function processResponse(msg) {
+        var fqdex, response, item;
+        for (fqdex = 0; fqdex < fetchQueue.length; ++fqdex)
+            if (fetchQueue[fqdex].seq === msg.seq) {
+                item = fetchQueue.splice(fqdex, 1)[0];
+                if (item.clientId && msg.init.status == 200)
+                    fetchPorts[item.clientId] = item.msgport;               // successful lookup
+                item.resolve(new Response(msg.body, msg.init));
+                break;
+            }
+    }
+})();
+
+
+/*
+ * Reaper
+ *
+ * Periodic cleanup returning a promise. This removes anything that is obsolete and resolve all
+ * associated transactions.
+ *
+ */
+
+function Reaper() {
+    var promise = self.clients.matchAll({
+        includeUncontrolled: true
+    }).then(function(clientList) {                                          // both sources and clients
+        var clients = {};
+        for (var clidex in clientList)
+            clients[clientList[clidex].id] = clientList[clidex];
+        for (var sourceId in msgPorts)
+            if (!clients[sourceId]) {
+                console.log("[1.0.5-2] source gone:", sourceId);
+                delete msgPorts[sourceId];                                  // no longer a source
+            }
+        for (var clientId in fetchPorts)
+            if (!clients[clientId]) {
+                console.log("[1.0.5-2] client gone:", clientId);
+                delete fetchPorts[clientId];                                // no longer a client
+            }
+        for (var fqdex = 0; fqdex < fetchQueue.length; ++fqdex) {
+            var item = fetchQueue[fqdex];
+            while (item.clientId && !clients[item.clientId]
+                || !Object.values(msgPorts).includes(item.msgport)) {
+                fetchQueue.splice(fqdex, 1);                                // either client or source missing
+                item.resolve(new Response(undefined, {
+                    status: 404,
+                    statusText: "Source Disappeared"                        // if client is gone this disappears too
+                }));
+                if (fqdex >= fetchQueue.length)
+                    break;                                                  // we've run out of transactions
+                item = fetchQueue[fqdex];
+            }
         }
     });
-});
+    return (promise);
+}
+
+
+/*
+ * ClearCaches
+ *
+ * Clear obsolete caches, returning a promise.
+ *
+ */
+
+function ClearCaches() {
+    var promise = caches.keys().then(function(cacheNames) {                 // delete old cache entries
+            return (Promise.all(
+                cacheNames.map(function(cacheName) {
+                    if (cacheName != CurrentCacheName) {
+                        console.log('[1.0.5-2] deleting old cache:', cacheName);
+                        return (caches.delete(cacheName));
+                    }
+                })
+            ));
+        }).then(function() {                                                // claim all clients
+            console.log('[1.0.5-2] claiming clients');
+            return (self.clients.claim());
+        });
+    return (promise);
+}
+
+
+/*
+ * GetClientResponse
+ *
+ *     Get message ports from the current clients if they can supply contents for the specified URL.
+ * This returns a promise that is resolved after the msgPorts object is updated. Note that the
+ * clientId argument is the requester, but the client that actually has the available data will
+ * often be a different window.
+ *     On the first request from a given client, every source is requested to respond to the URL
+ * pathname, and the first one that does is designated the official source for that client. While
+ * asking all the sources, this checks every second to see if any of them have gone away so that it
+ * does not hang waiting. A source can be unresponsive and delay the fetch forever, e.g. in the
+ * debugger, but if the window has closed then it will get reaped and the fetch will complete.
+ *     On subsequent requests, to a known source, this does not have a timeout. If the source goes
+ * away then the fetch will not complete, but the client is screwed anyway in that case. Eventually
+ * the reaper will be called for other reasons and it will clean things up then.
+ *
+ */
+
+function GetClientResponse(event, urlpath) {
+
+    // delegate
+    //
+    // Delegate to a source window and return a promise
+    //
+    function delegate(msgport, clientId) {
+        return (new Promise(function(resolve, reject) {
+            var seqno = fetchNextSeq++;
+            fetchQueue.push({
+                seq: seqno,
+                msgport: msgport,
+                clientId: clientId,
+                resolve: resolve
+            });
+            msgport.postMessage({
+                id: "fetch",
+                seq: seqno,
+                version: "1.0.5-2",
+                request: {
+                    method: event.request.method,
+                    url: event.request.url
+                },
+            });
+        }));
+    }
+
+    var clientId;
+    if (event.clientId !== "") {
+        clientId = event.clientId;
+        if (fetchPorts[clientId])
+            return (delegate(fetchPorts[clientId], false));
+    }
+    else
+        clientId = event.resultingClientId;                                 // new window
+    var pending = [];
+    var sourceIds = Object.getOwnPropertyNames(msgPorts);
+    for (var mdex = 0; mdex < sourceIds.length; ++mdex)
+    {
+        var sourceId = sourceIds[mdex];
+        pending.push(delegate(msgPorts[sourceId], clientId));
+    }
+    if (pending.length === 0)
+        return (Promise.resolve(new Response(undefined, {
+            status: 404,
+            statusText: "No Sources"
+        })));
+    var toid = setInterval(Reaper, 1000);                                   // reap every second while pending
+    return (Promise.all(pending).then(function(responses) {
+        clearInterval(toid);                                                // all completed
+        for (var idex in responses)
+            if (responses[idex].status == 200)
+                return (responses[idex]);                                   // this is the one we want
+        return (responses[0]);                                              // fall back to first
+    }));
+}
 
 
 /*
@@ -74,7 +317,7 @@ var promiseport = new Promise(function (resolve, reject) {
  */
 
 self.addEventListener('install', function(event) {
-    console.log("[1.0.2-1] install");
+    console.log("[1.0.5-2] install");
     self.skipWaiting();
 });
 
@@ -95,53 +338,30 @@ self.addEventListener('fetch', function(event) {
             var tid;
             var promise;
             var url = new URL(event.request.url);
-            var nocache = event.method != "GET" || url.search.length > 0;
-            if (nocache && url.pathname.startsWith("/run/")) {
-                if (msgport)
-                    promise = Promise.resolve(msgport);
-                else {
-                    promise = self.clients.matchAll({
-                        includeUncontrolled: true
-                    }).then(function(clientList) {
-                        console.log("[1.0.2-1] requesting new msgport");
-                        var urls = clientList.map(function(client) {
-                            client.postMessage({                                // tell client(s) we need a fetch source
-                                msg: "Naan_need_fetch_port"
-                            });
-                            return (client.url);
+            var nocache = event.request.method != "GET"
+                || url.search.length !== 0
+                || event.request.headers.get('range');
+            if (url.pathname.startsWith("/run/")) {
+                nocache = true;
+                if (waitingForInit) {
+                    promise = new Promise(function(resolve, reject) {
+                        waitingForInit.push(function(){
+                            resolve();                                      // init was complete
                         });
-                        console.log('[1.0.2-1] matching clients:', urls.join(', '));
-                        return (promiseport);
+                    }).then(function(){
+                        return (GetClientResponse(event, url.pathname));    // then get the response
                     });
-                }
-                promise = promise.then(function(mp) { return new Promise(function(resolve, reject) {
-                    /*
-                    tid = setTimeout( function() {
-                        reject("timeout!");
-                    }, 250); */
-                    var seqno = fetchNextSeq++;
-                    if (msgport) {
-                        fetchQueue.push({
-                            seq: seqno,
-                            resolve: resolve
-                        });
-                        msgport.postMessage({
-                            id: "fetch",
-                            seq: seqno,
-                            version: "1.0.2-1",
-                            request: {
-                                method: event.request.method,
-                                url: event.request.url
-                            },
-                        });
-                    } else {
-                        console.log("[1.0.2-1] fetch has no msgport");
-                        return (reject("fatal delegation error"));
-                    }
-                }) });
+                } else
+                    promise = GetClientResponse(event, url.pathname);
             }
             else
-                promise = fetch(event.request);
+                promise = fetch(event.request).catch(function (e) {
+                    console.log("[1.0.5-2] fetch failed", e);
+                    return (new Response(undefined, {
+                        status: 404,
+                        statusText: "Fetch Failed"
+                    }));
+                });
             return (promise.then(function(response) {
                 // delete tid timer ###
                 if (!nocache) {
@@ -165,28 +385,16 @@ self.addEventListener('fetch', function(event) {
  */
 
 self.addEventListener('activate', function(event) {
-    console.log("[1.0.2-1] activate");
+    console.log("[1.0.5-2] activate");
     self.clients.matchAll({                                                 // for debugging, list controlled clients           
         includeUncontrolled: true
     }).then(function(clientList) {
         var urls = clientList.map(function(client) {
             return (client.url);
         });
-        console.log('[1.0.2-1] matching clients:', urls.join(', '));
+        console.log('[1.0.5-2] matching clients:', urls.join(', '));
     });
-    var promise = caches.keys().then(function(cacheNames) {                 // delete old cache entries
-            return (Promise.all(
-                cacheNames.map(function(cacheName) {
-                    if (cacheName !== CurrentCacheName) {
-                        console.log('[1.0.2-1] deleting old cache:', cacheName);
-                        return (caches.delete(cacheName));
-                    }
-                })
-            ));
-        }).then(function() {                                                // claim all clients
-            console.log('[1.0.2-1] claiming clients for version', CurrentCacheName);
-            return (self.clients.claim());
-        });
+    var promise = ClearCaches();
     if (event.waitUntil)
         event.waitUntil(promise);
 });