@naanlang/naan 1.4.3 → 1.4.4

package/frameworks/browser/https_request.nlg

@@ -25,6 +25,7 @@
  *      range:          <string>                // sets range header
  *      headers:        [`(key, value)]         // add header(s) to the request, overriding defaults
  *      keepalive:      <boolean>               // true to send data at unload
+ *      retrier:        <function>              // retry selected failures
  *      debug:          <boolean>               // log errors and status results
  *  }
  *
@@ -39,11 +40,35 @@
  */
 
 closure HttpsApiRequest(url, options,
-    local urlq, fetchOptions, putdata, item, error, response, 
+    local delayf, urlq, fetchOptions, putdata, item, error, response, 
         contentType, content, headers) {
-    global(window)
-    if !options
-        options = { }
+    global(window, common)
+    //
+    // defRetrier -- backoff and retry selected errors
+    //
+    function defRetrier(error, headers, local retryAfter, retryms, waitms) {
+        if member(error.status, `(408, 500, 502, 503, 504)) {               // retriable errors
+            retryAfter = headers["retry-after"]
+            if retryAfter {
+                retryms = toint(retryAfter) * 1000
+                if !retryms
+                    retryms = Date(retryAfter).getTime() - Date.now()
+                if retryms < 0
+                    retryms = false
+            }
+            if !delayf
+                delayf = common.ExbackFactory(12,30)                        // 12s max backoff, 30s deadline
+            waitms = delayf()
+            if !waitms || waitms < retryms
+                waitms = retryms
+            if options.debug
+                debuglog("HttpsApiRequest.defRetrier: wait ${waitms} msec using retry-after ${retryAfter}")
+            if waitms                                                       // return false to give up
+                sleep(waitms)                                               // return true from sleep to retry
+        }
+    }
+
+    options = merge({ retrier: defRetrier }, options)
     if options.query
         urlq = url.concat(EncodeQuery("?", options.query))
     else
@@ -92,16 +117,23 @@ closure HttpsApiRequest(url, options,
                 debuglog("HttpsApiRequest error:", url, "error:", error)
             return (list(Error("HttpsApiRequest fetch failed:", error, url)))
         }
+        headers = { }
+        response.headers.forEach(function(value, key) {
+            headers[key.toLowerCase()] = value
+        })
         if response.status < 200 || response.status >= 299 {                // 299 is "cancelled"
             if options.debug
                 debuglog("HttpsApiRequest status:", url, response.statusCode)
             error = Error("HttpsApiRequest status:", url, response.status, {
                 status: response.status
             })
-            if options.retrier(error)
+            if options.retrier(error, headers)
                 continue                                                    // optional retry/delay logic
             else
-                return (list(error, false, { status: response.status }))
+                return (list(error, false, {
+                    headers: headers
+                    status: response.status
+                }))
         } else
             break
     }
@@ -125,10 +157,6 @@ closure HttpsApiRequest(url, options,
         `(error, content) = await(response.arrayBuffer())
     else
         `(error, content) = await(response.text())
-    headers = { }
-    response.headers.forEach(function(value, key) {
-        headers[key.toLowerCase()] = value
-    })
     list(error, content, {
         headers: headers
         status: response.status
@@ -149,6 +177,7 @@ function https_browserInit(local manifest) {
 
     Naan.module.build(module.id, "https_request", function(modobj, compobj) {
         require("browser.nlg")
+        common = require("../common/utils.nlg")
         compobj.manifest = manifest
         modobj.exports.HttpsApiRequest = HttpsApiRequest
     })

package/frameworks/browser/sworker.js

@@ -36,7 +36,7 @@
  *
  */
 
-var CurrentCacheName = "Naanlang-1.4.3+1";
+var CurrentCacheName = "Naanlang-1.4.4+1";
 
 
 //
@@ -92,18 +92,18 @@ var terminated;                                 // flag that this service worker
         var pubVersion = event.data.hereIsMyVersion;
         var sourceId = event.source.id;                                     // client id of sender of message
         msgPorts[sourceId] = msgport;
-        console.log("[1.4.3+1] received new msgport for", sourceId, pubID, "-", pubVersion);
-        if (pubVersion != "1.4.3+1") {
+        console.log("[1.4.4+1] received new msgport for", sourceId, pubID, "-", pubVersion);
+        if (pubVersion != "1.4.4+1") {
             if (upgradeMsgPorts) {                                          // if not activated
-                console.log("[1.4.3+1] update pending for", sourceId);
+                console.log("[1.4.4+1] update pending for", sourceId);
                 upgradeMsgPorts[sourceId] = msgport;
             }
             else {
                 msgport.postMessage({                                       // notify new version available
                     id: "upgrade",
-                    version: "1.4.3+1"
+                    version: "1.4.4+1"
                 });
-                console.log("[1.4.3+1] update sent for:", sourceId);
+                console.log("[1.4.4+1] update sent for:", sourceId);
             }
         }
         Reaper();                                                           // clean up obsolete info
@@ -134,10 +134,10 @@ var terminated;                                 // flag that this service worker
         msgport.onmessage = function(msg) {
             msg = msg.data;
             if (msg.id == "skipWaiting") {                                  // try to activate this SW now
-                console.log("[1.4.3+1] attempting skipWaiting");
+                console.log("[1.4.4+1] attempting skipWaiting");
                 self.skipWaiting();
             } else if (msg.id == "terminate") {                             // termiante this SW now
-                console.log("[1.4.3+1] terminated");
+                console.log("[1.4.4+1] terminated");
                 terminate();
             } else if (msg.id == "abortURL") {                              // abort an I/O transaction
                 var href = new URL(msg.url).href;
@@ -150,7 +150,7 @@ var terminated;                                 // flag that this service worker
             } else if (msg.id == "response")                                // response from fetch request
                 processResponse(msg);
             else if (msg.id == "text")                                      // just log some text
-                console.log("[1.4.3+1] msg received:", msg.text);
+                console.log("[1.4.4+1] msg received:", msg.text);
         };
         
         // send text to IDE log
@@ -161,7 +161,7 @@ var terminated;                                 // flag that this service worker
             // don't clutter the log
             msgport.postMessage({
                 id: "text",
-                text: "port received by 1.4.3+1",
+                text: "port received by 1.4.4+1",
             });
             */
         if (--pending === 0)
@@ -178,7 +178,7 @@ var terminated;                                 // flag that this service worker
         includeUncontrolled: true
     }).then(function(clientList) {
         clientList.every(function(client) {
-            console.log("[1.4.3+1] requesting new msgport for", client.id);
+            console.log("[1.4.4+1] requesting new msgport for", client.id);
             ++pending;
             client.postMessage({                                            // tell client(s) we need this fetch source
                 msg: "Naan_need_fetch_port",
@@ -224,12 +224,12 @@ function Reaper() {
             clients[clientList[clidex].id] = clientList[clidex];
         for (var sourceId in msgPorts)
             if (!clients[sourceId]) {
-                console.log("[1.4.3+1] source gone:", sourceId);
+                console.log("[1.4.4+1] source gone:", sourceId);
                 delete msgPorts[sourceId];                                  // no longer a source
             }
         for (var clientId in fetchPorts)
             if (!clients[clientId]) {
-                console.log("[1.4.3+1] client gone:", clientId);
+                console.log("[1.4.4+1] client gone:", clientId);
                 delete fetchPorts[clientId];                                // no longer a client
             }
         for (var fqdex = 0; fqdex < fetchQueue.length; ++fqdex) {
@@ -263,16 +263,16 @@ function ClearCaches() {
             return (Promise.all(
                 cacheNames.map(function(cacheName) {
                     if (cacheName != CurrentCacheName) {
-                        console.log('[1.4.3+1] deleting old cache:', cacheName);
+                        console.log('[1.4.4+1] deleting old cache:', cacheName);
                         return (caches.delete(cacheName));
                     }
                 })
             ));
         }).then(function() {                                                // claim all clients
-            console.log('[1.4.3+1] claiming clients');
+            console.log('[1.4.4+1] claiming clients');
             return (self.clients.claim());
         }).then(function() {
-            console.log('[1.4.3+1] clients claimed');
+            console.log('[1.4.4+1] clients claimed');
             return (Promise.resolve(true));
         });
     return (promise);
@@ -315,7 +315,7 @@ function GetClientResponse(event, urlpath) {
             msgport.postMessage({
                 id: "fetch",
                 seq: seqno,
-                version: "1.4.3+1",
+                version: "1.4.4+1",
                 request: {
                     method: event.request.method,
                     url: event.request.url
@@ -363,7 +363,7 @@ function GetClientResponse(event, urlpath) {
  */
 
 self.addEventListener('install', function(event) {
-    console.log("[1.4.3+1] install");
+    console.log("[1.4.4+1] install");
     self.skipWaiting();
 });
 
@@ -376,20 +376,20 @@ self.addEventListener('install', function(event) {
  */
 
 self.addEventListener('activate', function(event) {
-    console.log("[1.4.3+1] activate");
+    console.log("[1.4.4+1] activate");
     self.clients.matchAll({                                                 // for debugging, list controlled clients           
         includeUncontrolled: true
     }).then(function(clientList) {
         var urls = clientList.map(function(client) {
             return (client.url);
         });
-        console.log('[1.4.3+1] matching clients:', urls.join(', '));
+        console.log('[1.4.4+1] matching clients:', urls.join(', '));
     });
     var promise = ClearCaches().then(function() {
         for (var sourceId in upgradeMsgPorts) {
             upgradeMsgPorts[sourceId].postMessage({                         // notify new version available
                 id: "upgrade",
-                version: "1.4.3+1"
+                version: "1.4.4+1"
             });
         }
         upgradeMsgPorts = false;
@@ -421,7 +421,7 @@ self.addEventListener('fetch', function(event) {
             var promise;
             var url = new URL(request.url);
             var nocache = request.method != "GET"
-                || url.search.length !== 0 && url.searchParams.get("naanver") !== "3b6364f71856e32f6cb8edae0e67bbc2"
+                || url.search.length !== 0 && url.searchParams.get("naanver") !== "3a2321ac98091606ee7cf5cc8919f70c"
                 || request.headers.get('range');
             if (url.pathname.startsWith("/run/")) {
                 nocache = true;
@@ -455,7 +455,7 @@ self.addEventListener('fetch', function(event) {
                             statusText: "Request Cancelled"
                         }));
                     }
-                    console.log("[1.4.3+1] fetch failed", request.url, e);
+                    console.log("[1.4.4+1] fetch failed", request.url, e);
                     return (new Response(undefined, {
                         status: 404,
                         statusText: "Fetch Failed"

package/frameworks/browser/terminals.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2019-2024 by Richard C. Zulch
+ * Copyright (c) 2019-2025 by Richard C. Zulch
  *
  */
 
@@ -48,8 +48,8 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
     nlregex = RegExp("\\n", "g")
     target.workerID = options.workerID
     target.msgQueue = []
-    target.dispatcher = call(options.dispatcher, function(reply) { reply })
-        
+    target.dispatcher = call(options.dispatcher)
+
     // findWorker
     //
     function findWorker(testID, local instance) {
@@ -135,19 +135,20 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
 
         // dispatcher
         // Respond to a host request while preserving context to permit reply routing
-        closure dispatcher(message, local reply) {
-            reply = target.dispatcher(message.payload.data)
-            target.ws.send(window.JSON.stringify({
-                guid: api.guid
-                clientID: message.clientID
-                serverID: message.serverID
-                workerID: message.workerID
-                respOp: "msgout"
-                payload: {
-                    id: "targetout",
-                    data: reply
-                }
-            }))
+        closure dispatcher(message) {
+            target.dispatcher(message.payload.data, function(reply) {
+                target.ws.send(window.JSON.stringify({
+                    guid: api.guid
+                    clientID: message.clientID
+                    serverID: message.serverID
+                    workerID: message.workerID
+                    respOp: "msgout"
+                    payload: {
+                        id: "targetout",
+                        data: reply
+                    }
+                }))
+            })
         }
 
         // onerror
@@ -334,6 +335,7 @@ closure TermHost(track, api, name, options, local target, nlregex, xtarg)
     //
     
     target.debugWrite = function(text, level) {
+        text = text.replaceAll("\n", "\r\n")
         if (level >= 5)
             target.term.WriteLn(text, target.term.Cyan)                     // cyan for builtin logging
         else if (level >= 4)
@@ -596,6 +598,7 @@ closure TermLocal(track, name, options, local target, nlregex, listener)
     //
     
     target.debugWrite = function(text, level) {
+        text = text.replaceAll("\n", "\r\n")
         if (level >= 5)
             target.term.WriteLn(text, target.term.Cyan)                     // cyan for builtin logging
         else if (level >= 4)

package/frameworks/node/http_request.nlg

@@ -0,0 +1,103 @@
+/*
+ * http_request.nlg
+ *
+ *     Http operations for NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024-2025 by Richard C. Zulch
+ *
+ */
+
+
+/* HttpRequest
+ *
+ * Make a simple http request. This is not supported by the standard NaaN framework HttpsApiRequest
+ * because it is not SSL.
+ *
+ * Options:
+ *  {
+ *      method:         <string>                // HTTP method (defaults to GET)
+ *      headers:        <dictionary>            // key/data pairs
+ *      putdata:        <data>                  // data to put
+ *      json:           <boolean>               // ignore content-type of return and decode JSON
+ *  }
+ *
+ * The return is a result tuple comprising:
+ *      (<error>,                               -- false on success
+ *      <content>,                              -- returned data
+ *      {
+ *          status:     <integer>               -- http response status
+ *          headers:    <array>                 -- headers if available
+ *      })
+ *
+ */
+
+closure HttpRequest(url, options,
+    local pending, chunks, params, request) {
+    global(js, nodeHttp)
+    pending = new(nonce)
+    chunks = []
+    params = {
+        method: options.method || undefined
+        headers: options.headers || undefined
+    }
+    request = nodeHttp.request(url, params, closure (response) {
+        response.on("data", function (chunk) {
+            chunks.push(chunk)
+        })
+        response.on("end", function (local content, error) {
+            content = js.g.Buffer.concat(chunks)
+            if response.statusCode < 200 || response.statusCode >= 299 {    // 299 is "cancelled"
+                debuglog("httpRequest status:", url, response.statusCode, "message:", content)
+                pending.signal(list(Error("httpRequest statusCode:", url, response.statusCode, {
+                    status: response.statusCode
+                }), false, { status: response.statusCode }))
+                return
+            }
+            if response.complete {
+                content = content.toString()
+                if options.json
+                    `(error, content) = JsonParse(content)
+            }
+            else
+                error = Error("httpRequest terminated prematurely:", url, response.statusCode)
+            pending.signal(list(error, content, {
+                headers: response.headers
+                status: response.statusCode
+                contentType: response.headers["content-type"]
+            }))
+        })
+    })
+    request.on("error", function hre(error) {
+        debuglog("httpRequest error:", url, "error:", error)
+        error = Error("httpRequest failed:", url, error)
+        pending.signal(list(error))
+    })
+    if options.putdata {
+        if string(options.putdata)
+            request.write(options.putdata)
+        else
+            request.write(JSONstringify(options.putdata))
+    }
+    request.end()
+    pending.wait()
+};
+
+
+/*
+ * http_nodeInit
+ *
+ * Initialize HTTP request operations for NodeJS.
+ *
+ */
+
+function http_nodeInit(local manifest) {
+    manifest = `(HttpRequest, http_nodeInit)
+
+    Naan.module.build(module.id, "http_request", function(modobj, compobj) {
+        require("./node.nlg")
+        compobj.manifest = manifest
+        modobj.exports.HttpRequest = HttpRequest
+    })
+}();

package/frameworks/node/https_request.nlg

@@ -47,7 +47,8 @@
  */
 
 closure httpsApiReqOnce(url, options,
-    local urlq, reqOptions, putdata, item, xurl, pending, chunks, request) {
+    local urlq, reqOptions, putdata, error, item, xurl, pending, chunks, request) {
+    global(js, nodeHttps, nodeUrl)
     if options.query
         urlq = url.concat(EncodeQuery("?", options.query))
     else
@@ -126,7 +127,10 @@ closure httpsApiReqOnce(url, options,
                     debuglog("HttpsApiRequest status:", url, response.statusCode, "message:", content)
                 pending.signal(list(Error("HttpsApiRequest statusCode:", url, response.statusCode, {
                     status: response.statusCode
-                }), false, { status: response.statusCode }))
+                }), false, {
+                    headers: response.headers
+                    status: response.statusCode
+                }))
                 return
             }
             if response.complete {
@@ -234,28 +238,56 @@ closure sseParserFactory(message, local parser) {
  *
  */
 
-closure HttpsApiRequest(url, options, local parser, decoder, result) {
+closure HttpsApiRequest(url, options,
+    local delayf, parser, decoder, error, content, status) {
+    global(stringDecoder, common)
+    //
+    // defRetrier -- backoff and retry selected errors
+    //
+    function defRetrier(error, headers, local retryAfter, retryms, waitms) {
+        if member(error.status, `(408, 500, 502, 503, 504)) {               // retriable errors
+            retryAfter = headers["retry-after"]
+            if retryAfter {
+                retryms = toint(retryAfter) * 1000
+                if !retryms
+                    retryms = Date(retryAfter).getTime() - Date.now()
+                if retryms < 0
+                    retryms = false
+            }
+            if !delayf
+                delayf = common.ExbackFactory(12,30)                        // 12s max backoff, 30s deadline
+            waitms = delayf()
+            if !waitms || waitms < retryms
+                waitms = retryms
+            if options.debug
+                debuglog("HttpsApiRequest.defRetrier: wait ${waitms} msec using retry-after ${retryAfter}")
+            if waitms                                                       // return false to give up
+                sleep(waitms)                                               // return true from sleep to retry
+        }
+    }
+
+    options = merge({ retrier: defRetrier }, options)
     if options.sseHook {
         parser = sseParserFactory(options.sseHook)
         decoder = xnew(stringDecoder.StringDecoder, "utf8")
-        options = merge(options, {
+        options = merge({
             streamHook: function(chunk) {
                 if chunk
                     parser.write(decoder.write(chunk))                      // data arrived
                 else
                     parser.write(decoder.end())                             // end of data
             }
-        })
+        }, options)
     }
     loop {
-        result = httpsApiReqOnce(url, options)
-        if result.0 && result.0.status {                                    // check for retries
-            if !options.retrier(result.0)
+        `(error, content, status) = httpsApiReqOnce(url, options)
+        if error && error.status {                                          // check for retries
+            if !options.retrier(error, status.headers)
                 break
         } else
             break
     }
-    result
+    list(error, content, status)
 };
 
 
@@ -271,6 +303,7 @@ function https_nodeInit(local manifest) {
 
     Naan.module.build(module.id, "https_request", function(modobj, compobj) {
         require("./node.nlg")
+        common = require("../common/utils.nlg")
         compobj.manifest = manifest
         modobj.exports.HttpsApiRequest = HttpsApiRequest
     })

package/frameworks/node/node.nlg

@@ -131,6 +131,7 @@ function nodrInit(local manifest) {
             os = js.r("os")
             stream = js.r("stream")
             nodeHttps = js.r("https")
+            nodeHttp = js.r("http")
             nodeUrl= js.r("url")
             threads = js.r("worker_threads")
             stringDecoder = js.r("string_decoder")

package/frameworks/node/worker.nlg

@@ -642,6 +642,7 @@ closure workController(api, local workco) {
                 name: config.name
                 workerID: workerID
             }) }
+        list(false, worker)
     }
 
     // wsReceive

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.4.3+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.4.4+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info

package/frameworks/running/executors.nlg

@@ -22,6 +22,7 @@ closure ExecutorBase(track, type, name, local exec) {
     exec = new(object, this)
     exec.type = type
     exec.name = name
+    exec.contexts = { }
     
     // execReset
     //
@@ -57,6 +58,18 @@ closure ExecutorBase(track, type, name, local exec) {
     // The specified message was received from the instance, so signal it.
     //
     exec.execReceived = closure execReceived(data, local pending) {
+        if data.reverse {
+            if !exec.contexts[data.contextID]
+                return                                                      // ignore unknown context
+            if !exec.reverseDispatch
+                exec.reverseDispatch = require("./taskexec.nlg").TaskDispatcher(function(data) {
+                    data.reverse = true
+                    exec.PostMessage(data)
+                }, exec.contexts[data.contextID].coder)
+            data.reverse = false
+            exec.reverseDispatch(data)
+            return
+        }
         pending = exec.pending[data.xid]
         if !pending {                                                       // not waiting on message
             if data.xmsg == "xready"
@@ -110,7 +123,7 @@ closure ExecutorBase(track, type, name, local exec) {
     // persistent reconnection. Currently this does not ever reclaim old contexts on the instance
     // end of things, but it should. ###
     //
-    exec.context = closure context(state, local error, retries, ms, delta, excon) {
+    exec.context = closure context(state, coder, local error, retries, ms, delta, excon) {
         retries = 0
         ms = milliseconds()
         delta = 1
@@ -127,8 +140,14 @@ closure ExecutorBase(track, type, name, local exec) {
             sleep(delta)
             if delta < 100
                 delta *= 2 }
-        excon = ExecutorContext(exec, state)
-        excon.register()
+        excon = ExecutorContext(exec, state, coder)
+        `(error) = excon.register()
+        if error
+            list(error)
+        else {
+            exec.contexts[excon.state.contextID] = excon
+            list(false, excon)
+        }
     }
 
     // attention
@@ -160,7 +179,7 @@ closure ExecutorBase(track, type, name, local exec) {
  *
  */
 
-closure ExecutorContext(exec, state, local context, coder, util) {
+closure ExecutorContext(exec, state, coder, local context, util) {
     global(apply)
     context = new(object, this)
     if !state || !state.contextID
@@ -168,7 +187,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
             contextID: UUID()
         }
     context.state = state
-    coder = packageCoder(evalf)
+    context.coder = coder || packageCoder(evalf)
     util = modlist().Util.exports
 
     // getState
@@ -208,7 +227,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
         pkg = util.pkgSave(`expr, roots, {
             nsignore: true
             comments: true
-            objectEncoder: coder.encodeObject
+            objectEncoder: context.coder.encodeObject
         })
         data = {
             contextID: state.contextID
@@ -224,7 +243,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
                 localroots: false
                 intern: "symbols"
                 execute: false
-                objectDecoder: coder.decodeObject
+                objectDecoder: context.coder.decodeObject
             })))
         else
             result