@naanlang/naan 1.4.3 → 1.5.0

package/frameworks/node/gitter.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2025 by Richard C. Zulch
  *
  */
 
@@ -56,7 +56,7 @@ closure Gitter(fs, local gitt) {
             asyncResult(callback, false, gitt.roots[repopath])
         execOptions = {
             cmdargs: ["rev-parse", "--show-toplevel"]
-            execops: {
+            spawnops: {
                 cwd: repopath
             }
             outLE: "\x00"                                                   // lines delimited by nulls
@@ -117,7 +117,7 @@ closure Gitter(fs, local gitt) {
         repopath = fs.path.resolve(fs.rootpath, repopath)
         execOptions = {
             cmdargs: ["log", "-z", "--pretty=format:%H|%an|%ae|%aI|%B"]
-            execops: {
+            spawnops: {
                 cwd: repopath
             }
             outLE: "\x00"                                                   // lines delimited by nulls
@@ -168,10 +168,12 @@ closure Gitter(fs, local gitt) {
         if !string(repopath)
             return (asyncResult(callback, Error("invalid path:", repopath)))
         repopath = fs.path.resolve(fs.rootpath, repopath)
-        bridge = bridgePath(repopath)
+		bridge = fs.path.normalize(bridgePath(repopath))                    // ensure it works for our FS
+		if bridge == "."
+		    bridge = ""                                                     // no bridge in this case
         execOptions = {
             cmdargs: ["status", "-z", "--porcelain"]
-            execops: {
+            spawnops: {
                 cwd: repopath
             }
             outLE: "\x00"                                                   // lines delimited by nulls
@@ -219,13 +221,13 @@ closure Gitter(fs, local gitt) {
         repopath = fs.path.resolve(fs.rootpath, repopath)
         execOptions = {
             cmdargs: ["show", options.hash.concat(":", bridgePath(repopath), gitPath(options.pathspec))]
-            execops: {
+            spawnops: {
                 cwd: repopath
             }
         }
         nodecp.execFile("git",
             execOptions.cmdargs,
-            execOptions.execops,
+            execOptions.spawnops,
             function(err, stdout, stderr) {
                 if err
                     callback(Error("git failed", err, stderr))

package/frameworks/node/http_request.nlg

@@ -0,0 +1,103 @@
+/*
+ * http_request.nlg
+ *
+ *     Http operations for NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024-2025 by Richard C. Zulch
+ *
+ */
+
+
+/* HttpRequest
+ *
+ * Make a simple http request. This is not supported by the standard NaaN framework HttpsApiRequest
+ * because it is not SSL.
+ *
+ * Options:
+ *  {
+ *      method:         <string>                // HTTP method (defaults to GET)
+ *      headers:        <dictionary>            // key/data pairs
+ *      putdata:        <data>                  // data to put
+ *      json:           <boolean>               // ignore content-type of return and decode JSON
+ *  }
+ *
+ * The return is a result tuple comprising:
+ *      (<error>,                               -- false on success
+ *      <content>,                              -- returned data
+ *      {
+ *          status:     <integer>               -- http response status
+ *          headers:    <array>                 -- headers if available
+ *      })
+ *
+ */
+
+closure HttpRequest(url, options,
+    local pending, chunks, params, request) {
+    global(js, nodeHttp)
+    pending = new(nonce)
+    chunks = []
+    params = {
+        method: options.method || undefined
+        headers: options.headers || undefined
+    }
+    request = nodeHttp.request(url, params, closure (response) {
+        response.on("data", function (chunk) {
+            chunks.push(chunk)
+        })
+        response.on("end", function (local content, error) {
+            content = js.g.Buffer.concat(chunks)
+            if response.statusCode < 200 || response.statusCode >= 299 {    // 299 is "cancelled"
+                debuglog("httpRequest status:", url, response.statusCode, "message:", content)
+                pending.signal(list(Error("httpRequest statusCode:", url, response.statusCode, {
+                    status: response.statusCode
+                }), false, { status: response.statusCode }))
+                return
+            }
+            if response.complete {
+                content = content.toString()
+                if options.json
+                    `(error, content) = JsonParse(content)
+            }
+            else
+                error = Error("httpRequest terminated prematurely:", url, response.statusCode)
+            pending.signal(list(error, content, {
+                headers: response.headers
+                status: response.statusCode
+                contentType: response.headers["content-type"]
+            }))
+        })
+    })
+    request.on("error", function hre(error) {
+        debuglog("httpRequest error:", url, "error:", error)
+        error = Error("httpRequest failed:", url, error)
+        pending.signal(list(error))
+    })
+    if options.putdata {
+        if string(options.putdata)
+            request.write(options.putdata)
+        else
+            request.write(JSONstringify(options.putdata))
+    }
+    request.end()
+    pending.wait()
+};
+
+
+/*
+ * http_nodeInit
+ *
+ * Initialize HTTP request operations for NodeJS.
+ *
+ */
+
+function http_nodeInit(local manifest) {
+    manifest = `(HttpRequest, http_nodeInit)
+
+    Naan.module.build(module.id, "http_request", function(modobj, compobj) {
+        require("./node.nlg")
+        compobj.manifest = manifest
+        modobj.exports.HttpRequest = HttpRequest
+    })
+}();

package/frameworks/node/https_request.nlg

@@ -47,7 +47,8 @@
  */
 
 closure httpsApiReqOnce(url, options,
-    local urlq, reqOptions, putdata, item, xurl, pending, chunks, request) {
+    local urlq, reqOptions, putdata, error, item, xurl, pending, chunks, request) {
+    global(js, nodeHttps, nodeUrl)
     if options.query
         urlq = url.concat(EncodeQuery("?", options.query))
     else
@@ -126,7 +127,10 @@ closure httpsApiReqOnce(url, options,
                     debuglog("HttpsApiRequest status:", url, response.statusCode, "message:", content)
                 pending.signal(list(Error("HttpsApiRequest statusCode:", url, response.statusCode, {
                     status: response.statusCode
-                }), false, { status: response.statusCode }))
+                }), false, {
+                    headers: response.headers
+                    status: response.statusCode
+                }))
                 return
             }
             if response.complete {
@@ -234,28 +238,56 @@ closure sseParserFactory(message, local parser) {
  *
  */
 
-closure HttpsApiRequest(url, options, local parser, decoder, result) {
+closure HttpsApiRequest(url, options,
+    local delayf, parser, decoder, error, content, status) {
+    global(stringDecoder, common)
+    //
+    // defRetrier -- backoff and retry selected errors
+    //
+    function defRetrier(error, headers, local retryAfter, retryms, waitms) {
+        if member(error.status, `(408, 500, 502, 503, 504)) {               // retriable errors
+            retryAfter = headers["retry-after"]
+            if retryAfter {
+                retryms = toint(retryAfter) * 1000
+                if !retryms
+                    retryms = Date(retryAfter).getTime() - Date.now()
+                if retryms < 0
+                    retryms = false
+            }
+            if !delayf
+                delayf = common.ExbackFactory(12,30)                        // 12s max backoff, 30s deadline
+            waitms = delayf()
+            if !waitms || waitms < retryms
+                waitms = retryms
+            if options.debug
+                debuglog("HttpsApiRequest.defRetrier: wait ${waitms} msec using retry-after ${retryAfter}")
+            if waitms                                                       // return false to give up
+                sleep(waitms)                                               // return true from sleep to retry
+        }
+    }
+
+    options = merge({ retrier: defRetrier }, options)
     if options.sseHook {
         parser = sseParserFactory(options.sseHook)
         decoder = xnew(stringDecoder.StringDecoder, "utf8")
-        options = merge(options, {
+        options = merge({
             streamHook: function(chunk) {
                 if chunk
                     parser.write(decoder.write(chunk))                      // data arrived
                 else
                     parser.write(decoder.end())                             // end of data
             }
-        })
+        }, options)
     }
     loop {
-        result = httpsApiReqOnce(url, options)
-        if result.0 && result.0.status {                                    // check for retries
-            if !options.retrier(result.0)
+        `(error, content, status) = httpsApiReqOnce(url, options)
+        if error && error.status {                                          // check for retries
+            if !options.retrier(error, status.headers)
                 break
         } else
             break
     }
-    result
+    list(error, content, status)
 };
 
 
@@ -271,6 +303,7 @@ function https_nodeInit(local manifest) {
 
     Naan.module.build(module.id, "https_request", function(modobj, compobj) {
         require("./node.nlg")
+        common = require("../common/utils.nlg")
         compobj.manifest = manifest
         modobj.exports.HttpsApiRequest = HttpsApiRequest
     })

package/frameworks/node/node.nlg

@@ -131,6 +131,7 @@ function nodrInit(local manifest) {
             os = js.r("os")
             stream = js.r("stream")
             nodeHttps = js.r("https")
+            nodeHttp = js.r("http")
             nodeUrl= js.r("url")
             threads = js.r("worker_threads")
             stringDecoder = js.r("string_decoder")

package/frameworks/node/shell.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2024 by Richard C. Zulch
+ * Copyright (c) 2024-2025 by Richard C. Zulch
  *
  */
 
@@ -68,9 +68,9 @@ closure ShellConnect(options, local shob, error, fs, xparser, shelly) {
             }
         }
         if cwd
-            xlparams.execops = { cwd: cwd }
+            xlparams.spawnops = { cwd: cwd }
         else if options.cwd
-            xlparams.execops = { cwd: options.cwd }
+            xlparams.spawnops = { cwd: options.cwd }
         fs.execLines(cmd, xlparams, function(error, result) {
         	evalactive(list(`assign, `(compress, "$$"), report))            // assign to $$ in active namespace
             pending.signal(list(error, merge(result, report)))
@@ -80,7 +80,7 @@ closure ShellConnect(options, local shob, error, fs, xparser, shelly) {
 
     // shellparser
     // intercept command-line parsing and look for a !shell escape
-    function shellparser(source, grammar, operators, local token) {
+    function shellparser(source, grammar, operators, local token, text) {
         token = source.tokenread()
         if token.atom == `! {
             source.tokenlast(true)                                          // "consume" the ! token

package/frameworks/node/worker.nlg

@@ -642,6 +642,7 @@ closure workController(api, local workco) {
                 name: config.name
                 workerID: workerID
             }) }
+        list(false, worker)
     }
 
     // wsReceive

package/frameworks/project/projects.nlg

@@ -296,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.4.3+1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.5.0+1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -692,6 +692,7 @@ closure projConfig(projman, where, projID, local procon, fs, configpath) {
             erase: true
             overwrite: true
             follow: true
+            cacheControl: publish.cacheControlFiles
         })
     }
 

package/frameworks/running/executors.nlg

@@ -22,6 +22,7 @@ closure ExecutorBase(track, type, name, local exec) {
     exec = new(object, this)
     exec.type = type
     exec.name = name
+    exec.contexts = { }
     
     // execReset
     //
@@ -57,6 +58,18 @@ closure ExecutorBase(track, type, name, local exec) {
     // The specified message was received from the instance, so signal it.
     //
     exec.execReceived = closure execReceived(data, local pending) {
+        if data.reverse {
+            if !exec.contexts[data.contextID]
+                return                                                      // ignore unknown context
+            if !exec.reverseDispatch
+                exec.reverseDispatch = require("./taskexec.nlg").TaskDispatcher(function(data) {
+                    data.reverse = true
+                    exec.PostMessage(data)
+                }, exec.contexts[data.contextID].coder)
+            data.reverse = false
+            exec.reverseDispatch(data)
+            return
+        }
         pending = exec.pending[data.xid]
         if !pending {                                                       // not waiting on message
             if data.xmsg == "xready"
@@ -110,7 +123,7 @@ closure ExecutorBase(track, type, name, local exec) {
     // persistent reconnection. Currently this does not ever reclaim old contexts on the instance
     // end of things, but it should. ###
     //
-    exec.context = closure context(state, local error, retries, ms, delta, excon) {
+    exec.context = closure context(state, coder, local error, retries, ms, delta, excon) {
         retries = 0
         ms = milliseconds()
         delta = 1
@@ -127,8 +140,14 @@ closure ExecutorBase(track, type, name, local exec) {
             sleep(delta)
             if delta < 100
                 delta *= 2 }
-        excon = ExecutorContext(exec, state)
-        excon.register()
+        excon = ExecutorContext(exec, state, coder)
+        `(error) = excon.register()
+        if error
+            list(error)
+        else {
+            exec.contexts[excon.state.contextID] = excon
+            list(false, excon)
+        }
     }
 
     // attention
@@ -160,7 +179,7 @@ closure ExecutorBase(track, type, name, local exec) {
  *
  */
 
-closure ExecutorContext(exec, state, local context, coder, util) {
+closure ExecutorContext(exec, state, coder, local context, util) {
     global(apply)
     context = new(object, this)
     if !state || !state.contextID
@@ -168,7 +187,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
             contextID: UUID()
         }
     context.state = state
-    coder = packageCoder(evalf)
+    context.coder = coder || packageCoder(evalf)
     util = modlist().Util.exports
 
     // getState
@@ -208,7 +227,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
         pkg = util.pkgSave(`expr, roots, {
             nsignore: true
             comments: true
-            objectEncoder: coder.encodeObject
+            objectEncoder: context.coder.encodeObject
         })
         data = {
             contextID: state.contextID
@@ -224,7 +243,7 @@ closure ExecutorContext(exec, state, local context, coder, util) {
                 localroots: false
                 intern: "symbols"
                 execute: false
-                objectDecoder: coder.decodeObject
+                objectDecoder: context.coder.decodeObject
             })))
         else
             result

package/frameworks/running/taskexec.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2024 by Richard C. Zulch
+ * Copyright (c) 2021-2025 by Richard C. Zulch
  *
  */
 
@@ -19,27 +19,57 @@
  *
  */
 
-closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
+closure TaskDispatcher(replier, coder, local contexts, gohome, util) {
     global()
     contexts = { }
     gohome = makeActivator()
-    coder = packageCoder(evalf)
     util = modlist().Util.exports
-
-    // sendEval
+    
+    // ContextCoder
     //
-    // Send an evaluation request to the original execution context.
+    // A context-specific coder allowing remote-initiated communication.
     //
-    closure evalf(expr) {
-        debuglog("TaskDispatcher doesn't yet implement reverse proxy objects")
-        list(Error("### reverse object proxy not implemented"))
+    closure ContextCoder(contextID, _guid, dynreply, local exec, error, context) {
+        contexts[contextID].coder = packageCoder(closure evalf(expr) {
+            if !exec {
+                exec = require("./executors.nlg").ExecutorBase(false, "reverser", "TD")
+                exec.PostMessage = closure(data) {
+                    if data.xmsg == "xstart" {                              // report ready locally
+                        data.xmsg = "xready"
+                        exec.execReceived(data)
+                    } else {
+                        data.reverse = true
+                        data._guid = _guid || undefined
+                        dynreply(data)
+                    }
+                    list(false)
+                }
+                contexts[contextID].execReceived = exec.execReceived        // needed for registered reply
+                `(error, context) = exec.context({
+                    contextID: contextID
+                }, contexts[contextID].coder)
+                if error {
+                    ErrorDebuglog("TaskDispatcher.ContextCoder failed:", error)
+                    exec = false
+                    return (list(error))
+                }
+            }
+            context.eval(expr)
+        })
     }
 
     // dispatcher
     //
     // Return a new task dispatcher.
     //
-    closure dispatcher(data, local reply, restorens, context, expr, roots, proc, args, result) {
+    closure dispatcher(data, dynreply, local reply, restorens, context, expr, roots, proc, args, result) {
+        if data.reverse {
+            data.reverse = false
+            contexts[data.contextID].execReceived(data)
+            return
+        }
+        if !dynreply
+            dynreply = replier
         reply = {
             xmsg: "error"
             xid: data.xid
@@ -62,6 +92,10 @@ closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
                     namespace: nsactive()
                     activator: gohome
                 }
+                if coder
+                    context.coder = coder
+                else
+                    ContextCoder(data.contextID, data._guid, dynreply)
                 restorens()
                 reply.xmsg = "registered"
                 reply.contextID = data.contextID
@@ -85,11 +119,11 @@ closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
                         intern: true                                        // intern symbols
                         localroots: true                                    // make roots local if needed to intern
                         execute: true                                       // execute auto-running procedures
-                        objectDecoder: coder.decodeObject
+                        objectDecoder: context.coder.decodeObject
                     })
                     expr = caar(expr)
                     if expr.0 == `\.invoke\-method
-                        expr = coder.invokeMethod(expr.1, expr.2)           // object method, args
+                        expr = context.coder.invokeMethod(expr.1, expr.2)   // object method, args
                     else
                         expr = evalactive(expr)                             // evaluate expression
                     reply.error = undefined
@@ -104,7 +138,7 @@ closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
                     reply.result = util.pkgSave(`expr, roots, {
                         nsignore: true                                      // don't report namespace errors
                         comments: true
-                        objectEncoder: coder.encodeObject})                 // include comments
+                        objectEncoder: context.coder.encodeObject})         // include comments
                     if context.namespace !== nsactive().0
                     {                                                       // evaluation changed namespace
                         context.namespace = nsactive().0
@@ -154,7 +188,7 @@ closure TaskDispatcher(replier, local contexts, gohome, coder, util) {
         } finally {
             restorens()
         }
-        replier(reply)
+        dynreply(reply)
     }
 };
 

package/frameworks/storage/dbt_pouch.nlg

@@ -7,7 +7,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2017-2024 by Richard C. Zulch
+ * Copyright (c) 2017-2025 by Richard C. Zulch
  *
  */
 
@@ -76,21 +76,31 @@ closure dapoTable(database, rootpath, local table, prefix) {
 
     // add
     //
-    // Add a new record to the table with optional metadata. This will fail if the key already
-    // exists. Returns the new document record.
-
-    table.add = closure add(key, content, metadata, callback, local id, doc) {
+    // Add a new record to the table with options. This will fail if the key already exists. Optional
+    // metadata is stored with the record and cannot be changed without calling update. Metadata keys
+    // cannot begin with underscore ("_").
+    //
+    // The returned document, which can be used with update, has the following keys:
+    //      key         - the key name of the record
+    //      content     - the body content stored under in the record
+    //      _md5        - the MD5 hash of the content
+    //      _length     - the length of the content
+    //      _id         - the PouchDB ID
+    //      _rev        - the PouchDB revision
+    //      <...>       - various metadata keys
+
+    table.add = closure add(key, content, options, callback, local id, doc) {
         if !callback
-            return (syncAdapter(add, key, content, metadata))
+            return (syncAdapter(add, key, content, options))
         id = makeID(key)
         if !string(id)
             return (asyncResult(callback, id))                              // error from makeID
-        if metadata && !dictionary(metadata)
+        if  options.metadata && !dictionary(options.metadata)
             return (asyncResult(callback, Error("invalid arguments")))
         if !database.pouch
             return (asyncResult(callback, Error("database closed")))
-        if metadata {
-            doc = new(metadata)                                             // start with metadata
+        if options.metadata {
+            doc = new(options.metadata)                                     // start with metadata
             doc._rev = undefined }                                          // _rev not allowed in metadata
         else
             doc = { }
@@ -115,12 +125,13 @@ closure dapoTable(database, rootpath, local table, prefix) {
 
     // update
     //
-    // Update a record in the table using the document previously returned by add or get.
-    // This will fail if the record has been modified by someone else in the meantime.
+    // Update a record in the table using the document previously returned by add or get. Specify
+    // options to update those values, otherwise existing/default values are used. This will fail
+    // if the record has been modified by someone else in the meantime.
 
-    table.update = closure update(doc, callback) {
+    table.update = closure update(doc, options, callback) {
         if !callback
-            return (syncAdapter(update, doc))
+            return (syncAdapter(update, doc, options))
         if !doc._id || !doc._id.startsWith(table.prefix) || !doc._rev
             return (asyncResult(callback, Error("invalid argument")))
         if makeID(doc.key) != doc._id
@@ -130,6 +141,8 @@ closure dapoTable(database, rootpath, local table, prefix) {
         doc = new(doc)
         doc._md5 = undefined
         doc._length = undefined
+        if options.metadata
+            doc = merge(doc, options.metadata)
         database.pouch.put(doc, function(error, resp) {
             if error
                 error = Error("update failed", error)