@naanlang/naan 1.5.0 → 1.7.0

package/lib/node_worker.js

@@ -0,0 +1,36 @@
+/*
+ * node_worker.js
+ * naanide
+ *
+ *     Start a Naanide worker.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2017-2026 by Richard C. Zulch
+ *
+ */
+
+"use strict";
+
+
+/*
+ * Begin
+ *
+ */
+
+var workerData = require('node:worker_threads').workerData || { };          // passed from creator
+var jspath = require("path");
+var naanpath = jspath.resolve(require.resolve('@naanlang/naan'), '../../../');
+var envpath = jspath.resolve(naanpath, 'lib/env_nodeworker.js');
+var naanenv = require(envpath);
+
+naanenv.NaanWorkerActivate(function(naanworker) {
+    naanworker.setRequire(require);                                         // require relative to this file
+    naanworker.setImport(function(m) { return (import(m)); });              // import relative to this file
+    
+    naanworker.textCommand(""
+        + `App.naanpath = '${ naanpath.replace(/\\/g, "\\\\") }';;`         // path/to/our naan library
+        + "App.rootpath = js.d;;"                                           // path/to/our package.json
+        + "Naan.module.defineLoc('naanlib', App.naanpath);;"                // defines "naanlib:" prefix for require
+        + "Naan.module.nodeparse('lib/node_worker_init.nlg');;\n");         // Naan initialization
+});

package/lib/node_worker_init.nlg

@@ -0,0 +1,43 @@
+/*
+ * nide_worker_init.nlg
+ * NaanIDE
+ *
+ * Loaded automatically for Node.js before REPL turned over to the user.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2020-2026 by Richard C. Zulch
+ *
+ */
+
+
+loglevel(3);
+chns("Start");
+
+
+/*
+ * Debugging & Tools
+ *
+ */
+
+function init(local metadata) {
+    sprint()                                                                // no interruptions
+    if !module.owner.list.running.components.debugnub {
+        require("naanlib:frameworks/common/repltools.nlg").ImportReplTools()
+        require("naanlib:frameworks/running/debugnub.nlg").DebugNub()
+    }
+    App.shell = require("naanlib:frameworks/node/shell.nlg").ShellConnect({
+        cwd: js.d
+    })
+    metadata = new(js.t.metadata())
+    App.mainInstanceID = metadata.mainInstanceID
+    App.imp = require("naanlib:frameworks/service/imp.nlg").Imp({
+        name: metadata.name
+        type: "node-worker"
+    })
+    require("naanlib:frameworks/service/nubnode.nlg").NodeWorkerNub(App.imp, {
+        name: App.imp.us.name
+    })
+    if metadata.maintext
+        textparse(metadata.maintext)
+}();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@naanlang/naan",
-    "version": "1.5.0",
+    "version": "1.7.0",
     "author": "Richard C. Zulch",
     "description": "Naan™ software platform",
     "main": "./lib/core/naanlib.js",

package/plugins/openSearch/openSearch.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2026 by Richard C. Zulch
  *
  */
 
@@ -53,6 +53,7 @@ closure openSearchIndex(prosc, indexName, ioptions, local osin) {
     osin = new(object, this)
     if symbol(indexName)
         indexName = indexName.tostring()
+    osin.name = indexName
     osin.analyzers = {                                                      // map names, language to analyzer
         english: "english"
         hashtag: "hashtag"
@@ -335,7 +336,7 @@ closure openSearchIndex(prosc, indexName, ioptions, local osin) {
                 `(error, data) = prosc.request(indexName.concat("/_bulk"), {
                     method: "POST"
                     putdata: docs
-                    contentType: "application/json"
+                    contentType: "application/x-ndjson"
                 })
             if data
                 data = data.items.map(function(item) {

package/plugins/openSearch/os_dynamo.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023-2025 by Richard C. Zulch
+ * Copyright (c) 2023-2026 by Richard C. Zulch
  *
  */
 
@@ -41,6 +41,7 @@
 
 closure OSDynIndexer(index, table, options,
     local dbOptions, scanPending, indexPending, updatePending, output, error, records, data, ids) {
+    global()
     if !options.converter
         return (Error("OSDynIndexer: options.converter is required"))
     dbOptions = {
@@ -78,7 +79,7 @@ closure OSDynIndexer(index, table, options,
     //
     // Batch index documents into OpenSearch, returning a nonce signaled with ids array.
     //
-    closure batchIndex(records, local pending) {
+    closure batchIndex(records, local pending, docs) {
         pending = new(nonce)
         docs = asyncArray(records, 10, closure(record) {
             options.converter(record)

package/plugins/serviceAws/aws_cloudwatchlogs.nlg

@@ -18,7 +18,7 @@
  *
  */
 
-closure CloudWatchLogs(local cwlogs) {
+closure CloudWatchLogs(local cwlogs, delayf, backoffMS) {
     global(awsSDK)
     cwlogs = new(object, this)
 
@@ -35,7 +35,42 @@ closure CloudWatchLogs(local cwlogs) {
         })
         list(false, { ok: true })
     }
+
+    // retrier
+    //
+    //     Perform exponential backoff when throttled, returning true for retry or false otherwise.
+    //
+    function retrier(error, local waitms, newbo) {
+        if !error
+            delayf = false
+        else if error.name == "ThrottlingException" {                       // retriable errors
+            if backoffMS > milliseconds() {
+                debuglog("DynaTable.retrier: existing backoff", backoffMS - milliseconds(), "msec")
+                return (true)                                               // use existing backoff
+            }
+            if !delayf
+                delayf = ExbackFactory(6,60)                                // 6s max backoff, 60s deadline
+            waitms = delayf()
+            debuglog("DynaTable.retrier: new backoff:", waitms, "msec")
+            if waitms {                                                     // returns false for no retry
+                newbo = milliseconds() + waitms                             // tick when proceeding is allowed
+                if !(backoffMS > newbo)
+                    backoffMS = newbo
+                true
+            }
+        }
+    }
     
+    // delayer
+    //
+    //     Delay until the current backoff is complete.
+    //
+    function delayer(local waitms) {
+        waitms = backoffMS - milliseconds()
+        if waitms > 0
+            sleep(waitms + waitms * Math.random()/2)
+    }
+
     //
     // cwlQueryPager
     //
@@ -46,7 +81,14 @@ closure CloudWatchLogs(local cwlogs) {
         if !remaining
             remaining = 10                                              // default retrieve limit
         params.limit = remaining
-        QueryPager(cwlogs.aws, procname, params, function(data, local processed) {
+        delayer()
+        QueryPager(cwlogs.aws, procname, params, function(error, data, local processed) {
+            if error {
+                if !retrier(error)
+                    return (false)
+                delayer()
+                return (true)
+            }
             processed = doneCB(data)
             if processed
                 remaining -= processed

package/plugins/serviceAws/aws_dynamo.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2024 by Richard C. Zulch
+ * Copyright (c) 2021-2026 by Richard C. Zulch
  *
  */
 
@@ -562,7 +562,7 @@ closure DynaConverter(local conv) {
  *
  */
 
-closure DynaTable(dyna, tablename, options, local table) {
+closure DynaTable(dyna, tablename, options, local table, delayf, backoffMS) {
     global()
     table = new(object, this)
     table.name = tablename
@@ -577,6 +577,38 @@ closure DynaTable(dyna, tablename, options, local table) {
             debuglog("DynaTable: invalid (hashKey/rangeKey) option:", tablename, list(options.hashKey, options.rangeKey))
     }
 
+    // retrier
+    //
+    //     Perform exponential backoff when throttled, returning true for retry or false otherwise.
+    //
+    function retrier(error, local waitms, newbo) {
+        if !error
+            delayf = false
+        else if error.name == "ThrottlingException" {                       // retriable errors
+            if backoffMS > milliseconds()
+                return (true)                                               // use existing backoff
+            if !delayf
+                delayf = ExbackFactory(6,60)                                // 6s max backoff, 60s deadline
+            waitms = delayf()
+            if waitms {                                                     // returns false for no retry
+                newbo = milliseconds() + waitms                             // tick when proceeding is allowed
+                if !(backoffMS > newbo)
+                    backoffMS = newbo
+                true
+            }
+        }
+    }
+    
+    // delayer
+    //
+    //     Delay until the current backoff is complete.
+    //
+    function delayer(local waitms) {
+        waitms = backoffMS - milliseconds()
+        if waitms > 0
+            sleep(waitms + waitms * Math.random()/2)
+    }
+
     // info
     //
     //     Return info about the table, or an error, in standard (error, dictionary) tuple format.
@@ -694,16 +726,24 @@ closure DynaTable(dyna, tablename, options, local table) {
                 KeyType: "RANGE"
             })
         }
-        pending = new(nonce)
-        dyna.aws.createTable(params, function(error, data) {
-            if error {
-                dynaTableError = error
-                debuglog("aws.createTable failed", tablename, ErrorString(error))
-                error = Error("table.create(".concat(tablename, ") failed"), error)
-            }
-            pending.signal(list(error, data))
-        })
-        `(error, data) = pending.wait()
+
+        loop {
+            pending = new(nonce)
+            delayer()
+            dyna.aws.createTable(params, function(error, data) {
+                if error {
+                    dynaTableError = error
+                    if error.name != "ThrottlingException"
+                        debuglog("aws.createTable failed", tablename, ErrorString(error))
+                    error = Error("table.create(".concat(tablename, ") failed"), error)
+                }
+                pending.signal(list(error, data))
+            })
+            `(error, data) = pending.wait()
+            if !retrier(error)
+                break
+        }
+
         if error
             return(list(Error("aws.createTable failed", error)))
         start = milliseconds()
@@ -764,7 +804,8 @@ closure DynaTable(dyna, tablename, options, local table) {
     // The result is a standard result tuple `(error, record).
 
     //
-    table.getRecord = closure getRecord(hashValue, options, local params, rangeValue, pending) {
+    table.getRecord = closure getRecord(hashValue, options,
+        local params, rangeValue, pending, error, record) {
         if preflightError(this)
             return
         params = {
@@ -776,17 +817,25 @@ closure DynaTable(dyna, tablename, options, local table) {
         } else
             rangeValue = options
         params.Key = makeDynKey(list(hashValue, rangeValue))
-        pending = new(nonce)
-        dyna.aws.getItem(params, function(error, data) {
-            if error {
-                dynaTableError = error
-                debuglog("aws.getItem failed", tablename, ErrorString(error))
-                error = Error("table.getRecord(".concat(tablename, ") failed"), error)
-            } else
-                data = dyna.conv.recordDynToNaan(data.Item)
-            pending.signal(list(error, data))
-        })
-        pending.wait()
+
+        loop {
+            pending = new(nonce)
+            delayer()
+            dyna.aws.getItem(params, function(error, data) {
+                if error {
+                    dynaTableError = error
+                    if error.name != "ThrottlingException"
+                        debuglog("aws.getItem failed", tablename, ErrorString(error))
+                    error = Error("table.getRecord(".concat(tablename, ") failed"), error)
+                } else
+                    data = dyna.conv.recordDynToNaan(data.Item)
+                pending.signal(list(error, data))
+            })
+            `(error, record) = pending.wait()
+            if !retrier(error)
+                break
+        }
+        list(error, record)
     }
     
     //
@@ -794,23 +843,32 @@ closure DynaTable(dyna, tablename, options, local table) {
     //
     //     Put a record in the table, replacing any existing record with the same key.
         
-    table.putRecord = closure putRecord(record, local params, pending) {
+    table.putRecord = closure putRecord(record,
+        local params, pending, error) {
         if preflightError(this)
             return
         params = {
             TableName: tablename
             Item: dyna.conv.recordNaanToDyn(record)
         }
-        pending = new(nonce)
-        dyna.aws.putItem(params, function(error, data) {
-            if error {
-                dynaTableError = error
-                debuglog("aws.putItem failed", tablename, ErrorString(error))
-                error = Error("table.putRecord(".concat(tablename, ") failed"), error)
-            }
-            pending.signal(list(error, data))
-        })
-        pending.wait()
+
+        loop {
+            pending = new(nonce)
+            delayer()
+            dyna.aws.putItem(params, function(error, data) {
+                if error {
+                    dynaTableError = error
+                    if error.name != "ThrottlingException"
+                        debuglog("aws.putItem failed", tablename, ErrorString(error))
+                    error = Error("table.putRecord(".concat(tablename, ") failed"), error)
+                }
+                pending.signal(list(error, data))
+            })
+            `(error, record) = pending.wait()
+            if !retrier(error)
+                break
+        }
+        list(error, record)
     }
 
     //
@@ -831,7 +889,7 @@ closure DynaTable(dyna, tablename, options, local table) {
     // The result is a standard result tuple `(error, record).
         
     table.updateRecord = closure updateRecord(hashValue, rangeValue, options,
-        local error, params, pending) {
+        local error, params, pending, record) {
         if preflightError(this)
             return
         params = {
@@ -856,18 +914,25 @@ closure DynaTable(dyna, tablename, options, local table) {
             if !params.ReturnValues
                 params.ReturnValues = "NONE"
         }
-        pending = new(nonce)
-        dyna.aws.updateItem(params, function(error, data) {
-            if error {
-                dynaTableError = error
-                debuglog("aws.updateItem failed", tablename, ErrorString(error), Dialect.print(params))
-                error = Error("table.updateRecord(".concat(tablename, ") failed"), error)
-            }
-            else
-                data = dyna.conv.recordDynToNaan(data.Attributes)
-            pending.signal(list(error, data))
-        })
-        pending.wait()
+        loop {
+            pending = new(nonce)
+            delayer()
+            dyna.aws.updateItem(params, function(error, data) {
+                if error {
+                    dynaTableError = error
+                    if error.name != "ThrottlingException"
+                        debuglog("aws.updateItem failed", tablename, ErrorString(error), Dialect.print(params))
+                    error = Error("table.updateRecord(".concat(tablename, ") failed"), error)
+                }
+                else
+                    data = dyna.conv.recordDynToNaan(data.Attributes)
+                pending.signal(list(error, data))
+            })
+            `(error, record) = pending.wait()
+            if !retrier(error)
+                break
+        }
+        list(error, record)
     }
     
     //
@@ -881,7 +946,7 @@ closure DynaTable(dyna, tablename, options, local table) {
  
         
     table.deleteRecord = closure deleteRecord(hashValue, rangeValue, options,
-        local params, error, pending) {
+        local params, error, pending, record) {
         if preflightError(this)
             return
         params = {
@@ -902,16 +967,25 @@ closure DynaTable(dyna, tablename, options, local table) {
             if !params.ReturnValues
                 params.ReturnValues = "NONE"
         }
-        pending = new(nonce)
-        dyna.aws.deleteItem(params, function(error, data) {
-            if error {
-                dynaTableError = error
-                debuglog("aws.deleteItem failed", tablename, ErrorString(error))
-                error = Error("table.deleteRecord(".concat(tablename, ") failed"), error)
-            }
-            pending.signal(list(error, data))
-        })
-        pending.wait()
+
+        loop {
+            pending = new(nonce)
+            delayer()
+            dyna.aws.deleteItem(params, function(error, data) {
+                if error {
+                    dynaTableError = error
+                    if error.name != "ThrottlingException"
+                        debuglog("aws.deleteItem failed", tablename, ErrorString(error))
+                    error = Error("table.deleteRecord(".concat(tablename, ") failed"), error)
+                }
+                pending.signal(list(error, data))
+            })
+            pending.wait()
+            `(error, record) = pending.wait()
+            if !retrier(error)
+                break
+        }
+        list(error, record)
     }
 
     //
@@ -922,16 +996,23 @@ closure DynaTable(dyna, tablename, options, local table) {
     //
     closure dynQueryPager(procname, params, remaining, doneCB) {
         if !remaining
-            remaining = 10                                              // default retrieve limit
+            remaining = 10                                                  // default retrieve limit
         params.Limit = remaining
-        QueryPager(dyna.aws, procname, params, function(data, local processed) {
+        delayer()
+        QueryPager(dyna.aws, procname, params, function(error, data, local processed) {
+            if error {
+                if !retrier(error)
+                    return (false)
+                delayer()
+                return (true)
+            }
             processed = doneCB(data)
             if processed
                 remaining -= processed
             params.ExclusiveStartKey = data.LastEvaluatedKey
             if processed && remaining > 0 && data.LastEvaluatedKey {
-                params.Limit = remaining                                // set up for next iteration
-                true                                                    // continue looping
+                params.Limit = remaining                                    // set up for next iteration
+                true                                                        // continue looping
             }
         })
     }

package/plugins/serviceAws/aws_utils.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2025 by Richard C. Zulch
+ * Copyright (c) 2021-2026 by Richard C. Zulch
  *
  */
 
@@ -27,19 +27,14 @@ closure queryPager(apiobject, procname, params, xargs, doneCB, local pending, lo
         return (list(Error("queryPager method undefined:", procname)))
     pending = new(nonce)
     looping = true
-    while looping {                                                         // loop for multiple pages
+    while looping {                                                         // loop for multiple pages or retry
         pending.reset()
         xapply(apiobject, procname, append(cons(params, xargs), cons(function (error, data) {
-            if error {
-                qpPagerError = error
-                debuglog(procname, "failed", error)
-                looping = false
-                pending.signal(list(error))
-            }
-            else {
-                looping = doneCB(data)
+            looping = doneCB(error, data)
+            if looping || !error
                 pending.signal(list(false, {ok: true}))
-            }
+            else
+                pending.signal(list(error))
         })))
         result = pending.wait()
     }

package/plugins/serviceAws/serviceAws.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2022 by Richard C. Zulch
+ * Copyright (c) 2020-2026 by Richard C. Zulch
  *
  */
 
@@ -23,7 +23,8 @@ function sawsInit(local manifest) {
 
     Naan.module.build(module.id, "serviceAws", function(modobj, compobj) {
         compobj.manifest = manifest
-        require("naanlib:frameworks/common").LiveImport()
+        require("naanlib:frameworks/common/utils.nlg")                      // get ExbackFactory
+        require("naanlib:frameworks/common").LiveImport()                   // applies to all exports
         require("./aws_utils.nlg")
         function sawsPostload() {
             if js.g

package/plugins/serviceNexara/speechrec.nlg

@@ -100,7 +100,7 @@ closure NexaraSpeechRec(local nxspr) {
     // Convert a Nexara response to a Yandex SpeechKit structure. Hey, it was first.
     //
     function nexaraToYCSK(resp,
-        local output, wordx, segment, chunk, word, wordet) {
+        local output, wordx, segment, chunk, syncerror, word, wordet, wordtrim, tstart) {
         output = {
             \@type: "type.googleapis.com/yandex.cloud.ai.stt.v2.LongRunningRecognitionResponse"
             chunks: []
@@ -120,14 +120,18 @@ closure NexaraSpeechRec(local nxspr) {
                 ]
             }
             for word in segment.text.split(" ") {
+                if word == ""
+                    continue                                                // ignore extra spaces
                 wordet = resp.words[wordx++]
-                if wordet.word != word {
-                    debuglog("nexaraToYCSK: expected ${word} but got ${wordet.word}")
-                    return (false)
+                wordtrim = wordet.word.trim()                               // detected word without spaces
+                tstart = wordet.start.toFixed(3).concat("s")
+                if !syncerror && wordtrim != word {
+                    syncerror = true
+                    debuglog("nexaraToYCSK: warning: at ${tstart} expected '${word}' but got '${wordtrim}'")
                 }
                 chunk.alternatives.0.words.push({
-                    word: wordet.word
-                    startTime: wordet.start.toFixed(3).concat("s")
+                    word: wordtrim
+                    startTime: tstart
                     endTime: wordet.end.toFixed(3).concat("s")
                 })
             }

package/test/test_01_core.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2017-2023 by Richard C. Zulch
+ * Copyright (c) 2017-2026 by Richard C. Zulch
  *
  */
 
@@ -881,7 +881,7 @@ RegisterTestCategory("core", [
 
     [ "unintern(true) === true", "false"],
 
-    [ "length(stacktrace());", "55" ],
+    [ "length(stacktrace());", "57" ],
     
     //
     // relate builtin