@naanlang/naan 1.4.3 → 1.5.0

package/lib/env_node.js

@@ -110,11 +110,13 @@ exports.NaanNodeREPL = function NaanNodeREPL(options) {
     {                                                                       // Node REPL does too much crap
     }
     
-    if (process.stdin.isTTY && !options.replDisable)
+    if (process.stdin.isTTY)
     {                                                                       // must have TTY for local REPL
         replServer = repl.start({ prompt: '', eval: myEval});
     
         replServer.on('line', function(cmd) {                               // process raw command line
+            if (options.replDisable)
+                return;
             if (cmd.charAt(0) != ".") {
                 textToRemoteTerm("\x1b[90m\x1b[3m".concat(cmd, "\x1b[0m\n"));
                 naanlib.textLine(cmd + "\n");
@@ -126,9 +128,13 @@ exports.NaanNodeREPL = function NaanNodeREPL(options) {
         //
     
         replServer._events.SIGINT = function () {
-            naanlib.escape();
+            if (options.replDisable) {
+                console.log("^C");
+                process.exit();
+            } else
+                naanlib.escape();
         };
-    
+
         //
         // Hook an exit command
         //

package/lib/node_repl_init.nlg

@@ -6,11 +6,11 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2018-2021 by Richard C. Zulch
+ * Copyright (c) 2018-2025 by Richard C. Zulch
  *
  */
 
-loglevel(2);
+loglevel(3);
 
 
 /*

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@naanlang/naan",
-    "version": "1.4.3",
+    "version": "1.5.0",
     "author": "Richard C. Zulch",
     "description": "Naan™ software platform",
     "main": "./lib/core/naanlib.js",

package/plugins/nodeLib/node_util.nlg

@@ -5,7 +5,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2023 by Richard C. Zulch
+ * Copyright (c) 2021-2025 by Richard C. Zulch
  *
  */
 
@@ -13,12 +13,11 @@
 /*
  * MultipartBody
  *
- * column positioning:                          //                          //                      !
  *     Create a multipart Form body from the specified dictionary, returning a buffer. Any xobject in
  * the dictionary is coded as an octet. If you want a different filename, include an options object
  * that defines that filename.
  *
- *The result is a dictionary as follows:
+ *The return value is a dictionary as follows:
  *  {
  *      body: <Buffer>
  *      header: {
@@ -29,11 +28,17 @@
  *
  */
 
-function MultipartBody(dict, options, local boundary, parts, key, data, filename) {
+function MultipartBody(dict, options,
+    local boundary, parts, key, data, item, filename) {
     boundary = "xx-xx-xx-xx-xx"
     parts = []
     for `(key, data) in dict {
-        if xobject(data) {
+        if Array.isArray(data) {                                            // PHP-style arrays
+            for item in data
+                parts.push(xnew(js.g.Buffer.from, "--".concat(boundary, "\r\n",
+                    "Content-Disposition: form-data; name=\"", key, "[]",
+                    "\"; \r\n\r\n", item, "\r\n"), "utf8"))
+        } else if xobject(data) {                                           // binary file data
             if options[key]
                 filename = options[key]
             else
@@ -43,8 +48,7 @@ function MultipartBody(dict, options, local boundary, parts, key, data, filename
                 "Content-Type:application/octet-stream\r\n\r\n"), "utf8"))
             parts.push(xnew(js.g.Buffer.from, data, "binary"))
             parts.push(xnew(js.g.Buffer.from, "\r\n", "utf8"))
-        }
-        else {
+        } else {                                                            // ordinary data
             parts.push(xnew(js.g.Buffer.from, "--".concat(boundary, "\r\n",
                 "Content-Disposition: form-data; name=\"", key,
                 "\"; \r\n\r\n", data, "\r\n"), "utf8"))
@@ -165,8 +169,11 @@ function nodeUtilInit(local manifest) {
     Naan.module.build(module.id, "node_util", function(modobj, compobj) {
         require("nodeLib.nlg")
         compobj.manifest = manifest
+        modobj.exports.MultipartBody = MultipartBody
+        modobj.exports.EncodeQuery = EncodeQuery
         modobj.exports.JsonParse = JsonParse
         modobj.exports.JsonStringify = JsonStringify
+        modobj.exports.RandomBytes = RandomBytes
         modobj.exports.UUID = UUID
         function postload() {
             nodeFs = js.r("fs")

package/plugins/openSearch/os_dynamo.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2025 by Richard C. Zulch
  *
  */
 
@@ -65,7 +65,7 @@ closure OSDynIndexer(index, table, options,
     // Update ids into the database, returning a nonce signaled with error array.
     //
     closure updateRecordIDs(docs, ids, writeID) {
-        asyncArray(docs, 10, function(entry, index, cancel, local error) {
+        asyncArray(docs, 10, closure(entry, index, cancel, local error) {
             ignore(error)
             `(error) = table.updateRecord(
                 entry[table.hashKey],
@@ -78,11 +78,11 @@ closure OSDynIndexer(index, table, options,
     //
     // Batch index documents into OpenSearch, returning a nonce signaled with ids array.
     //
-    closure batchIndex(records, local pending, docs) {
+    closure batchIndex(records, local pending) {
         pending = new(nonce)
-        docs = records.map(function(record) {
+        docs = asyncArray(records, 10, closure(record) {
             options.converter(record)
-        })
+        }).wait()
         future(function(local error, ids) {
             `(error, ids) = index.write(docs)
             pending.signal(list(error, records, ids))

package/plugins/serviceAws/aws_cloudwatchlogs.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021-2023 by Richard C. Zulch
+ * Copyright (c) 2021-2025 by Richard C. Zulch
  *
  */
 
@@ -19,6 +19,7 @@
  */
 
 closure CloudWatchLogs(local cwlogs) {
+    global(awsSDK)
     cwlogs = new(object, this)
 
     //

package/plugins/serviceAws/aws_dynamo.nlg

@@ -946,7 +946,7 @@ closure DynaTable(dyna, tablename, options, local table) {
     //      range:          <key-expression>                                // range of keys to retrieve
     //      filter:         <key-expression>                                // conditions for items to return
     //      reverse:        <boolean>                                       // reverse order
-    //      project:        <array>                                         // project only specified attributes
+    //      project:        <array> | `count                                // project only specified attributes
     //      paging:         <exclusive start key>                           // for paging; updated each call
     //      consistentRead: <boolean>                                       // true to use consistent read
     //      limit:          <number>                                        // maximum count of results
@@ -962,6 +962,7 @@ closure DynaTable(dyna, tablename, options, local table) {
             TableName: tablename
         }
         hashKey = table.hashKey
+        output = []
         if options.indexName {
             params.IndexName = options.indexName
             if options.indexHashKey
@@ -989,15 +990,23 @@ closure DynaTable(dyna, tablename, options, local table) {
         }
         if options.reverse
             params.ScanIndexForward = false
-        if options.project
+        if options.project == `count {
+            params.Select = "COUNT"
+            output = {
+                Count: 0
+                ScannedCount: 0
+            }
+        } else if options.project
             params.ProjectionExpression = options.project.join(", ")
         if options.paging
             params.ExclusiveStartKey = options.paging
         if options.consistentRead
             params.ConsistentRead = options.consistentRead
-        output = []
         `(error) = dynQueryPager(`query, params, options.limit, function(data) {
-            if data.Count > 0
+            if dictionary(output) {
+                output.Count += data.Count
+                output.ScannedCount += data.ScannedCount
+            } else if data.Count > 0
                 output = output.concat(dyna.conv.recordDynToNaan(data.Items))
             data.Count
         })
@@ -1016,7 +1025,7 @@ closure DynaTable(dyna, tablename, options, local table) {
     //  {
     //      index:          <string>                                        // index name in table
     //      filter:         <key-expression>                                // conditions for items to return
-    //      project:        <array>                                         // project only specified attributes
+    //      project:        <array> | `count                                // project only specified attributes
     //      paging:         <exclusive start key>                           // for paging; updated each call
     //      consistentRead: <boolean>                                       // true to use consistent read
     //      limit:          <number>                                        // maximum count of results
@@ -1031,6 +1040,7 @@ closure DynaTable(dyna, tablename, options, local table) {
         params = {
             TableName: tablename
         }
+        output = []
         if options.index
             params.IndexName = options.index
         if options.filter {
@@ -1040,15 +1050,23 @@ closure DynaTable(dyna, tablename, options, local table) {
             if error
                 return (list(Error("table.scanRecords filter:", error)))
         }
-        if options.project
+        if options.project == `count {
+            params.Select = "COUNT"
+            output = {
+                Count: 0
+                ScannedCount: 0
+            }
+        } else if options.project
             params.ProjectionExpression = options.project.join(", ")
         if options.paging
             params.ExclusiveStartKey = options.paging
         if options.consistentRead
             params.ConsistentRead = options.consistentRead
-        output = []
         `(error) = dynQueryPager(`scan, params, options.limit, function(data) {
-            if data.Count >= 0
+            if dictionary(output) {
+                output.Count += data.Count
+                output.ScannedCount += data.ScannedCount
+            } else if data.Count >= 0
                 output = output.concat(dyna.conv.recordDynToNaan(data.Items))
             data.Count
         })

package/plugins/serviceAws/aws_s3.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2024 by Richard C. Zulch
+ * Copyright (c) 2020-2025 by Richard C. Zulch
  *
  */
 
@@ -22,6 +22,7 @@
  */
 
 closure S3bucket(s3, name, bucketOptions, local bucket) {
+    global()
     bucket = new(object, this)
     bucket.name = name
 
@@ -83,8 +84,6 @@ closure S3bucket(s3, name, bucketOptions, local bucket) {
     //  }
     //
     bucket.put = closure put(key, data, options, local params, pending) {
-        if !contentType
-            contentType = "application/octet-stream"
         params = {
             Bucket: bucket.name
             Key: key
@@ -246,6 +245,7 @@ closure S3bucket(s3, name, bucketOptions, local bucket) {
  */
 
 closure S3(local s3) {
+    global(awsSDK)
     s3 = new(object, this)
 
     //

package/plugins/serviceAws/aws_sqs.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2023 by Richard C. Zulch
+ * Copyright (c) 2023-2025 by Richard C. Zulch
  *
  */
 
@@ -19,6 +19,7 @@
  */
 
 closure SqsMQ(sqs, queueUrl, local sqsq) {
+    global()
     sqsq = new(object, this)
 
     //
@@ -55,6 +56,7 @@ closure SqsMQ(sqs, queueUrl, local sqsq) {
  */
 
 closure SQS(local sqs) {
+    global(awsSDK)
     sqs = new(object, this)
 
     //

package/plugins/serviceAws/aws_utils.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2021 by Richard C. Zulch
+ * Copyright (c) 2021-2025 by Richard C. Zulch
  *
  */
 
@@ -22,6 +22,7 @@
  */
 
 closure queryPager(apiobject, procname, params, xargs, doneCB, local pending, looping, result) {
+    global()
     if !apiobject[procname]
         return (list(Error("queryPager method undefined:", procname)))
     pending = new(nonce)
@@ -55,6 +56,7 @@ closure queryPager(apiobject, procname, params, xargs, doneCB, local pending, lo
  */
 
 function QueryPager args {
+    global()
     closure(local apiobject, procname, params, xargs, doneCB) {
         apiobject = pop(args)
         procname = pop(args)

package/plugins/serviceAws/aws_vm.nlg

@@ -0,0 +1,198 @@
+/*
+ * aws_vm.nlg
+ * serviceAws
+ *
+ * VM controller under AWS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024-2025 by Richard C. Zulch
+ *
+ */
+
+vmState = false;
+
+/*
+ * VmControl
+ *
+ *     VM control object for AWS.
+ *
+ */
+
+closure VmControl(creds, local dokon) {
+    global(js, http, vmState)
+    if vmState.dokon
+        return (vmState.dokon)                                              // singleton
+    dokon = new(object, this)
+    vmState = new(nonce)                                                    // don't save with state
+    vmState.dokon = dokon
+
+    // detachVM
+    //
+    // Tell AWS to detach a VM from its scalability group. Returns a tuple with:
+    //  {
+    //      Activities: [
+    //          {
+    //              ActivityId: "STRING_VALUE",
+    //              AutoScalingGroupName: "STRING_VALUE",
+    //              Description: "STRING_VALUE",
+    //              Cause: "STRING_VALUE",
+    //              StartTime: new Date("TIMESTAMP"),
+    //              EndTime: new Date("TIMESTAMP"),
+    //              StatusCode: "PendingSpotBidPlacement" || "WaitingForSpotInstanceRequestId"
+    //                  || "WaitingForSpotInstanceId" || "WaitingForInstanceId" || "PreInService"
+    //                  || "InProgress" || "WaitingForELBConnectionDraining" || "MidLifecycleAction"
+    //                  || "WaitingForInstanceWarmup" || "Successful" || "Failed" || "Cancelled"
+    //                  || "WaitingForConnectionDraining",
+    //              StatusMessage: "STRING_VALUE",
+    //              Progress: Number("int"),
+    //              Details: "STRING_VALUE",
+    //              AutoScalingGroupState: "STRING_VALUE",
+    //              AutoScalingGroupARN: "STRING_VALUE",
+    //          }
+    //      ]
+    //  }
+    closure detachVM(instanceID, local error, ascSdk, params, asclient, response) {
+        `(error, ascSdk) = await(js.i("@aws-sdk/client-auto-scaling"))
+        if error
+            return (list(error))
+        params = { }
+        params.region = creds.region
+        params.credentials = {
+            accessKeyId:        creds.keyID
+            secretAccessKey:    creds.keySecret
+        }
+        if creds.endpoint
+            params.endpoint = creds.endpoint
+        asclient = xnew(ascSdk.AutoScalingClient, params)
+        `(error, response) = await(asclient.send(xnew(ascSdk.DetachInstancesCommand, {
+            AutoScalingGroupName: creds.ec2AsgName
+            InstanceIds: [ instanceID ]
+            ShouldDecrementDesiredCapacity: false
+        })))
+        list(error, response)
+    }
+
+    // terminateVM
+    //
+    // Tell AWS to terminate a VM, ours in point of fact. Returns a tuple with:
+    //  {
+    //      TerminatingInstances: [
+    //          {
+    //              InstanceId: "STRING_VALUE"
+    //              CurrentState: { // InstanceState
+    //                  Code: Number("int"),
+    //                  Name: "pending" || "running" || "shutting-down" || "terminated"
+    //                      || "stopping" || "stopped",
+    //              }
+    //              PreviousState: {
+    //                  Code: Number("int"),
+    //                  Name: "pending" || "running" || "shutting-down" || "terminated"
+    //                      || "stopping" || "stopped",
+    //              }
+    //          }
+    //      ]
+    //  }
+    //
+    closure terminateVM(instanceID, local error, ec2sdk, params, ec2client, response) {
+        `(error, ec2sdk) = await(js.i("@aws-sdk/client-ec2"))
+        if error
+            return (list(error))
+        params = { }
+        params.region = creds.region
+        params.credentials = {
+            accessKeyId:        creds.keyID
+            secretAccessKey:    creds.keySecret
+        }
+        if creds.endpoint
+            params.endpoint = creds.endpoint
+        ec2client = xnew(ec2sdk.EC2Client, params)
+        `(error, response) = await(ec2client.send(xnew(ec2sdk.TerminateInstancesCommand,
+            {  InstanceIds: [ instanceID ] })))
+        list(error, response)
+    }
+
+    // getHostname
+    //
+    // Ask the VM for its FQDN.
+    //
+    //  TOKEN="`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`"
+    //  curl -H "X-aws-ec2-metadata-token: $TOKEN" 'http://169.254.169.254/latest/meta-data/hostname'
+    //      => ip-172-31-80-96.ec2.internal
+    //
+    dokon.getHostname = closure getHostname(local error, token, hostname) {
+        `(error, token) = http.HttpRequest("http://169.254.169.254/latest/api/token", {
+            headers: {
+                "X-aws-ec2-metadata-token-ttl-seconds": "2160"
+            }
+            method: "PUT"
+        })
+        if !error
+            `(error, hostname) = http.HttpRequest("http://169.254.169.254/latest/meta-data/hostname", {
+                headers: {
+                    "X-aws-ec2-metadata-token": token
+                }
+            })
+        if error
+            list(error)
+        else
+            list(false, hostname)
+    }
+
+    // vmShutdown
+    //
+    // Get our AWS EC2 VM instance ID and then request that it be shut down.
+    //
+    // How to get auth token:
+    //  curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"
+	//      =>  AQAEAJHVrb_zRdju_A3tYBJLtKHsjej5GFVsDu1VuBCErP8MNVOr0Q==
+    //
+    // How to get instance-id:
+    //  curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id
+    //      =>  i-008dc3846da316a93
+    //
+
+    dokon.shutdown = closure shutdown(local error, token, instanceID, response) {
+        `(error, token) = http.HttpRequest("http://169.254.169.254/latest/api/token", {
+            headers: {
+                "X-aws-ec2-metadata-token-ttl-seconds": "2160"
+            }
+            method: "PUT"
+        })
+        if !error
+            `(error, instanceID) = http.HttpRequest("http://169.254.169.254/latest/meta-data/instance-id", {
+                headers: {
+                    "X-aws-ec2-metadata-token": token
+                }
+            })
+        if !error
+            `(error, response) = detachVM(instanceID)
+        if !error
+            `(error, response) = terminateVM(instanceID)
+        if error
+            list(error)
+        else
+            list(false, response)
+    }
+
+    // finis
+    dokon
+};
+
+
+/*
+ * vmConInit
+ *
+ *     Initialize the component.
+ *
+ */
+
+function vmConInit(local manifest) {
+    manifest = `(VmControl, vmConInit)
+    Naan.module.build(module.id, "aws_vm", function(modobj, compobj) {
+        require("./serviceAws.nlg")
+        compobj.manifest = manifest
+        http = require("naanlib:frameworks/node/http_request.nlg")
+        module.exports.VmControl = VmControl
+    })
+} ();

package/plugins/serviceAws/dbt_aws.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2024 by Richard C. Zulch
+ * Copyright (c) 2020-2025 by Richard C. Zulch
  *
  */
 
@@ -22,6 +22,7 @@
  */
 
 closure dawsBucket(s3b, local bucket, s3) {
+    global(JSpath)
     bucket = new(object, this)
     s3 = s3b.s3
     bucket.name = s3b.bucketName
@@ -311,6 +312,7 @@ closure dawsBucket(s3b, local bucket, s3) {
  */
 
 closure dawsTable(database, path, tableOptions, local table) {
+    global(js, TextDecoder)
     table = new(object, this)
     table.type = database.type
     table.db = database
@@ -411,9 +413,10 @@ closure dawsTable(database, path, tableOptions, local table) {
 
     // add
     //
-    // Add a new record to the table with optional metadata. This will fail if the key already
-    // exists. The metadata is stored with the record and cannot be changed without calling update.
-    // Metadata keys cannot begin with underscore ("_").
+    // Add a new record to the table with options. This will fail if the key already exists. Optional
+    // metadata is stored with the record and cannot be changed without calling update. Metadata keys
+    // cannot begin with underscore ("_"). If optional cacheControl is not specified then the default
+    // tableOptions.cacheControl is used.
     //
     // The returned document, which can be used with update, has the following keys:
     //      key         - the key name of the record
@@ -421,26 +424,25 @@ closure dawsTable(database, path, tableOptions, local table) {
     //      md5         - the MD5 hash of the content
     //      <...>       - the various metadata keys
 
-    table.add = closure add(key, content, metadata, callback, local id, doc) {
+    table.add = closure add(key, content, options, callback, local id, doc) {
         if !callback
-            return (syncAdapter(add, key, content, metadata))
+            return (syncAdapter(add, key, content, options))
         id = makeID(key)
         if !string(id)
-            return (asyncResult(callback, id))                                 // error from makeID
-        if  metadata && !dictionary(metadata)
+            return (asyncResult(callback, id))                              // error from makeID
+        if  options.metadata && !dictionary(options.metadata)
             return (asyncResult(callback, Error("invalid arguments")))
-        if metadata
-            doc = new(metadata)                                             // start with metadata
+        if options.metadata
+            doc = new(options.metadata)                                     // start with metadata
         else
             doc = { }
         doc.key = key
         doc.id = id
         if content
             doc.content = content
-        metadata = metadata2s3(metadata)                                    // filter out any invalid metadata
         database.bucket.put(id, content, {
-            metadata: metadata
-            cacheControl: tableOptions.cacheControl
+            metadata: metadata2s3(options.metadata)                         // filter metadata for s3
+            cacheControl: options.cacheControl || tableOptions.cacheControl
         }, function(error, resp) {
             if error
                 error = Error("add failed", error, id, { status: error.status })
@@ -453,17 +455,17 @@ closure dawsTable(database, path, tableOptions, local table) {
 
     // update
     //
-    // Update a record in the table using the document previously returned by add or get.
+    // Update a record in the table using the document previously returned by add or get. Specify
+    // options to update those values, otherwise existing/default values are used.
 
-    table.update = closure update(doc, callback, local metadata) {
+    table.update = closure update(doc, options, callback) {
         if !callback
-            return (syncAdapter(update, doc))
+            return (syncAdapter(update, doc, options))
         if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc._md5
             return (asyncResult(callback, Error("invalid argument")))
-        metadata = metadata2s3(doc)
         database.bucket.put(doc._id, doc.content, {
-            metadata: metadata
-            cacheControl: tableOptions.cacheControl
+            metadata: metadata2s3(options.metadata || doc)
+            cacheControl: options.cacheControl || tableOptions.cacheControl
         }, function(error, resp) {
             if error
                 error = Error("update failed", error, doc._id, { status: error.status })
@@ -519,7 +521,7 @@ closure dawsTable(database, path, tableOptions, local table) {
                     resp.content = readableStreamToArrayBuffer(resp.content)
                 else if jsTypedArray(resp.content)
                     resp.content = xnew(TextDecoder, "utf-8").decode(resp.content)
-                else if jsInstanceOf(resp.content, js.w.ReadableStream)
+                else if jsInstanceOf(resp.content, js.w.ReadableStream)     // for browsers only
                     resp.content = xnew(TextDecoder, "utf-8").decode(readableStreamToArrayBuffer(resp.content))
                 resp._id = id }
             callback(error, resp)

package/plugins/serviceAws/psm_aws.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2024 by Richard C. Zulch
+ * Copyright (c) 2020-2025 by Richard C. Zulch
  *
  */
 
@@ -19,6 +19,7 @@
  */
 
 closure psmaConnector(psm, local connClassID, connector, watch) {
+    global(awsSDK)
     connClassID = "S3DB"
     require("./dbt_aws.nlg")
     connector = new(object, this)

package/plugins/serviceGC/gc_api.nlg

@@ -83,6 +83,7 @@ closure GcApi(options, local gcapi, cacert, renew, access_token, expires_at) {
         `(error, data, status) = https.HttpsApiRequest(url, {
             putdata: put || undefined
             cacert: cacert || undefined
+            debug: options.debug || undefined
             headers: [
                 `(Accept, "application/json"),
                 list(`Authorization, "Bearer ".concat(access_token))