@naanlang/naan 1.0.2 → 1.0.5

package/plugins/serviceAws/aws_dynextra.nlg

@@ -0,0 +1,294 @@
+/*
+ * aws_dynextra.nlg
+ * serviceAws
+ *
+ *     Extra DynamoDB services for AWS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * dynexBatchGetRecords
+ *
+ * Retrieve records from one or more tables using arrays of keys. The results are put back in a
+ * dictionary with the same shape as the source. The result is a standard `(error, data) tuple.
+ * The request/response layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          keys:   [<hash-range tuple>]    // records to get
+ *          data:   [<retrieved records>]   // received records in reply
+ *      },
+ *      { <next-table> }
+ *  ]
+ * The operations are processed in parallel batches, and the final result is a standard result
+ * tuple `(error, result). An error is only reported if we are unable to complete all operations.
+ *
+ */
+
+function dynexBatchGetRecords(dyna, request,
+    local index, titem, tname, error, batches, activity, params, batchsize, primary) {
+    if !dyna.aws
+        return (list(Error("DynaTable.batchGetRecords: not logged into AWS")))
+    request = new(request)                                                  // allow local modifications
+    index = { }
+    for titem in request {                                                  // preflight all tables
+        tname = titem.table.tablename
+        if index[tname]
+            return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
+        `(error) = titem.table.info()
+        if error
+            return (Error("DynaTable.batchGetRecords: table", tname, ":", error))
+        titem.data = []
+        index[tname] = titem
+    }
+    batches = []
+    activity = true
+    while activity {                                                        // loop until requests empty
+        params = {
+            RequestItems: { }
+        }
+        batchsize = 0
+        while :fillbatch activity {                                         // loop until batch full
+            activity = false
+            for `(tname, titem) in index {
+                if titem.keys.length > 0 {
+                    primary = titem.table.makeDynKey(titem.keys.unshift())
+                    if params.RequestItems[tname].Keys
+                        params.RequestItems[tname].Keys.push(primary)
+                    else
+                        params.RequestItems[tname] = {
+                            Keys: [primary]
+                        }
+                    activity = true
+                    if ++batchsize >= 100
+                        break :fillbatch                                    // our batch is full
+                } else
+                    item.keys = undefined                                   // clear for return result
+            }
+        }
+        if batchsize > 0
+            batches.push(params)
+    }
+    asyncArray(batches, 10, closure bgr(params, local delayer, pending, err, response, table) {
+        delayer = dyna.conv.genBackoffDelayer()
+        loop {
+            pending = new(nonce)
+            dyna.aws.batchGetItem(params, function(err, data) {
+                if err
+                    debuglog("aws.batchGetItem failed", err)
+                pending.signal(err, data)
+            })
+            `(err, response) = pending.wait()
+            for `(tname, table) in response.Responses
+                for titem in table
+                    index[tname].data.push(dyna.conv.recordDynToNaan(titem))
+            if response.UnprocessedKeys
+                params.RequestItems = response.UnprocessedKeys              // still more to do
+            else
+                break                                                       // total success or failure
+            if delayer() > dyna.timeout {
+                if !error                                                   // too slow
+                    error = err
+                break
+            }
+        }
+    }).wait()
+    if error
+        list(error)
+    else
+        list(false, request)
+};
+
+
+/*
+ * dynexBatchWriteRecords
+ *
+ * Put arrays of records into one or more tables, replacing or deleting any existing records with
+ * the same keys. This returns arrays of results coresponding to the input records, where each
+ * is a standard `(error, data) tuple. The request layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          delete: [<key tuples/records>]  // records to delete (needing only keys)
+ *          put:    [<records>]             // records to put
+ *      },
+ *      { <next-table> }
+ *  ]
+ * The operations are processed in parallel batches, and the final result is a standard result
+ * tuple `(error, result). An error is only reported if we are unable to complete all operations.
+ *
+ */
+
+closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
+    if !dyna.aws
+        return (list(Error("DynaTable.batchWriteRecords: not logged into AWS")))
+
+    //
+    // addOp
+    // Add a put or delete operation to the parameters
+    //
+    function addOp(params, table, del, record, local request) {
+        if del {                                                            // deleting a record
+            if tuple(record)
+                record = table.makeDynKey(record)                           // caller provided hash/range key pair
+            else
+                request = { DeleteRequest: { Key: dyna.conv.recordNaanToDyn(record) } }
+        }
+        else                                                                // putting a record
+            request = { PutRequest: { Item: dyna.conv.recordNaanToDyn(record) } }
+        if params.RequestItems[table.tablename]
+            params.RequestItems[table.tablename].push(request)
+        else
+            params.RequestItems[table.tablename] = [request]
+    }
+
+    // transform into batches
+    //
+    // This is tricky because it wants each batch to use multiple tables for maximum parallelism,
+    // just in case AWS isn't already optimizing this. It takes a request from each table and
+    // combines them into a single batch repeatedly, until the batch is full or the tables in the
+    // request have no more entries.
+    //
+    request = new(request)                                                  // allow local modifications
+    batches = []
+    activity = true
+    while activity
+        let (params, batchsize, table) {
+            params = {
+                RequestItems: { }
+            }
+            batchsize = 0
+            while :fillbatch activity {                                     // loop until no activity or batch full
+                activity = false
+                for table in request {
+                    if table.delete.length > 0
+                        addOp(params, table, true, table.delete.unshift())
+                    else if table.put.length > 0
+                        addOp(params, table, false, table.put.unshift())
+                    else
+                        continue
+                    activity = true
+                    if ++batchsize >= 25
+                        break :fillbatch                                    // our batch is full
+                }
+            }
+            if batchsize > 0
+                batches.push(params)
+        } ()
+    asyncArray(batches, 10, closure bwr(params, local delayer, pending, err, response) {
+        delayer = dyna.conv.genBackoffDelayer()
+        loop {
+            pending = new(nonce)
+            dyna.aws.batchWriteItem(params, function(err, data) {
+                if err
+                    debuglog("aws.batchWriteItem failed", err)
+                pending.signal(err, data)
+            })
+            `(err, response) = pending.wait()
+            if response.UnprocessedKeys
+                params.RequestItems = response.UnprocessedKeys              // still more to do
+            else
+                break                                                       // total success or failure
+            if delayer() > dyna.timeout {
+                if !error                                                   // too slow
+                    error = err
+                break
+            }
+        }
+    }).wait()
+    if error
+        list(error)
+    else
+        list(false, { ok: true })
+};
+
+
+
+/*
+ * dynexTransactGetRecords
+ *
+ * Retrieve records from one or more tables using arrays of keys as a single transaction. The
+ * results are put back in an array with the same shape as the source. The request/response
+ * layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          keys:   [<hash-range tuple>]    // records to get
+ *          data:   [<retrieved records>]   // received records in reply
+ *      },
+ *      { <next-table> }
+ *  ]
+ * This returns a standard result tuple `(error, result). An error is only reported if we are 
+ * unable to complete all operations.
+ *
+ */
+
+closure dynexTransactGetRecords(dyna, request,
+    local index, titem, tname, error, params, pending, response) {
+    if !dyna.aws
+        return (list(Error("DynaTable.transactGetRecords: not logged into AWS")))
+    request = new(request)                                                  // allow local modifications
+    index = { }
+    for titem in request {                                                  // preflight all tables
+        tname = titem.table.tablename
+        if index[tname]
+            return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
+        `(error) = titem.table.info()
+        if error
+            return (Error("DynaTable.batchGetRecords: table", tname, ":", error))
+        titem.data = []
+        index[tname] = titem
+    }
+    params = {
+        TransactItems: []
+    }
+    for `(tname, titem) in index {
+        while titem.keys.length > 0 {
+            params.TransactItems.push({
+                Get: {
+                    TableName: tname
+                    Key: makeDynKey(titem.keys.unshift())
+                }
+            })
+        }
+        titem.keys = undefined
+    }
+    pending = new(nonce)
+    dyna.aws.transactGetItems(params, function(err, data) {
+        if err
+            debuglog("aws.transactGetItems failed", err)
+        pending.signal(err, data)
+    })
+    `(error, response) = pending.wait()
+    if error
+        list(error)
+    else {
+        for titem in response.Responses {
+            tname = titem.Item.*.0
+            index[tname].data.push(dyna.conv.recordDynToNaan(titem.Item[tname]))
+        }
+        list(false, request)
+    }
+};
+
+
+/*
+ * dynexInit
+ *
+ *     Initialize the component.
+ *
+ */
+
+function dynexInit(local manifest) {
+    manifest = `(dynexBatchGetRecords, dynexBatchWriteRecords, dynexTransactGetRecords, dynexInit)
+
+    Naan.module.build(module.id, "aws_dynextra", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("./serviceAws.nlg")
+    })
+} ();

package/plugins/serviceAws/aws_lambda.nlg

@@ -26,10 +26,13 @@ closure Lambda(local lamb) {
     //
     lamb.login = closure login(creds) {
         lamb.aws = xnew(awsSDK.Lambda, {
-            apiVersion:         "2015-03-31",
-            accessKeyId:        creds.keyID,
-            secretAccessKey:    creds.keySecret,
-            region:             creds.region })
+            region:                 creds.region
+            credentials: {
+                accessKeyId:        creds.keyID
+                secretAccessKey:    creds.keySecret
+            }
+        })
+        list(false, { ok: true })
     }
     
     //
@@ -55,8 +58,12 @@ closure Lambda(local lamb) {
     //
     // updateFunctionCode
     //
+    // Note that as of 2022-05-17 AWS SDK v3 requires that the zipfile be a Uint8Array, not an
+    // ArrayBuffer or anything simple like that. What a #$*#(! waste of time.
+    //
     lamb.updateFunctionCode = closure updateFunctionCode(name, zipfile, local params, pending) {
         params = {
+            Architectures: [ "x86_64" ]
             FunctionName: name
             ZipFile: zipfile
         }

package/plugins/serviceAws/aws_s3.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2022 by Richard C. Zulch
  *
  */
 
@@ -81,15 +81,17 @@ closure S3(local s3) {
     s3.login = closure login(creds, local params) {
         if !awsSDK
             return (list(Error("S3.login: no AWS SDK available")))
-        params = {
-            apiVersion: "2012-08-10"
-        }
+        params = { }
         if creds {
-            params.accessKeyId = creds.keyID,
-            params.secretAccessKey = creds.keySecret,
             params.region = creds.region
+            params.credentials = {
+                accessKeyId:        creds.keyID
+                secretAccessKey:    creds.keySecret
+            }
         }
         s3.aws = xnew(awsSDK.S3, params)
+        if awsSDK.S3Client
+            s3.client = xnew(awsSDK.S3Client, params)
         `(false, { ok: true })
     }
     
@@ -99,16 +101,44 @@ closure S3(local s3) {
     //     Return a new bucket access object for the table of the specified name. The bucket may not 
     // yet exist; see S3bucket for methods.
     //
-    // parameters:
+    s3.bucket = closure bucket(name) {
+        S3bucket(s3, name)
+    }
+    
+    //
+    // getSignedURL
+    //
+    // This obtains a pre-signed URL for the specified command and its parameters. Options control
+    // how the URL is generated.
+    //
+    // Options:
     //  {
-    //      hash:       <name>                  // primary hash key attribute name (required)
-    //      hashType:   <type>                  // string | numeric (required)
-    //      range:      <name>                  // primary range key attribute name (optional)
-    //      rangeType:  <name>                  // string | numeric (required if range specified)
+    //      expiresIn:        <seconds>         // default expiration is 900 (15 minutes)
     //  }
     //
-    s3.bucket = closure bucket(name) {
-        S3bucket(s3, name)
+    s3.getSignedURL = closure getSignedURL(cmdname, cmdparams, local command, options, pending) {
+        if s3.client {
+            if cmdname == "putObject"
+                cmdname = "PutObjectCommand"
+            else if cmdname == "getObject"
+                cmdname = "GetObjectCommand"
+            else
+                return (list(Error("S3bucket.getSignedURL: command not present:", cmdname)))
+            if cmdparams.Expires {
+                cmdparams = new(cmdparams)
+                options.expiresIn = cmdparams.Expires
+                cmdparams.Expires = undefined
+            } else
+                options = { }
+            command = xnew(awsSDK.s3Commands[cmdname], cmdparams)
+            await(awsSDK.getSignedUrl(s3.client, command, options))
+        } else {
+            pending = new(nonce)
+            s3.aws.getSignedUrl(cmdname, cmdparams, function(error, data) {
+                pending.signal(list(error, data))
+            })
+            pending.wait()
+        }
     }
 
     // finis
@@ -126,7 +156,7 @@ closure S3(local s3) {
 function s3Init(local manifest) {
     manifest = `(S3bucket, S3, s3Init)
 
-    Naan.module.build(module.id, "S3", function(modobj, compobj) {
+    Naan.module.build(module.id, "aws_s3", function(modobj, compobj) {
         require("./serviceAws.nlg")
         compobj.manifest = manifest
         modobj.exports.S3 = S3

package/plugins/serviceAws/dbt_aws.nlg

@@ -6,23 +6,11 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2022 by Richard C. Zulch
  *
  */
 
 
-/*
- * dawsValidateName
- *
- *     True iff the specified name is valid here.
- *
- */
-
-function dawsValidateName(key, path) {
-    
-};
-
-
 /*
  * dawsBucket
  *
@@ -33,12 +21,12 @@ function dawsValidateName(key, path) {
  *
  */
 
-closure dawsBucket(s3, local bucket) {
+closure dawsBucket(s3b, local bucket, s3) {
     bucket = new(object, this)
-    bucket.s3 = s3
-    bucket.name = s3.config.params.Bucket
-    bucket.region = s3.config.region
-    if not s3 || !string(bucket.region) || bucket.region.length == 0 || !string(bucket.name) || bucket.bucket.length == 0
+    s3 = s3b.s3
+    bucket.name = s3b.bucketName
+    bucket.region = s3b.region
+    if !s3 || !string(bucket.region) || bucket.region.length == 0 || !string(bucket.name) || bucket.bucket.length == 0
         return (list(Error("invalid S3 parameter")))
 
     // asyncResult
@@ -56,7 +44,7 @@ closure dawsBucket(s3, local bucket) {
     // that the bucket is still open before beginning (or continuing) any operations.
     
     bucket.close = function close() {
-        bucket.s3 = false                                                   // just remove access
+        bucket.s3 = s3 = false                                              // just remove access
         list(false, { })
     }
 
@@ -70,6 +58,7 @@ closure dawsBucket(s3, local bucket) {
         if !s3
             return (asyncResult(callback, Error("database closed")))
         params = {
+            Bucket: bucket.name
             Key: key
         }
 
@@ -100,6 +89,7 @@ closure dawsBucket(s3, local bucket) {
         if !s3
             return (asyncResult(callback, Error("database closed")))
         params = {
+            Bucket: bucket.name
             Key: key
         }
         if data {
@@ -110,6 +100,7 @@ closure dawsBucket(s3, local bucket) {
                 mimeType = {
                     "css":  "text/css",
                     "csv":  "text/csv",
+                    "gif":  "image/gif",
                     "htm":  "text/html",
                     "html": "text/html",
                     "ico":  "image/x-icon",
@@ -170,6 +161,7 @@ closure dawsBucket(s3, local bucket) {
         if !s3
             return (asyncResult(callback, Error("database closed")))
         params = {
+            Bucket: bucket.name
             Key:    key
         }
         s3.deleteObject(params, function(error, resp) {
@@ -251,7 +243,9 @@ closure dawsBucket(s3, local bucket) {
 
         if !s3
             return (asyncResult(callback, Error("database closed")))
-        params = { }
+        params = {
+            Bucket: bucket.name
+        }
         if string(prefix)
             params.Prefix = prefix
         if onelevel
@@ -259,6 +253,8 @@ closure dawsBucket(s3, local bucket) {
         listOne()
     }
     
+    // finis
+
     list(false, bucket)
 
 };
@@ -400,7 +396,7 @@ closure dawsTable(database, path, local table) {
         metadata = metadata2s3(metadata)                                    // filter out any invalid metadata
         database.bucket.put(id, content, metadata, function(error, resp) {
             if error
-                error = Error("add failed", error, id, { status: error.statusCode })
+                error = Error("add failed", error, id, { status: error.$metadata.httpStatusCode })
             else {
                 doc.md5 = resp.md5                                          // update the original doc
                 doc.length = resp.length }
@@ -420,7 +416,7 @@ closure dawsTable(database, path, local table) {
         metadata = metadata2s3(doc)
         database.bucket.put(doc._id, doc.content, metadata, function(error, resp) {
             if error
-                error = Error("update failed", error, doc._id, { status: error.statusCode })
+                error = Error("update failed", error, doc._id, { status: error.$metadata.httpStatusCode })
             else {
                 doc.md5 = resp.md5                                          // update the original doc
                 doc.modified = undefined                                    // we don't know exact modify date
@@ -441,7 +437,7 @@ closure dawsTable(database, path, local table) {
             return (asyncResult(callback, Error("invalid argument")))
         database.bucket.delete(doc._id, function(error, resp) {
             if error
-                error = Error("delete failed", error, doc._id, { status: error.statusCode })
+                error = Error("delete failed", error, doc._id, { status: error.$metadata.httpStatusCode })
             callback(error, resp)
         })
     }
@@ -458,7 +454,7 @@ closure dawsTable(database, path, local table) {
             return (asyncResult(callback, id))                              // error from makeID
         database.bucket.get(id, false, function(error, resp) {
             if error
-                error = Error("get failed", error, id, { status: error.statusCode })
+                error = Error("get failed", error, id, { status: error.$metadata.httpStatusCode })
             else {
                 resp.key = key
                 resp = s3resp2doc(resp)
@@ -491,7 +487,7 @@ closure dawsTable(database, path, local table) {
             return (asyncResult(callback, id))                                 // error from makeID
         database.bucket.get(id, true, function(error, resp) {
             if error
-                error = Error("md5 failed", error, id, { status: error.statusCode })
+                error = Error("md5 failed", error, id, { status: error.$metadata.httpStatusCode })
             else if resp._md5
                 resp = resp._md5
             else
@@ -502,7 +498,10 @@ closure dawsTable(database, path, local table) {
     
     // head
     //
-    // Return the header of the specified record, i.e. everything except the content.
+    // Return the header of the specified record, i.e. everything except the content. This operates
+    // on directories (with names ending in slash) differently because AWS does not support calling
+    // headObject upon them. We simply want to confirm that the directory exists, so we list objects
+    // that have the name without the slash and we get a common prefix if it exists.
 
     table.head = closure head(key, callback, local id) {
         if !callback
@@ -510,15 +509,30 @@ closure dawsTable(database, path, local table) {
         id = makeID(key)
         if !string(id)
             return (asyncResult(callback, id))                                 // error from makeID
-        database.bucket.get(id, true, function(error, resp) {
-            if error
-                error = Error("head failed", error, id, { status: error.statusCode })
-            else {
-                resp.key = key
-                resp = s3resp2doc(resp)
-                resp._id = id }
-            callback(error, resp)
-        })
+        if id.slice(-1) == "/"
+            database.bucket.list(id.slice(0,-1), true, function blcb(error, resp) {
+                if error
+                    error = Error("head failed", error, id, { status: error.$metadata.httpStatusCode })
+                else if id != resp.CommonPrefixes.0.Prefix {
+                    error = Error("head not found", id, resp.CommonPrefixes.0.Prefix, { status: 404 })
+                    resp = false
+                }
+                else {
+                    resp.key = key
+                    resp = s3resp2doc(resp)
+                    resp._id = id }
+                callback(error, resp)
+            })
+        else
+            database.bucket.get(id, true, function bgcb(error, resp) {
+                if error
+                    error = Error("head failed", error, id, { status: error.$metadata.httpStatusCode })
+                else {
+                    resp.key = key
+                    resp = s3resp2doc(resp)
+                    resp._id = id }
+                callback(error, resp)
+            })
     }
 
     // copy
@@ -539,7 +553,7 @@ closure dawsTable(database, path, local table) {
                 error = Error("copy failed", {
                     source: srckey
                     destination: destkey
-                    status: error.statusCode
+                    status: error.$metadata.httpStatusCode
                 }, error)
             callback(error, resp)
         })
@@ -631,7 +645,7 @@ closure dawsTable(database, path, local table) {
             s3prefix = table.s3prefix
         database.bucket.list(s3prefix, !options.deep, function(error, objects, local obj, key) {
             if error
-                error = Error("records list failed", error, options.prefix, { status: error.statusCode })
+                error = Error("records list failed", error, options.prefix, { status: error.$metadata.httpStatusCode })
             else {
                 for obj in objects.Contents {
                     if obj.Key.startsWith(table.s3prefix)
@@ -674,13 +688,13 @@ closure dawsTable(database, path, local table) {
  *
  */
 
-closure S3DB(s3, local database, result) {
+closure S3DB(s3b, local database, result) {
     database = new(object, this)
-    result = dawsBucket(s3)
+    result = dawsBucket(s3b)
     if result.0
         return (result)                                                     // can't access bucket
     database.bucket = result.1
-    database.s3 = s3
+    database.s3b = s3b
     database.type = "Amazon S3"
 
     // isopen
@@ -700,7 +714,6 @@ closure S3DB(s3, local database, result) {
             return (list(Error("database closed")))
         database.bucket.close()
         database.bucket = false
-        database.s3 = false
         list(false, { ok: true })
     }
 
@@ -712,10 +725,10 @@ closure S3DB(s3, local database, result) {
         if !database.bucket
             return (list(Error("database closed")))
         result = {
-            type: database.type
+            type:           database.type
             info: {
-                region: s3.config.region
-                bucket: s3.config.params.Bucket
+                region:     s3b.region
+                bucket:     s3b.bucketName
             }
         }
         list(false, result)
@@ -758,7 +771,7 @@ closure S3DB(s3, local database, result) {
 
 function dawsInit(local manifest) {
 
-    manifest = `(dawsValidateName, dawsBucket, dawsTable, S3DB, dawsInit)
+    manifest = `(dawsBucket, dawsTable, S3DB, dawsInit)
 
     Naan.module.build(module.id, "dbt_aws", function(modobj, compobj) {
         require("./serviceAws.nlg")