@naanlang/naan 1.0.2 → 1.0.3

package/plugins/serviceAws/aws_dynextra.nlg

@@ -0,0 +1,294 @@
+/*
+ * aws_dynextra.nlg
+ * serviceAws
+ *
+ *     Extra DynamoDB services for AWS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * dynexBatchGetRecords
+ *
+ * Retrieve records from one or more tables using arrays of keys. The results are put back in a
+ * dictionary with the same shape as the source. The result is a standard `(error, data) tuple.
+ * The request/response layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          keys:   [<hash-range tuple>]    // records to get
+ *          data:   [<retrieved records>]   // received records in reply
+ *      },
+ *      { <next-table> }
+ *  ]
+ * The operations are processed in parallel batches, and the final result is a standard result
+ * tuple `(error, result). An error is only reported if we are unable to complete all operations.
+ *
+ */
+
+function dynexBatchGetRecords(dyna, request,
+    local index, titem, tname, error, batches, activity, params, batchsize, primary) {
+    if !dyna.aws
+        return (list(Error("DynaTable.batchGetRecords: not logged into AWS")))
+    request = new(request)                                                  // allow local modifications
+    index = { }
+    for titem in request {                                                  // preflight all tables
+        tname = titem.table.tablename
+        if index[tname]
+            return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
+        `(error) = titem.table.info()
+        if error
+            return (Error("DynaTable.batchGetRecords: table", tname, ":", error))
+        titem.data = []
+        index[tname] = titem
+    }
+    batches = []
+    activity = true
+    while activity {                                                        // loop until requests empty
+        params = {
+            RequestItems: { }
+        }
+        batchsize = 0
+        while :fillbatch activity {                                         // loop until batch full
+            activity = false
+            for `(tname, titem) in index {
+                if titem.keys.length > 0 {
+                    primary = titem.table.makeDynKey(titem.keys.unshift())
+                    if params.RequestItems[tname].Keys
+                        params.RequestItems[tname].Keys.push(primary)
+                    else
+                        params.RequestItems[tname] = {
+                            Keys: [primary]
+                        }
+                    activity = true
+                    if ++batchsize >= 100
+                        break :fillbatch                                    // our batch is full
+                } else
+                    item.keys = undefined                                   // clear for return result
+            }
+        }
+        if batchsize > 0
+            batches.push(params)
+    }
+    asyncArray(batches, 10, closure bgr(params, local delayer, pending, err, response, table) {
+        delayer = dyna.conv.genBackoffDelayer()
+        loop {
+            pending = new(nonce)
+            dyna.aws.batchGetItem(params, function(err, data) {
+                if err
+                    debuglog("aws.batchGetItem failed", err)
+                pending.signal(err, data)
+            })
+            `(err, response) = pending.wait()
+            for `(tname, table) in response.Responses
+                for titem in table
+                    index[tname].data.push(dyna.conv.recordDynToNaan(titem))
+            if response.UnprocessedKeys
+                params.RequestItems = response.UnprocessedKeys              // still more to do
+            else
+                break                                                       // total success or failure
+            if delayer() > dyna.timeout {
+                if !error                                                   // too slow
+                    error = err
+                break
+            }
+        }
+    }).wait()
+    if error
+        list(error)
+    else
+        list(false, request)
+};
+
+
+/*
+ * dynexBatchWriteRecords
+ *
+ * Put arrays of records into one or more tables, replacing or deleting any existing records with
+ * the same keys. This returns arrays of results coresponding to the input records, where each
+ * is a standard `(error, data) tuple. The request layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          delete: [<key tuples/records>]  // records to delete (needing only keys)
+ *          put:    [<records>]             // records to put
+ *      },
+ *      { <next-table> }
+ *  ]
+ * The operations are processed in parallel batches, and the final result is a standard result
+ * tuple `(error, result). An error is only reported if we are unable to complete all operations.
+ *
+ */
+
+closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
+    if !dyna.aws
+        return (list(Error("DynaTable.batchWriteRecords: not logged into AWS")))
+
+    //
+    // addOp
+    // Add a put or delete operation to the parameters
+    //
+    function addOp(params, table, del, record, local request) {
+        if del {                                                            // deleting a record
+            if tuple(record)
+                record = table.makeDynKey(record)                           // caller provided hash/range key pair
+            else
+                request = { DeleteRequest: { Key: dyna.conv.recordNaanToDyn(record) } }
+        }
+        else                                                                // putting a record
+            request = { PutRequest: { Item: dyna.conv.recordNaanToDyn(record) } }
+        if params.RequestItems[table.tablename]
+            params.RequestItems[table.tablename].push(request)
+        else
+            params.RequestItems[table.tablename] = [request]
+    }
+
+    // transform into batches
+    //
+    // This is tricky because it wants each batch to use multiple tables for maximum parallelism,
+    // just in case AWS isn't already optimizing this. It takes a request from each table and
+    // combines them into a single batch repeatedly, until the batch is full or the tables in the
+    // request have no more entries.
+    //
+    request = new(request)                                                  // allow local modifications
+    batches = []
+    activity = true
+    while activity
+        let (params, batchsize, table) {
+            params = {
+                RequestItems: { }
+            }
+            batchsize = 0
+            while :fillbatch activity {                                     // loop until no activity or batch full
+                activity = false
+                for table in request {
+                    if table.delete.length > 0
+                        addOp(params, table, true, table.delete.unshift())
+                    else if table.put.length > 0
+                        addOp(params, table, false, table.put.unshift())
+                    else
+                        continue
+                    activity = true
+                    if ++batchsize >= 25
+                        break :fillbatch                                    // our batch is full
+                }
+            }
+            if batchsize > 0
+                batches.push(params)
+        } ()
+    asyncArray(batches, 10, closure bwr(params, local delayer, pending, err, response) {
+        delayer = dyna.conv.genBackoffDelayer()
+        loop {
+            pending = new(nonce)
+            dyna.aws.batchWriteItem(params, function(err, data) {
+                if err
+                    debuglog("aws.batchWriteItem failed", err)
+                pending.signal(err, data)
+            })
+            `(err, response) = pending.wait()
+            if response.UnprocessedKeys
+                params.RequestItems = response.UnprocessedKeys              // still more to do
+            else
+                break                                                       // total success or failure
+            if delayer() > dyna.timeout {
+                if !error                                                   // too slow
+                    error = err
+                break
+            }
+        }
+    }).wait()
+    if error
+        list(error)
+    else
+        list(false, { ok: true })
+};
+
+
+
+/*
+ * dynexTransactGetRecords
+ *
+ * Retrieve records from one or more tables using arrays of keys as a single transaction. The
+ * results are put back in an array with the same shape as the source. The request/response
+ * layout is as follows:
+ *  [
+ *      {
+ *          table:  <DynaTable object>      // table to act upon
+ *          keys:   [<hash-range tuple>]    // records to get
+ *          data:   [<retrieved records>]   // received records in reply
+ *      },
+ *      { <next-table> }
+ *  ]
+ * This returns a standard result tuple `(error, result). An error is only reported if we are 
+ * unable to complete all operations.
+ *
+ */
+
+closure dynexTransactGetRecords(dyna, request,
+    local index, titem, tname, error, params, pending, response) {
+    if !dyna.aws
+        return (list(Error("DynaTable.transactGetRecords: not logged into AWS")))
+    request = new(request)                                                  // allow local modifications
+    index = { }
+    for titem in request {                                                  // preflight all tables
+        tname = titem.table.tablename
+        if index[tname]
+            return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
+        `(error) = titem.table.info()
+        if error
+            return (Error("DynaTable.batchGetRecords: table", tname, ":", error))
+        titem.data = []
+        index[tname] = titem
+    }
+    params = {
+        TransactItems: []
+    }
+    for `(tname, titem) in index {
+        while titem.keys.length > 0 {
+            params.TransactItems.push({
+                Get: {
+                    TableName: tname
+                    Key: makeDynKey(titem.keys.unshift())
+                }
+            })
+        }
+        titem.keys = undefined
+    }
+    pending = new(nonce)
+    dyna.aws.transactGetItems(params, function(err, data) {
+        if err
+            debuglog("aws.transactGetItems failed", err)
+        pending.signal(err, data)
+    })
+    `(error, response) = pending.wait()
+    if error
+        list(error)
+    else {
+        for titem in response.Responses {
+            tname = titem.Item.*.0
+            index[tname].data.push(dyna.conv.recordDynToNaan(titem.Item[tname]))
+        }
+        list(false, request)
+    }
+};
+
+
+/*
+ * dynexInit
+ *
+ *     Initialize the component.
+ *
+ */
+
+function dynexInit(local manifest) {
+    manifest = `(dynexBatchGetRecords, dynexBatchWriteRecords, dynexTransactGetRecords, dynexInit)
+
+    Naan.module.build(module.id, "aws_dynextra", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("./serviceAws.nlg")
+    })
+} ();

package/plugins/serviceAws/dbt_aws.nlg

@@ -6,7 +6,7 @@
  *
  * column positioning:                          //                          //                      !
  *
- * Copyright (c) 2020-2021 by Richard C. Zulch
+ * Copyright (c) 2020-2022 by Richard C. Zulch
  *
  */
 
@@ -259,6 +259,8 @@ closure dawsBucket(s3, local bucket) {
         listOne()
     }
     
+    // finis
+
     list(false, bucket)
 
 };
@@ -502,7 +504,10 @@ closure dawsTable(database, path, local table) {
     
     // head
     //
-    // Return the header of the specified record, i.e. everything except the content.
+    // Return the header of the specified record, i.e. everything except the content. This operates
+    // on directories (with names ending in slash) differently because AWS does not support calling
+    // headObject upon them. We simply want to confirm that the directory exists, so we list objects
+    // that have the name without the slash and we get a common prefix if it exists.
 
     table.head = closure head(key, callback, local id) {
         if !callback
@@ -510,15 +515,30 @@ closure dawsTable(database, path, local table) {
         id = makeID(key)
         if !string(id)
             return (asyncResult(callback, id))                                 // error from makeID
-        database.bucket.get(id, true, function(error, resp) {
-            if error
-                error = Error("head failed", error, id, { status: error.statusCode })
-            else {
-                resp.key = key
-                resp = s3resp2doc(resp)
-                resp._id = id }
-            callback(error, resp)
-        })
+        if id.slice(-1) == "/"
+            database.bucket.list(id.slice(0,-1), true, function blcb(error, resp) {
+                if error
+                    error = Error("head failed", error, id, { status: error.statusCode })
+                else if id != resp.CommonPrefixes.0.Prefix {
+                    error = Error("head not found", id, resp.CommonPrefixes.0.Prefix, { status: 404 })
+                    resp = false
+                }
+                else {
+                    resp.key = key
+                    resp = s3resp2doc(resp)
+                    resp._id = id }
+                callback(error, resp)
+            })
+        else
+            database.bucket.get(id, true, function bgcb(error, resp) {
+                if error
+                    error = Error("head failed", error, id, { status: error.statusCode })
+                else {
+                    resp.key = key
+                    resp = s3resp2doc(resp)
+                    resp._id = id }
+                callback(error, resp)
+            })
     }
 
     // copy

package/plugins/serviceAws/psm_aws.nlg

@@ -50,36 +50,26 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             key:            "bucketName" }
         ]
     }
-        
-    // hashResourceID
-    //
-    // Make a stable hash based on the resource credentials that we can persist or share between
-    // different domains accessing the same resource.
-    connector.hashResourceID = function hashResourceID(resource, local ident) {
-        ident = resource.region.concat(resource.region, resource.bucketName)
-        if resource.label
-            ident = ident.concat(resource.label)
-        HashSHA256(ident)
-    }
 
     // findResource
     //
-    // Find a resource with the specified contents, returning the resID or false. The loose criteria
-    // is: if added, would the specified contents be redundant to an existing resource?
+    // Find a resource with the specified contents, returning the resID or false.
     connector.findResource = function findResource(resource, local resID, creds) {
         for resID in listResources().1 {
             creds = access(resID).1
-            if resource.region == creds.region && resource.bucketName == creds.bucketName
-                && (!resource.label || resource.label == creds.name)
+            if ((!resource.region || resource.region == creds.region)
+                && (!resource.bucketName || resource.bucketName == creds.bucketName)
+                && (!resource.label || resource.label == creds.name))
                 return (resID)
         }
         false
     }
 
-    // addResource
+    // writeResource
     //
-    // Add credentials for a named resource. The label is a string. The resource is a dictionary
-    // comprising these keys:
+    // Add or update credentials for a named resource. If no resID is specified this adds a new
+    // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
+    // keys:
     //      label       - [optional] label we use for this resource
     //      keyID       - AWS IAM keyID
     //      keySecret   - AWS IAM keySecret
@@ -87,10 +77,13 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
     //      bucketName  - AWS bucketName
     //      hidden      - [optional] don't enumerate to user
     //      locked      - [optional] don't allow user delete
-    // The return value is (error, result) tuple. Error, if non-false, is a dictionary of field
-    // keys and error diagnostic strings. The special key "label" refers to the label string.
-    connector.addResource = function addResource(resource, local label, errors, creds, error, resID, data) {
-        function badField(str) { !string(str) || str.trim().length == 0 } 
+    // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
+    // keys and error diagnostic strings.
+    connector.writeResource = function writeResource(resource, resID,
+        local creds, errors, error, data, change) {
+        // badField - test for a bad string
+        function badField(str) { !string(str) || str.trim().length == 0 }
+
         creds = { 
             keyID: resource.keyID,
             keySecret: resource.keySecret,
@@ -104,6 +97,8 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             creds.label = resource.region.concat("-", resource.bucketName)
         if badField(creds.label)
             errors.label = "invalid label"
+        else if !resID && findResource({ label: creds.label })
+            errors.label = "duplicate name"
         if badField(resource.keyID)
             errors.keyID = "required field"
         if badField(resource.keySecret)
@@ -119,14 +114,35 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
                 creds.hidden = resource.hidden
             if resource.locked
                 creds.locked = resource.locked
-            resID = hashResourceID(resource)
-            `(error, data) = connector.vault.addResource(resID, creds)
+            if resID {
+                `(error, data) = connector.vault.updateResource(resID, creds)
+                change = { changed: [resID] }
+            } else {
+                resID = UUID()
+                `(error, data) = connector.vault.addResource(resID, creds)
+                change = { added: [resID] }
+            }
             if data {
-                watch.notify(connClassID, { added: [resID] })
-                data = resID }
+                watch.notify(connClassID, change)
+                data = resID
+            }
         }
         list(error, data)
     }
+
+    // addResource
+    //
+    // Add a new resource, returning a standard error tuple per writeResource.
+    connector.addResource = function addResource(resource) {
+        writeResource(resource)
+    }
+
+    // updateResource
+    //
+    // Update an existing resource, returning a standard error tuple per writeResource.
+    connector.updateResource = function updateResource(resID, resource) {
+        writeResource(resource, resID)
+    }
     
     // getResource
     //

package/plugins/serviceAws/serviceAws.nlg

@@ -24,6 +24,7 @@ function sawsInit(local manifest) {
     Naan.module.build(module.id, "serviceAws", function(modobj, compobj) {
         compobj.manifest = manifest
         require("frameworks/common").LiveImport()
+        require("./aws_utils.nlg")
         function sawsPostload() {
             if js.g
                 awsSDK = js.r("aws-sdk")

package/plugins/serviceGitHub/psm_github.nlg

@@ -45,41 +45,34 @@ closure phubConnector(psm, local connClassID, connector, watch) {
 
     // findResource
     //
-    // Find a resource with the specified contents, returning the resID or false. The loose criteria
-    // is: if added, would the specified contents be redundant to an existing resource?
+    // Find a resource with the specified contents, returning the resID or false.
     connector.findResource = function findResource(resource, local resID, creds) {
         for resID in listResources().1 {
             creds = access(resID).1
-            if resource.userName == creds.userName
+            if ((!resource.label || resource.label == creds.label)
+                && (!resource.userName || resource.userName == creds.userName))
                 return (resID)
         }
         false
     }
-    
-    // hashResourceID
-    //
-    // Make a stable hash based on the resource credentials that we can persist or share between
-    // different domains accessing the same resource.
-    connector.hashResourceID = function hashResourceID(resource, local ident) {
-        ident = resource.userName
-        if resource.label
-            ident = ident.concat(resource.label)
-        HashSHA256(ident)
-    }
 
-    // addResource
+    // writeResource
     //
-    // Add credentials for a named resource. The label is a string. The resource is a dictionary
-    // comprising these keys:
+    // Add or update credentials for a named resource. If no resID is specified this adds a new
+    // resource; otherwise it updates an existing one. The resource is a dictionary comprising these
+    // keys:
     //      label       - [optional] label we use for this resource
     //      userName    - GitHub user name
     //      userPat     - GitHUB Personal Access Token
     //      hidden      - [optional] don't enumerate to user
     //      locked      - [optional] don't allow user delete
-    // The return value is (error, result) tuple. Error, if non-false, is a dictionary of field
-    // keys and error diagnostic strings. The special key "label" refers to the label string.
-    connector.addResource = function addResource(resource, local label, errors, creds, error, resID, data) {
-        function badField(str) { !string(str) || str.trim().length == 0 } 
+    // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
+    // keys and error diagnostic strings.
+    connector.writeResource = function writeResource(resource, resID,
+        local label, errors, creds, error, data) {
+        // badField - test for a bad string
+        function badField(str) { !string(str) || str.trim().length == 0 }
+
         creds = { 
             userName: resource.userName,
             userPat: resource.userPat
@@ -91,6 +84,8 @@ closure phubConnector(psm, local connClassID, connector, watch) {
             creds.label = resource.region.concat("-", resource.bucketName)
         if badField(creds.label)
             errors.label = "invalid label"
+        else if !resID && findResource({ label: creds.label })
+            errors.label = "duplicate name"
         if badField(resource.userName)
             errors.userName = "required field"
         if badField(resource.userPat)
@@ -102,15 +97,36 @@ closure phubConnector(psm, local connClassID, connector, watch) {
                 creds.hidden = resource.hidden
             if resource.locked
                 creds.locked = resource.locked
-            resID = hashResourceID(resource)
-            `(error, data) = connector.vault.addResource(resID, creds)
+            if resID {
+                `(error, data) = connector.vault.updateResource(resID, creds)
+                change = { changed: [resID] }
+            } else {
+                resID = UUID()
+                `(error, data) = connector.vault.addResource(resID, creds)
+                change = { added: [resID] }
+            }
             if data {
-                watch.notify(connClassID, { added: [resID] })
-                data = resID }
+                watch.notify(connClassID, change)
+                data = resID
+            }
         }
         list(error, data)
     }
 
+    // addResource
+    //
+    // Add a new resource, returning a standard error tuple per writeResource.
+    connector.addResource = function addResource(resource) {
+        writeResource(resource)
+    }
+
+    // updateResource
+    //
+    // Update an existing resource, returning a standard error tuple per writeResource.
+    connector.updateResource = function updateResource(resID, resource) {
+        writeResource(resource, resID)
+    }
+
     // deleteResource
     //
     // Delete credentials for a resource