@naanlang/naan 1.0.6 → 1.0.8

package/plugins/serviceAws/aws/lambda_ws_init.nlg

@@ -22,7 +22,8 @@ function initPreload() {
     Naan.module.chns("Play")
     sudo(putproc(exception, false))                                         // ### need a better mechanism
 };
-initPreload();
+if symbol(NideBuild)
+    initPreload();                                                          // not building
 
 
 /*

package/plugins/serviceAws/aws_dynamo.nlg

@@ -515,6 +515,8 @@ closure DynaConverter(local conv) {
 closure DynaTable(dyna, tablename, local table) {
     table = new(object, this)
     table.name = tablename
+    if dyna.ycdb
+        table.ycdb = true                                                   // Yandex Cloud version of DynamoDB
 
     // info
     //
@@ -650,7 +652,7 @@ closure DynaTable(dyna, tablename, local table) {
     // Ensure that the table is ready before executing an operation. This returns an error tuple or
     // false to proceed.
     //
-    function preflightError(op) {
+    function preflightError(op, local result) {
         if !dyna.aws
             return (list(Error("DynaTable.".concat(op, ": not logged into AWS"))))
         if !table.hashKey {
@@ -665,7 +667,7 @@ closure DynaTable(dyna, tablename, local table) {
     // Make a dynamoDB key dictionary for the table using the specified primary key, which can be a
     // tuple of hash/range or just a hash if only a hash is needed for this table.
     //
-    conv.makeDynKey = function makeDynKey(primaryKey, local hashValue, rangeValue, key) {
+    table.makeDynKey = function makeDynKey(primaryKey, local hashValue, rangeValue, key) {
         if tuple(primaryKey)
             `(hashValue, rangeValue) = primaryKey
         else
@@ -681,7 +683,7 @@ closure DynaTable(dyna, tablename, local table) {
     //
     // Get a record from the table with the specified hash value, and optional range value.
     //
-    table.getRecord = closure getRecord(hashValue, rangeValue, local result, params, pending) {
+    table.getRecord = closure getRecord(hashValue, rangeValue, local params, pending) {
         if preflightError(this)
             return
         params = {
@@ -706,7 +708,7 @@ closure DynaTable(dyna, tablename, local table) {
     //
     //     Put a record in the table, replacing any existing record with the same key.
         
-    table.putRecord = closure putRecord(record, local result, params, pending) {
+    table.putRecord = closure putRecord(record, local params, pending) {
         if preflightError(this)
             return
         params = {
@@ -735,13 +737,15 @@ closure DynaTable(dyna, tablename, local table) {
     //      set:        [`(attribute, value)]   // set attributes to values
     //      sum:        [`(attribute, value)]   // attributes += values
     //      diff:       [`(attribute, value)]   // attributes -= values
+    //      add:        [`(attribute, value)]   // ADD(attributes, values) -- attrib need not exist
     //      remove:     [attribute]             // remove attributes from the record
     //      return:     "all_old" | "all_new" | "updated_old" | "updated_new"
     //                                          // return the record as chosen, otherwise { }
     //  }
     // The result is a standard result tuple `(error, record).
         
-    table.updateRecord = closure updateRecord(hashValue, rangeValue, options, local error, result, params, actions, pending) {
+    table.updateRecord = closure updateRecord(hashValue, rangeValue, options,
+        local error, params, actions, pending) {
         if preflightError(this)
             return
         params = {
@@ -750,13 +754,10 @@ closure DynaTable(dyna, tablename, local table) {
         params.Key = makeDynKey(list(hashValue, rangeValue))
         if options.condition {
             `(error) = dyna.conv.createKeyExpression(params, {
-                KeyConditionExpression: {
-                    key: table.hashKey
-                    "=": hashValue
-                }
+                ConditionExpression: options.condition
             })
             if error
-                return (list(Error("table.updateRecord:", error)))
+                return (list(Error("table.updateRecord condition:", error)))
         }
         dyna.conv.createUpdateExpression(params, options)
         if options.return {
@@ -793,7 +794,8 @@ closure DynaTable(dyna, tablename, local table) {
     //  }
  
         
-    table.deleteRecord = closure deleteRecord(hashValue, rangeValue, options, local result, params, pending) {
+    table.deleteRecord = closure deleteRecord(hashValue, rangeValue, options,
+        local params, pending) {
         if preflightError(this)
             return
         params = {
@@ -802,13 +804,10 @@ closure DynaTable(dyna, tablename, local table) {
         params.Key = makeDynKey(list(hashValue, rangeValue))
         if options.condition {
             `(error) = dyna.conv.createKeyExpression(params, {
-                KeyConditionExpression: {
-                    key: table.hashKey
-                    "=": hashValue
-                }
+                ConditionExpression: options.condition
             })
             if error
-                return (list(Error("table.deleteRecord:", error)))
+                return (list(Error("table.deleteRecord condition:", error)))
         }
         if options.return {
             params.ReturnValues = {
@@ -869,7 +868,7 @@ closure DynaTable(dyna, tablename, local table) {
     //
 
     table.queryRecords = closure queryRecords(hashValue, options
-        local error, result, params, hashKey, op, expr, output, error)
+        local error, params, hashKey, op, expr, output, error)
     {
         if preflightError(this)
             return
@@ -938,7 +937,7 @@ closure DynaTable(dyna, tablename, local table) {
     //
 
     table.scanRecords = closure scanRecords(options
-        local result, params, op, expr, output, error)
+        local params, op, expr, output, error)
     {
         if preflightError(this)
             return
@@ -994,14 +993,22 @@ closure DynamoDB(local dyna) {
 
     // login
     //
-    dyna.login = function login(creds) {
-        dyna.aws = xnew(awsSDK.DynamoDB, {
-            region:                 creds.region
-            credentials: {
+    dyna.login = function login(creds, local params) {
+        if !awsSDK
+            return (list(Error("DynamoDB.login: no AWS SDK available")))
+        params = { }
+        if creds {
+            params.region = creds.region
+            params.credentials = {
                 accessKeyId:        creds.keyID
                 secretAccessKey:    creds.keySecret
             }
-        })
+            if creds.dyna_endpoint {
+                params.endpoint = creds.dyna_endpoint
+                dyna.ycdb = true
+            }
+        }
+        dyna.aws = xnew(awsSDK.DynamoDB, params)
         list(false, { ok: true })
     }
     
@@ -1011,6 +1018,8 @@ closure DynamoDB(local dyna) {
     // yet exist; see DynaTable for methods.
     //
     dyna.table = function table(name) {
+        if !dyna.aws
+            debuglog("DynamoDB.table: database not open:", name)
         DynaTable(dyna, name)
     }
     
@@ -1032,9 +1041,9 @@ closure DynamoDB(local dyna) {
         dynexBatchGetRecords(dyna, request, options)
     }
     
-    // dynexBatchWriteRecords
+    // batchWriteRecords
     //
-    dyna.batchWriteRecords = function batchWriteRecords(request, options) {
+    dyna.batchWriteRecords = function loadBatchWriteRecords(request, options) {
         require("./aws_dynextra.nlg")
         dyna.batchWriteRecords = function bwr(request, options) {
             dynexBatchWriteRecords(dyna, request, options)
@@ -1042,7 +1051,7 @@ closure DynamoDB(local dyna) {
         dynexBatchWriteRecords(dyna, request, options)
     }
     
-    // dynexTransactGetRecords
+    // transactGetRecords
     //
     dyna.transactGetRecords = function loadTransactGetRecords(request, options) {
         require("./aws_dynextra.nlg")
@@ -1052,6 +1061,16 @@ closure DynamoDB(local dyna) {
         dynexTransactGetRecords(dyna, request, options)
     }
 
+    // transactWriteRecords
+    //
+    dyna.transactWriteRecords = function loadTransactWriteRecords(request, options) {
+        require("./aws_dynextra.nlg")
+        dyna.transactWriteRecords = function tgr(request, options) {
+            dynexTransactWriteRecords(dyna, request, options)
+        }
+        dynexTransactWriteRecords(dyna, request, options)
+    }
+
     // finis
     dyna
 }

package/plugins/serviceAws/aws_dynextra.nlg

@@ -37,7 +37,7 @@ function dynexBatchGetRecords(dyna, request,
     request = new(request)                                                  // allow local modifications
     index = { }
     for titem in request {                                                  // preflight all tables
-        tname = titem.table.tablename
+        tname = titem.table.name
         if index[tname]
             return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
         `(error) = titem.table.info()
@@ -57,7 +57,7 @@ function dynexBatchGetRecords(dyna, request,
             activity = false
             for `(tname, titem) in index {
                 if titem.keys.length > 0 {
-                    primary = titem.table.makeDynKey(titem.keys.unshift())
+                    primary = titem.table.makeDynKey(titem.keys.shift())
                     if params.RequestItems[tname].Keys
                         params.RequestItems[tname].Keys.push(primary)
                     else
@@ -81,7 +81,7 @@ function dynexBatchGetRecords(dyna, request,
             dyna.aws.batchGetItem(params, function(err, data) {
                 if err
                     debuglog("aws.batchGetItem failed", err)
-                pending.signal(err, data)
+                pending.signal(list(err, data))
             })
             `(err, response) = pending.wait()
             for `(tname, table) in response.Responses
@@ -124,7 +124,8 @@ function dynexBatchGetRecords(dyna, request,
  *
  */
 
-closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
+closure dynexBatchWriteRecords(dyna, request,
+    local index, group, tname, batches, error, activity) {
     if !dyna.aws
         return (list(Error("DynaTable.batchWriteRecords: not logged into AWS")))
 
@@ -132,19 +133,18 @@ closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
     // addOp
     // Add a put or delete operation to the parameters
     //
-    function addOp(params, table, del, record, local request) {
+    function addOp(params, table, del, record, primary, local request) {
         if del {                                                            // deleting a record
-            if tuple(record)
-                record = table.makeDynKey(record)                           // caller provided hash/range key pair
-            else
-                request = { DeleteRequest: { Key: dyna.conv.recordNaanToDyn(record) } }
+            if !tuple(record)
+                record = list(record[table.hashKey], record[table.rangeKey])
+            request = { DeleteRequest: { Key: table.makeDynKey(record) } }
         }
         else                                                                // putting a record
             request = { PutRequest: { Item: dyna.conv.recordNaanToDyn(record) } }
-        if params.RequestItems[table.tablename]
-            params.RequestItems[table.tablename].push(request)
+        if params.RequestItems[table.name]
+            params.RequestItems[table.name].push(request)
         else
-            params.RequestItems[table.tablename] = [request]
+            params.RequestItems[table.name] = [request]
     }
 
     // transform into batches
@@ -155,48 +155,68 @@ closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
     // request have no more entries.
     //
     request = new(request)                                                  // allow local modifications
+    index = { }
+    for group in request {                                                  // preflight all tables
+        tname = group.table.name
+        if index[tname]
+            return (Error("DynaTable.dynexBatchWriteRecords: duplicate table in request:", tname))
+        `(error) = group.table.info()
+        if error
+            return (Error("DynaTable.dynexBatchWriteRecords: table", tname, ":", error))
+        index[tname] = group
+        if group.put                                                        // ### in case these are xobjects
+            group.put = new(group.put)                                      // new() doesn't deep clone them
+        if group.delete
+            group.delete = new(group.delete)
+    }
+    request = new(request)                                                  // allow local modifications
     batches = []
     activity = true
     while activity
-        let (params, batchsize, table) {
+        let (params, batchsize) {
             params = {
                 RequestItems: { }
             }
             batchsize = 0
             while :fillbatch activity {                                     // loop until no activity or batch full
                 activity = false
-                for table in request {
-                    if table.delete.length > 0
-                        addOp(params, table, true, table.delete.unshift())
-                    else if table.put.length > 0
-                        addOp(params, table, false, table.put.unshift())
-                    else
-                        continue
-                    activity = true
-                    if ++batchsize >= 25
+                for group in request {
+                    if group.delete.length > 0 {
+                        addOp(params, group.table, true, group.delete.shift())
+                        activity = true
+                        ++batchsize
+                    }
+                    if group.put.length > 0 {
+                        addOp(params, group.table, false, group.put.shift())
+                        activity = true
+                        ++batchsize
+                    }
+                    if batchsize >= 25
                         break :fillbatch                                    // our batch is full
                 }
             }
             if batchsize > 0
                 batches.push(params)
         } ()
-    asyncArray(batches, 10, closure bwr(params, local delayer, pending, err, response) {
+    asyncArray(batches, 1, closure bwr(params, local delayer, pending, err, response) {
         delayer = dyna.conv.genBackoffDelayer()
         loop {
             pending = new(nonce)
-            dyna.aws.batchWriteItem(params, function(err, data) {
+            dyna.aws.batchWriteItem(params, closure(err, data) {
                 if err
                     debuglog("aws.batchWriteItem failed", err)
-                pending.signal(err, data)
+                pending.signal(list(err, data))
             })
             `(err, response) = pending.wait()
             if response.UnprocessedKeys
                 params.RequestItems = response.UnprocessedKeys              // still more to do
             else
                 break                                                       // total success or failure
+            debuglog("dynexBatchWriteRecords unprocessedKeys:", response.UnprocessedKeys)
             if delayer() > dyna.timeout {
                 if !error                                                   // too slow
                     error = err
+                debuglog("dynexBatchWriteRecords timeout", error, err)
                 break
             }
         }
@@ -208,7 +228,6 @@ closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
 };
 
 
-
 /*
  * dynexTransactGetRecords
  *
@@ -235,12 +254,12 @@ closure dynexTransactGetRecords(dyna, request,
     request = new(request)                                                  // allow local modifications
     index = { }
     for titem in request {                                                  // preflight all tables
-        tname = titem.table.tablename
+        tname = titem.table.name
         if index[tname]
-            return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
+            return (Error("DynaTable.transactGetRecords: duplicate table in request:", tname))
         `(error) = titem.table.info()
         if error
-            return (Error("DynaTable.batchGetRecords: table", tname, ":", error))
+            return (Error("DynaTable.transactGetRecords: table", tname, ":", error))
         titem.data = []
         index[tname] = titem
     }
@@ -252,7 +271,7 @@ closure dynexTransactGetRecords(dyna, request,
             params.TransactItems.push({
                 Get: {
                     TableName: tname
-                    Key: makeDynKey(titem.keys.unshift())
+                    Key: titem.table.makeDynKey(titem.keys.shift())
                 }
             })
         }
@@ -262,7 +281,7 @@ closure dynexTransactGetRecords(dyna, request,
     dyna.aws.transactGetItems(params, function(err, data) {
         if err
             debuglog("aws.transactGetItems failed", err)
-        pending.signal(err, data)
+        pending.signal(list(err, data))
     })
     `(error, response) = pending.wait()
     if error
@@ -277,6 +296,111 @@ closure dynexTransactGetRecords(dyna, request,
 };
 
 
+/*
+ * dynexTransactWriteRecords
+ *
+ * column positioning:                          //                          //                      !
+ * Write records to one or more tables using arrays of keys as a single transaction. The result is an
+ * error if anything failed or generic success if it worked. The request layout is as follows. Note
+ * that unlike the other dynex* calls, each table/key/operation item is separately specified.
+ *
+ *  [
+ *      {
+ *          table:  <DynaTable object>          // table to act upon
+ *          key:    <hash-range tuple>
+ *          put:    {
+ *              condition:  <range-expression>  // conditional expression
+ *              item:       <record-to-put>
+ *          }
+ *          update: {
+ *              condition:  <range-expression>      // conditional expression
+ *              set:        [`(attribute, value)]   // set attributes to values
+ *              sum:        [`(attribute, value)]   // attributes += values
+ *              diff:       [`(attribute, value)]   // attributes -= values
+ *              add:        [`(attribute, value)]   // ADD(attributes, values) -- attrib need not exist
+ *              remove:     [attribute]             // remove attributes from the record
+ *          }
+ *          delete: {
+ *              condition:  <range-expression>  // conditional expression
+ *          }
+ *          check:  {
+ *              condition:  <range-expression>  // conditional expression
+ *              return:     "all_old"           // optionally return record if condition fails
+ *          }
+ *      }
+ *      { <next-item> }
+ *  ]
+ * This returns a standard result tuple `(error, result). An error is only reported if we are 
+ * unable to complete all operations.
+ *
+ */
+
+closure dynexTransactWriteRecords(dyna, request,
+    local index, titem, tname, error, params, opset, spec, op, pending, response) {
+    if !dyna.aws
+        return (list(Error("DynaTable.transactWriteRecords: not logged into AWS")))
+    request = new(request)                                                  // allow local modifications
+    index = { }
+    for titem in request {                                                  // preflight all tables
+        tname = titem.table.name
+        if !index[tname] {
+            `(error) = titem.table.info()
+            if error
+                return (Error("DynaTable.transactWriteRecords: table", tname, ":", error))
+            index[tname] = titem
+        }
+    }
+    params = {
+        TransactItems: []
+    }
+    for titem in request {
+        opset = {
+            TableName: titem.table.name
+            Key: titem.table.makeDynKey(titem.key)
+        }
+        if titem.check {
+            spec = titem.check
+            op = `ConditionCheck
+            if spec.return
+                opset.ReturnValues = "ALL_OLD"
+        } else if titem.put {
+            spec = titem.put
+            op = `Put
+            opset.Item = dyna.conv.recordNaanToDyn(spec.item)
+            opset.Key = undefined                                           // key must be in item
+        } else if titem.update {
+            spec = titem.update
+            op = `Update
+            dyna.conv.createUpdateExpression(opset, spec)
+        } else if titem.delete {
+            spec = titem.delete
+            op = `Delete
+        }
+        if spec.condition {
+            `(error) = dyna.conv.createKeyExpression(opset, {
+                ConditionExpression: spec.condition
+            })
+            if error
+                return (list(Error("DynaTable.transactWriteRecords condition:", error)))
+        }
+        titem = { }
+        titem[op] = opset
+        params.TransactItems.push(titem)
+    }
+    pending = new(nonce)
+    dyna.aws.transactWriteItems(params, function(err, data) {
+        if err
+            debuglog("aws.transactWriteItems failed", err)
+        pending.signal(list(err, data))
+    })
+    `(error, response) = pending.wait()
+    if error
+        list(error)
+    else
+        list(false, response)
+};
+
+
 /*
  * dynexInit
  *
@@ -285,7 +409,8 @@ closure dynexTransactGetRecords(dyna, request,
  */
 
 function dynexInit(local manifest) {
-    manifest = `(dynexBatchGetRecords, dynexBatchWriteRecords, dynexTransactGetRecords, dynexInit)
+    manifest = `(dynexBatchGetRecords, dynexBatchWriteRecords, dynexTransactGetRecords,
+        dynexTransactWriteRecords, dynexInit)
 
     Naan.module.build(module.id, "aws_dynextra", function(modobj, compobj) {
         compobj.manifest = manifest

package/plugins/serviceAws/aws_lambda.nlg

@@ -46,10 +46,8 @@ closure Lambda(local lamb) {
             params.Qualifier = qualifier
         pending = new(nonce)
         lamb.aws.getFunction(params, function(error, data) {
-            if error {
-                lambdaError = error
-                debuglog("aws.getFunction failed", error)
-            }
+            if error
+                ErrorDebuglog("aws.getFunction failed", error)
             pending.signal(list(error, data))
         })
         pending.wait()
@@ -69,10 +67,8 @@ closure Lambda(local lamb) {
         }
         pending = new(nonce)
         lamb.aws.updateFunctionCode(params, function(error, data) {
-            if error {
-                lambdaError = error
-                debuglog("aws.updateFunctionCode failed", error)
-            }
+            if error
+                ErrorDebuglog("aws.updateFunctionCode failed", error)
             pending.signal(list(error, data))
         })
         pending.wait()