@naanlang/naan 1.0.6 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@naanlang/naan",
-    "version": "1.0.6",
+    "version": "1.0.7",
     "author": "Richard C. Zulch",
     "description": "Naan™ software platform",
     "main": "./lib/core/naanlib.js",
@@ -12,6 +12,7 @@
     },
     "scripts": {
         "start": "node lib/node_repl.js",
+        "debug": "node --inspect-brk lib/node_repl.js",
         "test": "node test/node_test.js"
     }
 }

package/plugins/serviceAws/aws_dynamo.nlg

@@ -515,6 +515,8 @@ closure DynaConverter(local conv) {
 closure DynaTable(dyna, tablename, local table) {
     table = new(object, this)
     table.name = tablename
+    if dyna.ycdb
+        table.ycdb = true                                                   // Yandex Cloud version of DynamoDB
 
     // info
     //
@@ -665,7 +667,7 @@ closure DynaTable(dyna, tablename, local table) {
     // Make a dynamoDB key dictionary for the table using the specified primary key, which can be a
     // tuple of hash/range or just a hash if only a hash is needed for this table.
     //
-    conv.makeDynKey = function makeDynKey(primaryKey, local hashValue, rangeValue, key) {
+    table.makeDynKey = function makeDynKey(primaryKey, local hashValue, rangeValue, key) {
         if tuple(primaryKey)
             `(hashValue, rangeValue) = primaryKey
         else
@@ -994,14 +996,22 @@ closure DynamoDB(local dyna) {
 
     // login
     //
-    dyna.login = function login(creds) {
-        dyna.aws = xnew(awsSDK.DynamoDB, {
-            region:                 creds.region
-            credentials: {
+    dyna.login = function login(creds, local params) {
+        if !awsSDK
+            return (list(Error("DynamoDB.login: no AWS SDK available")))
+        params = { }
+        if creds {
+            params.region = creds.region
+            params.credentials = {
                 accessKeyId:        creds.keyID
                 secretAccessKey:    creds.keySecret
             }
-        })
+            if creds.dyna_endpoint {
+                params.endpoint = creds.dyna_endpoint
+                dyna.ycdb = true
+            }
+        }
+        dyna.aws = xnew(awsSDK.DynamoDB, params)
         list(false, { ok: true })
     }
     
@@ -1011,6 +1021,8 @@ closure DynamoDB(local dyna) {
     // yet exist; see DynaTable for methods.
     //
     dyna.table = function table(name) {
+        if !dyna.aws
+            debuglog("DynamoDB.table: database not open:", name)
         DynaTable(dyna, name)
     }
     

package/plugins/serviceAws/aws_dynextra.nlg

@@ -37,7 +37,7 @@ function dynexBatchGetRecords(dyna, request,
     request = new(request)                                                  // allow local modifications
     index = { }
     for titem in request {                                                  // preflight all tables
-        tname = titem.table.tablename
+        tname = titem.table.name
         if index[tname]
             return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
         `(error) = titem.table.info()
@@ -57,7 +57,7 @@ function dynexBatchGetRecords(dyna, request,
             activity = false
             for `(tname, titem) in index {
                 if titem.keys.length > 0 {
-                    primary = titem.table.makeDynKey(titem.keys.unshift())
+                    primary = titem.table.makeDynKey(titem.keys.shift())
                     if params.RequestItems[tname].Keys
                         params.RequestItems[tname].Keys.push(primary)
                     else
@@ -81,7 +81,7 @@ function dynexBatchGetRecords(dyna, request,
             dyna.aws.batchGetItem(params, function(err, data) {
                 if err
                     debuglog("aws.batchGetItem failed", err)
-                pending.signal(err, data)
+                pending.signal(list(err, data))
             })
             `(err, response) = pending.wait()
             for `(tname, table) in response.Responses
@@ -124,7 +124,8 @@ function dynexBatchGetRecords(dyna, request,
  *
  */
 
-closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
+closure dynexBatchWriteRecords(dyna, request,
+    local index, group, tname, batches, error, activity) {
     if !dyna.aws
         return (list(Error("DynaTable.batchWriteRecords: not logged into AWS")))
 
@@ -132,19 +133,18 @@ closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
     // addOp
     // Add a put or delete operation to the parameters
     //
-    function addOp(params, table, del, record, local request) {
+    function addOp(params, table, del, record, primary, local request) {
         if del {                                                            // deleting a record
-            if tuple(record)
-                record = table.makeDynKey(record)                           // caller provided hash/range key pair
-            else
-                request = { DeleteRequest: { Key: dyna.conv.recordNaanToDyn(record) } }
+            if !tuple(record)
+                record = list(record[table.hashKey], record[table.rangeKey])
+            request = { DeleteRequest: { Key: table.makeDynKey(record) } }
         }
         else                                                                // putting a record
             request = { PutRequest: { Item: dyna.conv.recordNaanToDyn(record) } }
-        if params.RequestItems[table.tablename]
-            params.RequestItems[table.tablename].push(request)
+        if params.RequestItems[table.name]
+            params.RequestItems[table.name].push(request)
         else
-            params.RequestItems[table.tablename] = [request]
+            params.RequestItems[table.name] = [request]
     }
 
     // transform into batches
@@ -155,48 +155,68 @@ closure dynexBatchWriteRecords(dyna, request, local batches, error, activity) {
     // request have no more entries.
     //
     request = new(request)                                                  // allow local modifications
+    index = { }
+    for group in request {                                                  // preflight all tables
+        tname = group.table.name
+        if index[tname]
+            return (Error("DynaTable.dynexBatchWriteRecords: duplicate table in request:", tname))
+        `(error) = group.table.info()
+        if error
+            return (Error("DynaTable.dynexBatchWriteRecords: table", tname, ":", error))
+        index[tname] = group
+        if group.put                                                        // ### in case these are xobjects
+            group.put = new(group.put)                                      // new() doesn't deep clone them
+        if group.delete
+            group.delete = new(group.delete)
+    }
+    request = new(request)                                                  // allow local modifications
     batches = []
     activity = true
     while activity
-        let (params, batchsize, table) {
+        let (params, batchsize) {
             params = {
                 RequestItems: { }
             }
             batchsize = 0
             while :fillbatch activity {                                     // loop until no activity or batch full
                 activity = false
-                for table in request {
-                    if table.delete.length > 0
-                        addOp(params, table, true, table.delete.unshift())
-                    else if table.put.length > 0
-                        addOp(params, table, false, table.put.unshift())
-                    else
-                        continue
-                    activity = true
-                    if ++batchsize >= 25
+                for group in request {
+                    if group.delete.length > 0 {
+                        addOp(params, group.table, true, group.delete.shift())
+                        activity = true
+                        ++batchsize
+                    }
+                    if group.put.length > 0 {
+                        addOp(params, group.table, false, group.put.shift())
+                        activity = true
+                        ++batchsize
+                    }
+                    if batchsize >= 25
                         break :fillbatch                                    // our batch is full
                 }
             }
             if batchsize > 0
                 batches.push(params)
         } ()
-    asyncArray(batches, 10, closure bwr(params, local delayer, pending, err, response) {
+    asyncArray(batches, 1, closure bwr(params, local delayer, pending, err, response) {
         delayer = dyna.conv.genBackoffDelayer()
         loop {
             pending = new(nonce)
-            dyna.aws.batchWriteItem(params, function(err, data) {
+            dyna.aws.batchWriteItem(params, closure(err, data) {
                 if err
                     debuglog("aws.batchWriteItem failed", err)
-                pending.signal(err, data)
+                pending.signal(list(err, data))
             })
             `(err, response) = pending.wait()
             if response.UnprocessedKeys
                 params.RequestItems = response.UnprocessedKeys              // still more to do
             else
                 break                                                       // total success or failure
+            debuglog("dynexBatchWriteRecords unprocessedKeys:", response.UnprocessedKeys)
             if delayer() > dyna.timeout {
                 if !error                                                   // too slow
                     error = err
+                debuglog("dynexBatchWriteRecords timeout", error, err)
                 break
             }
         }
@@ -235,7 +255,7 @@ closure dynexTransactGetRecords(dyna, request,
     request = new(request)                                                  // allow local modifications
     index = { }
     for titem in request {                                                  // preflight all tables
-        tname = titem.table.tablename
+        tname = titem.table.name
         if index[tname]
             return (Error("DynaTable.batchGetRecords: duplicate table in request:", tname))
         `(error) = titem.table.info()
@@ -252,7 +272,7 @@ closure dynexTransactGetRecords(dyna, request,
             params.TransactItems.push({
                 Get: {
                     TableName: tname
-                    Key: makeDynKey(titem.keys.unshift())
+                    Key: makeDynKey(titem.keys.shift())
                 }
             })
         }
@@ -262,7 +282,7 @@ closure dynexTransactGetRecords(dyna, request,
     dyna.aws.transactGetItems(params, function(err, data) {
         if err
             debuglog("aws.transactGetItems failed", err)
-        pending.signal(err, data)
+        pending.signal(list(err, data))
     })
     `(error, response) = pending.wait()
     if error

package/plugins/serviceAws/aws_s3.nlg

@@ -13,17 +13,36 @@
 
 /*
  * S3bucket
- *
- *     S3 buckets.
+ * 
+ * Bucket ooptions:
+ *  {
+ *      cacheControl: "max-age=2592000"         - default cache control, e.g. to "max-age=2592000"
+ *  }
  *
  */
 
-closure S3bucket(s3, name, local bucket) {
+closure S3bucket(s3, name, bucketOptions, local bucket) {
     bucket = new(object, this)
     bucket.name = name
-    
+        
+    //
+    // head
+    //
+    bucket.head = closure head(key, local pending) {
+        pending = new(nonce)
+        s3.aws.headObject({
+            Bucket: bucket.name
+            Key: key
+        }, function (error, data) {
+            if error
+                error = Error("S3bucket.head failed:", error.name, "key:", key, "in", bucket.name)
+            pending.signal(list(error, data))
+        })
+        pending.wait()
+    }
+
     //
-    // list
+    // listObjects
     //
     bucket.listObjects = closure listObjects(local pending) {
         pending = new(nonce)
@@ -31,31 +50,46 @@ closure S3bucket(s3, name, local bucket) {
             Bucket: bucket.name
         }, function (error, data) {
             if error
-                error = Error("S3bucket.list failed", error)
+                error = Error("S3bucket.list failed:", error.name, "for", bucket.name)
             pending.signal(list(error, data))
         })
         pending.wait()
     }
 
     //
-    // upload
+    // put
     //
-    bucket.upload = closure upload(key, data, local pending) {
-        pending = new(nonce)
-        s3.aws.upload({
+    // options:
+    //  {
+    //      contentType: "text/plain"           - override contentType if specified
+    //      cacheControl: "max-age=2592000"     - default cache control, e.g. to "max-age=2592000"
+    //      metadata: { <items> }               - optional dictionary of keys to store
+    //  }
+    //
+    bucket.put = closure put(key, data, options, local params, pending) {
+        if !contentType
+            contentType = "application/octet-stream"
+        params = {
             Bucket: bucket.name
-            ACL: "public-read"
-            Body: data
             Key: key
-                ContentType: "application/octet-stream"                     // force download from browser
-        })
-        .on("httpUploadProgress", function(evt) {
-            debuglog("upload progress", totuple(evt))
-        })
-        .send(function(error, data) {
+            ContentMD5: HashMD5_base64(data)
+            Body: data
+        }
+        if options.contentType
+            params.ContentType = options.contentType
+        if options.metadata
+            params.Metadata = options.metadata
+        else
+            params.Metadata = "application/octet-stream"
+        if options.cacheControl
+            params.CacheControl = options.cacheControl
+        else if bucket.cacheControl
+            params.CacheControl = bucket.cacheControl
+        pending = new(nonce)
+        s3.aws.putObject(params, function(error, resp) {
             if error
-                error = Error("S3bucket.upload failed", error)
-            pending.signal(list(error, data))
+                error = Error("S3bucket.put failed:", error.name, "key:", key, "in", bucket.name)
+            pending.signal(list(error, resp))
         })
         pending.wait()
     }
@@ -88,6 +122,8 @@ closure S3(local s3) {
                 accessKeyId:        creds.keyID
                 secretAccessKey:    creds.keySecret
             }
+            if creds.endpoint
+                params.endpoint = creds.endpoint
         }
         s3.aws = xnew(awsSDK.S3, params)
         if awsSDK.S3Client
@@ -111,12 +147,23 @@ closure S3(local s3) {
     // This obtains a pre-signed URL for the specified command and its parameters. Options control
     // how the URL is generated.
     //
+    // CommandParams:
+    //  {
+    //      Bucket:             <string>        // bucket name
+    //      Key:                <string>        // object being operated upon
+    //      ContentType:        <string>        // video/mp4 or whatever
+    //      ResponseCacheControl:   <string>    // reportedly for getObject only
+    //      ACL:                <string>        // e.g. "public-read"
+    //      Body:               <data>          // fiik
+    //  }
+    //
     // Options:
     //  {
-    //      expiresIn:        <seconds>         // default expiration is 900 (15 minutes)
+    //      expiresIn:          <seconds>       // default expiration is 900 (15 minutes)
+    //      signingDate:        <epoch-secs>    // reference time for URL being created
     //  }
     //
-    s3.getSignedURL = closure getSignedURL(cmdname, cmdparams, local command, options, pending) {
+    s3.getSignedURL = closure getSignedURL(cmdname, cmdparams, options, local command, pending) {
         if s3.client {
             if cmdname == "putObject"
                 cmdname = "PutObjectCommand"
@@ -124,16 +171,15 @@ closure S3(local s3) {
                 cmdname = "GetObjectCommand"
             else
                 return (list(Error("S3bucket.getSignedURL: command not present:", cmdname)))
-            if cmdparams.Expires {
-                cmdparams = new(cmdparams)
-                options.expiresIn = cmdparams.Expires
-                cmdparams.Expires = undefined
-            } else
-                options = { }
             command = xnew(awsSDK.s3Commands[cmdname], cmdparams)
             await(awsSDK.getSignedUrl(s3.client, command, options))
         } else {
+            debuglog("s3.getSignedURL: warning: old API")
             pending = new(nonce)
+            if options.expiresIn {
+                cmdparams = new(cmdparams)
+                cmdparams.Expires = options.expiresIn
+            }
             s3.aws.getSignedUrl(cmdname, cmdparams, function(error, data) {
                 pending.signal(list(error, data))
             })

package/plugins/serviceAws/dbt_aws.nlg

@@ -37,6 +37,19 @@ closure dawsBucket(s3b, local bucket, s3) {
     closure asyncResult args {
         future(function() { apply(pop(args), args) }).run()
     }
+    
+    // hashFromETag
+    //
+    // Return the MD5 hash for an ETag
+    //
+    function hashFromETag(etag) {
+        if etag.startsWith("W/")
+            etag = etag.slice(2)                                            // "weak" from gzip with NGINX, apparently
+        if etag.length == 34                                                // multipart S3 ETag will is longer and will be ignored
+            etag.substring(1,33)
+        else
+            undefined
+    }
 
     // close
     //
@@ -65,8 +78,8 @@ closure dawsBucket(s3b, local bucket, s3) {
         function s3GetCB(error, resp) {
             if error
                 error = Error("can't get object", error, key)
-            else if resp.ETag.length == 34
-                resp.md5 = resp.ETag.substring(1,33)                        // ETags have to have quotes, but our hash doesn't
+            else
+                resp.md5 = hashFromETag(resp.ETag)
             callback(error, resp)
         }
 
@@ -79,12 +92,18 @@ closure dawsBucket(s3b, local bucket, s3) {
     // put
     //
     // Put an object in the bucket. Parameters include:
-    //      key         - the object name, which may not be empty or start with "/"
-    //      data        - optional content to store
-    //      metadata    - optional dictionary of keys to store
+    //  key                                     - the object name, which may not be empty or start with "/"
+    //  data                                    - optional content to store
+    //  options:
+    //  {
+    //      contentType: "text/plain"           - override contentType if specified
+    //      cacheControl: "max-age=2592000"     - default cache control, e.g. to "max-age=2592000"
+    //      metadata: { <items> }               - optional dictionary of keys to store
+    //  }
     
-    bucket.put = closure put(key, data, metadata, callback, local params, mimeType, md5) {
-        if !string(key) || length(key) == 0 || key.substring(0,1) == "/" || metadata && !dictionary(metadata)
+    bucket.put = closure put(key, data, options, callback,
+        local params, mimeType, md5) {
+        if !string(key) || length(key) == 0 || key.substring(0,1) == "/"
             return (asyncResult(callback, Error("invalid arguments")))
         if !s3
             return (asyncResult(callback, Error("database closed")))
@@ -128,9 +147,13 @@ closure dawsBucket(s3b, local bucket, s3) {
                     mimeType = mimeType.concat("; charset=UTF-8")
                 params.ContentType = mimeType
             }
+            if options.contentType
+                params.ContentType = options.contentType
         }
-        if metadata
-            params.Metadata = metadata
+        if options.metadata
+            params.Metadata = options.metadata
+        if options.cacheControl
+            params.CacheControl = options.cacheControl
         if data
             params.Body = data
         else
@@ -142,11 +165,10 @@ closure dawsBucket(s3b, local bucket, s3) {
                 error = Error("can't put object", error, key)
             else {
                 resp.length = length(data)
-                if resp.ETag.length == 34 {                                 // multipart S3 ETag will is longer and will be ignored
-                    resp.md5 = resp.ETag.substring(1,33)                    // ETags have to have quotes, but our hash doesn't
-                    if md5 != resp.md5 {
-                        error = Error("hash match failure", md5, resp.md5)  // has match failed on round-trip
-                        resp = false } } }
+                resp.md5 = hashFromETag(resp.ETag)
+                if resp.md5 && md5 != resp.md5 {
+                    error = Error("hash match failure", md5, resp.md5)      // has match failed on round-trip
+                    resp = false } }
             callback(error, resp)
         })
     }
@@ -285,12 +307,31 @@ closure dawsBucket(s3b, local bucket, s3) {
  *
  */
 
-closure dawsTable(database, path, local table) {
+closure dawsTable(database, path, tableOptions, local table) {
     table = new(object, this)
     table.type = database.type
     table.db = database
     table.path = path
     table.s3prefix = makePrefix(path)
+    
+    // readableStreamToArrayBuffer
+    //
+    // For browsers, convert a ReadableStream to an array buffer.
+    //
+    function readableStreamToArrayBuffer(stream, local result, reader, error, data, chunk) {
+        result = xnew(js.w.Uint8Array, 0)
+        reader = stream.getReader()
+        loop {
+            `(error, data) = await(reader.read())
+            if error || data.done
+                break
+            chunk = xnew(js.w.Uint8Array, result.length + data.value.length)
+            chunk.set(result)
+            chunk.set(data.value, result.length)
+            result = chunk
+        }
+        result
+    }
         
     // makePrefix
     //
@@ -344,7 +385,7 @@ closure dawsTable(database, path, local table) {
         if resp.ContentType
             doc.contentType = resp.ContentType
         if resp.md5
-            doc.md5 = resp.md5
+            doc._md5 = resp.md5
         doc
     }
     
@@ -394,11 +435,14 @@ closure dawsTable(database, path, local table) {
         if content
             doc.content = content
         metadata = metadata2s3(metadata)                                    // filter out any invalid metadata
-        database.bucket.put(id, content, metadata, function(error, resp) {
+        database.bucket.put(id, content, {
+            metadata: metadata
+            cacheControl: tableOptions.cacheControl
+        }, function(error, resp) {
             if error
                 error = Error("add failed", error, id, { status: error.$metadata.httpStatusCode })
             else {
-                doc.md5 = resp.md5                                          // update the original doc
+                doc._md5 = resp.md5                                         // update the original doc
                 doc.length = resp.length }
             callback(error, doc)
         })
@@ -411,14 +455,17 @@ closure dawsTable(database, path, local table) {
     table.update = closure update(doc, callback, local metadata) {
         if !callback
             return (syncAdapter(update, doc))
-        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc.md5
+        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc._md5
             return (asyncResult(callback, Error("invalid argument")))
         metadata = metadata2s3(doc)
-        database.bucket.put(doc._id, doc.content, metadata, function(error, resp) {
+        database.bucket.put(doc._id, doc.content, {
+            metadata: metadata
+            cacheControl: tableOptions.cacheControl
+        }, function(error, resp) {
             if error
                 error = Error("update failed", error, doc._id, { status: error.$metadata.httpStatusCode })
             else {
-                doc.md5 = resp.md5                                          // update the original doc
+                doc._md5 = resp.md5                                         // update the original doc
                 doc.modified = undefined                                    // we don't know exact modify date
                 doc.length = resp.length }
             callback(error, doc)
@@ -433,7 +480,7 @@ closure dawsTable(database, path, local table) {
     table.delete = closure delete(doc, callback) {
         if !callback
             return (syncAdapter(delete, doc))
-        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc.md5
+        if !doc._id || !doc._id.startsWith(table.s3prefix) || !(doc._md5 || doc._id.slice(-1) == "/")
             return (asyncResult(callback, Error("invalid argument")))
         database.bucket.delete(doc._id, function(error, resp) {
             if error
@@ -469,6 +516,8 @@ closure dawsTable(database, path, local table) {
                     { }                                                     // already "decoded", e.g. base64
                 else if jsTypedArray(resp.content)
                     resp.content = xnew(TextDecoder, "utf-8").decode(resp.content)
+                else if jsInstanceOf(resp.content, js.w.ReadableStream)
+                    resp.content = xnew(TextDecoder, "utf-8").decode(readableStreamToArrayBuffer(resp.content))
                 resp._id = id }
             callback(error, resp)
         })
@@ -748,12 +797,12 @@ closure S3DB(s3b, local database, result) {
     //
     // Return a table-access object for the table.
 
-    database.table = function table(path) {
+    database.table = function table(path, options) {
         if !database.bucket
             return (list(Error("database closed")))
         if !string(path)
             return (list(Error("invalid table path")))
-         dawsTable(database, path)
+         dawsTable(database, path, options)
     }
     
     // finis