@naanlang/naan 1.0.6 → 1.0.8

package/plugins/serviceAws/aws_s3.nlg

@@ -13,17 +13,36 @@
 
 /*
  * S3bucket
- *
- *     S3 buckets.
+ * 
+ * Bucket ooptions:
+ *  {
+ *      cacheControl: "max-age=2592000"         - default cache control, e.g. to "max-age=2592000"
+ *  }
  *
  */
 
-closure S3bucket(s3, name, local bucket) {
+closure S3bucket(s3, name, bucketOptions, local bucket) {
     bucket = new(object, this)
     bucket.name = name
-    
+        
+    //
+    // head
+    //
+    bucket.head = closure head(key, local pending) {
+        pending = new(nonce)
+        s3.aws.headObject({
+            Bucket: bucket.name
+            Key: key
+        }, function (error, data) {
+            if error
+                error = Error("S3bucket.head failed:", error.name, "key:", key, "in", bucket.name)
+            pending.signal(list(error, data))
+        })
+        pending.wait()
+    }
+
     //
-    // list
+    // listObjects
     //
     bucket.listObjects = closure listObjects(local pending) {
         pending = new(nonce)
@@ -31,31 +50,46 @@ closure S3bucket(s3, name, local bucket) {
             Bucket: bucket.name
         }, function (error, data) {
             if error
-                error = Error("S3bucket.list failed", error)
+                error = Error("S3bucket.list failed:", error.name, "for", bucket.name)
             pending.signal(list(error, data))
         })
         pending.wait()
     }
 
     //
-    // upload
+    // put
     //
-    bucket.upload = closure upload(key, data, local pending) {
-        pending = new(nonce)
-        s3.aws.upload({
+    // options:
+    //  {
+    //      contentType: "text/plain"           - override contentType if specified
+    //      cacheControl: "max-age=2592000"     - default cache control, e.g. to "max-age=2592000"
+    //      metadata: { <items> }               - optional dictionary of keys to store
+    //  }
+    //
+    bucket.put = closure put(key, data, options, local params, pending) {
+        if !contentType
+            contentType = "application/octet-stream"
+        params = {
             Bucket: bucket.name
-            ACL: "public-read"
-            Body: data
             Key: key
-                ContentType: "application/octet-stream"                     // force download from browser
-        })
-        .on("httpUploadProgress", function(evt) {
-            debuglog("upload progress", totuple(evt))
-        })
-        .send(function(error, data) {
+            ContentMD5: HashMD5_base64(data)
+            Body: data
+        }
+        if options.contentType
+            params.ContentType = options.contentType
+        if options.metadata
+            params.Metadata = options.metadata
+        else
+            params.Metadata = "application/octet-stream"
+        if options.cacheControl
+            params.CacheControl = options.cacheControl
+        else if bucket.cacheControl
+            params.CacheControl = bucket.cacheControl
+        pending = new(nonce)
+        s3.aws.putObject(params, function(error, resp) {
             if error
-                error = Error("S3bucket.upload failed", error)
-            pending.signal(list(error, data))
+                error = Error("S3bucket.put failed:", error.name, "key:", key, "in", bucket.name)
+            pending.signal(list(error, resp))
         })
         pending.wait()
     }
@@ -88,6 +122,8 @@ closure S3(local s3) {
                 accessKeyId:        creds.keyID
                 secretAccessKey:    creds.keySecret
             }
+            if creds.endpoint
+                params.endpoint = creds.endpoint
         }
         s3.aws = xnew(awsSDK.S3, params)
         if awsSDK.S3Client
@@ -111,12 +147,23 @@ closure S3(local s3) {
     // This obtains a pre-signed URL for the specified command and its parameters. Options control
     // how the URL is generated.
     //
+    // CommandParams:
+    //  {
+    //      Bucket:             <string>        // bucket name
+    //      Key:                <string>        // object being operated upon
+    //      ContentType:        <string>        // video/mp4 or whatever
+    //      ResponseCacheControl:   <string>    // reportedly for getObject only
+    //      ACL:                <string>        // e.g. "public-read"
+    //      Body:               <data>          // fiik
+    //  }
+    //
     // Options:
     //  {
-    //      expiresIn:        <seconds>         // default expiration is 900 (15 minutes)
+    //      expiresIn:          <seconds>       // default expiration is 900 (15 minutes)
+    //      signingDate:        <epoch-secs>    // reference time for URL being created
     //  }
     //
-    s3.getSignedURL = closure getSignedURL(cmdname, cmdparams, local command, options, pending) {
+    s3.getSignedURL = closure getSignedURL(cmdname, cmdparams, options, local command, pending) {
         if s3.client {
             if cmdname == "putObject"
                 cmdname = "PutObjectCommand"
@@ -124,16 +171,15 @@ closure S3(local s3) {
                 cmdname = "GetObjectCommand"
             else
                 return (list(Error("S3bucket.getSignedURL: command not present:", cmdname)))
-            if cmdparams.Expires {
-                cmdparams = new(cmdparams)
-                options.expiresIn = cmdparams.Expires
-                cmdparams.Expires = undefined
-            } else
-                options = { }
             command = xnew(awsSDK.s3Commands[cmdname], cmdparams)
             await(awsSDK.getSignedUrl(s3.client, command, options))
         } else {
+            debuglog("s3.getSignedURL: warning: old API")
             pending = new(nonce)
+            if options.expiresIn {
+                cmdparams = new(cmdparams)
+                cmdparams.Expires = options.expiresIn
+            }
             s3.aws.getSignedUrl(cmdname, cmdparams, function(error, data) {
                 pending.signal(list(error, data))
             })

package/plugins/serviceAws/aws_sqs.nlg

@@ -0,0 +1,113 @@
+/*
+ * aws_sqs.nlg
+ * serviceAws
+ *
+ *     Access to SQS (Simple Queue Service) for AWS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2023 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * SqsMQ
+ *
+ *     SqsMQ, a container for holding the queue name and sending messages to it.
+ *
+ */
+
+closure SqsMQ(sqs, queueUrl, local sqsq) {
+    sqsq = new(object, this)
+
+    //
+    // sendMessage
+    //
+    sqsq.sendMessage = closure sendMessage(msgBody) {
+        sqs.sendMessage(queueUrl, msgBody)
+    }
+    
+    // finis
+    sqsq
+}
+
+
+/*
+ * SQS
+ *
+ *     SQS service functions.
+ *
+ */
+
+closure SQS(local sqs) {
+    sqs = new(object, this)
+
+    //
+    // login
+    //
+    sqs.login = closure login(creds, local params) {
+        if !awsSDK
+            return (list(Error("SQS.login: no AWS SDK available")))
+        params = {
+        }
+        if creds {
+            params.region = creds.region
+            params.credentials = {
+                accessKeyId:        creds.keyID
+                secretAccessKey:    creds.keySecret
+            }
+            if creds.sqs_endpoint
+                params.endpoint = creds.sqs_endpoint
+        }
+        sqs.aws = xnew(awsSDK.SQS, params)
+        `(false, { ok: true })
+    }
+    
+    //
+    // sendMessage
+    //
+    sqs.sendMessage = closure sendMessage(queueUrl, msgBody, local params, pending) {
+        debuglog("SQS.sendMessage:", awsSDK, sqs.aws, sqs.aws.sendMessage)
+        params = {
+            MessageBody: msgBody
+            QueueUrl: queueUrl
+        }
+        pending = new(nonce)
+        sqs.aws.sendMessage(params, function(error, data) {
+            if error
+                ErrorDebuglog("aws.sendMessage failed", error)
+            pending.signal(list(error, data))
+        })
+        pending.wait()
+    }
+    
+    //
+    // mq
+    //
+    sqs.mq = function(queueUrl) {
+        SqsMQ(sqs, queueUrl)
+    }
+
+    // finis
+    sqs
+}
+
+
+/*
+ * sqsInit
+ *
+ *     Initialize the SQS module.
+ *
+ */
+
+function sqsInit(local manifest) {
+    manifest = `(SqsMQ, SQS, sqsInit)
+
+    Naan.module.build(module.id, "aws_sqs", function(modobj, compobj) {
+        require("./serviceAws.nlg")
+        compobj.manifest = manifest
+        modobj.exports.SQS = SQS
+    })
+
+} ();

package/plugins/serviceAws/dbt_aws.nlg

@@ -37,6 +37,19 @@ closure dawsBucket(s3b, local bucket, s3) {
     closure asyncResult args {
         future(function() { apply(pop(args), args) }).run()
     }
+    
+    // hashFromETag
+    //
+    // Return the MD5 hash for an ETag
+    //
+    function hashFromETag(etag) {
+        if etag.startsWith("W/")
+            etag = etag.slice(2)                                            // "weak" from gzip with NGINX, apparently
+        if etag.length == 34                                                // multipart S3 ETag will is longer and will be ignored
+            etag.substring(1,33)
+        else
+            undefined
+    }
 
     // close
     //
@@ -65,8 +78,8 @@ closure dawsBucket(s3b, local bucket, s3) {
         function s3GetCB(error, resp) {
             if error
                 error = Error("can't get object", error, key)
-            else if resp.ETag.length == 34
-                resp.md5 = resp.ETag.substring(1,33)                        // ETags have to have quotes, but our hash doesn't
+            else
+                resp.md5 = hashFromETag(resp.ETag)
             callback(error, resp)
         }
 
@@ -79,12 +92,18 @@ closure dawsBucket(s3b, local bucket, s3) {
     // put
     //
     // Put an object in the bucket. Parameters include:
-    //      key         - the object name, which may not be empty or start with "/"
-    //      data        - optional content to store
-    //      metadata    - optional dictionary of keys to store
+    //  key                                     - the object name, which may not be empty or start with "/"
+    //  data                                    - optional content to store
+    //  options:
+    //  {
+    //      contentType: "text/plain"           - override contentType if specified
+    //      cacheControl: "max-age=2592000"     - default cache control, e.g. to "max-age=2592000"
+    //      metadata: { <items> }               - optional dictionary of keys to store
+    //  }
     
-    bucket.put = closure put(key, data, metadata, callback, local params, mimeType, md5) {
-        if !string(key) || length(key) == 0 || key.substring(0,1) == "/" || metadata && !dictionary(metadata)
+    bucket.put = closure put(key, data, options, callback,
+        local params, mimeType, md5) {
+        if !string(key) || length(key) == 0 || key.substring(0,1) == "/"
             return (asyncResult(callback, Error("invalid arguments")))
         if !s3
             return (asyncResult(callback, Error("database closed")))
@@ -128,9 +147,13 @@ closure dawsBucket(s3b, local bucket, s3) {
                     mimeType = mimeType.concat("; charset=UTF-8")
                 params.ContentType = mimeType
             }
+            if options.contentType
+                params.ContentType = options.contentType
         }
-        if metadata
-            params.Metadata = metadata
+        if options.metadata
+            params.Metadata = options.metadata
+        if options.cacheControl
+            params.CacheControl = options.cacheControl
         if data
             params.Body = data
         else
@@ -142,11 +165,10 @@ closure dawsBucket(s3b, local bucket, s3) {
                 error = Error("can't put object", error, key)
             else {
                 resp.length = length(data)
-                if resp.ETag.length == 34 {                                 // multipart S3 ETag will is longer and will be ignored
-                    resp.md5 = resp.ETag.substring(1,33)                    // ETags have to have quotes, but our hash doesn't
-                    if md5 != resp.md5 {
-                        error = Error("hash match failure", md5, resp.md5)  // has match failed on round-trip
-                        resp = false } } }
+                resp.md5 = hashFromETag(resp.ETag)
+                if resp.md5 && md5 != resp.md5 {
+                    error = Error("hash match failure", md5, resp.md5)      // has match failed on round-trip
+                    resp = false } }
             callback(error, resp)
         })
     }
@@ -285,12 +307,31 @@ closure dawsBucket(s3b, local bucket, s3) {
  *
  */
 
-closure dawsTable(database, path, local table) {
+closure dawsTable(database, path, tableOptions, local table) {
     table = new(object, this)
     table.type = database.type
     table.db = database
     table.path = path
     table.s3prefix = makePrefix(path)
+    
+    // readableStreamToArrayBuffer
+    //
+    // For browsers, convert a ReadableStream to an array buffer.
+    //
+    function readableStreamToArrayBuffer(stream, local result, reader, error, data, chunk) {
+        result = xnew(js.w.Uint8Array, 0)
+        reader = stream.getReader()
+        loop {
+            `(error, data) = await(reader.read())
+            if error || data.done
+                break
+            chunk = xnew(js.w.Uint8Array, result.length + data.value.length)
+            chunk.set(result)
+            chunk.set(data.value, result.length)
+            result = chunk
+        }
+        result
+    }
         
     // makePrefix
     //
@@ -344,7 +385,7 @@ closure dawsTable(database, path, local table) {
         if resp.ContentType
             doc.contentType = resp.ContentType
         if resp.md5
-            doc.md5 = resp.md5
+            doc._md5 = resp.md5
         doc
     }
     
@@ -394,11 +435,14 @@ closure dawsTable(database, path, local table) {
         if content
             doc.content = content
         metadata = metadata2s3(metadata)                                    // filter out any invalid metadata
-        database.bucket.put(id, content, metadata, function(error, resp) {
+        database.bucket.put(id, content, {
+            metadata: metadata
+            cacheControl: tableOptions.cacheControl
+        }, function(error, resp) {
             if error
                 error = Error("add failed", error, id, { status: error.$metadata.httpStatusCode })
             else {
-                doc.md5 = resp.md5                                          // update the original doc
+                doc._md5 = resp.md5                                         // update the original doc
                 doc.length = resp.length }
             callback(error, doc)
         })
@@ -411,14 +455,17 @@ closure dawsTable(database, path, local table) {
     table.update = closure update(doc, callback, local metadata) {
         if !callback
             return (syncAdapter(update, doc))
-        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc.md5
+        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc._md5
             return (asyncResult(callback, Error("invalid argument")))
         metadata = metadata2s3(doc)
-        database.bucket.put(doc._id, doc.content, metadata, function(error, resp) {
+        database.bucket.put(doc._id, doc.content, {
+            metadata: metadata
+            cacheControl: tableOptions.cacheControl
+        }, function(error, resp) {
             if error
                 error = Error("update failed", error, doc._id, { status: error.$metadata.httpStatusCode })
             else {
-                doc.md5 = resp.md5                                          // update the original doc
+                doc._md5 = resp.md5                                         // update the original doc
                 doc.modified = undefined                                    // we don't know exact modify date
                 doc.length = resp.length }
             callback(error, doc)
@@ -433,7 +480,7 @@ closure dawsTable(database, path, local table) {
     table.delete = closure delete(doc, callback) {
         if !callback
             return (syncAdapter(delete, doc))
-        if !doc._id || !doc._id.startsWith(table.s3prefix) || !doc.md5
+        if !doc._id || !doc._id.startsWith(table.s3prefix) || !(doc._md5 || doc._id.slice(-1) == "/")
             return (asyncResult(callback, Error("invalid argument")))
         database.bucket.delete(doc._id, function(error, resp) {
             if error
@@ -469,6 +516,8 @@ closure dawsTable(database, path, local table) {
                     { }                                                     // already "decoded", e.g. base64
                 else if jsTypedArray(resp.content)
                     resp.content = xnew(TextDecoder, "utf-8").decode(resp.content)
+                else if jsInstanceOf(resp.content, js.w.ReadableStream)
+                    resp.content = xnew(TextDecoder, "utf-8").decode(readableStreamToArrayBuffer(resp.content))
                 resp._id = id }
             callback(error, resp)
         })
@@ -748,12 +797,12 @@ closure S3DB(s3b, local database, result) {
     //
     // Return a table-access object for the table.
 
-    database.table = function table(path) {
+    database.table = function table(path, options) {
         if !database.bucket
             return (list(Error("database closed")))
         if !string(path)
             return (list(Error("invalid table path")))
-         dawsTable(database, path)
+         dawsTable(database, path, options)
     }
     
     // finis

package/plugins/serviceAws/psm_aws.nlg

@@ -47,7 +47,11 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
         {
             label:          "Bucket"
             placeholder:    "e.g. myBucketName"
-            key:            "bucketName" }
+            key:            "bucketName" },
+        {
+            label:          "Endpoint"
+            placeholder:    "custom URL"
+            key:            "endpoint" }
         ]
     }
 
@@ -59,6 +63,7 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             `(error, creds) = connector.vault.accessResource(resID)
             if creds && (!resource.region || resource.region == creds.region)
                 && (!resource.bucketName || resource.bucketName == creds.bucketName)
+                && (!resource.endpoint || resource.endpoint == creds.endpoint)
                 && (!resource.label || resource.label == creds.label)
                 return (resID)
         }
@@ -75,6 +80,7 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
     //      keySecret   - AWS IAM keySecret
     //      region      - AWS region
     //      bucketName  - AWS bucketName
+    //      endpoint    - custom URL
     //      hidden      - [optional] don't enumerate to user
     //      locked      - [optional] don't allow user delete
     // The return value is an (error, resID) tuple. Error, if non-false, is a dictionary of field
@@ -90,6 +96,8 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             region: resource.region,
             bucketName: resource.bucketName
         }
+        if resource.endpoint
+            creds.endpoint = resource.endpoint
         errors = { }
         if resource.label
             creds.label = resource.label
@@ -107,6 +115,8 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             errors.region = "required field"
         if badField(resource.bucketName)
             errors.bucketName = "required field"
+        if resource.endpoint && badField(resource.endpoint)
+            errors.endpoint = "invalid field"
         if length(errors) > 0
             error = errors
         else {
@@ -164,6 +174,8 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             resource.region = creds.region
         if string(creds.bucketName)
             resource.bucketName = creds.bucketName
+        if string(creds.endpoint)
+            resource.endpoint = creds.endpoint
         list(false, resource)
     }
 
@@ -209,22 +221,28 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
     //
     // Return an S3 access object, which is the credentials to access an AWS S3 bucket.
 
-    connector.access = function access(resID, local error, creds, s3b) {
+    connector.access = function access(resID, local error, creds, params, s3b) {
         `(error, creds) = connector.vault.accessResource(resID)
         if error
             return (list(error))                                            // can't access that resource
         if !awsSDK.S3
             return (list(Error("S3 connector: no AWS SDK available")))
+        params = {
+            region:                     creds.region
+            credentials: {
+                accessKeyId:            creds.keyID
+                secretAccessKey:        creds.keySecret
+            }
+        }
         s3b = {
-            s3: xnew(awsSDK.S3, {
-                region:                 creds.region
-                credentials: {
-                    accessKeyId:        creds.keyID
-                    secretAccessKey:    creds.keySecret
-                }})
             bucketName:                 creds.bucketName
             region:                     creds.region
         }
+        if creds.endpoint {
+            params.endpoint = creds.endpoint
+            s3b.endpoint = creds.endpoint
+        }
+        s3b.s3 = xnew(awsSDK.S3, params)
         list(false, s3b)
     }
 
@@ -232,10 +250,11 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
     //
     // Connect to an AWS bucket filesystem, returning a view.
     
-    connector.connect = function connect(resID, service, args, local error,  s3b, db, table) {
+    connector.connect = function connect(resID, service, args,
+        local error,  s3b, db, rootpath, table) {
         `(error, s3b) = connector.access(resID)
         if !error
-            `(error, db) = S3DB(s3b)                                        // get S3 bucket as a database
+            `(error, db) = S3DB(s3b, args.1)                                // get S3 bucket as a database
         if error
             list(error)
         else if service == "NideDB"
@@ -244,7 +263,7 @@ closure psmaConnector(psm, local connClassID, connector, watch) {
             rootpath = args.0                                               // connect to a NideFS filesystem
             if !rootpath
                 return (Error("mandatory rootpath missing"))
-            `(error, table) = db.table(rootpath)
+            `(error, table) = db.table(rootpath, args.1)                    // args.1 is table options; see dawsTable
             if error
                 list(error)
             else

package/plugins/serviceAws/serviceAws.nlg

@@ -26,10 +26,8 @@ function sawsInit(local manifest) {
         require("frameworks/common").LiveImport()
         require("./aws_utils.nlg")
         function sawsPostload() {
-            if js.lambda
-                awsSDK = js.r("/opt/aws-sdk-node.min.js")                   // SDK is a layer
-            else if js.g
-                awsSDK = js.r("aws-sdk-node.min")                           // SDK must be in search path
+            if js.g
+                awsSDK = js.r(JSpath.resolve(js.d, module.locpath, "aws/aws-sdk-node.min.js"))
             else
                 awsSDK = JsLoadScript(JSpath.join(module.locpath, "./aws/aws-sdk.min.js"), "awsSdk")
             module.exports.awsSDK = awsSDK

package/plugins/serviceYC/generic_api.yaml

@@ -0,0 +1,36 @@
+openapi: 3.0.0
+info:
+  title: tutor-backend API
+  version: 1.0.0
+servers:
+- url: https://d5dhrgakpe39e11lqefr.apigw.yandexcloud.net
+paths:
+  /{proxy+}:
+    options:
+      parameters:
+      - name: proxy
+        in: path
+        required: true
+        schema:
+          type: string
+      x-yc-apigateway-integration:
+        type: dummy
+        http_code: 200
+        http_headers:
+          Access-Control-Allow-Headers: "*"
+          Access-Control-Allow-Methods: "DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT"
+          Access-Control-Allow-Origin: "*"
+        content:
+          "application/json": ""
+    x-yc-apigateway-any-method:
+      parameters:
+      - name: proxy
+        in: path
+        required: true
+        schema:
+          type: string
+      x-yc-apigateway-integration:
+        type: cloud_functions
+        function_id: d4ehjjcfk8ffe43o5t7j
+        tag: $latest
+        service_account_id: ajeuftvbtshn348ert80