@naanlang/naan 1.2.0 → 1.3.0

package/plugins/serviceAws/aws_dynamo.nlg

@@ -191,21 +191,23 @@ closure DynaConverter(local conv) {
     }
 
     // datumNaanToDyn
-    // Convert a Naan datum to DynamoDB format.
+    // Convert a Naan datum to DynamoDB format. If no valid conversion is known then it becomes NULL
+    // in DynamoDB and a debug warning is logged. Note that this is not called for arbitrary symbols
+    // from genAttributeValueAdder() because it uses symbols to refer to actual attributes by name.
     //
     conv.datumNaanToDyn = function datumNaanToDyn(datum, local output, ntype, dtype) {
         output = { }
         if tuple(datum)
             datum = datum.toarray
-        if array(datum) {
+        if Array.isArray(datum) {
             dtype = "L"                                                     // heterogenous list
             datum = datum.map(function(item) {
                 datumNaanToDyn(item)
             })
         } else {
             ntype = typeof(datum)
-            if ntype == xobject
-                dtype = "B"                                                 // assume binary
+            if ntype == xobject && jsTypedArray(datum)
+                dtype = "B"                                                 // binary/blob
             else if ntype == dictionary {
                 dtype = "M"
                 datum = recordNaanToDyn(datum)
@@ -217,21 +219,19 @@ closure DynaConverter(local conv) {
             else if ntype == string
                 dtype = "S"
             else if ntype == symbol {
-                if datum === null {
-                    dtype = "NULL"
-                    datum = "true"
-                }
-                else if datum === true || datum === false
+                if datum === true || datum === false
                     dtype = "BOOL"
-                else if datum === undefined
-                    return (undefined)                                      // Zen!
-                else
-                    return (datum.tostring)
-                    
-            } else
+                else if datum !== null {
+                    debuglog("DynaConverter.datumNaanToDyn: nonstandard symbol", datum)
+                    datum = null }
+            } else {
                 debuglog("DynaConverter.datumNaanToDyn: invalid type", ntype)
+                datum = null }
         }
-        output[dtype] = datum
+        if datum === null
+            output["NULL"] = true
+        else
+            output[dtype] = datum
         output
     }
 
@@ -277,7 +277,7 @@ closure DynaConverter(local conv) {
     // Convert a DynamoDB typed item dictionary to Naan format.
     //
     conv.recordDynToNaan = function recordDynToNaan(ditem, local output, key, data, dtype, dvalue) {
-        if ditem.constructor === Array.prototype.constructor || array(ditem) {        // process an array
+        if Array.isArray(ditem) {                                           // process an array
             output = []
             for data in ditem
                 output.push(recordDynToNaan(data))
@@ -429,7 +429,7 @@ closure DynaConverter(local conv) {
         attval = genAttributeValueAdder(params)
         attname = genAttributeNameAdder(params)
         for `(rangename, ranges) in conds {
-            if !array(ranges)
+            if !Array.isArray(ranges)
                 ranges = [ranges]
             for range in ranges {
                 rangex = false
@@ -930,7 +930,7 @@ closure DynaTable(dyna, tablename, options, local table) {
                 remaining -= processed
             params.ExclusiveStartKey = data.LastEvaluatedKey
             if processed && remaining > 0 && data.LastEvaluatedKey {
-                params.limit = remaining                                // set up for next iteration
+                params.Limit = remaining                                // set up for next iteration
                 true                                                    // continue looping
             }
         })

package/plugins/serviceAws/dbt_aws.nlg

@@ -513,7 +513,7 @@ closure dawsTable(database, path, tableOptions, local table) {
                             resp._error = "JSON decoding failed"
                     } }
                 else if resp.contentType.startsWith("image/")
-                    { }                                                     // already "decoded", e.g. base64
+                    resp.content = readableStreamToArrayBuffer(resp.content)
                 else if jsTypedArray(resp.content)
                     resp.content = xnew(TextDecoder, "utf-8").decode(resp.content)
                 else if jsInstanceOf(resp.content, js.w.ReadableStream)

package/plugins/serviceYC/generic_api.yaml

@@ -0,0 +1,36 @@
+openapi: 3.0.0
+info:
+  title: tutor-backend API
+  version: 1.0.0
+servers:
+- url: https://d5dhrgakpe39e11lqefr.apigw.yandexcloud.net
+paths:
+  /{proxy+}:
+    options:
+      parameters:
+      - name: proxy
+        in: path
+        required: true
+        schema:
+          type: string
+      x-yc-apigateway-integration:
+        type: dummy
+        http_code: 200
+        http_headers:
+          Access-Control-Allow-Headers: "*"
+          Access-Control-Allow-Methods: "DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT"
+          Access-Control-Allow-Origin: "*"
+        content:
+          "application/json": ""
+    x-yc-apigateway-any-method:
+      parameters:
+      - name: proxy
+        in: path
+        required: true
+        schema:
+          type: string
+      x-yc-apigateway-integration:
+        type: cloud_functions
+        function_id: d4ehjjcfk8ffe43o5t7j
+        tag: $latest
+        service_account_id: ajeuftvbtshn348ert80

package/plugins/serviceYC/naanide-relay-index.js

@@ -0,0 +1,148 @@
+/*
+ * index.js
+ * naanide-relay
+ * 
+ * Yandex Cloud Function for proxying API requests from NaanIDE in the browser.
+ *
+ * This function acts as a relay (mitm) proxy so that a browser may access Yandex Cloud APIs without
+ * needing a transient access token and without CORS problems. Security is provided by a pre-shared
+ * static key, configured as an environment variable in the YCF and known to the browser code.
+ *
+ * The request to the proxy should be precisely what is intended for the target API, except that the
+ * following headers should be included:
+ *
+ *      x-naanide-auth:     <pre-shared-key>
+ *      x-naanide-ep:       <target-url-with-path-and-query-variables>
+ *      x-naanide-hdrs:     <optional headers>
+ *
+ * The response in the browser contains some remapped headers, but is otherwise the API's response.
+ *
+ * Assuming the pre-shared key is "foo", the following gets a list of YDB databases in a folder:
+
+curl -H "x-naanide-auth: foo" \
+     -H "x-naanide-ep: https://ydb.api.cloud.yandex.net/ydb/v1/databases?folderId=b1gjjahqm5u9tjolom9l" \
+     'https://functions.yandexcloud.net/d4e5gicpajcqdbj91jt8'
+
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ * 
+ */
+
+
+/*
+ * index.handler
+ *
+ * Forward a request and respond with the result.
+ *
+ */
+
+module.exports.handler = async function (event, context) {
+    "use strict";
+    const authorizationKey = process.env.NAANIDE_AUTH_KEY;
+    
+    //
+    // OPTIONS request
+    //
+    // Allow our headers for CORS
+    //
+
+    if (event.httpMethod == "OPTIONS")
+        return {
+            statusCode: 200,
+            headers: {
+                "Access-Control-Allow-Headers": "*"
+            }
+        };
+    
+    //
+    // Failed authentication
+    //
+
+    if (event.headers["X-Naanide-Auth"] != authorizationKey) {
+        return {
+            statusCode: 401
+        };
+    }
+    
+    //
+    // Authenticated
+    //
+
+    let https = require("https");
+    let url = event.headers["X-Naanide-Ep"];
+    let headers = event.headers["X-Naanide-Hdrs"];
+    if (headers) {
+        try {
+            headers = JSON.parse(headers);
+        } catch (e) {
+            console.log("X-Naanide-Hdrs parse error", e);
+            headers = false;
+        }
+    }
+    if (!headers)
+        headers = { };
+    if (!headers[Object.keys(headers).find(key => key.toLowerCase() === "authorization")])
+        headers.Authorization = context.token.token_type.concat(" ", context.token.access_token);
+    let options = {
+        method: event.httpMethod,
+        headers: headers
+    };
+    let chunks = [];
+    let content = false;
+    let finalResponse = {
+        statusCode: 500
+    };
+
+    //
+    // forward_request
+    //
+    async function forward_request() {
+        return new Promise(function(resolve, reject) {
+            let req = https.request(url, options, function(resp) {
+                finalResponse.statusCode = resp.statusCode;
+                // data event
+                resp.on("data", function (chunk) {
+                    chunks.push(chunk);
+                });
+                // end event
+                resp.on("end", function () {
+                    if (resp.complete)
+                    {
+                        let content = Buffer.concat(chunks);
+                        let contentType = resp.headers["content-type"];
+                        finalResponse.body = content.toString();
+                        if (resp.headers["x-request-id"]) {
+                            resp.headers["x-upstream-request-id"] = resp.headers["x-request-id"];
+                            delete resp.headers["x-request-id"];
+                        }
+                        finalResponse.headers = resp.headers;
+                    }
+                    else {
+                        console.log("request terminated before full response received");
+                        finalResponse.statusCode = 502;
+                    }
+                    resolve();
+                });
+            });
+            // error event
+            req.on("error", function (err) {
+                console.log("upstream request error:", err);
+                finalResponse.statusCode = 502;
+                reject(err);
+            });
+            console.log("event.body", event.body, event.isBase64Encoded);
+            if (event.body) {
+                if (event.isBase64Encoded)
+                    req.write(decodeURIComponent(escape(atob(event.body))));
+                else
+                    req.write(event.body);
+            }
+            req.end();
+        });
+    }
+    
+    await forward_request();
+    return finalResponse;
+}

package/plugins/serviceYC/serviceYC.nlg

@@ -0,0 +1,60 @@
+/*
+ * serviceYC.nlg
+ * serviceYC
+ *
+ *     ServiceYC module configuration and management for browser and NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * YcRequest
+ *
+ *     Make a request to a Yandex Cloud API using a Yandex Cloud Function acting as a relay proxy.
+ * This works like frameworks/browser/HttpsApiRequest and frameworks/node/HttpsApiRequest except
+ * that it takes a credentials argument for access to the relay. Technically this is not required
+ * when we are going from node, but I'm keeping it for consistency.
+ *
+ */
+
+closure YcRequest(url, creds, options, local headers) {
+    if !options
+        options = { }
+    else
+        options = new(options)
+    headers = [
+        list("x-naanide-auth", creds.relayAuthKey)
+        list("x-naanide-ep", url)
+    ]
+    if options.headers
+        headers.push(list("x-naanide-hdrs", options.headers.join("\n")))
+    options.headers = headers
+    https.HttpsApiRequest(creds.relayURL, options)
+};
+
+
+/*
+ * seycInit
+ *
+ *     Initialize the YC module.
+ *
+ */
+
+function seycInit(local manifest) {
+    manifest = `(YcRequest, seycInit)
+
+    Naan.module.build(module.id, "serviceYC", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("naanlib:frameworks/common").LiveImport()
+        if js.g {
+            nodeHttp = js.r("http")
+            https = require("naanlib:frameworks/node/https_request.nlg") }
+        else
+            https = require("naanlib:frameworks/browser/https_request.nlg")
+        module.exports.YcRequest = YcRequest
+    })
+} ();

package/plugins/serviceYC/yc_apigateway.nlg

@@ -0,0 +1,205 @@
+/*
+ * yc_apigateway.nlg
+ * serviceYC
+ *
+ *     Support for YC API Gateway and websockets.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * Gateway
+ *
+ *     Handler for incoming websocket messages from the YC API Gateway.
+ *
+ */
+
+closure Gateway(local apig) {
+    global(https)
+    apig = new(object, this)
+
+    // open
+    //
+    // Initialize for a new gateway, but we don't know anything about it yet. This should ensure we
+    // can talk with the cloud, returning a result tuple. The first message that comes in will have
+    // headers that identify the gateway resource specifically.
+    //
+    apig.open = closure open() {
+        apig.ycvm = VmControl()
+    }
+
+    // close
+    //
+    // Close this handler because no more requests will come in.
+    //
+    apig.close = closure close() {
+        apig.ycvm = false
+    }
+
+    // requested
+    //
+    // A new websocket operation was requested. The ExpressJS request/response structures are passed.
+    //
+    // The requested return value for connect:
+    //  {
+    //      op: "connect"
+    //      connID: <string>
+    //      connAt: <integer>
+    //  }
+    // The requested return value for disconnect:
+    //  {
+    //      op: "disconnect"
+    //      connID: <string>
+    //      code: <integer>
+    //      reason: <string>
+    //  }
+    // The requested return value for message:
+    //  {
+    //      op: "message"
+    //      connID: <string>
+    //      message: <string>
+    //      contType: <string>
+    //  }
+    //
+    // Example request headers from /wsgw endpoint:
+    //  {
+    //      user-agent          : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    //      accept-encoding     : "gzip",
+    //      accept-language     : "en-US,en;q=0.9,ru;q=0.8",
+    //      cache-control       : "no-cache",
+    //      host                : "tutorbe-1.ru-central1.internal",
+    //      origin              : "https://d5di07c52f5hpi7h8kb8.apigw.yandexcloud.net",
+    //      pragma              : "no-cache",
+    //      sec-websocket-extensions: "permessage-deflate; client_max_window_bits",
+    //      uber-trace-id       : "0000000000000000014a60f3326789e4:1a4ee8a332acff59:0:01",
+    //      x-api-gateway-function-id: "d4e16asgl4t3i2j1m7t4",
+    //      x-envoy-external-address: "64.201.120.56",
+    //      x-envoy-original-path: "/?guid=1a5d070b-6aaf-4421-aa66-b13d15a7eaa3",
+    //      x-forwarded-for     : "64.201.120.56",
+    //      x-forwarded-proto   : "https",
+    //      x-real-remote-address: "64.201.120.56",
+    //      x-request-id        : "ceaa9f57-4cb8-4fce-8081-c065b8073130",
+    //      x-serverless-certificate-ids: "{}",
+    //      x-serverless-gateway-id: "d5di07c52f5hpi7h8kb8",
+    //      x-trace-id          : "3e393eb7-4680-4d71-8b3d-824e5b768255",
+    //      x-yc-apigateway-websocket-connected-at: "1725131521366",
+    //      x-yc-apigateway-websocket-connection-id: "c05188b1jdifd7gsn7bqqochiv94l622u",
+    //      x-yc-apigateway-websocket-event-type: "CONNECT",
+    //      x-naanide-guid      : "1a5d070b-6aaf-4421-aa66-b13d15a7eaa3",
+    //      connection          : "close"
+    //  }
+    //
+    apig.requested = closure requested(req, res, local data, eventType) {
+        if !apig.gatewayID
+            apig.gatewayID = req.get("x-serverless-gateway-id")
+        data = {
+            connID: req.get("x-yc-apigateway-websocket-connection-id")
+            connAt: toint(req.get("x-yc-apigateway-websocket-connected-at"))
+            guid: req.get("x-naanide-guid") || undefined
+        }
+        eventType = req.get("x-yc-apigateway-websocket-event-type")
+        if eventType == "CONNECT" {
+            data.op = "connect"
+        }
+        else if eventType == "DISCONNECT" {
+            data.op = "disconnect"
+            data.code = req.get("X-Yc-Apigateway-Websocket-Disconnect-Status-Code")
+            data.reason = req.get("X-Yc-Apigateway-Websocket-Disconnect-Reason")
+        }
+        else if eventType == "MESSAGE" {
+            data.op = "message"
+            data.message = new(req.body)
+            data.contType = req.get('content-type')
+        } else
+            debuglog("serviceYC::Gateway.requested: invalid event type", eventType)
+        list(false, data)
+    }
+    
+    // send
+    //
+    // Send a websocket message via the API Gateway.
+    //
+    apig.send = closure send(connID, msg, local error, authHeader, data) {
+        `(error, authHeader) = apig.ycvm.authHeader()
+        if !string(msg)
+            msg = JSONstringify(msg)
+        data = {
+            data: EncodeBase64(msg)
+            type: "TEXT"                                                    // allegedly "BINARY" is an option
+        }
+        if !error
+            `(error, data) = https.HttpsApiRequest("https://apigateway-connections.api.cloud.yandex.net/apigateways/websocket/v1/connections/${connID}:send", {
+                method: "POST"
+                headers: [ authHeader ]
+                putdata: data
+                debug: true
+            })
+        if error {
+            ErrorDebuglog("serviceYC::Gateway.send failed:", error)
+            list(error)
+        } else
+            list(false, data)
+    }
+    
+    // get
+    //
+    // Get websocket connection status from the API Gateway.
+    //
+    apig.get = closure get(connID, local error, authHeader, data) {
+        `(error, authHeader) = apig.ycvm.authHeader()
+        if !error
+            `(error, data) = https.HttpsApiRequest("https://apigateway-connections.api.cloud.yandex.net/apigateways/websocket/v1/connections/${connID}", {
+                method: "GET"
+                headers: [ authHeader ]
+            })
+        if error {
+            ErrorDebuglog("serviceYC::Gateway.get failed:", error)
+            list(error)
+        } else
+            list(false, data)
+    }
+    
+    // disconnect
+    //
+    // Disconnect a websocket connection from the API Gateway.
+    //
+    apig.disconnect = closure disconnect(connID, local error, authHeader, data) {
+        `(error, authHeader) = apig.ycvm.authHeader()
+        if !error
+            `(error, data) = https.HttpsApiRequest("https://apigateway-connections.api.cloud.yandex.net/apigateways/websocket/v1/connections/${connID}", {
+                method: "DELETE"
+                headers: [ authHeader ]
+            })
+        if error {
+            ErrorDebuglog("serviceYC::Gateway.disconnect failed:", error)
+            list(error)
+        } else
+            list(false, data)
+    }
+
+    // finis
+    
+    apig
+};
+
+
+/*
+ * ycagInit
+ *
+ *     Initialize the YC module.
+ *
+ */
+
+function ycagInit(local manifest) {
+    manifest = `(Gateway, ycagInit)
+
+    Naan.module.build(module.id, "yc_apigateway", function(modobj, compobj) {
+        require("./yc_vm.nlg")
+        compobj.manifest = manifest
+        module.exports.Gateway = Gateway
+    })
+} ();