@naanlang/naan 1.3.1 → 1.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@naanlang/naan",
-    "version": "1.3.1",
+    "version": "1.3.2",
     "author": "Richard C. Zulch",
     "description": "Naan™ software platform",
     "main": "./lib/core/naanlib.js",

package/plugins/serviceAws/dbt_aws.nlg

@@ -128,10 +128,13 @@ closure dawsBucket(s3b, local bucket, s3) {
                     "jpg":  "image/jpeg",
                     "js":   "text/javascript",
                     "json": "application/json",
+                    "mov":  "video/mp4",
                     "mp3":  "audio/mpeg",
                     "mp4":  "video/mp4",
                     "nlg":  "text/plain",
                     "npk":  "application/json",
+                    "ogg":  "video/ogg",
+                    "png":  "image/png",
                     "svg":  "image/svg+xml",
                     "text": "text/plain",
                     "txt":  "text/plain",

package/plugins/serviceGC/gc_api.nlg

@@ -0,0 +1,263 @@
+/*
+ * gc_api.nlg
+ * serviceGC
+ *
+ *     NodeJS REST API client for serviceGC.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * GcApi
+ *
+ * GC API object. This handles access token acquisition and refresh.
+ *
+ * options:
+ *  {
+ *      preview:    <boolean>                   // true for preview models
+ *      debug:      <boolean>                   // https requests have additional diagnostics
+ *      sseHook:    <proc>                      // call procedure with SSE messages
+ *  }
+ *
+ */
+
+closure GcApi(options, local gcapi, cacert, renew, access_token, expires_at) {
+    global(https, JSpath)
+    gcapi = new(object, this)
+    if options.preview {
+        gcapi.urlbase = "https://gigachat-preview.devices.sberbank.ru/api/v1/"
+        gcapi.model_suffix = "-preview"
+    }
+    else
+        gcapi.urlbase = "https://gigachat.devices.sberbank.ru/api/v1/"
+
+    // auth
+    //
+    // Authenticate the API and get an access token and expiration date. The creds dictionary will be
+    // modified, so pass a copy if desired. Returns standard error tuple.
+    //
+    closure auth(creds, local path, error, headers, data) {
+        if defined(creds, `rootCert)
+            cacert = creds.rootCert                                         // use supplied cacert or false
+        else {                                                              // load our default
+            path = JSpath.resolve(js.d, module.locpath, "russian_trusted_root_ca_pem.crt")
+            `(error, cacert) = require("../nodeLib/node_fs.nlg").ReadFile(path, "binary")
+            if error
+                return (list(Error("gc: unable to load root cert", error)))
+            creds.rootCert = cacert                                         // save for renewal
+        }
+        gcapi.rqUID = UUID()
+        headers = [
+            `("Content-Type", "application/x-www-form-urlencoded")
+            `("Accept", "application/json")
+            list("RqUID", gcapi.rqUID)
+            list("Authorization", "${creds.authType} ${creds.authKey}")
+        ]
+        `(error, data) = https.HttpsApiRequest(creds.url, {
+            putdata: "scope=${creds.scope}"
+            headers: headers
+            cacert: cacert || undefined
+            debug: options.debug || undefined })
+        if error
+            return (list(Error("gc: unable to authenticate api", error)))
+        renew = creds
+        access_token = data.access_token
+        expires_at = data.expires_at
+        list(false, { ok: true })
+    }
+
+    // request
+    //
+    // Make an HTTPS request on the API, renewing the access token first if needed.
+    //
+    closure request(url, put, local error, data, status) {
+        if !(milliseconds() < expires_at - 60000) {
+            `(error, data) = auth(renew)
+            if error
+                return (list(Error("gc: unable to renew access")))
+        }
+        `(error, data, status) = https.HttpsApiRequest(url, {
+            putdata: put || undefined
+            cacert: cacert || undefined
+            headers: [
+                `(Accept, "application/json"),
+                list(`Authorization, "Bearer ".concat(access_token))
+            ]
+            sseHook: options.sseHook || undefined
+        })
+        if error
+            list(error)
+        else
+            list(false, new(data))
+    }
+
+    // login
+    //
+    // Login to the GC API with the specified credentials.
+    //
+    // creds:
+    //  {
+    //      authType:   <string>                // "Basic"
+    //      authKey:    <string>                // "NTdkM...Zg=="
+    //    [ url:        <string> ]              // "https://ngw.devices.sberbank.ru:9443/api/v2/oauth"
+    //    [ scope:      <string> ]              // GIGACHAT_API_PERS | GIGACHAT_API_B2B | GIGACHAT_API_CORP
+    //    [ rootCert:   <Uint8Array> ]          // optional root_ca_pem.crt, omit for default, or false for none
+    //  }
+    //
+    gcapi.login = function login(creds) {
+        if !string(creds.authType) || !string(creds.authKey)
+            return (list(Error("gc: invalid credentials")))
+        creds = merge({
+            scope: "GIGACHAT_API_CORP"
+            url: "https://ngw.devices.sberbank.ru:9443/api/v2/oauth"
+        }, creds)
+        auth(creds)
+    }
+
+    // modelList
+    //
+    // Return a list of available models.
+    //
+    gcapi.modelList = closure modelList() {
+        request(gcapi.urlbase.concat("models"))
+    }
+
+    // chatCompletion
+    //
+    // Return a chat completion.
+    //
+    //  query defined by GC:
+    //  {
+    //      model:              <string>            // desired model from modelList
+    //      messages:   [
+    //          {
+    //              role:       <string>            // system | user | assistant | function
+    //              content:    <string> | <dictionary>     // prompts | function arguments
+    //          }
+    //          ...
+    //      ]
+    //      attachments: [
+    //          <string>                            // name of an attachment
+    //          ...
+    //      ]
+    //      temp:               <float>
+    //      top_p:              <float>
+    //      max_tokens:         <integer>
+    //      repetition_penalty: <float>
+    //      stream:             <boolean>
+    //      update_interval:    <float>
+    //  }
+    //
+    //  response defined by GC:
+    //  {
+    //      choice: [
+    //          message: {
+    //              role:       <string>            // system | user | assistant | function
+    //              content:    <string> | <dictionary>     // prompts | function arguments
+    //          }
+    //          index:          <integer>
+    //          finish_reason:  <string>            // stop | length | function_call | blacklist
+    //      ]
+    //      create:             <integer>
+    //      model:              <string>
+    //      usage: {
+    //          prompt_tokens:  <integer>           // role: user
+    //          completion_tokens:  <integer>       // role: assistant
+    //          total_tokens:   <integer>
+    //      }
+    //      object:             <string>
+    //  }
+    //
+    gcapi.chatCompletion = closure chatCompletion(query) {
+        if options.preview {
+            query = new(query)
+            if !query.model.endsWith("-preview")
+                query.model = query.model.concat("-preview")
+        }
+        request(gcapi.urlbase.concat("chat/completions"), query)
+    }
+    
+    // uploadFile
+    //
+    gcapi.uploadFile = closure uploadFile(image) {
+        request(gcapi.urlbase.concat("files"), image)
+    }
+    
+    // listFiles
+    //
+    gcapi.listFiles = closure listFiles() {
+        request(gcapi.urlbase.concat("files"))
+    }
+        
+    // fileInfo
+    //
+    gcapi.fileInfo = closure fileInfo(fileID) {
+        request(gcapi.urlbase.concat("files/:${fileID}"))
+    }
+
+    // downloadFile
+    //
+    gcapi.downloadFile = closure downloadFile(fileID) {
+        request(gcapi.urlbase.concat("files/:${fileID}/content"))
+    }
+
+    // createEmbedding
+    //
+    //  query:
+    //  {
+    //    [ model:              <string> ]                      // model name
+    //      input:              <string> || [ <string>... ]     // string or array thereof
+    //  }
+    //
+    gcapi.createEmbedding = closure createEmbedding(query) {
+        if !query.model {
+            query = new(query)
+            query.model = "Embeddings"
+        }
+        request(gcapi.urlbase.concat("embeddings"), query)
+    }
+
+    // countTokens
+    //
+    //  input defined by gc:
+    //  {
+    //      model:              <string>                        // model name
+    //      input:              <string> || [ <string>... ]
+    //  }
+    //
+    //  output defined by gc:
+    //  {
+    //      object:             <string>                        // "tokens"
+    //      tokens:             <integer>
+    //      characters:         <integer>
+    //  }
+    //
+    gcapi.countTokens = closure countTokens(request) {
+        request(gcapi.urlbase.concat("tokens/count"), request)
+    }
+
+    // fin
+    gcapi
+};
+
+
+/*
+ * gcapiInit
+ *
+ *     Initialize the module.
+ *
+ */
+
+function gcapiInit(local manifest) {
+    manifest = `(GcApi, gcapiInit)
+
+    Naan.module.build(module.id, "gc_api", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("./serviceGC.nlg")
+        module.exports.GcApi = GcApi
+    })
+} ();

package/plugins/serviceGC/russian_trusted_root_ca_pem.crt

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFwjCCA6qgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx
+PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu
+ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg
+Q0EwHhcNMjIwMzAxMjEwNDE1WhcNMzIwMjI3MjEwNDE1WjBwMQswCQYDVQQGEwJS
+VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg
+YW5kIENvbW11bmljYXRpb25zMSAwHgYDVQQDDBdSdXNzaWFuIFRydXN0ZWQgUm9v
+dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMfFOZ8pUAL3+r2n
+qqE0Zp52selXsKGFYoG0GM5bwz1bSFtCt+AZQMhkWQheI3poZAToYJu69pHLKS6Q
+XBiwBC1cvzYmUYKMYZC7jE5YhEU2bSL0mX7NaMxMDmH2/NwuOVRj8OImVa5s1F4U
+zn4Kv3PFlDBjjSjXKVY9kmjUBsXQrIHeaqmUIsPIlNWUnimXS0I0abExqkbdrXbX
+YwCOXhOO2pDUx3ckmJlCMUGacUTnylyQW2VsJIyIGA8V0xzdaeUXg0VZ6ZmNUr5Y
+Ber/EAOLPb8NYpsAhJe2mXjMB/J9HNsoFMBFJ0lLOT/+dQvjbdRZoOT8eqJpWnVD
+U+QL/qEZnz57N88OWM3rabJkRNdU/Z7x5SFIM9FrqtN8xewsiBWBI0K6XFuOBOTD
+4V08o4TzJ8+Ccq5XlCUW2L48pZNCYuBDfBh7FxkB7qDgGDiaftEkZZfApRg2E+M9
+G8wkNKTPLDc4wH0FDTijhgxR3Y4PiS1HL2Zhw7bD3CbslmEGgfnnZojNkJtcLeBH
+BLa52/dSwNU4WWLubaYSiAmA9IUMX1/RpfpxOxd4Ykmhz97oFbUaDJFipIggx5sX
+ePAlkTdWnv+RWBxlJwMQ25oEHmRguNYf4Zr/Rxr9cS93Y+mdXIZaBEE0KS2iLRqa
+OiWBki9IMQU4phqPOBAaG7A+eP8PAgMBAAGjZjBkMB0GA1UdDgQWBBTh0YHlzlpf
+BKrS6badZrHF+qwshzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qwshzAS
+BgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF
+AAOCAgEAALIY1wkilt/urfEVM5vKzr6utOeDWCUczmWX/RX4ljpRdgF+5fAIS4vH
+tmXkqpSCOVeWUrJV9QvZn6L227ZwuE15cWi8DCDal3Ue90WgAJJZMfTshN4OI8cq
+W9E4EG9wglbEtMnObHlms8F3CHmrw3k6KmUkWGoa+/ENmcVl68u/cMRl1JbW2bM+
+/3A+SAg2c6iPDlehczKx2oa95QW0SkPPWGuNA/CE8CpyANIhu9XFrj3RQ3EqeRcS
+AQQod1RNuHpfETLU/A2gMmvn/w/sx7TB3W5BPs6rprOA37tutPq9u6FTZOcG1Oqj
+C/B7yTqgI7rbyvox7DEXoX7rIiEqyNNUguTk/u3SZ4VXE2kmxdmSh3TQvybfbnXV
+4JbCZVaqiZraqc7oZMnRoWrXRG3ztbnbes/9qhRGI7PqXqeKJBztxRTEVj8ONs1d
+WN5szTwaPIvhkhO3CO5ErU2rVdUr89wKpNXbBODFKRtgxUT70YpmJ46VVaqdAhOZ
+D9EUUn4YaeLaS8AjSF/h7UkjOibNc4qVDiPP+rkehFWM66PVnP1Msh93tc+taIfC
+EYVMxjh8zNbFuoc7fzvvrFILLe7ifvEIUqSVIC/AzplM/Jxw7buXFeGP1qVCBEHq
+391d/9RAfaZ12zkwFsl+IKwE/OZxW8AHa9i1p4GO0YSNuczzEm4=
+-----END CERTIFICATE-----

package/plugins/serviceGC/serviceGC.nlg

@@ -0,0 +1,31 @@
+/*
+ * serviceGC.nlg
+ * serviceGC
+ *
+ *     ServiceGC module configuration and management for NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * segcInit
+ *
+ *     Initialize the GC module.
+ *
+ */
+
+function segcInit(local manifest) {
+    manifest = `(segcInit)
+
+    Naan.module.build(module.id, "serviceGC", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("naanlib:frameworks/common").LiveImport()
+        if !js.g
+            throw("serviceGC: NodeJS required")
+        https = require("naanlib:frameworks/node/https_request.nlg")
+    })
+} ();

package/plugins/serviceYC/yc_speechrec.nlg

@@ -0,0 +1,138 @@
+/*
+ * yc_speechrec.nlg
+ * serviceYC
+ *
+ *     Access to Yandex Speechkit recognition.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2024 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * YCSpeechRec
+ *
+ * Speech recognition object.
+ *
+ */
+
+closure YCSpeechRec(options, local ycspr) {
+    ycspr = new(object, this)
+    ycspr.recogURL = "https://transcribe.api.cloud.yandex.net/speech/stt/v2/longRunningRecognize"
+    ycspr.checkURL = "https://operation.api.cloud.yandex.net/operations/"
+
+    // login
+    //
+    ycspr.login = closure login(creds) {
+        ycspr.creds = creds
+        list(false, { ok: true })
+    }
+
+    // startRecog
+    //
+    // Start a speech recognition task using audio from the specified YC Cloud Storage object.
+    //
+    // Available options are:
+    //  {
+    //      language:   <string>                // e.g. "ru-RU", "en-US", or false for auto-detect
+    //      deferred:   <boolean>               // up to 24 hours for 1/4 the price
+    //      debug:      <boolean>               // extra debug logging for https request
+    //  }
+    //
+    // This returns an Operation object containing the ID of the asynchronous recognition operation:
+    //  {
+    //      done:       <boolean>               // recognition complete
+    //      recogID:    <string>                // operationID to poll for completion
+    //      createdAt:  <date>                  // when operation started
+    //      createdBy:  <string>                // YC ID of user who started it
+    //      modifiedAt: <date>                  // last modified
+    //      // following items only when operation complete:
+    //      response: {
+    //          "@type":    <string>            // gRPC type
+    //          chunks:     <array>             // array of chunks
+    //      }
+    //  }
+    //
+    ycspr.startRecog = closure startRecog(audioURI, options, local error, data) {
+        params = {
+            config: {
+                specification: {
+                    languageCode: options.language || "auto"                // e.g. "ru-RU", "en-US"
+                    model: options.deferred && "deferred-general" || "general"
+                    audioEncoding: "MP3"
+                    // profanityFilter: false                               // default value
+                    // literature_text: false                               // default for language "auto"
+                    // sampleRateHertz: 48000                               // default value for LPCM format
+                    // audioChannelCount: ...                               // only for LPCM format
+                    // rawResults: false                                    // write numbers as digits
+                }
+            }
+            audio: {
+                uri: audioURI
+            }
+        }
+        `(error, data) = https.HttpsApiRequest(ycspr.recogURL, {
+            putdata: params
+            headers: [
+                list(`Authorization, "Api-Key ".concat(ycspr.creds.apiKeySecret))
+            ]
+            debug: options.debug || undefined })
+        if error
+            list(Error("YCSpeechRec.startRecog failed:", error))
+        else
+            list(false, {
+                done: data.done
+                recogID: data.id
+                createdAt: Date(data.createdAt)
+                createdBy: data.createdBy
+                modifiedAt: Date(data.modifiedAt)
+            })
+    }
+
+    // checkRecog
+    //
+    // Check if a recognition operation, identified by recogID, has completed. If called too often
+    // this can encounter an API rate limit, which is 2500/hour total as of 20241007.
+    //
+    ycspr.checkRecog = closure checkRecog(recogID, local error, data) {
+        `(error, data) = https.HttpsApiRequest(ycspr.checkURL.concat(recogID), {
+            headers: [
+                list(`Authorization, "Api-Key ".concat(ycspr.creds.apiKeySecret))
+            ]})
+        if error
+            list(Error("YCSpeechRec.checkRecog failed:", error))
+        else
+            list(false, {
+                done: data.done
+                recogID: data.id
+                createdAt: Date(data.createdAt)
+                createdBy: data.createdBy
+                modifiedAt: Date(data.modifiedAt)
+                response: data.response || undefined
+            })
+    }
+
+    // finis
+    ycspr
+}
+
+
+/*
+ * ycSpeechRecInit
+ *
+ *     Initialize the module.
+ *
+ */
+
+function ycSpeechRecInit(local manifest) {
+    manifest = `(YCSpeechRec, ycSpeechRecInit)
+
+    Naan.module.build(module.id, "yc_speechrec", function(modobj, compobj) {
+        require("./serviceYC.nlg")
+        compobj.manifest = manifest
+        modobj.exports.YCSpeechRec = YCSpeechRec
+    })
+
+} ();