@naanlang/naan 1.0.3 → 1.0.6

package/frameworks/node/https_request.nlg

@@ -0,0 +1,119 @@
+/*
+ * https_node.nlg
+ *
+ *     Https operations for NodeJS.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2021-2022 by Richard C. Zulch
+ *
+ */
+
+
+/*
+ * HttpsApiRequest
+ *
+ * Perform an API request from in NodeJS. This operates the same as HttpsApiRequest for browsers.
+ *
+ * Options:
+ *  {
+ *      method:         <string>                // HTTP method (defaults to GET/PUT)
+ *      query:          <dictionary>            // query variables added to the URL
+ *      putdata:        <data>                  // data to put
+ *      contentType:    <string>                // sets Content-Type header, otherwise auto-determined
+ *      range:          <string>                // sets range header
+ *      headers:        [`(key, value)]         // add header(s) to the request, overriding defaults
+ *  }
+ *
+ */
+
+closure HttpsApiRequest(url, options, local urlq, reqOptions, putdata, item, pending, chunks, request) {
+    if options.query
+        urlq = url.concat(EncodeQuery("?", options.query))
+    else
+        urlq = url
+    putdata = options.putdata
+    reqOptions = { }
+    if options.method
+        reqOptions.method = options.method
+    else if putdata
+        reqOptions.method = "POST"
+    else
+        reqOptions.method = "GET"
+    reqOptions.headers = { }
+    if putdata {
+        if options.contentType
+            reqOptions.headers['Content-Type'] = options.contentType
+        else if jsTypedArray(options.putdata)
+            reqOptions.headers['Content-Type'] = "application/octet-stream"
+        else if member(typeof(options.putdata), `(dictionary, array, xobject)) {
+            reqOptions.headers['Content-Type'] = "application/json"
+            `(error, putdata) = JsonStringify(options.putdata)
+            if error
+                return (list(Error("HttpsApiRequest encode:", url)))
+        }
+        fetchOptions.body = putdata
+    }
+    if array(options.headers)
+        for item in options.headers
+            reqOptions.headers[item.0] = item.1
+    pending = new(nonce)
+    chunks = []
+    request = nodeHttps.request(url, reqOptions, function (response) {
+        if response.statusCode < 200 || response.statusCode >= 300 {
+            pending.signal(list(Error("HttpsApiRequest statusCode:", url, response.statusCode, {
+                status: response.statusCode
+            })))
+            return
+        }
+        response.on("data", function (chunk) {
+            chunks.push(chunk)
+        })
+        response.on("end", function (local content, error) {
+            content = Buffer.concat(chunks)
+            if response.complete {
+                if response.headers["content-type"].startsWith("application/json") {
+                    `(error, content) = JsonParse(content.toString())
+                    if error
+                        error = Error("HttpsApiRequest decode:", url, error)
+                    else
+                        content = new(content)
+                    if array(content) {                                     // deencapsulate to get API result
+                        content = totuple(content)
+                        error = content.0
+                        content = content.1
+                    }
+                }
+            }
+            else
+                error = Error("HttpsApiRequest terminated prematurely:", url)
+            pending.signal(list(error, content))
+        })
+    })
+    request.on("error", function(error) {
+        error = Error("HttpsApiRequest failed:", url, error)
+        pending.signal(list(error))
+    })
+    if data
+        request.write(data)
+    request.end()
+    pending.wait()
+};
+
+
+/*
+ * https_nodeInit
+ *
+ * Initialize HTTPS request operations for NodeJS.
+ *
+ */
+
+function https_nodeInit(local manifest) {
+    manifest = `(HttpsGetJson, HttpsPutJson, https_nodeInit)
+
+    Naan.module.build(module.id, "https_request", function(modobj, compobj) {
+        require("./node.nlg")
+        compobj.manifest = manifest
+        modobj.exports.HttpsApiRequest = HttpsApiRequest
+    })
+}();

package/frameworks/node/node.nlg

@@ -75,6 +75,7 @@ function nodrInit(local manifest) {
             fs = js.r("fs")
             os = js.r("os")
             stream = js.r("stream")
+            nodeHttps = js.r("https")
         }()
         module.reload = nodrReload
     })

package/frameworks/node/psm_server.nlg

@@ -27,6 +27,7 @@ closure psmsFsView(rootpath, local view, gitter) {
     view = view.1
     
     // inherited
+    //  --  view.folderPath(folderid, callback)
     //  --  view.winDrives(callback)
     //  --  view.testHidden(path, callback)
     //  --  view.readLines(path, options, callback)
@@ -86,16 +87,27 @@ closure psmsFsView(rootpath, local view, gitter) {
             }
             res.send(new(array, result))
         }
-                 
+                                 
+        // op: folderPath
+        function folderPathApi() {
+            res.send(new(array, view.folderPath(req.query.folderid)))
+        }
+ 
         // op: readLines
         function readLinesApi() {
             res.send(new(array, view.readLines(path, req.query)))
         }
 
         // op: readFile
-        function readFileApi(local options) {
+        function readFileApi(local error, localpath, options) {
+            `(error, localpath) = view.localPath(path)
+            if error {
+                res.send(new(array, list(error)))
+                return
+            }
             options = {
-                dotfiles: "allow" }
+                dotfiles: "allow"
+            }
             if req.query.encoding == "binary"
                 options.headers = {
                     "Content-Type": "application/octet-stream"
@@ -104,7 +116,7 @@ closure psmsFsView(rootpath, local view, gitter) {
                 options.headers = {                                         // ### is this always the right default?
                     "Content-Type": "text/html; charset=UTF-8"
                 }
-            res.sendFile(path, options, function (err, local status) {
+            res.sendFile(localpath, options, function (err, local status) {
                 if err {
                     if err.code == "ENOENT"
                         status = 404
@@ -187,11 +199,12 @@ closure psmsFsView(rootpath, local view, gitter) {
         
         // op: tree
         function treeApi() {
-            res.send(new(array, view.tree(path, req.query.depthlimit)))
+            res.send(new(array, view.tree(path, toint(req.query.depthlimit))))
         }
                 
         // op: dirSearch
         function dirSearchApi() {
+            req.query.excludeDirs = JSONparse(req.query.excludeDirs)
             res.send(new(array, view.dirSearch(path, req.query)))
         }
 
@@ -227,6 +240,7 @@ closure psmsFsView(rootpath, local view, gitter) {
         
         // op: grepLines
         function grepLinesApi() {
+            req.query.excludeDirs = JSONparse(req.query.excludeDirs)
             res.send(new(array, view.grepLines(path, req.query)))
         }
         
@@ -250,6 +264,7 @@ closure psmsFsView(rootpath, local view, gitter) {
 
         proc = {
             pathOp:         pathOpApi
+            folderPath:     folderPathApi
             readFile:       readFileApi
             readLines:      readLinesApi
             writeFile:      writeFileApi
@@ -276,8 +291,15 @@ closure psmsFsView(rootpath, local view, gitter) {
             gitLog:         gitLogApi
             gitShowFile:    gitShowFileApi
         }[req.query.op]
-        if proc
-            proc()
+        if proc {
+            try {
+                proc()
+            } catch {
+                if true {
+                    res.status(400).send("psm op failed: ".concat(req.query.op, ": ", exception))
+                }
+            }
+        }
         else
             res.status(400).send("invalid psm op: ".concat(req.query.op))
     }

package/frameworks/project/build.nlg

@@ -1024,13 +1024,14 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     //      } ... ]
     //  }
     //
-    build.make = closure make(debugMake, local sink, xchan, entry, outext, error) {
+    build.make = closure make(options, local sink, xchan, entry, outext, error) {
         sink = new(textstream, "", `string)                                 // log all output
         xchan = textstreams(list(sink, sink))
         try {
             resetlist()
-            analyze(debugMake)
-            if debugMake {
+            analyze(options.debugMake)
+            if options.debugMake {
+                printline("Debugging make for ", options.buildnum, "-", options.stage)
                 if build.errorlist.length == 0 {
                     printline("build operations (", build.filelist.length, ") for ", operation)
                     for entry in build.filelist
@@ -1052,6 +1053,8 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
             "\ndate: ", Date(),
             "\nfrom: ", JSpath.join(fs.rootpath, build.srcpath),
             "\n  to: ", build.destpath,
+            "\ntype: ", options.stage,
+            "\nvers: ", options.buildnum,
             "\n", sink.taketext())
         `(error) = fs.writeFile(JSpath.join(build.destpath, "build_log.txt"), outext)
         if error
@@ -1153,14 +1156,17 @@ closure Builder(rules, operation, fs, srcpath, destpath, local build, group, new
     //
     // build.refresh
     //
-    // Refresh after updating the v-site. This should probably update all types of executions. ###
+    // Refresh after updating the v-site, returning true iff an existing vsite was found.
     //
-    build.refresh = function refresh(track local main, url_origin) {
+    build.refresh = function refresh(track local main, url_origin, found) {
         if build.runway."v-site" {
             url_origin = js.w.location.origin.concat("/run", build.runway."v-site")
-            for target in track.enumlist()
-                target.vsiteRefresh(url_origin)
+            for target in track.enumlist() {
+                if target.vsiteRefresh(url_origin)
+                    found = true
+            }
         }
+        found
     }
 
     //

package/frameworks/project/proj_cliser.nlg

@@ -27,7 +27,7 @@ closure proj_cliserInstance(projman, where, projID, local project) {
     project.name = where.name
     project.where = where
     project.projID = projID
-    project.config = projConfig(projman, where)
+    project.config = projConfig(projman, where, projID)
     
     // create
     //
@@ -63,8 +63,16 @@ closure proj_cliserInstance(projman, where, projID, local project) {
     //
     //     Run the project by building and then launching.
     //
-    project.run = closure run(buildonly) {
-        project.config.run(buildonly)
+    project.run = closure run(stage, buildonly) {
+        project.config.run(stage, buildonly)
+    }
+
+    // publish
+    //
+    //     Publish the project according to the stage.
+    //
+    project.publish = closure publish(stage) {
+        project.config.publish(stage)
     }
 
     // close()

package/frameworks/project/proj_console.nlg

@@ -27,7 +27,7 @@ closure proj_consoleInstance(projman, where, projID, local project) {
     project.name = where.name
     project.where = where
     project.projID = projID
-    project.config = projConfig(projman, where)
+    project.config = projConfig(projman, where, projID)
     project.runPageDebug = true                                             // switch to Debug page on run
 
     // create
@@ -60,9 +60,18 @@ closure proj_consoleInstance(projman, where, projID, local project) {
 
     // run
     //
-    // Run the project by building and then launching.
-    project.run = closure run(buildonly) {
-        project.config.run(buildonly)
+    //     Run the project by building and then launching.
+    //
+    project.run = closure run(stage, buildonly) {
+        project.config.run(stage, buildonly)
+    }
+
+    // publish
+    //
+    //     Publish the project according to the stage.
+    //
+    project.publish = closure publish(stage) {
+        project.config.publish(stage)
     }
 
     // close

package/frameworks/project/proj_folder.nlg

@@ -61,11 +61,20 @@ closure proj_folderInstance(projman, where, projID, local project) {
     // run
     //
     // Run the project by building and then launching.
-    project.run = closure run(buildonly) {
+    project.run = closure run(stage, buildonly) {
         debuglog("folder run project stub")
         list(false, { ok: true })
     }
 
+    // publish
+    //
+    //     Publish the project according to the stage.
+    //
+    project.publish = closure publish(stage) {
+        debuglog("folder publish project stub")
+        list(false, { ok: true })
+    }
+
     // close
     //
     // Close the project, releasing resources.

package/frameworks/project/proj_lambda.nlg

@@ -27,7 +27,7 @@ closure proj_lambdaInstance(projman, where, projID, local project) {
     project.name = where.name
     project.where = where
     project.projID = projID
-    project.config = projConfig(projman, where)
+    project.config = projConfig(projman, where, projID)
     project.runPageDebug = true                                             // switch to Debug page on run
 
     // create
@@ -60,9 +60,18 @@ closure proj_lambdaInstance(projman, where, projID, local project) {
 
     // run
     //
-    // Run the project by building and then launching.
-    project.run = closure run(buildonly) {
-        project.config.run(buildonly)
+    //     Run the project by building and then launching.
+    //
+    project.run = closure run(stage, buildonly) {
+        project.config.run(stage, buildonly)
+    }
+
+    // publish
+    //
+    //     Publish the project according to the stage.
+    //
+    project.publish = closure publish(stage) {
+        project.config.publish(stage)
     }
 
     // close

package/frameworks/project/proj_static.nlg

@@ -26,7 +26,7 @@ closure proj_staticInstance(projman, where, projID, local project) {
     project.name = where.name
     project.where = where
     project.projID = projID
-    project.config = projConfig(projman, where)
+    project.config = projConfig(projman, where, projID)
     
     // create
     //
@@ -62,8 +62,16 @@ closure proj_staticInstance(projman, where, projID, local project) {
     //
     //     Run the project by building and then launching.
     //
-    project.run = closure run(buildonly) {
-        project.config.run(buildonly)
+    project.run = closure run(stage, buildonly) {
+        project.config.run(stage, buildonly)
+    }
+
+    // publish
+    //
+    //     Publish the project according to the stage.
+    //
+    project.publish = closure publish(stage) {
+        project.config.publish(stage)
     }
 
     // close()

package/frameworks/project/projects.nlg

@@ -23,6 +23,7 @@ closure ProjectManager(locate, track, dbname, local error, proj) {
     `(error, proj) = storageResources.ResourceManager(dbname, "Projects/")
     if error
         return(list(error))
+    proj.configLocal = proj.vault("configlocal")                            // local config storage
     proj.locate = locate                                                    // storage locations
     proj.track = track                                                      // execution tracking
     proj.types = []
@@ -226,7 +227,9 @@ closure projConnector(projman, projtype, local connector, watch) {
     // deleteResource
     //
     // Delete information for a project. The return value is (error, result) tuple.
-    connector.deleteResource = function deleteResource(resID) {
+    connector.deleteResource = function deleteResource(resID, local localID) {
+        localID = resID.concat("-data")
+        projman.configLocal.deleteResource(localID)                         // remove local storage if there is any
         connector.vault.deleteResource(resID)
         watch.notify(projtype.classID, { deleted: [resID] })
         list(false, { ok: true})
@@ -293,7 +296,7 @@ closure projConnector(projman, projtype, local connector, watch) {
  *
  * Nide.proj/
  *      nide_cliser.cfg                 - JSON definition of the project (see below)
- *      NaanIDE_version.txt             - [default] substitution file defining 1.0.3-1 etc.
+ *      NaanIDE_version.txt             - [default] substitution file defining 1.0.6-1 etc.
  *      NaanIDE_version_builds.txt      - [default] contains current build number as decimal string
  *      build/                          - [optional] default build output location
  *      NaanIDE.lic                     - [optional] defines Zulch Laboratories, Inc. and other license info
@@ -306,6 +309,10 @@ closure projConnector(projman, projtype, local connector, watch) {
  *      build:      <build instructions dictionary>
  *  }
  *
+ *     A project configuration also stores local data that is not in any file, which is useful for
+ * items like accounts & locations, or credentials that should not be "on disk". Project data from
+ * the .cfg file is available in config.configDict, while local data is in config.localDict.
+ *
  *     Given a new projConfig instance, you can either load (which fails of the project is not valid)
  * or create (which fails if the project already exists.) The load operation will update the where
  * argument with the actual name and type found. Unless other noted the methods all return standard
@@ -316,9 +323,12 @@ closure projConnector(projman, projtype, local connector, watch) {
 proj_configDataFolder = "Nide.proj/";
 proj_cliserConfigFile = "nide_cliser.cfg";
 
-closure projConfig(projman, where, local procon, fs, configpath) {
+closure projConfig(projman, where, projID, local procon, fs, configpath) {
     procon = new(object, this)
     procon.name = where.name
+    procon.dirName = proj_configDataFolder.slice(0,-1)                      // remove terminal "/"
+    if projID
+        procon.localID = projID.concat("-data")
     
     // validate
     //
@@ -376,6 +386,15 @@ closure projConfig(projman, where, local procon, fs, configpath) {
                 return (list(Error("can't parse project configuration", where.path, exception))) }
         procon.where_name = where.name                                      // copy name to config
         procon.where_typeID = where.typeID                                  // copy typeID to config
+        if procon.localID {
+            `(error, data) = projman.configLocal.accessResource(procon.localID)
+            if data
+                procon.localDict = data
+            else {
+                procon.localDict = { }
+                projman.configLocal.addResource(procon.localID, procon.localDict)
+            }
+        }
         list(false, procon.configDict)
     }
     
@@ -384,6 +403,11 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     // Write procon.configDict to our configuration file, returning a result tuple.
     //
     closure writeConfig(local error, info, data) {
+        if procon.localID {
+            `(error, data) = projman.configLocal.updateResource(procon.localID, procon.localDict)
+            if error
+                debuglog("projConfig.writeConfig cannot write local storage:", ErrorString(error))
+        }
         `(error, data) = fs.writeFile(configpath, JSONstringify(procon.configDict))
         if !error {
             procon.configMD5 = data
@@ -402,6 +426,8 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     // not already be loaded. The provided name overrides any name in the where argument above, but
     // path and locationID must be specified. Returns a standard result tuple.
     //
+    // ### we need to update this for optional multiple-configuraiton builds
+    //
     procon.create = closure create(name, local error, data, configpath) {
         if procon.loaded
             return (list(Error("Project.create: project already loaded")))
@@ -457,7 +483,9 @@ closure projConfig(projman, where, local procon, fs, configpath) {
                 error = Error("project configuration not a directory", where.path)
             else
                 `(error, data) = readConfig() }
-        if error
+        if error.code == "ENOENT"
+            return (list(Error("Project <b>".concat(where.name, "</b>:<br>not found at ", where.path))))
+        else if error
             return (list(Error("Project.load failed", error)))
         procon.loaded = true
         list(false, procon.configDict)
@@ -496,15 +524,63 @@ closure projConfig(projman, where, local procon, fs, configpath) {
             procon.configDict = newconfig 
         `(error, data) = writeConfig()
     }
-    
+        
+    // stages
+    //
+    //     Return a tuple of stages in the project configuration.
+    //
+    procon.stages = function stages() {
+        procon.configDict."build-rules".*
+    }
+
+    // currentStage
+    //
+    //     Validate and return the current stage of the project, or the first one if there isn't a
+    // previous selection, or false if no valid stages exist.
+    //
+    procon.currentStage = function currentStage(local stage, stagelist) {
+        stage = procon.localDict.currentStage
+        stagelist = stages()
+        if stage && !member(stage, stagelist)
+            stage = false
+        if stagelist && !stage
+            stage = stagelist.0
+        stage
+    }
+
+    // setCurrentStage
+    //
+    //     Set the current stage.
+    //
+    procon.setCurrentStage = function setCurrentStage(stage) {
+        if member(stage, stages()) {
+            procon.localDict.currentStage = stage
+            update()                                                        // ### errors here are never reported
+        }
+    }
+
     // buildrules
     //
-    //     Get the current build rules.
+    //     Get the build rules for the named stage.
     //
-    function buildrules(local rules) {
-        rules = procon.configDict."build-rules".default                     // "default" will be switchable later
+    function buildrules(stage, local rules, common, rule, key) {
+        rules = procon.configDict."build-rules"[stage]
         if !rules
             rules = procon.configDict.build                                 // fallback to old config structure
+        rules = new(rules)
+        common = procon.configDict."build-common"
+        if common {
+            for rule in common
+                if !rules[rule]
+                    rules[rule] = common[rule]                              // common rule was not overridden
+                else if array(common[rule], rules[rule])
+                    rules[rule] = common[rule].concat(rules[rule])          // merge common array with rule
+                else if dictionary(common[rule], rules[rule]) {
+                    for key in common[rule]                                 // merge common dictionary with rule
+                        if !rules[rule][key]
+                            rules[rule][key] = common[rule][key]
+                }
+        }
         rules
     }
 
@@ -512,16 +588,18 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     //
     //     Run the project by building and then launching.
     //
-    procon.run = closure run(buildonly, local rules, builder, buildpath, error, buildnum) {
+    procon.run = closure run(stage, buildonly, local rules, builder, buildpath, error, buildnum) {
         if !procon.loaded
             return (list(Error("project not loaded")))
-        rules = buildrules()
+        rules = buildrules(stage)
         if !rules || !rules.products
             return (list(Error("build rules missing or incomplete")))
         buildpath = fs.path.join(proj_configDataFolder, rules.products)
         rules.name = procon.where_name
         rules.typeID = procon.where_typeID
-        `(error, builder) = Builder(rules, "debug", fs, "", buildpath)
+        if !rules.operation
+            rules.operation = "debug"
+        `(error, builder) = Builder(rules, rules.operation, fs, "", buildpath)
         if error
             return (list(error))
         `(error, buildnum) = builder.increment()
@@ -529,11 +607,14 @@ closure projConfig(projman, where, local procon, fs, configpath) {
             ErrorDebuglog("build number increment failed", error)
             return (list(error))
         }
-        `(error) = builder.make()
+        `(error) = builder.make({
+            stage: stage
+            buildnum: buildnum
+        })
         if !error {
-            if buildonly
-                builder.refresh(projman.track)
-            else
+            if builder.refresh(projman.track)
+                buildonly = !buildonly                                      // reverse if refresh
+            if !buildonly
                 `(error) = builder.run(projman.track)
         }
         builder.destroy()
@@ -547,12 +628,13 @@ closure projConfig(projman, where, local procon, fs, configpath) {
     //
     //     Publish the project to its configured location.
     //
-    procon.publish = closure publish(local rules, publish, error, outfs, inpath) {
-        rules = buildrules()
-        publish = procon.configDict.publish
-        if !rules || !publish.where
+    procon.publish = closure publish(stage, local rules, where, publish, error, outfs, inpath) {
+        rules = buildrules(stage)
+        publish = rules.publish
+        where = procon.localDict.publish[stage].where
+        if !rules || !where
             return (list(Error("no project publish location configured")))
-        `(error, outfs) = projman.locate.connect(publish.where.locationID, "NideFS", publish.where.path)
+        `(error, outfs) = projman.locate.connect(where.locationID, "NideFS", where.path)
         if error
             return (list(Error("can't access publishing filesystem", error)))
         inpath = fs.path.join(proj_configDataFolder, rules.products, publish.input)